From ec4336bcc445db4d0e9721e363615141a4f24a7f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 15:11:41 +0800 Subject: [PATCH 0001/2585] - support for name, port & shutdown parameters for the different types of ECE instances (fai_{presentation,editor,search}_{name,port,shutdown} - next steps string is now accumulated throughout the lifespan of one ece-install run. - added generic method for updating/setting a setting in a .conf/.properties file - made set_ece_instance_conf use this new method --- usr/sbin/ece-install | 135 +++++++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 49 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 93279362..275aad33 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -23,7 +23,7 @@ java_home=/usr/lib/jvm/java-6-sun id="[$(basename $0)]" pid_file=/var/run/$(basename $0).pid download_dir=/tmp/ece-downloads -log=/tmp/$(basename $0).log +log=/var/log/$(basename $0).log conf_file=$HOME/.ece-install.conf common_nursery_dir=/etc/escenic/engine/common @@ -475,6 +475,24 @@ org.apache.catalina.core.ContainerBase.[Catalina].[localhost].handlers = 2localh EOF } +# Parameters: +# $1 is the property +# $2 is the value +# $3 is the file +function set_conf_file_value() +{ + if [ -r $3 ]; then + if [ $(grep $1 $3 | wc -l) -gt 0 ]; then + # TODO update existing values of the property instead of + # removing & appending it. + cat $3 | grep -v ^$1 > $3.tmp + mv $3.tmp $3 + fi + fi + + echo "$1=$2" >> $3 +} + # The function accepts the following parameters: # $1 is the property # $2 is the value @@ -484,17 +502,7 @@ EOF function set_ece_instance_conf() { instance_conf_file=/etc/escenic/ece-$instance_name.conf - - if [ -r $instance_conf_file ]; then - if [ $(grep $1 $instance_conf_file | wc -l) -gt 0 ]; then - # TODO update existing values of the property instead of - # removing & appending it. - cat $instance_conf_file | grep -v ^$1 > $instance_conf_file.tmp - mv $instance_conf_file.tmp $instance_conf_file - fi - fi - - echo "$1=$2" >> $instance_conf_file + set_conf_file_value $1 $2 $instance_conf_file } # Installs third party packages needed by the ECE (i.e. Java related). @@ -543,22 +551,31 @@ function set_up_app_server print "Or enter: , e.g.: '8180 8105'" echo -n "Your choice [8080 8005]> " read user_ports + + if [ -z "$user_ports" ]; then + appserver_port=$(echo $user_ports | cut -d' ' -f1) + shutdown_port=$(echo $user_ports | cut -d' ' -f2) + fi else - user_ports=$(get_conf_value fai_editor_port) - if [ -n "${user_ports}" ]; then - user_ports=$user_ports" "$(get_conf_value fai_editor_shutdown) + if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + appserver_port=$(get_conf_value fai_editor_port) + shutdown_port=$(get_conf_value fai_editor_shutdown) + elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + appserver_port=$(get_conf_value fai_presentation_port) + shutdown_port=$(get_conf_value fai_presentation_shutdown) + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + appserver_port=$(get_conf_value fai_search_port) + shutdown_port=$(get_conf_value fai_search_shutdown) fi fi - if [ -z "$user_ports" ]; then + if [ -z "$appserver_port" ]; then appserver_port=8080 - shutdown_port=8005 - else - appserver_port=$(echo $user_ports | cut -d' ' -f1) - shutdown_port=$(echo $user_ports | cut -d' ' -f2) + elif [ -z "$appserver_shutdown" ]; then + appserver_shutdown=8005 fi - debug "user_ports=[${user_ports}] appserver_port=$appserver_port shutdown_port=$shutdown_port" + debug "appserver_port=$appserver_port shutdown_port=$shutdown_port" if [ $fai_enabled -eq 0 ]; then print "Another question: Where does the database run?" @@ -822,29 +839,12 @@ function print_status_and_next_steps() print "The basic setup of $instance_name is now completed!" print "It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s - if [ $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then - echo "The cache server up and running at http://${HOSTNAME}:80/" - fi + print "You're next steps on ${HOSTNAME} will be:" + echo $next_steps - if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER -o \ - $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - print "Your app server is running on"\ "http://${HOSTNAME}:$appserver_port" - print "" - versions="ece -i $instance_name -t $type versions" - print "You can now switch to the $ece_user and type:" - print " $versions" - print "to see the Escenic software I have installed for you." - - print "" - print "You're next steps on ${HOSTNAME} will be:" - print "- configure the instances you want to be started at boot time" - print " in /etc/default/ece" - print "- include $instance_name in your RMI hub configuration." - print "" - print "Enjoy your time with Escenic Content Engine!" - print "" - print "-Vizrt Online" - fi + print "Enjoy your time with Escenic Content Engine!" + print "" + print "-Vizrt Online" } # Based on Erik Mogensen's work: @@ -1167,7 +1167,7 @@ function read_user_input() { echo "Hi, which server profile do you wish to install?" echo "" - echo "Select 1-7 and press ENTER" + echo "Select 1-9 and press ENTER" echo "" echo " $PROFILE_ALL_IN_ONE - All in one, the full stack on one host. " echo " Suitable for development and test environments." @@ -1215,6 +1215,11 @@ function assert_correct_runtime_environment() else echo $BASHPID > $pid_file fi + + if [ ! -e "$conf_file" ]; then + print $conf_file "doesn't exist, I cannot live without it :-(" + exit 1 + fi } function common_pre_install() @@ -1305,6 +1310,15 @@ function install_cache_server() fi set_up_varnish $backend_servers + + add_next_step "The cache server up and running at http://${HOSTNAME}:80/" +} + +# Parameters: +# $1 : your added line +function add_next_step() +{ + next_steps=${next_steps}${NEW_LINE}${1} } function install_database_server() @@ -1368,6 +1382,14 @@ function ask_for_instance_name() print "Press ENTER to accept the default instance name, $1." echo -n "Your choice [$1]> " read instance_name + else + if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + instance_name=$(get_conf_value fai_editor_name) + elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + instance_name=$(get_conf_value fai_presentation_name) + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + instance_name=$(get_conf_value fai_search_name) + fi fi if [ -z "$instance_name" ]; then @@ -1405,6 +1427,11 @@ EOF set_correct_permissions assemble_deploy_and_restart_type + + add_next_step "New ECE instance $instance_name installed." + add_next_step "It's running at http://$HOSTNAME:$appserver_port" + add_next_step "You can view all installed plugin & engine versions with:" + add_next_step "ece -i $instance_name versions" } function install_presentation_server() @@ -1502,15 +1529,26 @@ function install_rmi_hub() file=$file/neo/io/managers/HubConnectionManager.properties make_dir $(basename $file) - echo "hub=rmi://${hub_host}:1099/hub/Hub" > $file + set_conf_file_value hub \ + "rmi://${hub_host}:1099/hub/Hub" \ + $file cat > common/io/api/EventManager.properties <>$log 2>>$log } # reads the value of the desired setting from $conf_file @@ -1521,8 +1559,7 @@ function get_conf_value() { if [ ! -e "$conf_file" ]; then print $conf_file "doesn't exist." - echo "" - return + exit 1 fi if [ $(grep $1 $conf_file | grep -v ^# | wc -l) -gt 0 ]; then From 1df7bf3a33752445db66476ce08770d7f92456d3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 16:29:53 +0800 Subject: [PATCH 0002/2585] - made exit_on_error better by accepting more than one argument as the error string. --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index 24575a55..4fade9de 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -71,7 +71,7 @@ java_home function exit_on_error() { if [ $? -eq 1 ]; then - print $1 "FAILED, exiting :-(" + print $@ "FAILED, exiting :-(" exit 1 fi } From 049850a52dd8f296dab343f159cae23d6dd4d35a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 16:39:57 +0800 Subject: [PATCH 0003/2585] - made drop-and-create-ecedb safer by putting everything into functions. - it's now sourced by ece-install and create_ecedb is called explicitly --- usr/sbin/drop-and-create-ecedb | 151 ++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 59 deletions(-) diff --git a/usr/sbin/drop-and-create-ecedb b/usr/sbin/drop-and-create-ecedb index de52738c..7f6b1414 100644 --- a/usr/sbin/drop-and-create-ecedb +++ b/usr/sbin/drop-and-create-ecedb @@ -14,12 +14,12 @@ # -torstein@escenic.com drop_db_first=0 -user=ece5user -password=ece5password -db=ece5db -host=localhost +db_user=ece5user +db_password=ece5password +db_schema=ece5db +db_host=localhost ece_home=/opt/escenic/engine -dbproduct=mysql +db_product=mysql id="[`basename $0`]" # oracle specific settings @@ -28,17 +28,35 @@ tablespace_data=ece5_data tablespace_index=ece5_index oracle_data_dir=/home/oracle/app/oracle/oradata/orcl +debug=0 + +function debug() +{ + if [ $debug -eq 1 ]; then + echo $id $1 + fi + +} + +function exit_on_error() +{ + if [ $? -eq 1 ]; then + echo $id $@ "FAILED, exiting :-(" + exit 1 + fi +} + function create_oracle_ece_user() { sqlplus /nolog << EOF connect /as sysdba; - create user $user - identified by $password + create user $db_user + identified by $db_password default tablespace $tablespace_data quota unlimited on $tablespace_data; - grant connect to $user; - grant resource to $user; - grant create any view to $user; - grant execute on ctx_ddl to $user; + grant connect to $db_user; + grant resource to $db_user; + grant create any view to $db_user; + grant execute on ctx_ddl to $db_user; EOF } @@ -46,62 +64,68 @@ function run_db_scripts() { for el in $db_fn_list; do file=$1/$el.sql - echo $id "running $file ..." + debug "running $file ..." if [ -e $1/$el.sql ]; then - if [ $dbproduct = "oracle" ]; then - sqlplus $user/$password @$file + if [ $db_product = "oracle" ]; then + sqlplus $db_user/$db_password @$file else - mysql -u $user -p$password -h $host $db < $file + mysql -u $db_user -p$db_password -h $db_host $db_schema < $file fi fi + exit_on_error "running $el" done } -if [ $create_oracle_user -eq 1 ]; then - create_oracle_ece_user -fi +function pre_install_new_ecedb() +{ + if [ $create_oracle_user -eq 1 ]; then + create_oracle_ece_user + fi -if [ $drop_db_first -eq 1 ]; then - echo $id "dropping and re-creating $db on $host ..." - if [ $dbproduct = "mysql" ]; then - mysql -h $host << EOF - drop database $db; + if [ $drop_db_first -eq 1 ]; then + debug "dropping and re-creating $db_schema on $db_host ..." + if [ $db_product = "mysql" ]; then + mysql -h $db_host << EOF +drop database $db_schema; EOF - else - sqlplus /nolog << EOF - connect /as sysdba; - drop tablespace $tablespace_data including contents; - drop tablespace $tablespace_index including contents; + else + sqlplus /nolog << EOF +connect /as sysdba; +drop tablespace $tablespace_data including contents; +drop tablespace $tablespace_index including contents; EOF + fi fi -fi - -# we first create the DB (or, if drop_db_first is 1, we've just -# dropped it above) before running the SQL scripts. -if [ $dbproduct = "mysql" ]; then - mysql -h $host << EOF - create database $db character set utf8 collate utf8_general_ci; - grant all on $db.* to $user@'%' identified by '$password'; - grant all on $db.* to $user@'localhost' identified by '$password'; -EOF -else - sqlplus /nolog << EOF - connect /as sysdba; - - create tablespace $tablespace_data - datafile '$oracle_data_dir/${tablespace_data}01.dbf' - size 200M reuse - autoextend on next 50M maxsize 2047M - extent management local autoallocate; - - create tablespace $tablespace_index - datafile '$oracle_data_dir/${tablespace_index}01.dbf' - size 100M reuse - autoextend on next 50M maxsize 2047M - extent management local autoallocate; +} + +function create_schema() +{ + # we first create the DB (or, if drop_db_first is 1, we've just + # dropped it above) before running the SQL scripts. + if [ $db_product = "mysql" ]; then + mysql -h $db_host << EOF +create database $db_schema character set utf8 collate utf8_general_ci; +grant all on $db_schema.* to $db_user@'%' identified by '$db_password'; +grant all on $db_schema.* to $db_user@'localhost' identified by '$db_password'; EOF -fi + else + sqlplus /nolog << EOF +connect /as sysdba; + +create tablespace $tablespace_data +datafile '$oracle_data_dir/${tablespace_data}01.dbf' +size 200M reuse +autoextend on next 50M maxsize 2047M +extent management local autoallocate; +create tablespace $tablespace_index +datafile '$oracle_data_dir/${tablespace_index}01.dbf' +size 100M reuse +autoextend on next 50M maxsize 2047M +extent management local autoallocate; +EOF + fi +} db_fn_list=" tables @@ -114,10 +138,19 @@ indexes history " -run_db_scripts $ece_home/database/$dbproduct +function create_ecedb() +{ + create_schema + run_db_scripts $ece_home/database/$db_product + + if [ -e $ece_home/plugins ]; then + for el in `find -L $ece_home/plugins -name $db_product`; do + run_db_scripts $el + done + fi -for el in `find -L $ece_home/plugins -name $dbproduct`; do - run_db_scripts $el -done + echo "${id} ${db_product}://${db_host}/${db_schema} is now ready for ${db_user}/${db_password}" +} -echo "${id} ${dbproduct}://${host}/${db} is now ready for ${user}/${password}" +# pre_install_new_ecedb +# create_ecedb From 51d5272ad388d5dfebb7db64be70369aa616df8b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 17:12:11 +0800 Subject: [PATCH 0004/2585] - final status is now written on debug --- usr/sbin/drop-and-create-ecedb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/drop-and-create-ecedb b/usr/sbin/drop-and-create-ecedb index 7f6b1414..f10a5d14 100644 --- a/usr/sbin/drop-and-create-ecedb +++ b/usr/sbin/drop-and-create-ecedb @@ -149,7 +149,7 @@ function create_ecedb() done fi - echo "${id} ${db_product}://${db_host}/${db_schema} is now ready for ${db_user}/${db_password}" + debug "${id} ${db_product}://${db_host}/${db_schema} is now ready for ${db_user}/${db_password}" } # pre_install_new_ecedb From a794a9b42816bf7e97e2e5e3ceae8d18b50bdf7a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 17:45:25 +0800 Subject: [PATCH 0005/2585] - added exit_on_fail - more user/installation reports - get_asterixes (to be used later for (not) displaying the password) --- usr/sbin/ece-install | 72 +++++++++++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 18 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 275aad33..31f0d69b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -55,6 +55,14 @@ function print() echo $id $@ } +function exit_on_error() +{ + if [ $? -eq 1 ]; then + echo $id $@ "FAILED, exiting :-(" + exit 1 + fi +} + technet_download_list=" http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip @@ -357,11 +365,32 @@ function set_up_ecedb() ln -s $directory fi done + + source /usr/sbin/drop-and-create-ecedb + # TODO in future, it'll be possible to override db_user, + # db_password and db_schema here by reading these via + # get_conf_value + create_ecedb + exit_on_error "create_ecedb" - bash /usr/sbin/drop-and-create-ecedb \ - 1>>$log 2>>$log cd ~/ rm -rf /opt/escenic/engine/plugins + + add_next_step "- DB is now set up on $HOSTNAME:3306" +} + +# returns a string of asterixes with the same lenght as the inputted +# string. +function get_asterixes() +{ + s="" + j=$(echo $1 | wc -c) + + # wc -c always returns the length + 1, hence we start on index=1 + for (( i=1; i<$j; i++ )); do + s=${s}"-" + done + echo "$s" } function set_up_basic_nursery_configuration() @@ -836,12 +865,11 @@ function print_status_and_next_steps() minutes=$(( seconds_left / 60 )) seconds_left=$(( seconds_left - $minutes * 60 )) - print "The basic setup of $instance_name is now completed!" - print "It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s + print "The installation is now completed!" + print "- It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s + echo "$next_steps" - print "You're next steps on ${HOSTNAME} will be:" - echo $next_steps - + print "" print "Enjoy your time with Escenic Content Engine!" print "" print "-Vizrt Online" @@ -1311,14 +1339,18 @@ function install_cache_server() set_up_varnish $backend_servers - add_next_step "The cache server up and running at http://${HOSTNAME}:80/" + add_next_step "- Cache server is up and running at http://${HOSTNAME}:80/" } # Parameters: # $1 : your added line function add_next_step() { - next_steps=${next_steps}${NEW_LINE}${1} + if [ -n "$next_steps" ]; then + next_steps=${next_steps}${NEW_LINE}${id}" "${1} + else + next_steps=${id}" "${1} + fi } function install_database_server() @@ -1427,11 +1459,12 @@ EOF set_correct_permissions assemble_deploy_and_restart_type - - add_next_step "New ECE instance $instance_name installed." - add_next_step "It's running at http://$HOSTNAME:$appserver_port" - add_next_step "You can view all installed plugin & engine versions with:" - add_next_step "ece -i $instance_name versions" + + admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin + add_next_step "- New ECE instance $instance_name installed." + add_next_step "- Its admin interface is available at $admin_uri" + add_next_step "- You can view all installed plugin & engine versions with:" + add_next_step " ece -i $instance_name versions" } function install_presentation_server() @@ -1504,6 +1537,8 @@ function create_publication() # above. create_publication_in_db $publication_war assemble_deploy_and_restart_type + + add_next_step "- a new publication $publication_name has been created" } function install_editorial_server() @@ -1538,12 +1573,10 @@ clientConfiguration=/neo/io/services/HubConnection pingTime=10000 EOF - next_step="- Restart all your instances to make the hub seem them" - next_steps=${next_steps}${NEW_LINE}${next_step} + add_next_step "- Restart all your instances to make the hub seem them:" for el in /etc/escenic/engine/instance/*; do instance_name=$(basename $el) - next_step=" * $ ece -i ${instance_name} restart" - next_steps=${next_steps}${NEW_LINE}${next_step} + add_next_step " - $ ece -i ${instance_name} restart" done print "Starting the RMI-hub on $HOSTNAME ..." @@ -1665,6 +1698,9 @@ EOF fi cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ + + add_next_step "- Widget Framework has been installed into your local" + add_next_step " Maven repository and its ECE components have been installed" } function install_search_instance() From 57b5f44f3492a1f1b15f4a4d61ac079c1d13dda7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 17:46:23 +0800 Subject: [PATCH 0006/2585] - changed enable_rmi_hub to use 0/1 as the other switches. --- etc/default/ece | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etc/default/ece b/etc/default/ece index f7b2368b..a6983116 100644 --- a/etc/default/ece +++ b/etc/default/ece @@ -4,13 +4,13 @@ # instances. If you only have one # instance, use "default" as its # name. If you don't have any instances of that type on this host, # then just set the variable to an empty string. -engine_instance_list="app1 app2" +engine_instance_list="editor1 web1" search_instance_list="search1" -analysis_instance_list="analysis1" +analysis_instance_list="" # You only need one hub in an ECE cluster, so only set this one to -# true on one host. -enable_rmi_hub=true +# true on one host. 1 means on, 0 means off. +enable_rmi_hub=0 # The location of the ece script, default is /usr/bin/ece ece_script=/usr/bin/ece From 88c85d08005e1bed827712a75d621670ffb9ceb1 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 17:46:31 +0800 Subject: [PATCH 0007/2585] - changed enable_rmi_hub to use 0/1 as the other switches. --- etc/init.d/ece | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/etc/init.d/ece b/etc/init.d/ece index dc2a515c..1ee0751e 100755 --- a/etc/init.d/ece +++ b/etc/init.d/ece @@ -37,12 +37,14 @@ dir_list=" ece_script=/usr/bin/ece -# the values above may be overidden a file named the same as this -# init.d script, located in: +# The values above may be overidden a file named the same as this +# init.d script. This init.d configuration must also hold the +# variables controlling which ECE instances to start. The list of +# locations per default caters (at least) for Debian & Gentoo based +# systems: conf_file_location_list=" /etc/default /etc/conf.d -/tmp " function read_conf_file() @@ -85,7 +87,7 @@ function ensure_ece_script_is_ok() function execute_command() { - if [ $enable_rmi_hub = "true" ]; then + if [ $enable_rmi_hub -eq 1 ]; then su - $ece_unix_user -c "$ece_script -t rmi-hub $1" fi From 89dac88d79dc6de46942718c7ddfbc7524bb648c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 17:47:05 +0800 Subject: [PATCH 0008/2585] - updated installation guide with more details on the DB installation. --- usr/share/doc/escenic/ece-install-guide.html | 295 +++++++++++++++---- usr/share/doc/escenic/ece-install-guide.org | 137 ++++++--- usr/share/doc/escenic/ece-install-guide.txt | 218 ++++++++++---- 3 files changed, 503 insertions(+), 147 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 659f7c18..e8837245 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -213,7 +213,7 @@

Welcome to the /usr/sbin/ece-install Guide

Table of Contents

-

1 Supported operating systems

+

1 Supported Operating Systems

For the best experience, run this script on Debian Squeeze (or newer), @@ -291,7 +306,7 @@

1.2 Other GNU/Linux and U

-

2 A note on running ece-install on non-GNU/Linux systems

+

2 A Note on Running Ece-Install On Non-GNU/Linux Systems

Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp @@ -305,7 +320,7 @@

2 A note on running ece-ins

-

3 Downloading the ece-install script

+

3 Downloading the ece-install Script

The ece-install script can be downloaded from: @@ -326,7 +341,7 @@

3 Downloading the ece-insta

-

4 Running the ece-install script

+

4 Running the ece-install Script

You must run the script as the root user. If you start the script as @@ -362,12 +377,93 @@

4 Running the ece-install s + +

+You will need access to http://technet.escenic.com and put these +credentials in the root user's $HOME/.ece-install.conf, normally this +means root.ece-install.conf, the script will also tell you this if +you forget to provide such a configuration file: +

+ + +
[ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-(
+
+
+ + + + +

+As for the technet credentials, the script will also tell you if it +cannot find what it's looking for: +

+ + +
[ece-install] Be sure to set technet_user and technet_password
+[ece-install] in /root/.ece-install.conf
+
+
+ + + + +

+The minimum .ece-install.conf file is thus: +

+ + +
technet_user=<user>
+technet_password=<password>
+
+
+ + + + +

+If you're installing WF as well, you'll need additional credentials +for this, see the Widget Framwork section below, and if you want to +runn a fully automatic install without any user interaction, you'll +need to specify some FAI specific parameters, see the FAI section +below. +

+

+With the configuration file in place, you're now ready to run it: +

+ + +
# bash ece-install [-v|--verbose]
+
+
+ + + + +

+If something goes astray with the script, and the log file cannot give +you enough clues, the ultimate way of debugging this is to run the +BASH interpreter with the -x flag: +

+ + +
# bash -x ece-install
+
+
+ + + + +

+Here you can see everything that BASH does when running the script, +how the wildcard variables expand and so on. Normally, having a tail +on /var/log/ece-install.log should be sufficient, though. +

-

5 Available server profiles

+

5 Available Server Profiles

When starting the script, it will ask you which server profile you @@ -398,16 +494,27 @@

5 Available server profiles

-

5.1 Common for editorial, presentation and search instances

+

5.1 Common for Editorial, Presentation and Search Instances

-
    -
  • Sun Java 6 -
  • -
  • Apache Tomcat application server: + +
+ +
+

5.1.1 Sun Java 6

+
+ +
+ +
+ +
+

5.1.2 Apache Tomcat application server

+
+
  • Compression of textual content (previously, this was typically set - up with Apache and its mod_deflate module). + up with Apache and its mod_deflate module).
  • pooling set up tweaked for high read/write performance.
  • @@ -415,31 +522,72 @@

    5.1 Common for editorial,
  • routing information in the cookies
  • -
  • access log - +
  • application server access log
- -
  • APR, native library for optimal Tomcat I/O performance -
  • -
  • Escenic Assembly environment -
  • -
  • Database driver +
  • - -
  • Compression of content (what was previously accomplished with - Apache/mod_deflate) -
  • - +
    + +
    +

    5.1.3 Basic Escenic Nursery configuration

    +
    + +

    The basic Nursery configuration is taken care of for you, including RMI, + database, search and application server/URIs. +

    + +
    + +
    +

    5.1.4 APR, native library for optimal Tomcat I/O performance

    +
    + +
    + +
    + +
    +

    5.1.5 Escenic Assembly environment

    +
    +

    The reason why ece-install sets this up on each host, is to make the +installation process as smooth as possible. The assembly environment +may be removed after the installation if you want to. +

    +
    +
    +

    5.1.6 Database driver

    +
    + + +
    + +
    + +
    +

    5.1.7 Compression of content

    +
    + +

    This was was previously accomplished by having a web server in fron t +of the application server (or cache server if you used ESI). A typical +system architecture would contain Apache with mod_deflate. However, +this is no longer necessary as Varnish can handles ESI parsing of +compressed content (and many other things that we before needed Apache +for). Thus, we'll let the application server do the compression for us +now. +

    +
    +
    +
    -

    5.2 1 - Full stack on one host

    +

    5.2 1 - Full Stack on One Host

    This profile is suitable for developers and admins wanting to set up a @@ -457,7 +605,7 @@

    5.2 1 - Full stack on one

    -

    5.3 2 - Editorial server (publication server)

    +

    5.3 2 - Editorial Server (Publication Server)

    @@ -467,7 +615,7 @@

    5.3 2 - Editorial server

    -

    5.4 3 - Presentation server

    +

    5.4 3 - Presentation Server

    This will set up a typical presentation server: @@ -488,7 +636,7 @@

    5.4 3 - Presentation serv

    -

    5.5 4 - Database server

    +

    5.5 4 - Database Server

    If ece-install is run on a support version of Debian or Ubuntu, this @@ -506,15 +654,26 @@

    5.5 4 - Database server < directory as the ece-intall script itself.

    -If you wish to change the DB's host, user or password, you must update -the drop-and-create-ecedb script prior to running ece-install. +The script will fail by itself if the DB already exists:

    + + +
    [ece-install] Setting up the ECE database schema ...
    +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    +ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists
    +[ece-install] running tables FAILED, exiting :-(
    +
    +
    + + + +

    -

    5.6 5 - Cache server

    +

    5.6 5 - Cache Server

    If ece-install is run on a support version of Debian or Ubuntu, it @@ -578,12 +737,12 @@

    5.7 8 - Install Widget Fr these, please contact support@escenic.com.

    -If you don't have these ready in your .escenicrc, ece-install will +If you don't have these ready in your .ece-install.conf, ece-install will complain:

    -
    [ece-install] Be sure to set wf_user and wf_password in /root/.escenicrc
    +
    [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf
     [ece-install] If you don't have these, please contact support@escenic.com
     
     
    @@ -696,7 +855,7 @@

    6 Full Automatic Install (F

    -

    7 Running more than one installation process

    +

    7 Running More Than One Installation Process

    If the script believes there's already an ece-intall process running, @@ -719,7 +878,7 @@

    7 Running more than one ins

    -

    8 Re-running ece-install

    +

    8 Re-running ece-install (and How To Speed It Up)

    Although the initial thought behind ece-install, is to run it on a @@ -759,7 +918,7 @@

    8 Re-running ece-install
    -

    9 Overview of file paths used by the ece-install script

    +

    9 Overview of File Paths Used by the ece-install script

    There are of course other paths involved when setting up your system, @@ -811,9 +970,29 @@

    9 Overview of file paths us

    -

    10 Uninstalling everything that the ece-install set up

    +

    10 Assumptions

    + +
    + +
    +

    10.1 /etc/esecenic is shared

    +
    + +

    It's assumed that the /etc/escenic directory is either on a shared +file system or rsync-ed among the hosts that are running the ECE +instances (the exception being database and cache servers). +

    +
    +
    + +
    + +
    +

    11 Uninstalling everything That the ece-install Set Up

    +
    +

    WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your host. However, this may be useful if you're installing a clean @@ -830,9 +1009,9 @@

    10 Uninstalling everything

    -
    -

    11 Example Output

    -
    +
    +

    12 Example Output

    +

    Below is an example output of running ece-install, installing the all-in-one profile. @@ -846,7 +1025,7 @@

    11 Example Output


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-08-25 22:02:19 CST + Date: 2011-08-26 17:45:45 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 9cd730a5..19141e0b 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -1,6 +1,6 @@ Welcome to the /usr/sbin/ece-install Guide -* Supported operating systems +* Supported Operating Systems For the best experience, run this script on Debian Squeeze (or newer), the latest Ubuntu LTS or most other Debian based operating systems, but it should work without any problems on any Unix like system that @@ -21,14 +21,14 @@ You may also look inside the ece-install script for all calls to the 'assert\_pre\_prequesite' method and you'll get a list of the binaries ece-install assumes are present. -* A note on running ece-install on non-GNU/Linux systems +* A Note on Running Ece-Install On Non-GNU/Linux Systems Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp & tar. If you're running a system which has a different set of core tools, such as any of the BSDs (including Mac OS X) or Solaris, make sure that GNU's tools take precedence in the command PATH offered to ece-install. -* Downloading the ece-install script +* Downloading the ece-install Script The ece-install script can be downloaded from: https://raw.github.com/skybert/ece-scripts/master/usr/sbin/ece-install or be downloaded together with the other ece-scripts using git: @@ -36,7 +36,7 @@ or be downloaded together with the other ece-scripts using git: $ git clone git@github.com:skybert/ece-scripts.git #+END_SRC -* Running the ece-install script +* Running the ece-install Script You must run the script as the root user. If you start the script as a regular user, it will complain: #+BEGIN_SRC sh @@ -50,7 +50,51 @@ On all other Unix like system, do: #+BEGIN_SRC sh $ su #+END_SRC -* Available server profiles + +You will need access to http://technet.escenic.com and put these +credentials in the root user's $HOME/.ece-install.conf, normally this +means /root/.ece-install.conf, the script will also tell you this if +you forget to provide such a configuration file: +#+BEGIN_SRC sh +[ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-( +#+END_SRC + +As for the technet credentials, the script will also tell you if it +cannot find what it's looking for: +#+BEGIN_SRC sh +[ece-install] Be sure to set technet_user and technet_password +[ece-install] in /root/.ece-install.conf +#+END_SRC + +The minimum .ece-install.conf file is thus: +#+BEGIN_SRC sh +technet_user= +technet_password= +#+END_SRC + +If you're installing WF as well, you'll need additional credentials +for this, see the Widget Framwork section below, and if you want to +runn a fully automatic install without any user interaction, you'll +need to specify some FAI specific parameters, see the FAI section +below. + +With the configuration file in place, you're now ready to run it: +#+BEGIN_SRC sh +# bash ece-install [-v|--verbose] +#+END_SRC + +If something goes astray with the script, and the log file cannot give +you enough clues, the ultimate way of debugging this is to run the +BASH interpreter with the -x flag: +#+BEGIN_SRC sh +# bash -x ece-install +#+END_SRC + +Here you can see everything that BASH does when running the script, +how the wildcard variables expand and so on. Normally, having a tail +on /var/log/ece-install.log should be sufficient, though. + +* Available Server Profiles When starting the script, it will ask you which server profile you want to install: #+BEGIN_SRC sh @@ -69,24 +113,36 @@ Select 1-7 and press ENTER 8 - Install Widget Framework. This will also set up a WF based publication for you (repo.escenic.com user/pass required). #+END_SRC -** Common for editorial, presentation and search instances -- Sun Java 6 -- Apache Tomcat application server: - * Compression of textual content (previously, this was typically set - up with Apache and its mod\_deflate module). - * pooling set up tweaked for high read/write performance. - * proper logging configuration directing solr messages to its own log. - * routing information in the cookies - * access log - -- APR, native library for optimal Tomcat I/O performance -- Escenic Assembly environment -- Database driver - -- Compression of content (what was previously accomplished with - Apache/mod\_deflate) - -** 1 - Full stack on one host +** Common for Editorial, Presentation and Search Instances +*** Sun Java 6 +*** Apache Tomcat application server +- Compression of textual content (previously, this was typically set + up with Apache and its mod\_deflate module). +- pooling set up tweaked for high read/write performance. +- proper logging configuration directing solr messages to its own log. +- routing information in the cookies +- application server access log +*** Basic Escenic Nursery configuration +The basic Nursery configuration is taken care of for you, including RMI, + database, search and application server/URIs. +*** APR, native library for optimal Tomcat I/O performance +*** Escenic Assembly environment +The reason why ece-install sets this up on each host, is to make the +installation process as smooth as possible. The assembly environment +may be removed after the installation if you want to. + +*** Database driver + +*** Compression of content +This was was previously accomplished by having a web server in fron t +of the application server (or cache server if you used ESI). A typical +system architecture would contain Apache with mod\_deflate. However, +this is no longer necessary as Varnish can handles ESI parsing of +compressed content (and many other things that we before needed Apache +for). Thus, we'll let the application server do the compression for us +now. + +** 1 - Full Stack on One Host This profile is suitable for developers and admins wanting to set up a test environment. It will install the full stack including caching server, application server, ECE, assembly host, database, Widget @@ -96,10 +152,10 @@ For further details on each of the different server components, see the different profile descriptions bellow. -** 2 - Editorial server (publication server) +** 2 - Editorial Server (Publication Server) -** 3 - Presentation server +** 3 - Presentation Server This will set up a typical presentation server: - Memcached, distributed memory cache - Escenic Assembly environment (which may be removed after the @@ -107,7 +163,7 @@ This will set up a typical presentation server: - Deployment setup to only deploy escenic-admin and the publication(s). -** 4 - Database server +** 4 - Database Server If ece-install is run on a support version of Debian or Ubuntu, this will install the excellent Percona distribution of MySQL with their set of high performance patches. @@ -120,10 +176,15 @@ SQL files contained inside the distribution bundles. To accomplish this, the script will make a call to drop-and-create-ecedb in the same directory as the ece-intall script itself. -If you wish to change the DB's host, user or password, you must update -the drop-and-create-ecedb script prior to running ece-install. +The script will fail by itself if the DB already exists: +#+BEGIN_SRC sh +[ece-install] Setting up the ECE database schema ... +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists +ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists +[ece-install] running tables FAILED, exiting :-( +#+END_SRC -** 5 - Cache server +** 5 - Cache Server If ece-install is run on a support version of Debian or Ubuntu, it will install the latest Varnish 3.0 caching server from the Varnish APT repository. @@ -161,10 +222,10 @@ repo.escenic.com Maven repository. You should get these credentials when you bought Widget Framework. If you for some reason do not have these, please contact support@escenic.com. -If you don't have these ready in your .escenicrc, ece-install will +If you don't have these ready in your .ece-install.conf, ece-install will complain: #+BEGIN_SRC sh -[ece-install] Be sure to set wf_user and wf_password in /root/.escenicrc +[ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf [ece-install] If you don't have these, please contact support@escenic.com #+END_SRC ** 9 - Create Publication @@ -229,7 +290,7 @@ When FAI is enabled, ece-install will report: [ece-install] All user input will be read from /root/.ece-install.conf #+END_SRC -* Running more than one installation process +* Running More Than One Installation Process If the script believes there's already an ece-intall process running, it will abort: #+BEGIN_SRC sh @@ -239,7 +300,7 @@ it will abort: [ece-install] run ece-install again. #+END_SRC -* Re-running ece-install +* Re-running ece-install (and How To Speed It Up) Although the initial thought behind ece-install, is to run it on a clean system to get up and running as soon as possible. However, you may want to re-run ece-install on the same host, for instance to add @@ -263,7 +324,7 @@ following variables: - wf\_download\_list - tomcat\_download -* Overview of file paths used by the ece-install script +* Overview of File Paths Used by the ece-install script There are of course other paths involved when setting up your system, but these should be the most interesting ones. @@ -300,7 +361,13 @@ but these should be the most interesting ones. *) Applies only to Debian based systems. -* Uninstalling everything that the ece-install set up +* Assumptions +** /etc/esecenic is shared +It's assumed that the /etc/escenic directory is either on a shared +file system or rsync-ed among the hosts that are running the ECE +instances (the exception being database and cache servers). + +* Uninstalling everything That the ece-install Set Up WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your host. However, this may be useful if you're installing a clean diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index 830ef88c..abfecb97 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,35 +2,44 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-08-25 22:02:21 CST +Date: 2011-08-26 17:45:48 CST Table of Contents ================= -1 Supported operating systems +1 Supported Operating Systems 1.1 Debian based operating systems 1.2 Other GNU/Linux and Unix systems -2 A note on running ece-install on non-GNU/Linux systems -3 Downloading the ece-install script -4 Running the ece-install script -5 Available server profiles - 5.1 Common for editorial, presentation and search instances - 5.2 1 - Full stack on one host - 5.3 2 - Editorial server (publication server) - 5.4 3 - Presentation server - 5.5 4 - Database server - 5.6 5 - Cache server +2 A Note on Running Ece-Install On Non-GNU/Linux Systems +3 Downloading the ece-install Script +4 Running the ece-install Script +5 Available Server Profiles + 5.1 Common for Editorial, Presentation and Search Instances + 5.1.1 Sun Java 6 + 5.1.2 Apache Tomcat application server + 5.1.3 Basic Escenic Nursery configuration + 5.1.4 APR, native library for optimal Tomcat I/O performance + 5.1.5 Escenic Assembly environment + 5.1.6 Database driver + 5.1.7 Compression of content + 5.2 1 - Full Stack on One Host + 5.3 2 - Editorial Server (Publication Server) + 5.4 3 - Presentation Server + 5.5 4 - Database Server + 5.6 5 - Cache Server 5.7 8 - Install Widget Framework 5.8 9 - Create Publication 6 Full Automatic Install (FAI) -7 Running more than one installation process -8 Re-running ece-install -9 Overview of file paths used by the ece-install script -10 Uninstalling everything that the ece-install set up -11 Example Output +7 Running More Than One Installation Process +8 Re-running ece-install (and How To Speed It Up) +9 Overview of File Paths Used by the ece-install script +10 Assumptions + 10.1 /etc/esecenic is shared +11 Uninstalling everything That the ece-install Set Up +12 Example Output -1 Supported operating systems +1 Supported Operating Systems ------------------------------ For the best experience, run this script on Debian Squeeze (or newer), the latest Ubuntu LTS or most other Debian based operating systems, @@ -54,7 +63,7 @@ You may also look inside the ece-install script for all calls to the 'assert\_pre\_prequesite' method and you'll get a list of the binaries ece-install assumes are present. -2 A note on running ece-install on non-GNU/Linux systems +2 A Note on Running Ece-Install On Non-GNU/Linux Systems --------------------------------------------------------- Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp & tar. If you're running a system which has a different set of core @@ -62,7 +71,7 @@ tools, such as any of the BSDs (including Mac OS X) or Solaris, make sure that GNU's tools take precedence in the command PATH offered to ece-install. -3 Downloading the ece-install script +3 Downloading the ece-install Script ------------------------------------- The ece-install script can be downloaded from: [https://raw.github.com/skybert/ece-scripts/master/usr/sbin/ece-install] @@ -74,7 +83,7 @@ or be downloaded together with the other ece-scripts using git: -4 Running the ece-install script +4 Running the ece-install Script --------------------------------- You must run the script as the root user. If you start the script as a regular user, it will complain: @@ -98,7 +107,66 @@ On all other Unix like system, do: -5 Available server profiles + +You will need access to [http://technet.escenic.com] and put these +credentials in the root user's $HOME/.ece-install.conf, normally this +means /root/.ece-install.conf, the script will also tell you this if +you forget to provide such a configuration file: + + + [ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-( + + + + +As for the technet credentials, the script will also tell you if it +cannot find what it's looking for: + + + [ece-install] Be sure to set technet_user and technet_password + [ece-install] in /root/.ece-install.conf + + + + +The minimum .ece-install.conf file is thus: + + + technet_user= + technet_password= + + + + +If you're installing WF as well, you'll need additional credentials +for this, see the Widget Framwork section below, and if you want to +runn a fully automatic install without any user interaction, you'll +need to specify some FAI specific parameters, see the FAI section +below. + +With the configuration file in place, you're now ready to run it: + + + # bash ece-install [-v|--verbose] + + + + +If something goes astray with the script, and the log file cannot give +you enough clues, the ultimate way of debugging this is to run the +BASH interpreter with the -x flag: + + + # bash -x ece-install + + + + +Here you can see everything that BASH does when running the script, +how the wildcard variables expand and so on. Normally, having a tail +on /var/log/ece-install.log should be sufficient, though. + +5 Available Server Profiles ---------------------------- When starting the script, it will ask you which server profile you want to install: @@ -121,25 +189,49 @@ want to install: -5.1 Common for editorial, presentation and search instances +5.1 Common for Editorial, Presentation and Search Instances ============================================================ -- Sun Java 6 -- Apache Tomcat application server: - * Compression of textual content (previously, this was typically set - up with Apache and its mod\_deflate module). - * pooling set up tweaked for high read/write performance. - * proper logging configuration directing solr messages to its own log. - * routing information in the cookies - * access log - -- APR, native library for optimal Tomcat I/O performance -- Escenic Assembly environment -- Database driver - -- Compression of content (what was previously accomplished with - Apache/mod\_deflate) - -5.2 1 - Full stack on one host + +5.1.1 Sun Java 6 +~~~~~~~~~~~~~~~~~ + +5.1.2 Apache Tomcat application server +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- Compression of textual content (previously, this was typically set + up with Apache and its mod\_deflate module). +- pooling set up tweaked for high read/write performance. +- proper logging configuration directing solr messages to its own log. +- routing information in the cookies +- application server access log + +5.1.3 Basic Escenic Nursery configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The basic Nursery configuration is taken care of for you, including RMI, + database, search and application server/URIs. + +5.1.4 APR, native library for optimal Tomcat I/O performance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +5.1.5 Escenic Assembly environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The reason why ece-install sets this up on each host, is to make the +installation process as smooth as possible. The assembly environment +may be removed after the installation if you want to. + +5.1.6 Database driver +~~~~~~~~~~~~~~~~~~~~~~ + +5.1.7 Compression of content +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This was was previously accomplished by having a web server in fron t +of the application server (or cache server if you used ESI). A typical +system architecture would contain Apache with mod\_deflate. However, +this is no longer necessary as Varnish can handles ESI parsing of +compressed content (and many other things that we before needed Apache +for). Thus, we'll let the application server do the compression for us +now. + +5.2 1 - Full Stack on One Host =============================== This profile is suitable for developers and admins wanting to set up a test environment. It will install the full stack including caching @@ -150,11 +242,11 @@ For further details on each of the different server components, see the different profile descriptions bellow. -5.3 2 - Editorial server (publication server) +5.3 2 - Editorial Server (Publication Server) ============================================== -5.4 3 - Presentation server +5.4 3 - Presentation Server ============================ This will set up a typical presentation server: - Memcached, distributed memory cache @@ -163,7 +255,7 @@ This will set up a typical presentation server: - Deployment setup to only deploy escenic-admin and the publication(s). -5.5 4 - Database server +5.5 4 - Database Server ======================== If ece-install is run on a support version of Debian or Ubuntu, this will install the excellent Percona distribution of MySQL with their @@ -177,10 +269,18 @@ SQL files contained inside the distribution bundles. To accomplish this, the script will make a call to drop-and-create-ecedb in the same directory as the ece-intall script itself. -If you wish to change the DB's host, user or password, you must update -the drop-and-create-ecedb script prior to running ece-install. +The script will fail by itself if the DB already exists: + + + [ece-install] Setting up the ECE database schema ... + ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists + ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists + [ece-install] running tables FAILED, exiting :-( -5.6 5 - Cache server + + + +5.6 5 - Cache Server ===================== If ece-install is run on a support version of Debian or Ubuntu, it will install the latest Varnish 3.0 caching server from the Varnish @@ -220,11 +320,11 @@ repo.escenic.com Maven repository. You should get these credentials when you bought Widget Framework. If you for some reason do not have these, please contact support@escenic.com. -If you don't have these ready in your .escenicrc, ece-install will +If you don't have these ready in your .ece-install.conf, ece-install will complain: - [ece-install] Be sure to set wf_user and wf_password in /root/.escenicrc + [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf [ece-install] If you don't have these, please contact support@escenic.com @@ -298,7 +398,7 @@ When FAI is enabled, ece-install will report: -7 Running more than one installation process +7 Running More Than One Installation Process --------------------------------------------- If the script believes there's already an ece-intall process running, it will abort: @@ -312,8 +412,8 @@ it will abort: -8 Re-running ece-install -------------------------- +8 Re-running ece-install (and How To Speed It Up) +-------------------------------------------------- Although the initial thought behind ece-install, is to run it on a clean system to get up and running as soon as possible. However, you may want to re-run ece-install on the same host, for instance to add @@ -337,7 +437,7 @@ following variables: - wf\_download\_list - tomcat\_download -9 Overview of file paths used by the ece-install script +9 Overview of File Paths Used by the ece-install script -------------------------------------------------------- There are of course other paths involved when setting up your system, but these should be the most interesting ones. @@ -373,7 +473,17 @@ but these should be the most interesting ones. *) Applies only to Debian based systems. -10 Uninstalling everything that the ece-install set up +10 Assumptions +--------------- + +10.1 /etc/esecenic is shared +============================= +It's assumed that the /etc/escenic directory is either on a shared +file system or rsync-ed among the hosts that are running the ECE +instances (the exception being database and cache servers). + + +11 Uninstalling everything That the ece-install Set Up ------------------------------------------------------- WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your @@ -386,7 +496,7 @@ function, it has copy and pastable commands for undoing most/all things set up by the script. -11 Example Output +12 Example Output ------------------ Below is an example output of running ece-install, installing the all-in-one profile. From 6cb3348cd11a94f9919a380f137341bea2cf1b98 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 18:29:42 +0800 Subject: [PATCH 0009/2585] - updated install guide with FAI output - minor wording change to ece-install output --- usr/sbin/ece-install | 2 +- usr/share/doc/escenic/ece-install-guide.html | 87 ++++++++++++++++++-- usr/share/doc/escenic/ece-install-guide.org | 17 +++- usr/share/doc/escenic/ece-install-guide.txt | 81 ++++++++++++++++-- 4 files changed, 169 insertions(+), 18 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 31f0d69b..e3863e58 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -865,7 +865,7 @@ function print_status_and_next_steps() minutes=$(( seconds_left / 60 )) seconds_left=$(( seconds_left - $minutes * 60 )) - print "The installation is now completed!" + print "The installation is now complete!" print "- It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s echo "$next_steps" diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index e8837245..975b9bb5 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -257,7 +257,7 @@

    Table of Contents

  • 11 Uninstalling everything That the ece-install Set Up
  • -
  • 12 Example Output
  • +
  • 12 Example Output FAI
  • @@ -1010,22 +1010,95 @@

    11 Uninstalling everything

    -

    12 Example Output

    +

    12 Example Output FAI

    -

    Below is an example output of running ece-install, installing the -all-in-one profile. +

    The ece-install will often be run in FAI mode where all the settings +are read from ~/.ece-install.conf. Given the following entries in this +file:

    + + +
    technet_user=apples
    +technet_password=andoranges
    +wf_user=foo
    +wf_password=bar
    +
    +fai_enabled=1
    +fai_all_install=1
    +
    +
    + + + +

    -CANNOT INCLUDE FILE /tmp/ece-install.out +The following output is produced from ece-install:

    + + +
    root@ubiquitous:~# bash ece-install 
    +[ece-install] Full Automatic Install (FAI) enabled.
    +[ece-install] All user input will be read from /root/.ece-install.conf
    +[ece-install] I'm logging to /var/log/ece-install.log
    +[ece-install] Setting correct permissions on all ECE related directories ...
    +[ece-install] Stopping conflicting processes ...
    +[ece-install] Installing 3rd party packages needed by ece-install ...
    +[ece-install] Setting up the ece UNIX scripts ...
    +[ece-install] Adding the ece init.d script to the default run levels...
    +[ece-install] Setting up the escenic user's UNIX environment ...
    +[ece-install] Installing an all-in-one environment on ubiquitous ...
    +[ece-install] Installing the database server on ubiquitous ...
    +[ece-install] Installing the Percona database ...
    +[ece-install] Downloading Escenic software from technet.escenic.com ...
    +[ece-install] Setting up the Escenic Content Engine & its plugins ...
    +[ece-install] Setting up the ECE database schema ...
    +[ece-install] Installing 3rd party packages needed by ECE ...
    +[ece-install] Setting up the Assembly Tool ...
    +[ece-install] Setting up the basic Nursery configuration ...
    +[ece-install] Setting up the application server ...
    +[ece-install] Setting up proper log4j & Java logging configuration ...
    +[ece-install] Setting correct permissions on all ECE related directories ...
    +[ece-install] Assembling, deploying & starting dev1 ...
    +[ece-install] Installing a caching server on ubiquitous ...
    +[ece-install] Setting up Varnish to match your environment ...
    +[ece-install] Setting up solr ...
    +[ece-install] Creating a Maven settings file: /root/.m2/settings.xml ...
    +[ece-install] Downloading Widget Framework from technet.escenic.com ...
    +[ece-install] Installing Widget Framework into your local Maven repository ...
    +[ece-install] Getting ready to create a new publiation ...
    +[ece-install] Setting up the mypub publication ...
    +[ece-install] Basing mypub.war on the Widget Framework Demo ...
    +[ece-install] Creating publication mypub using dev1 ...
    +[ece-install] Assembling, deploying & starting dev1 ...
    +[ece-install] The installation is now complete!
    +[ece-install] - It took 0d 0h 29m 40s
    +[ece-install] - DB is now set up on ubiquitous:3306
    +[ece-install] - New ECE instance dev1 installed.
    +[ece-install] - Its admin interface is available at http://ubiquitous:8080/escenic-admin
    +[ece-install] - You can view all installed plugin & engine versions with:
    +[ece-install]   ece -i dev1 versions
    +[ece-install] - Cache server is up and running at http://ubiquitous:80/
    +[ece-install] - Widget Framework has been installed into your local
    +[ece-install]   Maven repository and its ECE components have been installed
    +[ece-install] - a new publication mypub has been created
    +[ece-install]
    +[ece-install] Enjoy your time with Escenic Content Engine!
    +[ece-install]
    +[ece-install] -Vizrt Online
    +
    +
    + + + +

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-08-26 17:45:45 CST + Date: 2011-08-26 18:28:21 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 19141e0b..7dd12512 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -379,9 +379,20 @@ function, it has copy and pastable commands for undoing most/all things set up by the script. -* Example Output -Below is an example output of running ece-install, installing the -all-in-one profile. +* Example Output FAI +The ece-install will often be run in FAI mode where all the settings +are read from ~/.ece-install.conf. Given the following entries in this +file: +#+BEGIN_SRC sh +technet_user=apples +technet_password=andoranges +wf_user=foo +wf_password=bar + +fai_enabled=1 +fai_all_install=1 +#+END_SRC +The following output is produced from ece-install: #+INCLUDE: "/tmp/ece-install.out" src text diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index abfecb97..37809d3e 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-08-26 17:45:48 CST +Date: 2011-08-26 18:28:17 CST Table of Contents @@ -36,7 +36,7 @@ Table of Contents 10 Assumptions 10.1 /etc/esecenic is shared 11 Uninstalling everything That the ece-install Set Up -12 Example Output +12 Example Output FAI 1 Supported Operating Systems @@ -496,10 +496,77 @@ function, it has copy and pastable commands for undoing most/all things set up by the script. -12 Example Output ------------------- -Below is an example output of running ece-install, installing the -all-in-one profile. +12 Example Output FAI +---------------------- +The ece-install will often be run in FAI mode where all the settings +are read from ~/.ece-install.conf. Given the following entries in this +file: + + + technet_user=apples + technet_password=andoranges + wf_user=foo + wf_password=bar + + fai_enabled=1 + fai_all_install=1 + + + + +The following output is produced from ece-install: + + + root@ubiquitous:~# bash ece-install + [ece-install] Full Automatic Install (FAI) enabled. + [ece-install] All user input will be read from /root/.ece-install.conf + [ece-install] I'm logging to /var/log/ece-install.log + [ece-install] Setting correct permissions on all ECE related directories ... + [ece-install] Stopping conflicting processes ... + [ece-install] Installing 3rd party packages needed by ece-install ... + [ece-install] Setting up the ece UNIX scripts ... + [ece-install] Adding the ece init.d script to the default run levels... + [ece-install] Setting up the escenic user's UNIX environment ... + [ece-install] Installing an all-in-one environment on ubiquitous ... + [ece-install] Installing the database server on ubiquitous ... + [ece-install] Installing the Percona database ... + [ece-install] Downloading Escenic software from technet.escenic.com ... + [ece-install] Setting up the Escenic Content Engine & its plugins ... + [ece-install] Setting up the ECE database schema ... + [ece-install] Installing 3rd party packages needed by ECE ... + [ece-install] Setting up the Assembly Tool ... + [ece-install] Setting up the basic Nursery configuration ... + [ece-install] Setting up the application server ... + [ece-install] Setting up proper log4j & Java logging configuration ... + [ece-install] Setting correct permissions on all ECE related directories ... + [ece-install] Assembling, deploying & starting dev1 ... + [ece-install] Installing a caching server on ubiquitous ... + [ece-install] Setting up Varnish to match your environment ... + [ece-install] Setting up solr ... + [ece-install] Creating a Maven settings file: /root/.m2/settings.xml ... + [ece-install] Downloading Widget Framework from technet.escenic.com ... + [ece-install] Installing Widget Framework into your local Maven repository ... + [ece-install] Getting ready to create a new publiation ... + [ece-install] Setting up the mypub publication ... + [ece-install] Basing mypub.war on the Widget Framework Demo ... + [ece-install] Creating publication mypub using dev1 ... + [ece-install] Assembling, deploying & starting dev1 ... + [ece-install] The installation is now complete! + [ece-install] - It took 0d 0h 29m 40s + [ece-install] - DB is now set up on ubiquitous:3306 + [ece-install] - New ECE instance dev1 installed. + [ece-install] - Its admin interface is available at http://ubiquitous:8080/escenic-admin + [ece-install] - You can view all installed plugin & engine versions with: + [ece-install] ece -i dev1 versions + [ece-install] - Cache server is up and running at http://ubiquitous:80/ + [ece-install] - Widget Framework has been installed into your local + [ece-install] Maven repository and its ECE components have been installed + [ece-install] - a new publication mypub has been created + [ece-install] + [ece-install] Enjoy your time with Escenic Content Engine! + [ece-install] + [ece-install] -Vizrt Online + + -CANNOT INCLUDE FILE /tmp/ece-install.out From c122e6f2182e50195de8f6cbde6509e0af5c3517 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 26 Aug 2011 18:32:28 +0800 Subject: [PATCH 0010/2585] - getting pernickety here --- usr/share/doc/escenic/ece-install-guide.html | 8 ++++---- usr/share/doc/escenic/ece-install-guide.org | 2 +- usr/share/doc/escenic/ece-install-guide.txt | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 975b9bb5..b6c4647a 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -256,7 +256,7 @@

    Table of Contents

  • 10.1 /etc/esecenic is shared
  • -
  • 11 Uninstalling everything That the ece-install Set Up
  • +
  • 11 Uninstalling Everything That the ece-install Set Up
  • 12 Example Output FAI
  • @@ -990,7 +990,7 @@

    10.1 /etc/esecenic is sh

    -

    11 Uninstalling everything That the ece-install Set Up

    +

    11 Uninstalling Everything That the ece-install Set Up

    WARNING: this is potentially dangerous as some of these components may @@ -1098,7 +1098,7 @@

    12 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-08-26 18:28:21 CST + Date: 2011-08-26 18:32:09 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 7dd12512..81aef24d 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -367,7 +367,7 @@ It's assumed that the /etc/escenic directory is either on a shared file system or rsync-ed among the hosts that are running the ECE instances (the exception being database and cache servers). -* Uninstalling everything That the ece-install Set Up +* Uninstalling Everything That the ece-install Set Up WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your host. However, this may be useful if you're installing a clean diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index 37809d3e..83a22be5 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-08-26 18:28:17 CST +Date: 2011-08-26 18:32:11 CST Table of Contents @@ -35,7 +35,7 @@ Table of Contents 9 Overview of File Paths Used by the ece-install script 10 Assumptions 10.1 /etc/esecenic is shared -11 Uninstalling everything That the ece-install Set Up +11 Uninstalling Everything That the ece-install Set Up 12 Example Output FAI @@ -483,7 +483,7 @@ file system or rsync-ed among the hosts that are running the ECE instances (the exception being database and cache servers). -11 Uninstalling everything That the ece-install Set Up +11 Uninstalling Everything That the ece-install Set Up ------------------------------------------------------- WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your From 3821b13668323347795c0d3cfdeee2b191e2b46b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Sep 2011 13:21:56 +0800 Subject: [PATCH 0011/2585] - added unzip to the pre-requisite binaries - added script specific time stamps - removed publication creation in the editorial server profile, now it corresponds to the help screen. --- usr/sbin/ece-install | 53 +++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index e3863e58..16c5f631 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -4,6 +4,10 @@ # five minutes. The setup the user is getting is suitable for a # production environment, except for the fact that with this script, # all ECE components are set up on the same host. +# +# BUG: if creating a publication and WF is downloaded, but not +# installed, then the pub is built from WF, but WF is not added to the +# plugins. This _is_ a corner case, but should still be fixed. # by tkj@vizrt.com @@ -14,13 +18,13 @@ ece_user=escenic ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 +tomcat_download=http://ftp.nsysu.edu.tw/Apache/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz # don't touch this one if you're installing on a Debian based system # as it'll be set up for you by the script itself. java_home=/usr/lib/jvm/java-6-sun ##################################################################### -id="[$(basename $0)]" pid_file=/var/run/$(basename $0).pid download_dir=/tmp/ece-downloads log=/var/log/$(basename $0).log @@ -43,6 +47,13 @@ NEW_LINE=" # components. next_steps="" + +function get_id() +{ + id="[$(basename $0)-$(get_seconds_since_start)]" + echo $id +} + function debug() { if [ $debug -eq 1 ]; then @@ -52,7 +63,7 @@ function debug() function print() { - echo $id $@ + echo $(get_id) $@ } function exit_on_error() @@ -80,7 +91,6 @@ http://technet.escenic.com/downloads/release/53/xml-editor-dist-2.1.0.0.zip wf_download_list=" http://technet.escenic.com/downloads/widget-framework/widget-framework-core-1.10.0.0.zip " -tomcat_download=http://ftp.nsysu.edu.tw/Apache/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz PROFILE_ALL_IN_ONE=1 PROFILE_CACHE_SERVER=5 @@ -193,13 +203,13 @@ function install_common_os_packages() git_package=git fi - packages="curl $git_package wget" + packages="curl $git_package wget unzip" install_packages_if_missing $packages fi - assert_pre_prequesite curl - assert_pre_prequesite wget - assert_pre_prequesite git + for el in curl wget git unzip; do + assert_pre_prequesite $el + done } make_dir $ece_directories @@ -822,7 +832,7 @@ EOF # last, give the control back to the ECE user & group function set_correct_permissions() { - print "Setting correct permissions on all ECE related directories ..." + print "Setting correct permissions on ECE related directories ..." if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then # TODO add support for useradd @@ -850,6 +860,13 @@ function set_correct_permissions() fi } +function get_seconds_since_start() +{ + now=`date +%s` + started=`stat -c %Y $pid_file` + seconds=$(( now - started )) + echo "$seconds" +} function print_status_and_next_steps() { @@ -1202,8 +1219,7 @@ function read_user_input() echo " It will install caching server, ECE, assembly host, database &" echo " Widget Framework as well as set up a new publication." echo " $PROFILE_EDITORIAL_SERVER - Editorial (publication) server." - echo " This will install ECE, an assembly host & as well as " - echo " creating and set up a new publication." + echo " This will install ECE and an assembly host." echo -n " $PROFILE_PRESENTATION_SERVER - Presentation server " echo "(ECE, memcached)." echo " $PROFILE_DB_SERVER - Database server." @@ -1235,13 +1251,14 @@ function assert_correct_runtime_environment() fi if [ -e $pid_file ]; then - print "There's already one $(basename $0) process running. If you believe" - print "this is wrong, e.g. if a previous run of $(basename $0) was aborted" - print "before it completed, you may delete ${pid_file} and " - print "run $(basename $0) again." + print "There's already one $(basename $0) process running." + print "If you blelieve this is wrong, e.g. if a previous run of" + print "$(basename $0) was aborted before it completed, you" + print "may remove ${pid_file} and run $(basename $0) again." exit 1 else echo $BASHPID > $pid_file + started=`stat -c %Y $pid_file` fi if [ ! -e "$conf_file" ]; then @@ -1347,9 +1364,9 @@ function install_cache_server() function add_next_step() { if [ -n "$next_steps" ]; then - next_steps=${next_steps}${NEW_LINE}${id}" "${1} + next_steps=${next_steps}${NEW_LINE}$(get_id)" "${1} else - next_steps=${id}" "${1} + next_steps=$(get_id)" "${1} fi } @@ -1546,10 +1563,6 @@ function install_editorial_server() print "Installing an editorial server on $HOSTNAME ..." install_ece_instance "editor1" 0 - type=engine - ece_command="ece -i $instance_name -t $type assemble deploy restart" - su - $ece_user -c "$ece_command" 1>>$log 2>>$log - create_publication } function install_rmi_hub() From 4cfd1aa0a4bcfca27794c9485d5f0c0a9d940988 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Sep 2011 13:36:25 +0800 Subject: [PATCH 0012/2585] - added support for host names containing dashes (Varnish doesn't accept backend names with dashes). This is highly relevant for (at least) Amazon cloud hosts which get assigned host names containing dashes. Thanks to mogsie for this fix: c588e2d91f9eaebe3ba7a84f62a154d9aa26254f --- usr/sbin/ece-install | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 16c5f631..563bc3a6 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1095,10 +1095,7 @@ function set_up_varnish() /etc/init.d/varnish stop 1>>$log 2>>$log file=/etc/default/varnish - cat $file | \ - sed 's/6081/80/g' \ - > $file.tmp - mv $file.tmp $file + sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file cat > /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < Date: Thu, 8 Sep 2011 14:52:33 +0800 Subject: [PATCH 0013/2585] - common_pre_install now reads .ece-install.conf by sourcing it. This is far easier than cat/grep/cut-ing the values. It also makes it easy for the user to override any defaults, such as tomcat_download and git_source. This change is inspired by mogsie's changes to allow using a fork-ed ece-install. --- usr/sbin/ece-install | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 563bc3a6..97a36eea 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -30,6 +30,7 @@ download_dir=/tmp/ece-downloads log=/var/log/$(basename $0).log conf_file=$HOME/.ece-install.conf common_nursery_dir=/etc/escenic/engine/common +git_source=git://github.com/skybert/ece-scripts.git # globals will be set to correct values in run-time. appserver_port=8080 @@ -224,7 +225,6 @@ function set_up_assembly_tool() if [ -e $download_dir/assemblytool*zip ]; then unzip -u $download_dir/assemblytool*zip \ 1>>$log 2>>$log - mv $download_dir/assemblytool*zip /tmp/ fi # adding an instance layer to the Nursery configuration @@ -238,7 +238,6 @@ fileSystemRoot = /etc/escenic/engine/instance/\${com.escenic.instance}/ EOF echo "" >> Nursery.properties echo "layer.06 = /layers/instance/Layer" >> Nursery.properties - echo "come here" > /tmp/tkj # set up which plugins to use cd /opt/escenic/assemblytool/ @@ -295,16 +294,20 @@ function set_up_engine_and_plugins() fi ln -s engine-* engine - mv $download_dir/engine*.zip /tmp else debug "engine- is already there, skipping to next step." fi - # now, there's only plugins left in the download dir # we extract them in /opt/escenic as we want to re-use # them between minor updates of ECE. cd /opt/escenic/ for el in $download_dir/*.zip; do + if [ "${el/engine*}" == "" ] ; then + continue + elif [ "${el/assemblytool*}" == "" ] ; then + continue + fi + unzip -u $el \ 1>>$log 2>>$log done @@ -321,7 +324,7 @@ function set_up_ece_scripts() (cd ece-scripts git pull 1>>$log 2>>$log) else - git clone git://github.com/skybert/ece-scripts.git \ + git clone $git_source \ 1>>$log 2>>$log fi @@ -1260,7 +1263,8 @@ function assert_correct_runtime_environment() fi if [ ! -e "$conf_file" ]; then - print $conf_file "doesn't exist, I cannot live without it :-(" + print $conf_file "doesn't exist." + print "I cannot live without it, so I'm exiting :-(" exit 1 fi } @@ -1269,8 +1273,7 @@ function common_pre_install() { print "I'm logging to $log" - technet_user=`grep technet_user $conf_file 2>/dev/null | cut -d'=' -f2` - technet_password=`grep technet_password $conf_file 2>/dev/null | cut -d'=' -f2` + source $conf_file if [ -z "$technet_user" -o -z "$technet_password" ]; then print "Be sure to set technet_user and technet_password " From 14622abb20b210d32a1e379818c9a22945e68303 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Sep 2011 15:50:17 +0800 Subject: [PATCH 0014/2585] - added hardening to (almost) all commands executed from within the script. They now use a wrapping function, cmd() which will call exit_on_error after the command is executed. This in effect, means that ece-install will exit as soon as an error is spotted during its execution. The script will output messages like: [ece-install-1] The command [adduser does -p does] FAILED, exiting :-( [ece-install-1] See /var/log/ece-install.log for further details. --- usr/sbin/ece-install | 161 +++++++++++++++++++++---------------------- 1 file changed, 78 insertions(+), 83 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 97a36eea..a99d8f1e 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -12,7 +12,8 @@ # by tkj@vizrt.com ##################################################################### -# User definable variables (the defaults are fine in most cases). +# User definable variables (the defaults are fine in most cases). Any +# of these may be overridden in the .ece-isntall.conf file ##################################################################### ece_user=escenic ece_group=escenic @@ -70,11 +71,18 @@ function print() function exit_on_error() { if [ $? -eq 1 ]; then - echo $id $@ "FAILED, exiting :-(" + print "The command ["$@"] FAILED, exiting :-(" + print "See $log for further details." exit 1 fi } +function run() +{ + $@ 1>>$log 2>>$log + exit_on_error $@ +} + technet_download_list=" http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip @@ -121,7 +129,7 @@ $common_nursery_dir function make_dir() { if [ ! -d $1 ]; then - mkdir -p $1 + run mkdir -p $1 fi } @@ -132,7 +140,7 @@ done function make_ln() { if [ -e $1 -a ! -h $(basename $1) ]; then - ln -s $1 + run ln -s $1 elif [ ! -e $1 ]; then print "Tried to make a symlink to $1, but it doesn't exist" exit 1 @@ -156,7 +164,7 @@ function download_escenic_components() continue fi - wget --continue \ + run wget --continue \ --http-user $technet_user \ --http-password $technet_password \ $el \ @@ -187,7 +195,7 @@ install_packages_if_missing() return fi - apt-get install -y $@ 1>>$log 2>>$log + run apt-get install -y $@ 1>>$log 2>>$log fi } @@ -223,7 +231,7 @@ function set_up_assembly_tool() cd /opt/escenic/assemblytool/ if [ -e $download_dir/assemblytool*zip ]; then - unzip -u $download_dir/assemblytool*zip \ + run unzip -u $download_dir/assemblytool*zip \ 1>>$log 2>>$log fi @@ -261,18 +269,13 @@ EOF make_ln $directory done - cd /opt/escenic/assemblytool/ - ant -q initialize \ + run cd /opt/escenic/assemblytool/ + run ant -q initialize \ 1>>$log 2>>$log - cat assemble.properties | \ - sed 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ - > assemble.properties.tmp - mv assemble.properties.tmp assemble.properties - - cat assemble.properties | \ - sed 's/\#\# plugins\ =\ \/path\/to\/plugins/plugins=\/opt\/escenic\/assemblytool\/plugins/g' \ - > assemble.properties.tmp - mv assemble.properties.tmp assemble.properties + run sed -i 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ + assemble.properties + run sed -i 's/\#\# plugins\ =\ \/path\/to\/plugins/plugins=\/opt\/escenic\/assemblytool\/plugins/g' \ + assemble.properties } function set_up_engine_and_plugins() @@ -287,13 +290,13 @@ function set_up_engine_and_plugins() cd /opt/escenic/ if [ ! -d engine-* ]; then - unzip -u $download_dir/engine*.zip \ + run unzip -u $download_dir/engine*.zip \ 1>>$log 2>>$log if [ -h engine ]; then - rm engine + run rm engine fi - ln -s engine-* engine + run ln -s engine-* engine else debug "engine- is already there, skipping to next step." fi @@ -308,7 +311,7 @@ function set_up_engine_and_plugins() continue fi - unzip -u $el \ + run unzip -u $el \ 1>>$log 2>>$log done @@ -319,17 +322,17 @@ function set_up_ece_scripts() { print "Setting up the ece UNIX scripts ..." - cd $download_dir + run cd $download_dir if [ -d ece-scripts ]; then (cd ece-scripts git pull 1>>$log 2>>$log) else - git clone $git_source \ + run git clone $git_source \ 1>>$log 2>>$log fi - cp -r ece-scripts/usr/* /usr/ - cp -r ece-scripts/etc/* /etc/ + run cp -r ece-scripts/usr/* /usr/ + run cp -r ece-scripts/etc/* /etc/ # no need to add init.d scripts to the runlevel(s) for these # profiles @@ -342,7 +345,7 @@ function set_up_ece_scripts() if [ $on_debian_or_derivative ]; then print "Adding the ece init.d script to the default run levels..." - update-rc.d ece defaults \ + run update-rc.d ece defaults \ 1>>$log 2>>$log else print "You remember to add /etc/intit.d/ece to the desired runlevels" @@ -358,7 +361,7 @@ function set_up_ecedb() print "Setting up the ECE database schema ..." make_dir /opt/escenic/engine/plugins - cd /opt/escenic/engine/plugins + run cd /opt/escenic/engine/plugins find ../../ -maxdepth 1 -type d | \ grep -v assemblytool | \ @@ -375,7 +378,7 @@ function set_up_ecedb() fi if [ ! -h $(basename $directory) ]; then - ln -s $directory + run ln -s $directory fi done @@ -384,10 +387,9 @@ function set_up_ecedb() # db_password and db_schema here by reading these via # get_conf_value create_ecedb - exit_on_error "create_ecedb" cd ~/ - rm -rf /opt/escenic/engine/plugins + run rm -rf /opt/escenic/engine/plugins add_next_step "- DB is now set up on $HOSTNAME:3306" } @@ -410,9 +412,9 @@ function set_up_basic_nursery_configuration() { print "Setting up the basic Nursery configuration ..." - cp -r /opt/escenic/engine/siteconfig/config-skeleton/* \ + run cp -r /opt/escenic/engine/siteconfig/config-skeleton/* \ $common_nursery_dir/ - cp -r /opt/escenic/engine/security/ \ + run cp -r /opt/escenic/engine/security/ \ $common_nursery_dir/ make_dir /etc/escenic/engine/instance @@ -422,25 +424,21 @@ function set_up_basic_nursery_configuration() continue fi - cp -r $el/misc/siteconfig/* $common_nursery_dir/ - + run cp -r $el/misc/siteconfig/* $common_nursery_dir/ done - cat > $common_nursery_dir/ServerConfig.properties < $common_nursery_dir/ServerConfig.properties < $common_nursery_dir/neo/io/managers/ContentManager.properties < $common_nursery_dir/neo/io/managers/ContentManager.properties < $file.tmp - mv $file.tmp $file + sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file file=$common_nursery_dir/com/escenic/webstart/StudioConfig.properties cat >> $file < $el/RMI.properties + run echo "port=$rmi_port" > $el/RMI.properties fi done @@ -468,7 +466,7 @@ function set_up_instance_specific_nursery_configuration() # we don't touch it if the file already exists. if [ ! -e $file ]; then - echo "hostname=$HOSTNAME" >> $file + run echo "hostname=$HOSTNAME" >> $file fi } @@ -486,7 +484,7 @@ log4j.appender.ECELOG.layout.ConversionPattern=%d %5p [%t] %x (%c) %m%n EOF cd $tomcat_base/lib/ make_ln $common_nursery_dir/trace.properties - ln -sf trace.properties log4j.properies + run ln -sf trace.properties log4j.properies cat > $tomcat_base/conf/logging.properties <>$log 2>>$log - cd /opt/ - - tar xzf $download_dir/apache-tomcat*.tar.gz - ln -sf apache-tomcat* tomcat + run cd /opt/ + run tar xzf $download_dir/apache-tomcat*.tar.gz + run ln -sf apache-tomcat* tomcat tomcat_base=/opt/tomcat-${instance_name} make_dir $tomcat_base - cp -r /opt/apache-tomcat*/conf/ $tomcat_base + run cp -r /opt/apache-tomcat*/conf/ $tomcat_base for el in bin escenic/lib lib work logs temp webapps; do make_dir $tomcat_base/$el done @@ -682,13 +679,11 @@ function set_up_app_server set_ece_instance_conf tomcat_base $tomcat_base set_ece_instance_conf tomcat_home /opt/tomcat - cd $tomcat_base/lib + run cd $tomcat_base/lib make_ln $jdbc_driver - cat $tomcat_base/conf/catalina.properties | \ - sed 's/common.loader=/common.loader=\$\{catalina.base\}\/escenic\/lib\/\*\.jar\,/g' \ - > $tomcat_base/conf/catalina.properties.tmp - mv $tomcat_base/conf/catalina.properties.tmp $tomcat_base/conf/catalina.properties + run sed -i 's/common.loader=/common.loader=\$\{catalina.base\}\/escenic\/lib\/\*\.jar\,/g' \ + > $tomcat_base/conf/catalina.properties cat > $tomcat_base/conf/server.xml < @@ -839,26 +834,26 @@ function set_correct_permissions() if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then # TODO add support for useradd - adduser $ece_user + run adduser $ece_user -p $ece_user fi if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then - addgroup $ece_group + run addgroup $ece_group fi for el in $dir_list; do if [ -d $el ]; then - chown -R ${ece_user}:${ece_group} $el \ + run chown -R ${ece_user}:${ece_group} $el \ 1>>$log 2>>$log fi done if [ -d "$tomcat_base" ]; then - chown -R ${ece_user}:${ece_group} $tomcat_base \ + run chown -R ${ece_user}:${ece_group} $tomcat_base \ 1>>$log 2>>$log fi if [ $(ls /tmp | grep ^ece | wc -l) -gt 0 ]; then - chown -R ${ece_user}:${ece_group} /tmp/ece-* \ + run chown -R ${ece_user}:${ece_group} /tmp/ece-* \ 1>>$log 2>>$log fi } @@ -912,14 +907,14 @@ function create_publication_in_db() exit 1; fi - curl --silent \ + run curl --silent \ -F "type=webapp" \ -F "resourceFile=@${1}" \ --cookie JSESSIONID="$cookie" \ "${ece_admin_uri}/do/publication/resource" \ 1>>$log 2>>$log - curl --silent \ + run curl --silent \ -F "name=${publication_name}" \ -F "publisherName=Escenic" \ -F "adminPassword=${publication_name}" \ @@ -948,7 +943,7 @@ function create_publication_definition_and_war() print "Setting up the ${publication_name} publication ..." make_dir /opt/escenic/assemblytool/publications/ - cd /opt/escenic/assemblytool/publications/ + run cd /opt/escenic/assemblytool/publications/ cat > /opt/escenic/assemblytool/publications/${publication_name}.properties <>$log 2>>$log - cp target/demo-core-*.war ${publication_war} + run cp target/demo-core-*.war ${publication_war} else print "Basing your ${publication_name}.war on ECE/demo-clean ..." - cp /opt/escenic/engine/contrib/wars/demo-clean.war ${publication_war} + run cp /opt/escenic/engine/contrib/wars/demo-clean.war ${publication_war} fi } @@ -1018,10 +1013,10 @@ function set_up_user_enviornment() # TODO get this from ece.conf and other magic if [ $(grep JAVA_HOME /home/$ece_user/.bashrc | wc -l) -lt 1 ]; then - echo JAVA_HOME=$java_home >> /home/$ece_user/.bashrc + run echo JAVA_HOME=$java_home >> /home/$ece_user/.bashrc fi if [ $(grep JAVA_HOME /root/.bashrc | wc -l) -lt 1 ]; then - echo JAVA_HOME=$java_home >> /root/.bashrc + run echo JAVA_HOME=$java_home >> /root/.bashrc fi if [ $on_debian_or_derivative -eq 1 ]; then @@ -1030,7 +1025,7 @@ function set_up_user_enviornment() if [ $(grep bash_completion.d/ece /home/$ece_user/.bashrc | wc -l) -lt 1 ] then - echo ". /etc/bash_completion.d/ece" \ + run echo ". /etc/bash_completion.d/ece" \ >> /home/$ece_user/.bashrc fi } @@ -1039,13 +1034,13 @@ function set_up_solr() { print "Setting up solr ..." if [ ! -d /etc/escenic/solr ]; then - cp -r /opt/escenic/engine/solr/conf /etc/escenic/solr + run cp -r /opt/escenic/engine/solr/conf /etc/escenic/solr fi make_dir /var/lib/escenic/solr/ - cd /var/lib/escenic/solr/ + run cd /var/lib/escenic/solr/ if [ ! -h conf ]; then - ln -s /etc/escenic/solr conf + run ln -s /etc/escenic/solr conf fi } @@ -1098,7 +1093,7 @@ function set_up_varnish() /etc/init.d/varnish stop 1>>$log 2>>$log file=/etc/default/varnish - sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file + run sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file cat > /etc/varnish/default.vcl <> $escenic_sources - apt-get update 1>>$log 2>>$log + run apt-get update 1>>$log 2>>$log elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then echo "$@" >> $escenic_sources - apt-get update 1>>$log 2>>$log + run apt-get update 1>>$log 2>>$log fi } @@ -1325,7 +1320,7 @@ function install_cache_server() print "Installing a caching server on $HOSTNAME ..." if [ $on_debian_or_derivative -eq 1 ]; then - curl http://repo.varnish-cache.org/debian/GPG-key.txt 2>>$log | \ + run curl http://repo.varnish-cache.org/debian/GPG-key.txt 2>>$log | \ apt-key add - 1>>$log 2>>$log fi @@ -1389,17 +1384,17 @@ function install_database_server() # some how, this is to install Percona 5.5 if [ -e /var/lib/mysql/debian-*.flag ]; then - rm /var/lib/mysql/debian-*.flag + run rm /var/lib/mysql/debian-*.flag fi if [ $supported_code_name -eq 1 ]; then print "Installing the Percona database ..." if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then - gpg --keyserver hkp://keys.gnupg.net \ + run gpg --keyserver hkp://keys.gnupg.net \ --recv-keys 1C4CBDCDCD2EFD2A \ 1>>$log 2>>$log - gpg -a --export CD2EFD2A | apt-key add - \ + run gpg -a --export CD2EFD2A | apt-key add - \ 1>>$log 2>>$log fi add_apt_source "deb http://repo.percona.com/apt ${code_name} main" @@ -1570,7 +1565,7 @@ function install_rmi_hub() { make_dir /etc/escenic/rmi-hub - cp -r /opt/escenic/engine/contrib/rmi-hub/config/* \ + run cp -r /opt/escenic/engine/contrib/rmi-hub/config/* \ /etc/escenic/rmi-hub/ hub_host=$HOSTNAME From 7e9470cf662ee21323f2e6d86eb628200c21886b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Sep 2011 15:54:39 +0800 Subject: [PATCH 0015/2585] - added doc about fail fast --- usr/share/doc/escenic/ece-install-guide.org | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 81aef24d..e764d43d 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -83,9 +83,16 @@ With the configuration file in place, you're now ready to run it: # bash ece-install [-v|--verbose] #+END_SRC -If something goes astray with the script, and the log file cannot give -you enough clues, the ultimate way of debugging this is to run the -BASH interpreter with the -x flag: +ece-install will try to "fail fast", exiting as soon as it detects an +error during its execution: +#+BEGIN_SRC sh +[ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-( +[ece-install-1] See /var/log/ece-install.log for further details. +#+END_SRC + +As a last resort, if something goes astray with the script, and the +log file cannot give you enough clues, the ultimate way of debugging +this is to run the BASH interpreter with the -x flag: #+BEGIN_SRC sh # bash -x ece-install #+END_SRC From 528b01b16a8c3249e0284c2b6e04028897d67978 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Sep 2011 15:55:33 +0800 Subject: [PATCH 0016/2585] - updated generated TXT and HTML --- usr/share/doc/escenic/ece-install-guide.html | 82 +++++--------------- usr/share/doc/escenic/ece-install-guide.txt | 73 ++++------------- 2 files changed, 35 insertions(+), 120 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index b6c4647a..f50bec49 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -440,9 +440,23 @@

    4 Running the ece-install S

    -If something goes astray with the script, and the log file cannot give -you enough clues, the ultimate way of debugging this is to run the -BASH interpreter with the -x flag: +ece-install will try to "fail fast", exiting as soon as it detects an +error during its execution: +

    + + +
    [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-(
    +[ece-install-1] See /var/log/ece-install.log for further details.
    +
    +
    + + + + +

    +As a last resort, if something goes astray with the script, and the +log file cannot give you enough clues, the ultimate way of debugging +this is to run the BASH interpreter with the -x flag:

    @@ -1034,71 +1048,15 @@

    12 Example Output FAI

    The following output is produced from ece-install: +CANNOT INCLUDE FILE /tmp/ece-install.out

    - - -
    root@ubiquitous:~# bash ece-install 
    -[ece-install] Full Automatic Install (FAI) enabled.
    -[ece-install] All user input will be read from /root/.ece-install.conf
    -[ece-install] I'm logging to /var/log/ece-install.log
    -[ece-install] Setting correct permissions on all ECE related directories ...
    -[ece-install] Stopping conflicting processes ...
    -[ece-install] Installing 3rd party packages needed by ece-install ...
    -[ece-install] Setting up the ece UNIX scripts ...
    -[ece-install] Adding the ece init.d script to the default run levels...
    -[ece-install] Setting up the escenic user's UNIX environment ...
    -[ece-install] Installing an all-in-one environment on ubiquitous ...
    -[ece-install] Installing the database server on ubiquitous ...
    -[ece-install] Installing the Percona database ...
    -[ece-install] Downloading Escenic software from technet.escenic.com ...
    -[ece-install] Setting up the Escenic Content Engine & its plugins ...
    -[ece-install] Setting up the ECE database schema ...
    -[ece-install] Installing 3rd party packages needed by ECE ...
    -[ece-install] Setting up the Assembly Tool ...
    -[ece-install] Setting up the basic Nursery configuration ...
    -[ece-install] Setting up the application server ...
    -[ece-install] Setting up proper log4j & Java logging configuration ...
    -[ece-install] Setting correct permissions on all ECE related directories ...
    -[ece-install] Assembling, deploying & starting dev1 ...
    -[ece-install] Installing a caching server on ubiquitous ...
    -[ece-install] Setting up Varnish to match your environment ...
    -[ece-install] Setting up solr ...
    -[ece-install] Creating a Maven settings file: /root/.m2/settings.xml ...
    -[ece-install] Downloading Widget Framework from technet.escenic.com ...
    -[ece-install] Installing Widget Framework into your local Maven repository ...
    -[ece-install] Getting ready to create a new publiation ...
    -[ece-install] Setting up the mypub publication ...
    -[ece-install] Basing mypub.war on the Widget Framework Demo ...
    -[ece-install] Creating publication mypub using dev1 ...
    -[ece-install] Assembling, deploying & starting dev1 ...
    -[ece-install] The installation is now complete!
    -[ece-install] - It took 0d 0h 29m 40s
    -[ece-install] - DB is now set up on ubiquitous:3306
    -[ece-install] - New ECE instance dev1 installed.
    -[ece-install] - Its admin interface is available at http://ubiquitous:8080/escenic-admin
    -[ece-install] - You can view all installed plugin & engine versions with:
    -[ece-install]   ece -i dev1 versions
    -[ece-install] - Cache server is up and running at http://ubiquitous:80/
    -[ece-install] - Widget Framework has been installed into your local
    -[ece-install]   Maven repository and its ECE components have been installed
    -[ece-install] - a new publication mypub has been created
    -[ece-install]
    -[ece-install] Enjoy your time with Escenic Content Engine!
    -[ece-install]
    -[ece-install] -Vizrt Online
    -
    -
    - - - -

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-08-26 18:32:09 CST + Date: 2011-09-08 15:54:44 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index 83a22be5..5fe2f9cd 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-08-26 18:32:11 CST +Date: 2011-09-08 15:54:43 CST Table of Contents @@ -152,9 +152,19 @@ With the configuration file in place, you're now ready to run it: -If something goes astray with the script, and the log file cannot give -you enough clues, the ultimate way of debugging this is to run the -BASH interpreter with the -x flag: +ece-install will try to "fail fast", exiting as soon as it detects an +error during its execution: + + + [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-( + [ece-install-1] See /var/log/ece-install.log for further details. + + + + +As a last resort, if something goes astray with the script, and the +log file cannot give you enough clues, the ultimate way of debugging +this is to run the BASH interpreter with the -x flag: # bash -x ece-install @@ -515,58 +525,5 @@ file: The following output is produced from ece-install: - - - root@ubiquitous:~# bash ece-install - [ece-install] Full Automatic Install (FAI) enabled. - [ece-install] All user input will be read from /root/.ece-install.conf - [ece-install] I'm logging to /var/log/ece-install.log - [ece-install] Setting correct permissions on all ECE related directories ... - [ece-install] Stopping conflicting processes ... - [ece-install] Installing 3rd party packages needed by ece-install ... - [ece-install] Setting up the ece UNIX scripts ... - [ece-install] Adding the ece init.d script to the default run levels... - [ece-install] Setting up the escenic user's UNIX environment ... - [ece-install] Installing an all-in-one environment on ubiquitous ... - [ece-install] Installing the database server on ubiquitous ... - [ece-install] Installing the Percona database ... - [ece-install] Downloading Escenic software from technet.escenic.com ... - [ece-install] Setting up the Escenic Content Engine & its plugins ... - [ece-install] Setting up the ECE database schema ... - [ece-install] Installing 3rd party packages needed by ECE ... - [ece-install] Setting up the Assembly Tool ... - [ece-install] Setting up the basic Nursery configuration ... - [ece-install] Setting up the application server ... - [ece-install] Setting up proper log4j & Java logging configuration ... - [ece-install] Setting correct permissions on all ECE related directories ... - [ece-install] Assembling, deploying & starting dev1 ... - [ece-install] Installing a caching server on ubiquitous ... - [ece-install] Setting up Varnish to match your environment ... - [ece-install] Setting up solr ... - [ece-install] Creating a Maven settings file: /root/.m2/settings.xml ... - [ece-install] Downloading Widget Framework from technet.escenic.com ... - [ece-install] Installing Widget Framework into your local Maven repository ... - [ece-install] Getting ready to create a new publiation ... - [ece-install] Setting up the mypub publication ... - [ece-install] Basing mypub.war on the Widget Framework Demo ... - [ece-install] Creating publication mypub using dev1 ... - [ece-install] Assembling, deploying & starting dev1 ... - [ece-install] The installation is now complete! - [ece-install] - It took 0d 0h 29m 40s - [ece-install] - DB is now set up on ubiquitous:3306 - [ece-install] - New ECE instance dev1 installed. - [ece-install] - Its admin interface is available at http://ubiquitous:8080/escenic-admin - [ece-install] - You can view all installed plugin & engine versions with: - [ece-install] ece -i dev1 versions - [ece-install] - Cache server is up and running at http://ubiquitous:80/ - [ece-install] - Widget Framework has been installed into your local - [ece-install] Maven repository and its ECE components have been installed - [ece-install] - a new publication mypub has been created - [ece-install] - [ece-install] Enjoy your time with Escenic Content Engine! - [ece-install] - [ece-install] -Vizrt Online - - - +CANNOT INCLUDE FILE /tmp/ece-install.out From fd0aa2a5e300c65b86ae48b7ed399be5ae1eaef5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Sep 2011 16:19:05 +0800 Subject: [PATCH 0017/2585] - set_correct_permissions will now create a user non interactively if ece_user doesn't exist on the system. the user will be prompted to set the password for the given user at the end of the installation process. Note that the new user is already usable as he can be used with SSH keys or via "su - $ece_user -c , as is the case with /etc/init.d/ece. This should close https://github.com/skybert/ece-scripts/issues/7, thanks goes to mogsie for pointing this one out. --- usr/sbin/ece-install | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a99d8f1e..375bf12f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -834,7 +834,10 @@ function set_correct_permissions() if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then # TODO add support for useradd - run adduser $ece_user -p $ece_user + run adduser $ece_user \ + --disabled-password \ + --gecos "Escenic-user,Room,Work,Home,Other" + add_next_step "- Set a password for the new $ece_user (passwd $ece_user)" fi if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then run addgroup $ece_group From 2ab70f1b8571f3a3ba6dc07ca2d3b67c9fe228bc Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Sep 2011 17:34:06 +0800 Subject: [PATCH 0018/2585] - added set_up_engine_directories to install_ece_instance - added /var/lib/escenic/solr/data/index to the dir_list, this should fix https://github.com/skybert/ece-scripts/issues/10 - changed sed stanzas from using cmd() to instead have a manual call to exit_on_error, passing string arguments inside sub calls got too difficult for me :-) --- usr/sbin/ece-install | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 375bf12f..9b9c82ec 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -113,7 +113,7 @@ PROFILE_CREATE_PUBLICATION=9 # Because of issue VF-3559, we also create the default family and host # directories. -dir_list=" +engine_dir_list=" $common_nursery_dir /etc/escenic/engine/family/default /etc/escenic/engine/host/localhost @@ -121,6 +121,7 @@ $common_nursery_dir /var/cache/escenic /var/crash/escenic /var/lib/escenic +/var/lib/escenic/solr/data/index /var/log/escenic /var/run/escenic /var/spool/escenic/migration @@ -133,10 +134,6 @@ function make_dir() fi } -for el in $dir_list; do - make_dir $el -done - function make_ln() { if [ -e $1 -a ! -h $(basename $1) ]; then @@ -147,6 +144,13 @@ function make_ln() fi } +function set_up_engine_directories() +{ + for el in $engine_dir_list; do + make_dir $el + done +} + # TODO download documentation to # /usr/share/doc/escenic/content-engine-/ @@ -272,10 +276,15 @@ EOF run cd /opt/escenic/assemblytool/ run ant -q initialize \ 1>>$log 2>>$log - run sed -i 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ - assemble.properties - run sed -i 's/\#\# plugins\ =\ \/path\/to\/plugins/plugins=\/opt\/escenic\/assemblytool\/plugins/g' \ - assemble.properties + sed -i 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ + assemble.properties \ + 1>>$log 2>>$log + exit_on_error "sed on assemble.properties" + + sed -i 's/\#\# plugins\ =\ \/path\/to\/plugins/plugins=\/opt\/escenic\/assemblytool\/plugins/g' \ + assemble.properties \ + 1>>$log 2>>$log + exit_on_error "sed on assemble.properties" } function set_up_engine_and_plugins() @@ -439,6 +448,7 @@ EOF file=$common_nursery_dir/com/escenic/community/CommunityEngine.properties sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file + exit_on_error "sed on $file" file=$common_nursery_dir/com/escenic/webstart/StudioConfig.properties cat >> $file < $tomcat_base/conf/catalina.properties cat > $tomcat_base/conf/server.xml <>$log 2>>$log file=/etc/default/varnish - run sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file + sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file + exit_on_error "sed on $file" cat > /etc/varnish/default.vcl < Date: Thu, 8 Sep 2011 17:56:55 +0800 Subject: [PATCH 0019/2585] - fixed sed bug in catalina.properties (improving old code often introduces new bugs, funny that). - (re)added hint on editing /etc/default/ece to set the ECE/search/RMI hub instances you want to start. ece-install will now include this in the next step list: [ece-install-38] - Edit /etc/default/ece to configure the instances [ece-install-39] on ubiquitous-0-0-7 you want to start at boot time. --- usr/sbin/ece-install | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9b9c82ec..70b8d64f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -692,8 +692,11 @@ function set_up_app_server run cd $tomcat_base/lib make_ln $jdbc_driver + file=$tomcat_base/conf/catalina.properties sed -i 's/common.loader=/common.loader=\$\{catalina.base\}\/escenic\/lib\/\*\.jar\,/g' \ - > $tomcat_base/conf/catalina.properties + $file \ + 1>>$log 2>>$log + exit_on_error "sed on $file" cat > $tomcat_base/conf/server.xml < @@ -893,8 +896,8 @@ function print_status_and_next_steps() minutes=$(( seconds_left / 60 )) seconds_left=$(( seconds_left - $minutes * 60 )) - print "The installation is now complete!" - print "- It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s + s="- The installation is now complete!" + print $s" It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s echo "$next_steps" print "" @@ -1490,9 +1493,12 @@ EOF admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin add_next_step "- New ECE instance $instance_name installed." - add_next_step "- Its admin interface is available at $admin_uri" + add_next_step "- Its admin interface is available at" + add_next_step " $admin_uri" add_next_step "- You can view all installed plugin & engine versions with:" add_next_step " ece -i $instance_name versions" + add_next_step "- Edit /etc/default/ece to configure the instances " + add_next_step " on $HOSTNAME you want to start at boot time." } function install_presentation_server() From 12bcb6cae7a30c235a84194e4480a17406ea8a8b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 9 Sep 2011 14:30:52 +0800 Subject: [PATCH 0020/2585] - added update_type_instances_to_start_up, based on idea & code by mogsie: 70d00024f5091a76468f33dd9aa79630fa5b11ad. I've added support for the search instance list too (and analysis & rmi hub should follow at some point). - improved set_conf_file_value, it now updates current values in place, instead of removing the old value and appending the new to the end of the file. "sed -i" rocks! --- usr/sbin/ece-install | 53 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 70b8d64f..efe3e0ff 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -532,15 +532,12 @@ EOF function set_conf_file_value() { if [ -r $3 ]; then - if [ $(grep $1 $3 | wc -l) -gt 0 ]; then - # TODO update existing values of the property instead of - # removing & appending it. - cat $3 | grep -v ^$1 > $3.tmp - mv $3.tmp $3 + if [ $(grep ^$1 $3 | wc -l) -gt 0 ]; then + sed -i "s/$1=.*/$1=$2/g" $3 + else + echo "$1=$2" >> $3 fi fi - - echo "$1=$2" >> $3 } # The function accepts the following parameters: @@ -1460,6 +1457,46 @@ function ask_for_instance_name() make_dir /etc/escenic/engine/instance/${instance_name} } +# Will update /etc/default/instance with the type instances which are +# installed on the given host. This method should work for engine, +# search and analysis instances. +function update_type_instances_to_start_up() +{ + if [ $type = "engine" ]; then + for el in $(/etc/escenic/ece-*.conf); do + if [ $(echo el | grep analysis | wc -l) -gt 0 ]; then + continue + elif [ $(echo el | grep search | wc -l) -gt 0 ]; then + continue + fi + + current_instance=$(basename ${el} | \ + sed -e 's/ece-//g' -e 's/engine-//g' | \ + cut -d'.' -f1) + all_instances="$all_instances $current_instance" + + set_conf_file_value engine_instance_list \ + "$all_instances" \ + /etc/default/ece + done + elif [ $type = "search" ]; then + for el in $(/etc/escenic/ece-*.conf); do + if [ $(echo el | grep search | wc -l) -lt 1 ]; then + continue + fi + + current_instance=$(basename ${el} | \ + sed -e 's/ece-//g' -e 's/search-//g' | \ + cut -d'.' -f1) + all_instances="$all_instances $current_instance" + + set_conf_file_value search_instance_list \ + "$all_instances" \ + /etc/default/ece + done + fi +} + # $1= # $2= function install_ece_instance() @@ -1490,6 +1527,8 @@ EOF set_correct_permissions assemble_deploy_and_restart_type + update_type_instances_to_start_up + admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin add_next_step "- New ECE instance $instance_name installed." From 85cc17324bcfaf9584998f77761216222f95b35c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 9 Sep 2011 19:04:26 +0800 Subject: [PATCH 0021/2585] - fixed problem with multi value fields in set_conf_file_value - added switching on Varnish versions, this should make the configuration work for both the latest 2 and 3 versions, as well as making such switching easier later on. - fixed a bug in exit_on_error as some programs (notably tar) return 2 and not 1 when they fail. This provides more fail fast behaviour. - update default tomcat bundle to the latest stable 6.x version - fixed bug in set_up_engine_directories - fixed some weeeiiird bug with the script $id always returning 'l' until the fifth function. - fixed a weird bug with wget which all of a sudden started to use IPv6 on my Ubuntu test system. - fixed bugs introduced with the hardening of all commands (sigh!) - fixed bug in update_type_instances_to_start_up (although it still seemed to work). - added some more hardening to install_presentation_server - removed TODO, doesn't apply anymore. --- usr/sbin/ece-install | 81 +++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index efe3e0ff..8da8b7d8 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -19,7 +19,7 @@ ece_user=escenic ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 -tomcat_download=http://ftp.nsysu.edu.tw/Apache/tomcat/tomcat-6/v6.0.32/bin/apache-tomcat-6.0.32.tar.gz +tomcat_download=http://apache.stu.edu.tw/tomcat/tomcat-6/v6.0.33/bin/apache-tomcat-6.0.33.tar.gz # don't touch this one if you're installing on a Debian based system # as it'll be set up for you by the script itself. @@ -50,10 +50,18 @@ NEW_LINE=" next_steps="" +function get_seconds_since_start() +{ + now=`date +%s` + started=`stat -c %Y $pid_file` + seconds=$(( now - started )) + echo "$seconds" +} + function get_id() { - id="[$(basename $0)-$(get_seconds_since_start)]" - echo $id + timestamp=$(get_seconds_since_start) + echo "[$(basename $0)-${timestamp}]" } function debug() @@ -70,7 +78,7 @@ function print() function exit_on_error() { - if [ $? -eq 1 ]; then + if [ $? -gt 0 ]; then print "The command ["$@"] FAILED, exiting :-(" print "See $log for further details." exit 1 @@ -169,10 +177,10 @@ function download_escenic_components() fi run wget --continue \ + --inet4-only \ --http-user $technet_user \ --http-password $technet_password \ - $el \ - 1>>$log 2>>$log + $el done } @@ -436,12 +444,12 @@ function set_up_basic_nursery_configuration() run cp -r $el/misc/siteconfig/* $common_nursery_dir/ done - run cat > $common_nursery_dir/ServerConfig.properties < $common_nursery_dir/ServerConfig.properties < $common_nursery_dir/neo/io/managers/ContentManager.properties < $common_nursery_dir/neo/io/managers/ContentManager.properties <> $3 + echo "$1=\"$2\"" >> $3 fi + else + echo "$1=\"$2\"" >> $3 fi } @@ -668,13 +678,12 @@ function set_up_app_server fi run cd $download_dir - run wget --continue $tomcat_download \ - 1>>$log 2>>$log + run wget --continue --inet4-only $tomcat_download run cd /opt/ run tar xzf $download_dir/apache-tomcat*.tar.gz run ln -sf apache-tomcat* tomcat - + tomcat_base=/opt/tomcat-${instance_name} make_dir $tomcat_base @@ -853,7 +862,7 @@ function set_correct_permissions() run addgroup $ece_group fi - for el in $dir_list; do + for el in $engine_dir_list; do if [ -d $el ]; then run chown -R ${ece_user}:${ece_group} $el \ 1>>$log 2>>$log @@ -871,14 +880,6 @@ function set_correct_permissions() fi } -function get_seconds_since_start() -{ - now=`date +%s` - started=`stat -c %Y $pid_file` - seconds=$(( now - started )) - echo "$seconds" -} - function print_status_and_next_steps() { # su - $ece_user -c "ece versions" 1>>$log 2>>$log @@ -1088,9 +1089,6 @@ function un_install_ece() sun-java6-jdk apt-get clean - - # prepare for new install - mv /tmp/*.zip /tmp/ece-downloads/ } function stop_conflicting_processes() @@ -1105,6 +1103,10 @@ function set_up_varnish() print "Setting up Varnish to match your environment ..." /etc/init.d/varnish stop 1>>$log 2>>$log + # we need to swap standard err and standard out here as varnishd + # -V for some reason writes to standard error. + using_varnish_3=$(varnishd -V 3>&1 1>&2 2>&3 | grep varnish-3 | wc -l) + file=/etc/default/varnish sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file exit_on_error "sed on $file" @@ -1134,9 +1136,8 @@ acl adactus { } EOF - for el in $backend_servers; do - appserver_id=$(echo $el | cut -d':' -f1 | sed 's/-/_/g' ) + appserver_id=$(echo $el | cut -d':' -f1 | sed 's/-/_/g') appserver_host=$(echo $el | cut -d':' -f1) appserver_port=$(echo $el | cut -d':' -f2) @@ -1206,7 +1207,18 @@ sub vcl_deliver { * URL has been fetched from cache or not. */ if (obj.hits > 0) { +EOF + if [ $using_varnish_3 -gt 0 ]; then + cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <>$log 2>>$log + exit_on_error "su - $ece_user -c \"$ece_command\"" } function ensure_that_instance_is_running() @@ -1701,7 +1712,6 @@ function install_widget_framework() fi print "Creating a Maven settings file: $HOME/.m2/settings.xml ..." - # TODO consider //${ece_user} instead make_dir $HOME/.m2 cat > $HOME/.m2/settings.xml < @@ -1741,6 +1751,7 @@ EOF for el in $wf_download_list; do cd $download_dir wget --continue \ + --inet4-only \ --http-user $technet_user \ --http-password $technet_password \ $el \ From 820a0744bb9677ccecb3035e69e4839814be3114 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 11:49:01 +0800 Subject: [PATCH 0022/2585] - set_up_user_enviornment would print to error messages if the ece_user or root didn't have an .bashrc. This should fix https://github.com/skybert/ece-scripts/issues/14 - removed TODO which has been done (java_home can be read from the configuration file). --- usr/sbin/ece-install | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8da8b7d8..d18b2eb5 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1024,21 +1024,19 @@ function set_up_user_enviornment() { print "Setting up the ${ece_user} user's UNIX environment ..." - # TODO get this from ece.conf and other magic - - if [ $(grep JAVA_HOME /home/$ece_user/.bashrc | wc -l) -lt 1 ]; then + if [ $(grep JAVA_HOME /home/$ece_user/.bashrc 2>/dev/null | wc -l) \ + -lt 1 ]; then run echo JAVA_HOME=$java_home >> /home/$ece_user/.bashrc fi - if [ $(grep JAVA_HOME /root/.bashrc | wc -l) -lt 1 ]; then + + if [ $(grep JAVA_HOME /root/.bashrc 2>/dev/null | wc -l) -lt 1 ]; then run echo JAVA_HOME=$java_home >> /root/.bashrc fi - if [ $on_debian_or_derivative -eq 1 ]; then - export JAVA_HOME=$java_home - fi + export JAVA_HOME=$java_home - if [ $(grep bash_completion.d/ece /home/$ece_user/.bashrc | wc -l) -lt 1 ] - then + if [ $(grep bash_completion.d/ece /home/$ece_user/.bashrc 2>/dev/null | \ + wc -l) -lt 1 ]; then run echo ". /etc/bash_completion.d/ece" \ >> /home/$ece_user/.bashrc fi From 8bf004389c430e406249503f895e54b3b989cf9a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 12:39:20 +0800 Subject: [PATCH 0023/2585] - the skip-tests for assemblytool and engine in set_up_engine_and_plugins wasn't working and hence extracted assemblytool in /opt/escenic as well as re-extracting engine. The skip-tests have been re-written. This should fix https://github.com/skybert/ece-scripts/issues/16 - added comment inside set_up_engine_and_plugins to clarify that we're only extracting plugins in the for loop. --- usr/sbin/ece-install | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index d18b2eb5..6ec43229 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -317,15 +317,15 @@ function set_up_engine_and_plugins() else debug "engine- is already there, skipping to next step." fi - - # we extract them in /opt/escenic as we want to re-use - # them between minor updates of ECE. + + # we now extract all the plugins. We extract them in /opt/escenic + # as we want to re-use them between minor updates of ECE. cd /opt/escenic/ for el in $download_dir/*.zip; do - if [ "${el/engine*}" == "" ] ; then - continue - elif [ "${el/assemblytool*}" == "" ] ; then - continue + if [ $(basename $el | grep ^engine-.*.zip | wc -l) -gt 0 ]; then + continue + elif [ $(basename $el | grep ^assemblytool-.*.zip | wc -l) -gt 0 ]; then + continue fi run unzip -u $el \ From dca92e0365808789b466bc8878c2887b6c73386e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 13:34:56 +0800 Subject: [PATCH 0024/2585] - all ece-install profiles now get [ece-install-116] - The installation is now complete! It took 0d 0h 1m 56s [ece-install-116] - See /usr/share/doc/escenic/ece-install-guide.txt for [ece-install-116] a full overview of files and directories set up by [ece-install-116] ece-install [ece-install-116] - For further reading, in-depth guides on installing, [ece-install-116] configuring and developing Escenic web sites can be [ece-install-116] fount at http://documentation.vizrt.com/ece-5.3.html - ECE instance profiles additionally get: [ece-install-114] - New ECE instance editor1 installed. [ece-install-114] - Its admin interface is available at [ece-install-115] http://ubiquitous-7:8180/escenic-admin [ece-install-115] - You can view all installed plugin & engine versions with: [ece-install-115] ece -i editor1 versions [ece-install-115] - type 'ece help' to see all the options of this script [ece-install-115] which allows you to control all the Escenic components. [ece-install-115] Also, see /usr/share/doc/escenic/ece-guide.txt for [ece-install-116] further information on the ece script. [ece-install-115] - Verify that /etc/default/ece lists all the instances [ece-install-115] on ubiquitous-7 you wish to start at boot time. --- usr/sbin/ece-install | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 6ec43229..c74ae23f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -882,8 +882,6 @@ function set_correct_permissions() function print_status_and_next_steps() { - # su - $ece_user -c "ece versions" 1>>$log 2>>$log - now=`date +%s` started=`stat -c %Y $pid_file` seconds=$(( now - started )) @@ -896,6 +894,18 @@ function print_status_and_next_steps() s="- The installation is now complete!" print $s" It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s + add_next_step "- See /usr/share/doc/escenic/ece-install-guide.txt for" + add_next_step " a full overview of files and directories set up by" + add_next_step " $(basename $0)" + add_next_step "- type 'ece help' to see all the options of this script" + add_next_step " which allows you to control all the Escenic components." + add_next_step " Also, see /usr/share/doc/escenic/ece-guide.txt for" + add_next_step " further information on the ece script." + add_next_step "- For further reading, in-depth guides on installing, " + add_next_step " configuring and developing Escenic web sites can be " + add_next_step " fount at http://documentation.vizrt.com/ece-5.3.html" + + echo "$next_steps" print "" @@ -1544,6 +1554,10 @@ EOF add_next_step " $admin_uri" add_next_step "- You can view all installed plugin & engine versions with:" add_next_step " ece -i $instance_name versions" + add_next_step "- type 'ece help' to see all the options of this script" + add_next_step " which allows you to control all the Escenic components." + add_next_step " Also, see /usr/share/doc/escenic/ece-guide.txt for" + add_next_step " addtional information on the ece script." add_next_step "- Verify that /etc/default/ece lists all the instances " add_next_step " on $HOSTNAME you wish to start at boot time." } @@ -1813,6 +1827,7 @@ EOF set_up_solr assemble_deploy_and_restart_type + } # useful for development and test environments. From 468d4c0aa2aa07465c9f9e8245618972c391c4ee Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 16:58:26 +0800 Subject: [PATCH 0025/2585] - fixed bug in set_conf_file_value which would fail to update values with slashes '/': sed: -e expression #1, char 32: unknown option to `s' The fix was to use a different delimiter for the sed fields. I chose tilde, '~' as this is the most unlikely character I could think of during my massive two minute brain search. --- usr/sbin/ece-install | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c74ae23f..34763e26 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -444,10 +444,26 @@ function set_up_basic_nursery_configuration() run cp -r $el/misc/siteconfig/* $common_nursery_dir/ done + public_host_name=$HOSTNAME:${appserver_port} + if [ $fai_enabled -eq 1 ]; then + if [ -n "$fai_public_host_name" ]; then + public_host_name=$fai_public_host_name + fi + else + print "What is the public address of your website?" + print "Press ENTER to use the default ($public_host_name)" + echo -n "Your choice [$public_host_name]> " + read user_host_name + fi + + if [ -n "$user_host_name" ]; then + public_host_name=$user_host_name + fi + cat > $common_nursery_dir/ServerConfig.properties < $common_nursery_dir/neo/io/managers/ContentManager.properties <> $3 fi @@ -604,12 +620,12 @@ function set_up_app_server if [ $fai_enabled -eq 0 ]; then print "On which ports do you wish to run the app server on?" - print "Press ENTER to accept the default (port 8080, shutdown port 8005)" + print "Press ENTER to accept port 8080 and shutdown port 8005" print "Or enter: , e.g.: '8180 8105'" echo -n "Your choice [8080 8005]> " read user_ports - if [ -z "$user_ports" ]; then + if [ -n "$user_ports" ]; then appserver_port=$(echo $user_ports | cut -d' ' -f1) shutdown_port=$(echo $user_ports | cut -d' ' -f2) fi From 48acf0da4da42e2d8c1ee1a1da1fc02d1054215f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 17:31:43 +0800 Subject: [PATCH 0026/2585] - added note on the new fai_public_host_name option. --- usr/share/doc/escenic/ece-install-guide.org | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index e764d43d..f72ec349 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -264,6 +264,7 @@ $HOME/.ece-install.conf file of the root user: | fai\_editor\_port | 8080 | HTTP port of the editor instance | | fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | | fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | | fai\_presentation\_install | 0 | Install the presentation server profile | | fai\_presentation\_name | web1 | Name of the presentation server instance | | fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | From 77faae287d6bcf12912cb2e62b6bac36820eec00 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 17:33:12 +0800 Subject: [PATCH 0027/2585] - updated txt and html guides --- usr/share/doc/escenic/ece-install-guide.html | 15 ++++++++++++--- usr/share/doc/escenic/ece-install-guide.txt | 10 ++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index f50bec49..aa572bec 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -814,6 +814,7 @@

    6 Full Automatic Install (F fai_editor_port8080HTTP port of the editor instance fai_editor_shutdown8005Shutdown port of the editor instance fai_enabled0Whether or not to run ece-install in FAI mode +fai_public_host_name${HOSTNAME}:8080The public address for your website fai_presentation_install0Install the presentation server profile fai_presentation_nameweb1Name of the presentation server instance fai_presentation_port8080HTTP port of the presentation server instance @@ -1048,15 +1049,23 @@

    12 Example Output FAI

    The following output is produced from ece-install: -CANNOT INCLUDE FILE /tmp/ece-install.out

    + + +
    hei
    +
    +
    + + + +

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-09-08 15:54:44 CST + Date: 2011-09-12 17:32:07 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index 5fe2f9cd..a198d473 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-09-08 15:54:43 CST +Date: 2011-09-12 17:32:03 CST Table of Contents @@ -370,6 +370,7 @@ $HOME/.ece-install.conf file of the root user: fai\_editor\_port 8080 HTTP port of the editor instance fai\_editor\_shutdown 8005 Shutdown port of the editor instance fai\_enabled 0 Whether or not to run ece-install in FAI mode + fai\_public\_host\_name ${HOSTNAME}:8080 The public address for your website fai\_presentation\_install 0 Install the presentation server profile fai\_presentation\_name web1 Name of the presentation server instance fai\_presentation\_port 8080 HTTP port of the presentation server instance @@ -525,5 +526,10 @@ file: The following output is produced from ece-install: -CANNOT INCLUDE FILE /tmp/ece-install.out + + + hei + + + From e4a162e77ab560c1df8ab39439cdf41677ddf9d7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 17:34:03 +0800 Subject: [PATCH 0028/2585] - renamed exit_on_(the_first)_error and my_id so that sourcing and running the script doesn't lead to name conflict with ece-install. --- usr/sbin/drop-and-create-ecedb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/usr/sbin/drop-and-create-ecedb b/usr/sbin/drop-and-create-ecedb index f10a5d14..9eb9c574 100644 --- a/usr/sbin/drop-and-create-ecedb +++ b/usr/sbin/drop-and-create-ecedb @@ -20,7 +20,7 @@ db_schema=ece5db db_host=localhost ece_home=/opt/escenic/engine db_product=mysql -id="[`basename $0`]" +my_id="[`basename $0`]" # oracle specific settings create_oracle_user=0 @@ -33,15 +33,15 @@ debug=0 function debug() { if [ $debug -eq 1 ]; then - echo $id $1 + echo $my_id $1 fi } -function exit_on_error() +function exit_on_the_first_error() { if [ $? -eq 1 ]; then - echo $id $@ "FAILED, exiting :-(" + echo $my_id $@ "FAILED, exiting :-(" exit 1 fi } @@ -72,7 +72,7 @@ function run_db_scripts() mysql -u $db_user -p$db_password -h $db_host $db_schema < $file fi fi - exit_on_error "running $el" + exit_on_the_first_error "running $el" done } From 48bd4da2189dc463e3a0c3f2c6755c6daebd13b6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 17:55:27 +0800 Subject: [PATCH 0029/2585] - added checking of the user not being root to sanity_check. ece will now complain if you try to run it as root: [ece#engine] Sorry, you cannot be root when running ece [ece#engine] The root user can only use /etc/init.d/ece --- usr/bin/ece | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 4fade9de..09f6a536 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -380,6 +380,12 @@ function set_instance_settings() function sanity_check() { + if [ $(whoami) = "root" ]; then + print "Sorry, you cannot be root when running $(basename $0)" + print "The root user can only use /etc/init.d/ece" + exit 1 + fi + verify_that_directory_and_file_are_writeable $log_file verify_that_directory_and_file_are_writeable $pid_file From 241557f42055340e6ded312c93f4653f04fc7005 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 18:15:13 +0800 Subject: [PATCH 0030/2585] - added installation section with overview of the different files used by ece - added note on running this script as a non-root user --- usr/share/doc/escenic/ece-guide.org | 56 +++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index f4f8e808..0b50e445 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -81,3 +81,59 @@ The following commands are available: *) only applicable if type is 'engine' #+END_SRC + +* Installation +The ece script and ece.conf may be used on any Unix like system that +has a fairly recent version of BASH installed. + +** Overview of File Paths Used by the ece script +These are recommended files and locations for using the ece script: + +|----------------------------------+-----------------------------| +| Path | Explanation | +|----------------------------------+-----------------------------| +| /usr/bin/ece | The script itself | +| /etc/escenic/ece.conf | The main configuration file | +| /etc/escenic/ece-.conf | Instance specific settings | +|----------------------------------+-----------------------------| + +As you can see in ece.conf, there are a number of default locations +dealing with log files, pid files, crash files as well as application +server files. The defaults all follow the File Hierarchy Standard, but +you may of course change these to your liking. + +If you wish to put the .conf files in other places, you may like to know +that the ece script has preset list of locations where it looks for +the .conf files mentioned above, namely: + +- current working directory +- /etc/escenic//instance/ +- /etc/escenic//host/ +- /etc/escenic//common +- /etc/escenic/ +- /etc/escenic +- /../etc + +You may override this list of locations by setting the +this environment variable in your .bashrc or similar: +#+BEGIN_SRC sh +ECE_CONF_LOCATIONS +#+END_SRC + +The reason for having so many options is because various Escenic +consultants, partners and customers have requested these locations to +fit their systems. As you can see, fitting everyone's fancy adds up +over time :-) + +* Running the ece script +You must be normal user to run the ece script, otherwise it will +complain: +#+BEGIN_SRC sh +[ece#engine] Sorry, you cannot be root when running ece +[ece#engine] The root user can only use /etc/init.d/ece +#+END_SRC + +As it mentions, the root user may use the init.d script and the +accompanying /etc/default/ece to command the different ECE, EAE and +RMI hub instances on your system. + From 4fe3054d9e16f58b404dfd1486514224f8e1dc6a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 18:15:35 +0800 Subject: [PATCH 0031/2585] - update html and text --- usr/share/doc/escenic/ece-guide.html | 127 +++++++++++++++++++++++++-- usr/share/doc/escenic/ece-guide.txt | 124 ++++++++++++++++++++++++-- 2 files changed, 236 insertions(+), 15 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.html b/usr/share/doc/escenic/ece-guide.html index c9a95efc..c0903d8c 100644 --- a/usr/share/doc/escenic/ece-guide.html +++ b/usr/share/doc/escenic/ece-guide.html @@ -7,7 +7,7 @@ Guide for the /usr/bin/ece Script - + @@ -221,6 +221,12 @@

    Table of Contents

  • 2 Getting an overview of all available options
  • +
  • 3 Installation + +
  • +
  • 4 Running the ece script
  • @@ -363,14 +369,123 @@

    2 Getting an overview of al +

    + -
    -

    Date: 2011-08-25 12:08:05 CST

    -

    Author: Torstein Krause Johansen

    -

    Org version 7.6 with Emacs version 23

    -Validate XHTML 1.0 + +
    +

    3 Installation

    +
    + +

    The ece script and ece.conf may be used on any Unix like system that +has a fairly recent version of BASH installed. +

    +
    + +
    +

    3.1 Overview of File Paths Used by the ece script

    +
    + +

    These are recommended files and locations for using the ece script: +

    + + ++ + + + + + + + + +
    PathExplanation
    /usr/bin/eceThe script itself
    /etc/escenic/ece.confThe main configuration file
    /etc/escenic/ece-<instance>.confInstance specific settings
    + + +

    +As you can see in ece.conf, there are a number of default locations +dealing with log files, pid files, crash files as well as application +server files. The defaults all follow the File Hierarchy Standard, but +you may of course change these to your liking. +

    +

    +If you wish to put the .conf files in other places, you may like to know +that the ece script has preset list of locations where it looks for +the .conf files mentioned above, namely: +

    +
      +
    • current working directory +
    • +
    • /etc/escenic/<type>/instance/<instance name> +
    • +
    • /etc/escenic/<type>/host/<hostname> +
    • +
    • /etc/escenic/<type>/common +
    • +
    • /etc/escenic/<type> +
    • +
    • etc/escenic - <current working direcotry>../etc +
    • +
    + + +

    +You may override this list of locations by setting the +this environment variable in your .bashrc or similar: +

    + + +
    ECE_CONF_LOCATIONS 
    +
    +
    + + + + +

    +The reason for having so many options is because various Escenic +consultants, partners and customers have requested these locations to +fit their systems. As you can see, fitting everyone's fancy adds up +over time :-) +

    +
    +
    + +
    + +
    +

    4 Running the ece script

    +
    + +

    You must be normal user to run the ece script, otherwise it will +complain: +

    + + +
    [ece#engine] Sorry, you cannot be root when running ece
    +[ece#engine] The root user can only use /etc/init.d/ece
    +
    +
    + + + + +

    +As it mentions, the root user may use the init.d script and the +accompanying /etc/default/ece to command the different ECE, EAE and +RMI hub instances on your system. +

    +
    +
    +
    +
    +

    + Author: Torstein Krause Johansen (tkj@vizrt.com) + Date: 2011-09-12 18:14:44 CST +

    diff --git a/usr/share/doc/escenic/ece-guide.txt b/usr/share/doc/escenic/ece-guide.txt index d4fefd5d..d3ee7c8d 100644 --- a/usr/share/doc/escenic/ece-guide.txt +++ b/usr/share/doc/escenic/ece-guide.txt @@ -1,8 +1,8 @@ - Guide for the ece Script - ======================== + Guide for the /usr/bin/ece Script + ================================= Author: Torstein Krause Johansen -Date: 2011-08-24 13:30:04 CST +Date: 2011-09-12 18:14:41 CST Table of Contents @@ -12,6 +12,9 @@ Table of Contents 1.2 Completing types of servers the ece scripts can operate on 1.3 Completing the publication resources available 2 Getting an overview of all available options +3 Installation + 3.1 Overview of File Paths Used by the ece script +4 Running the ece script 1 TAB completion @@ -26,7 +29,6 @@ You will find yourself using this regularly, both for speed, but also to remember all the different options and their correct wording: - $ ece TAB TAB applog deploy log start threaddump assemble help outlog status update @@ -34,7 +36,6 @@ to remember all the different options and their correct wording: - The commands are all described in detail in "ece help" 1.2 Completing types of servers the ece scripts can operate on @@ -56,19 +57,124 @@ To help selecting the correct publication resource, you can make ece list and complete the available resource names: - $ ece -p mypub -r TAB TAB content-type image-version layout-group feature layout menu - 2 Getting an overview of all available options ----------------------------------------------- $ ece help - - + Usage: /home/torstein/bin/ece [-t ] [-i ] + + DESCRIPTION + -t --type + The following types are available: + engine - The Escenic Content Engine, this is the default + and is the assumed type if none is specified. + search - A standalone search indexer and solr instance + rmi-hub - The RMI hub responsible for the internal + communication between the ECE instances. + analysis - The Escenic Analysis Engine also knows as 'Stats' + + -i --instance + The type instance, such as editor1, web1 or search1 + + -p --publication + Needed only for updating publication resources + + -r --resource + Used for updating publication resources. + Must be one of: content-type, feature, layout, layout-group + image-version, menu + + -v --verbose + Prints out debug statements, useful for debugging. + + The following commands are available: + applog the type's app server log + assemble runs the Assembly Tool *) + clean removes temporary files created by /home/torstein/bin/ece *) + deploy deploys the assembled EAR *) + help prints this help screen + kill uses force to stop the type + log the type's Log4J log + outlog the [ece#engine] script log (system out log) + restart restarts the type + start starts the type + status checks if the type is running + stop stops the type + threaddump write a thread dump to standard out (system out log) + update update publication resources + versions lists the ECE component versions + + *) only applicable if type is 'engine' + + + + +3 Installation +--------------- +The ece script and ece.conf may be used on any Unix like system that +has a fairly recent version of BASH installed. + +3.1 Overview of File Paths Used by the ece script +================================================== +These are recommended files and locations for using the ece script: + + Path Explanation + ----------------------------------+----------------------------- + /usr/bin/ece The script itself + /etc/escenic/ece.conf The main configuration file + /etc/escenic/ece-.conf Instance specific settings + +As you can see in ece.conf, there are a number of default locations +dealing with log files, pid files, crash files as well as application +server files. The defaults all follow the File Hierarchy Standard, but +you may of course change these to your liking. + +If you wish to put the .conf files in other places, you may like to know +that the ece script has preset list of locations where it looks for +the .conf files mentioned above, namely: + +- current working directory +- /etc/escenic//instance/ +- /etc/escenic//host/ +- /etc/escenic//common +- /etc/escenic/ +- /etc/escenic +- /../etc + +You may override this list of locations by setting the +this environment variable in your .bashrc or similar: + + + ECE_CONF_LOCATIONS + + + + +The reason for having so many options is because various Escenic +consultants, partners and customers have requested these locations to +fit their systems. As you can see, fitting everyone's fancy adds up +over time :-) + +4 Running the ece script +------------------------- +You must be normal user to run the ece script, otherwise it will +complain: + + + [ece#engine] Sorry, you cannot be root when running ece + [ece#engine] The root user can only use /etc/init.d/ece + + + + +As it mentions, the root user may use the init.d script and the +accompanying /etc/default/ece to command the different ECE, EAE and +RMI hub instances on your system. From 16d6fb604c56ee3a57b299687a1f2b9dcdf79df6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 19:00:04 +0800 Subject: [PATCH 0032/2585] - escaped an underscore in the .ece-install.conf table --- usr/share/doc/escenic/ece-install-guide.org | 56 ++++++++++----------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index f72ec349..d44993cb 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -250,34 +250,34 @@ The ece-install script has support for doing a full automatic install The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user: -|--------------------------------+------------------+-----------------------------------------------| -| Parameter | Default | Description | -|--------------------------------+------------------+-----------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use_instance | dev1 | Name of local instance to use for creation | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|--------------------------------+------------------+-----------------------------------------------| +|---------------------------------+------------------+-----------------------------------------------| +| Parameter | Default | Description | +|---------------------------------+------------------+-----------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|---------------------------------+------------------+-----------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 49bce2b7e45f801553edba42dea4e76b8e6b56a3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 19:01:01 +0800 Subject: [PATCH 0033/2585] - removed capitalising ece-script. --- usr/share/doc/escenic/ece-install-guide.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index d44993cb..12b6f5be 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -21,7 +21,7 @@ You may also look inside the ece-install script for all calls to the 'assert\_pre\_prequesite' method and you'll get a list of the binaries ece-install assumes are present. -* A Note on Running Ece-Install On Non-GNU/Linux Systems +* A Note on Running ece-install On Non-GNU/Linux Systems Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp & tar. If you're running a system which has a different set of core tools, such as any of the BSDs (including Mac OS X) or Solaris, make From 8503d8a9cd93875894bda56b375fe37576923592 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 19:03:50 +0800 Subject: [PATCH 0034/2585] - newly generated with FAI output included so that readers can see what information ece-install gives back. --- usr/share/doc/escenic/ece-install-guide.html | 76 +++++++++++- usr/share/doc/escenic/ece-install-guide.txt | 123 ++++++++++++++----- 2 files changed, 163 insertions(+), 36 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index aa572bec..d4ad2eca 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -222,7 +222,7 @@

    Table of Contents

    -
  • 2 A Note on Running Ece-Install On Non-GNU/Linux Systems
  • +
  • 2 A Note on Running ece-install On Non-GNU/Linux Systems
  • 3 Downloading the ece-install Script
  • 4 Running the ece-install Script
  • 5 Available Server Profiles @@ -306,7 +306,7 @@

    1.2 Other GNU/Linux and U
    -

    2 A Note on Running Ece-Install On Non-GNU/Linux Systems

    +

    2 A Note on Running ece-install On Non-GNU/Linux Systems

    Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp @@ -821,7 +821,7 @@

    6 Full Automatic Install (F fai_presentation_shutdown8005Shutdown port of the presentation instance fai_publication_create0Create a new publication fai_publication_namemypubName of the publication -fai_publication_useinstancedev1Name of local instance to use for creation +fai_publication_use_instancedev1Name of local instance to use for creation fai_rmi_install0Install RMI hub profile fai_search_namesearch1Name of the search instance fai_search_port8080HTTP port of the search instance @@ -1052,7 +1052,71 @@

    12 Example Output FAI

    -
    hei
    +
    root@ubiquitous-7:~# bash ece-install
    +[ece-install-0] Full Automatic Install (FAI) enabled.
    +[ece-install-0] All user input will be read from /root/.ece-install.conf
    +[ece-install-0] I'm logging to /var/log/ece-install.log
    +[ece-install-1] Stopping conflicting processes ...
    +[ece-install-1] Installing 3rd party packages needed by ece-install ...
    +[ece-install-2] Setting up the ece UNIX scripts ...
    +[ece-install-5] Adding the ece init.d script to the default run levels...
    +[ece-install-5] Setting up the doesnt user's UNIX environment ...
    +[ece-install-5] Installing an all-in-one environment on ubiquitous-7 ...
    +[ece-install-5] Installing the database server on ubiquitous-7 ...
    +[ece-install-5] Installing the Percona database ...
    +[ece-install-65] Downloading Escenic software from technet.escenic.com ...
    +[ece-install-66] Setting up the Escenic Content Engine & its plugins ...
    +[ece-install-76] Setting up the ECE database schema ...
    +[ece-install-90] Installing 3rd party packages needed by ECE ...
    +[ece-install-106] Setting up the Assembly Tool ...
    +[ece-install-110] Setting up the basic Nursery configuration ...
    +[ece-install-110] Setting up the application server ...
    +[ece-install-116] Setting up proper log4j & Java logging configuration ...
    +[ece-install-116] Setting correct permissions on ECE related directories ...
    +[ece-install-116] Assembling, deploying & starting dev1 ...
    +[ece-install-183] Installing a caching server on ubiquitous-7 ...
    +[ece-install-215] Setting up Varnish to match your environment ...
    +[ece-install-220] Setting up solr ...
    +[ece-install-221] Creating a Maven settings file: /root/.m2/settings.xml ...
    +[ece-install-221] Downloading Widget Framework from technet.escenic.com ...
    +[ece-install-259] Installing Widget Framework into your local Maven repository ...
    +[ece-install-318] Getting ready to create a new publiation ...
    +[ece-install-318] Setting up the mypub publication ...
    +[ece-install-318] Basing mypub.war on the Widget Framework Demo ...
    +[ece-install-344] Creating publication mypub using dev1 ...
    +[ece-install-382] Assembling, deploying & starting dev1 ...
    +[ece-install-503] - The installation is now complete! It took 0d 0h 8m 22s
    +[ece-install-90] - DB is now set up on ubiquitous-7:3306
    +[ece-install-181] - New ECE instance dev1 installed.
    +[ece-install-181] - Its admin interface is available at
    +[ece-install-181]   http://ubiquitous-7:8080/escenic-admin
    +[ece-install-181] - You can view all installed plugin & engine versions with:
    +[ece-install-181]   ece -i dev1 versions
    +[ece-install-182] - type 'ece help' to see all the options of this script
    +[ece-install-182]   which allows you to control all the Escenic components.
    +[ece-install-182]   Also, see /usr/share/doc/escenic/ece-guide.txt for
    +[ece-install-182]   addtional information on the ece script.
    +[ece-install-183] - Verify that /etc/default/ece lists all the instances 
    +[ece-install-183]   on ubiquitous-7 you wish to start at boot time.
    +[ece-install-220] - Cache server is up and running at http://ubiquitous-7:80/
    +[ece-install-317] - Widget Framework has been installed into your local
    +[ece-install-318]   Maven repository and its ECE components have been installed
    +[ece-install-502] - a new publication mypub has been created
    +[ece-install-503] - See /usr/share/doc/escenic/ece-install-guide.txt for
    +[ece-install-503]   a full overview of files and directories set up by
    +[ece-install-504]   ece-install
    +[ece-install-504] - type 'ece help' to see all the options of this script
    +[ece-install-504]   which allows you to control all the Escenic components.
    +[ece-install-504]   Also, see /usr/share/doc/escenic/ece-guide.txt for
    +[ece-install-505]   further information on the ece script.
    +[ece-install-505] - For further reading, in-depth guides on installing, 
    +[ece-install-505]   configuring and developing Escenic web sites can be 
    +[ece-install-505]   fount at http://documentation.vizrt.com/ece-5.3.html
    +[ece-install-506]
    +[ece-install-506] Enjoy your time with Escenic Content Engine!
    +[ece-install-506]
    +[ece-install-506] -Vizrt Online
    +
     
     
    @@ -1065,7 +1129,7 @@

    12 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-09-12 17:32:07 CST + Date: 2011-09-12 19:02:48 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index a198d473..ce9cb86a 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-09-12 17:32:03 CST +Date: 2011-09-12 19:02:40 CST Table of Contents @@ -10,7 +10,7 @@ Table of Contents 1 Supported Operating Systems 1.1 Debian based operating systems 1.2 Other GNU/Linux and Unix systems -2 A Note on Running Ece-Install On Non-GNU/Linux Systems +2 A Note on Running ece-install On Non-GNU/Linux Systems 3 Downloading the ece-install Script 4 Running the ece-install Script 5 Available Server Profiles @@ -63,7 +63,7 @@ You may also look inside the ece-install script for all calls to the 'assert\_pre\_prequesite' method and you'll get a list of the binaries ece-install assumes are present. -2 A Note on Running Ece-Install On Non-GNU/Linux Systems +2 A Note on Running ece-install On Non-GNU/Linux Systems --------------------------------------------------------- Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp & tar. If you're running a system which has a different set of core @@ -357,32 +357,32 @@ The ece-install script has support for doing a full automatic install The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user: - Parameter Default Description - --------------------------------+------------------+----------------------------------------------- - fai\_all\_install 0 Install all components on your server. - fai\_cache\_install 0 Install cache server profile - fai\_cache\_backends ${HOSTNAME}:8080 Space separated, e.g. "app1:8080 app2:8080" - fai\_db\_install 0 Install db profile - fai\_db\_host $HOSTNAME Useful for editor & presentation profiles - fai\_db\_port 3306 Useful for editor & presentation profiles - fai\_editor\_install 0 Install the editorial profile - fai\_editor\_name editor1 Name of the editor instance - fai\_editor\_port 8080 HTTP port of the editor instance - fai\_editor\_shutdown 8005 Shutdown port of the editor instance - fai\_enabled 0 Whether or not to run ece-install in FAI mode - fai\_public\_host\_name ${HOSTNAME}:8080 The public address for your website - fai\_presentation\_install 0 Install the presentation server profile - fai\_presentation\_name web1 Name of the presentation server instance - fai\_presentation\_port 8080 HTTP port of the presentation server instance - fai\_presentation\_shutdown 8005 Shutdown port of the presentation instance - fai\_publication\_create 0 Create a new publication - fai\_publication\_name mypub Name of the publication - fai\_publication\_use_instance dev1 Name of local instance to use for creation - fai\_rmi\_install 0 Install RMI hub profile - fai\_search\_name search1 Name of the search instance - fai\_search\_port 8080 HTTP port of the search instance - fai\_search\_shutdown 8005 Shutdown port of the search instance - fai\_wf\_install 0 Install Widget Framework profile + Parameter Default Description + ---------------------------------+------------------+----------------------------------------------- + fai\_all\_install 0 Install all components on your server. + fai\_cache\_install 0 Install cache server profile + fai\_cache\_backends ${HOSTNAME}:8080 Space separated, e.g. "app1:8080 app2:8080" + fai\_db\_install 0 Install db profile + fai\_db\_host $HOSTNAME Useful for editor & presentation profiles + fai\_db\_port 3306 Useful for editor & presentation profiles + fai\_editor\_install 0 Install the editorial profile + fai\_editor\_name editor1 Name of the editor instance + fai\_editor\_port 8080 HTTP port of the editor instance + fai\_editor\_shutdown 8005 Shutdown port of the editor instance + fai\_enabled 0 Whether or not to run ece-install in FAI mode + fai\_public\_host\_name ${HOSTNAME}:8080 The public address for your website + fai\_presentation\_install 0 Install the presentation server profile + fai\_presentation\_name web1 Name of the presentation server instance + fai\_presentation\_port 8080 HTTP port of the presentation server instance + fai\_presentation\_shutdown 8005 Shutdown port of the presentation instance + fai\_publication\_create 0 Create a new publication + fai\_publication\_name mypub Name of the publication + fai\_publication\_use\_instance dev1 Name of local instance to use for creation + fai\_rmi\_install 0 Install RMI hub profile + fai\_search\_name search1 Name of the search instance + fai\_search\_port 8080 HTTP port of the search instance + fai\_search\_shutdown 8005 Shutdown port of the search instance + fai\_wf\_install 0 Install Widget Framework profile As you've probably have guessed, 0 means "false" and 1 means "true" :-) @@ -528,7 +528,70 @@ file: The following output is produced from ece-install: - hei + root@ubiquitous-7:~# bash ece-install + [ece-install-0] Full Automatic Install (FAI) enabled. + [ece-install-0] All user input will be read from /root/.ece-install.conf + [ece-install-0] I'm logging to /var/log/ece-install.log + [ece-install-1] Stopping conflicting processes ... + [ece-install-1] Installing 3rd party packages needed by ece-install ... + [ece-install-2] Setting up the ece UNIX scripts ... + [ece-install-5] Adding the ece init.d script to the default run levels... + [ece-install-5] Setting up the doesnt user's UNIX environment ... + [ece-install-5] Installing an all-in-one environment on ubiquitous-7 ... + [ece-install-5] Installing the database server on ubiquitous-7 ... + [ece-install-5] Installing the Percona database ... + [ece-install-65] Downloading Escenic software from technet.escenic.com ... + [ece-install-66] Setting up the Escenic Content Engine & its plugins ... + [ece-install-76] Setting up the ECE database schema ... + [ece-install-90] Installing 3rd party packages needed by ECE ... + [ece-install-106] Setting up the Assembly Tool ... + [ece-install-110] Setting up the basic Nursery configuration ... + [ece-install-110] Setting up the application server ... + [ece-install-116] Setting up proper log4j & Java logging configuration ... + [ece-install-116] Setting correct permissions on ECE related directories ... + [ece-install-116] Assembling, deploying & starting dev1 ... + [ece-install-183] Installing a caching server on ubiquitous-7 ... + [ece-install-215] Setting up Varnish to match your environment ... + [ece-install-220] Setting up solr ... + [ece-install-221] Creating a Maven settings file: /root/.m2/settings.xml ... + [ece-install-221] Downloading Widget Framework from technet.escenic.com ... + [ece-install-259] Installing Widget Framework into your local Maven repository ... + [ece-install-318] Getting ready to create a new publiation ... + [ece-install-318] Setting up the mypub publication ... + [ece-install-318] Basing mypub.war on the Widget Framework Demo ... + [ece-install-344] Creating publication mypub using dev1 ... + [ece-install-382] Assembling, deploying & starting dev1 ... + [ece-install-503] - The installation is now complete! It took 0d 0h 8m 22s + [ece-install-90] - DB is now set up on ubiquitous-7:3306 + [ece-install-181] - New ECE instance dev1 installed. + [ece-install-181] - Its admin interface is available at + [ece-install-181] http://ubiquitous-7:8080/escenic-admin + [ece-install-181] - You can view all installed plugin & engine versions with: + [ece-install-181] ece -i dev1 versions + [ece-install-182] - type 'ece help' to see all the options of this script + [ece-install-182] which allows you to control all the Escenic components. + [ece-install-182] Also, see /usr/share/doc/escenic/ece-guide.txt for + [ece-install-182] addtional information on the ece script. + [ece-install-183] - Verify that /etc/default/ece lists all the instances + [ece-install-183] on ubiquitous-7 you wish to start at boot time. + [ece-install-220] - Cache server is up and running at http://ubiquitous-7:80/ + [ece-install-317] - Widget Framework has been installed into your local + [ece-install-318] Maven repository and its ECE components have been installed + [ece-install-502] - a new publication mypub has been created + [ece-install-503] - See /usr/share/doc/escenic/ece-install-guide.txt for + [ece-install-503] a full overview of files and directories set up by + [ece-install-504] ece-install + [ece-install-504] - type 'ece help' to see all the options of this script + [ece-install-504] which allows you to control all the Escenic components. + [ece-install-504] Also, see /usr/share/doc/escenic/ece-guide.txt for + [ece-install-505] further information on the ece script. + [ece-install-505] - For further reading, in-depth guides on installing, + [ece-install-505] configuring and developing Escenic web sites can be + [ece-install-505] fount at http://documentation.vizrt.com/ece-5.3.html + [ece-install-506] + [ece-install-506] Enjoy your time with Escenic Content Engine! + [ece-install-506] + [ece-install-506] -Vizrt Online From 22ddfb33129c38465d33425378d7d2b6e39c2773 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 12 Sep 2011 19:13:45 +0800 Subject: [PATCH 0035/2585] no need to have time stamps in the next step messages. --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 34763e26..8f3ba037 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1409,9 +1409,9 @@ function install_cache_server() function add_next_step() { if [ -n "$next_steps" ]; then - next_steps=${next_steps}${NEW_LINE}$(get_id)" "${1} + next_steps=${next_steps}${NEW_LINE}"[$(basename $0)] "${1} else - next_steps=$(get_id)" "${1} + next_steps="[$(basename $0)] "${1} fi } From 10f959078615438ed52495463d3c6cf156f015d5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Sep 2011 16:17:28 +0800 Subject: [PATCH 0036/2585] - added munin node for all installation profiles (called in common_post_install) - the install_munin_node method download's mogise's escenic_jstat plugin and patches it for use with the new ece scripts. - added munin gatherer to the web cache profile - added web server to the web cache profile. This web server will be used for the Adactus/Mobilze access point, optionally serving static content as well as accessing the munin data. --- usr/sbin/ece-install | 186 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 154 insertions(+), 32 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8f3ba037..c74fa89d 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -4,10 +4,6 @@ # five minutes. The setup the user is getting is suitable for a # production environment, except for the fact that with this script, # all ECE components are set up on the same host. -# -# BUG: if creating a publication and WF is downloaded, but not -# installed, then the pub is built from WF, but WF is not added to the -# plugins. This _is_ a corner case, but should still be fixed. # by tkj@vizrt.com @@ -21,8 +17,10 @@ jdbc_driver=/usr/share/java/mysql.jar debug=0 tomcat_download=http://apache.stu.edu.tw/tomcat/tomcat-6/v6.0.33/bin/apache-tomcat-6.0.33.tar.gz -# don't touch this one if you're installing on a Debian based system -# as it'll be set up for you by the script itself. +# The script will install the sun-java6-jdk package on Debian based +# systems and this is the path of the JAVA home with this package. If +# you're using a different system or have other preferences, change +# java_home here. java_home=/usr/lib/jvm/java-6-sun ##################################################################### @@ -49,7 +47,6 @@ NEW_LINE=" # components. next_steps="" - function get_seconds_since_start() { now=`date +%s` @@ -144,11 +141,20 @@ function make_dir() function make_ln() { - if [ -e $1 -a ! -h $(basename $1) ]; then - run ln -s $1 - elif [ ! -e $1 ]; then - print "Tried to make a symlink to $1, but it doesn't exist" - exit 1 + if [ $2 ]; then + if [ -e $1 -a ! -h $2 ]; then + run ln -s $1 $2 + elif [ ! -e $1 ]; then + print "Tried to make a symlink to $1, but it doesn't exist" + exit 1 + fi + else + if [ -e $1 -a ! -h $(basename $1) ]; then + run ln -s $1 + elif [ ! -e $1 ]; then + print "Tried to make a symlink to $1, but it doesn't exist" + exit 1 + fi fi } @@ -1102,14 +1108,15 @@ function un_install_ece() /var/run/ece-install.pid apt-get --yes --purge remove \ - varnish \ - percona* \ - maven2 \ ant \ ant-contrib \ ant-optional \ libmysql-java \ + maven2 \ memcached \ + munin* \ + percona* \ + varnish \ sun-java6-jdk apt-get clean @@ -1563,6 +1570,8 @@ EOF set_correct_permissions assemble_deploy_and_restart_type update_type_instances_to_start_up + set_conf_file_value ece_unix_user $ece_user /etc/default/ece + set_conf_file_value ece_unix_group $ece_group /etc/default/ece admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin add_next_step "- New ECE instance $instance_name installed." @@ -1581,7 +1590,7 @@ EOF function install_presentation_server() { print "Installing a presentation server on $HOSTNAME ..." - + type=engine install_ece_instance "web1" 1 } @@ -1589,10 +1598,6 @@ function assemble_deploy_and_restart_type() { print "Assembling, deploying & starting $instance_name ..." - if [ -z "$type" ]; then - type=engine - fi - # TODO clear strategy on when to add 'clean' to the command list # For now, we'll omit it to make the install script run as fast as # possible. It should in most cases work just fine, the exception @@ -1656,7 +1661,7 @@ function create_publication() function install_editorial_server() { print "Installing an editorial server on $HOSTNAME ..." - + type=engine install_ece_instance "editor1" 0 } @@ -1813,7 +1818,7 @@ EOF function install_search_instance() { - type=search1 + type=search install_ece_instance "search1" 0 file=/etc/escenic/ece-${instance_name}.conf @@ -1850,18 +1855,136 @@ EOF function install_all_in_one_environment() { print "Installing an all-in-one environment on $HOSTNAME ..." - + type=engine install_database_server install_ece_instance "dev1" 0 install_cache_server + install_web_server + install_munin_gatherer set_up_solr install_widget_framework create_publication } +function install_web_server() +{ + print "Installing a web server on $HOSTNAME ..." + + if [ $on_debian_or_derivative -eq 1 ]; then + packages="nginx" + install_packages_if_missing $packages + else + debug "Web server installation not supported on your system." + return + fi + + file=/etc/nginx/sites-available/default + run mv $file $file.orig + cat > $file <>$log 2>>$log + + add_next_step "- Web server is install on http://${HOSTNAME}:81/" + add_next_step " http://${HOSTNAME}:81/ gives you the munin interface" + add_next_step " http://${HOSTNAME}:81/binary gives you the Adactus endpoint" +} + +function install_munin_gatherer() +{ + print "Installing a Munin gatherer on $HOSTNAME ..." + + if [ $on_debian_or_derivative -eq 1 ]; then + packages="munin" + install_packages_if_missing $packages + else + print "Munin gatherer installation not supported on your system :-(" + print "You will have to install it manually." + return + fi + + # TODO ask user/read fai_munin_node_list + # TODO has a hard dependency on install_web_server, should perhaps + # be called from this one? + + add_next_step "- Munin interface available at http://$HOSTNAME:81/" +} + +function install_munin_node() +{ + print "Installing a Munin node on $HOSTNAME ..." + + if [ $on_debian_or_derivative -eq 1 ]; then + packages="munin-node munin-plugins-extra munin-java-plugins" + install_packages_if_missing $packages + else + print "Munin node installation not supported on your system" + print "You will have to install it manually." + return + fi + + # install the escenic_jstat munin plugin + file=/usr/share/munin/plugins/escenic_jstat_ + wget https://github.com/mogsie/escenic-munin/raw/master/escenic_jstat_ \ + -O $file \ + 1>>$log 2>>$log + run chmod 755 $file + + # TODO hack to get escenic_jstat to work with the new ece scripts. + cd $(dirname $file) + run patch -p1 < $download_dir/escenic_jstat_.diff + + escenic_jstat_modules="_gc _gcoverhead _heap _uptime" + for el in /etc/escenic/ece-*.conf; do + current_instance=$(basename ${el} | \ + sed -e 's/ece-//g' -e 's/engine-//g' | \ + cut -d'.' -f1) + for module in $escenic_jstat_modules; do + cd /usr/share/munin/plugins/ + make_ln escenic_jstat_ escenic_jstat_${current_instance}${module} + done + + # we need to hack a bit since escenic_jstat_ looks for + # instance PIDs in /var/run/escenic ece-.pid. It's + # now -.pid + file=/var/run/escenic/$type-${instance_name}.pid + if [ ! -e $file ]; then + run touch $file + fi + + # enabling the instance specific munin entries: + for el in /usr/share/munin/plugins/escenic_jstat_[a-z]*; do + run cd /etc/munin/plugins + make_ln $el + done + done + + if [ $on_debian_or_derivative -eq 1 ]; then + service munin-node restart \ + 1>>$log 2>>$log + fi + + add_next_step "- A Munin node has been installed on $HOSTNAME" +} + function common_post_install() { + install_munin_node print_status_and_next_steps rm $pid_file } @@ -1880,47 +2003,44 @@ if [ $fai_enabled -eq 1 ]; then print "Full Automatic Install (FAI) enabled." print "All user input will be read from $conf_file" + common_pre_install + if [ $(get_boolean_conf_value fai_all_install) -eq 1 ]; then install_profile_number=$PROFILE_ALL_IN_ONE - common_pre_install install_all_in_one_environment common_post_install elif [ $(get_boolean_conf_value fai_editor_install) -eq 1 ]; then install_profile_number=$PROFILE_EDITORIAL_SERVER - common_pre_install install_editorial_server common_post_install elif [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then install_profile_number=$PROFILE_DB_SERVER - common_pre_install install_database_server common_post_install elif [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then install_profile_number=$PROFILE_CACHE_SERVER - common_pre_install install_cache_server - common_post_install + install_web_server + install_munin_gatherer elif [ $(get_boolean_conf_value fai_rmi_install) -eq 1 ]; then install_profile_number=$PROFILE_RMI_HUB common_pre_install install_rmi_hub - common_post_install elif [ $(get_boolean_conf_value fai_wf_install) -eq 1 ]; then install_profile_number=$PROFILE_WIDGET_FRAMEWORK common_pre_install install_widget_framework - common_post_install elif [ $(get_boolean_conf_value fai_presentation_install) -eq 1 ]; then install_profile_number=$PROFILE_PRESENTATION_SERVER common_pre_install install_presentation_server - common_post_install elif [ $(get_boolean_conf_value fai_publication_create) -eq 1 ]; then install_profile_number=$PROFILE_CREATE_PUBLICATION common_pre_install create_publication - common_post_install fi + + common_post_install else read_user_input common_pre_install @@ -1930,6 +2050,8 @@ else ;; $PROFILE_CACHE_SERVER) install_cache_server + install_web_server + install_munin_gatherer ;; $PROFILE_DB_SERVER) install_database_server From 80ba4894494a8c21139cc3321392847fdce84fae Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Sep 2011 13:02:04 +0200 Subject: [PATCH 0037/2585] - one of the PGP keys for Percona was not added to the APT key chain, at least under some circumstances. --- usr/sbin/ece-install | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c74fa89d..86435d81 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1448,7 +1448,9 @@ function install_database_server() if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then run gpg --keyserver hkp://keys.gnupg.net \ - --recv-keys 1C4CBDCDCD2EFD2A \ + -a \ + --export 1C4CBDCDCD2EFD2A | \ + apt-key add - \ 1>>$log 2>>$log run gpg -a --export CD2EFD2A | apt-key add - \ 1>>$log 2>>$log From 34db77269e0e73932c885b866ff2922f9cc671e0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Sep 2011 15:28:37 +0800 Subject: [PATCH 0038/2585] - commands with pipes cannot be wrapped with the run function(), this caused problems especially with the GPG/apt-key handling - added removal of GPG keys to the un_install_ece method - all commands passed to the run() wrapper should _not_ re-direct standard out and standard error as run() does this for them. This hid some error messages. This is fixed now. - moved run level code into its own function, add_server_to_runlevels, and the code is now run from common_post_install instead of common_pre_install - fixed problem in un_install_ece if you haven't installed all profiles on one box. This shouldn't have affected most users, though. - made curl silent. --- usr/sbin/ece-install | 129 ++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 86435d81..c844233f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -213,7 +213,7 @@ install_packages_if_missing() return fi - run apt-get install -y $@ 1>>$log 2>>$log + run apt-get install -y $@ fi } @@ -249,8 +249,7 @@ function set_up_assembly_tool() cd /opt/escenic/assemblytool/ if [ -e $download_dir/assemblytool*zip ]; then - run unzip -u $download_dir/assemblytool*zip \ - 1>>$log 2>>$log + run unzip -u $download_dir/assemblytool*zip fi # adding an instance layer to the Nursery configuration @@ -288,8 +287,7 @@ EOF done run cd /opt/escenic/assemblytool/ - run ant -q initialize \ - 1>>$log 2>>$log + run ant -q initialize sed -i 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ assemble.properties \ 1>>$log 2>>$log @@ -313,8 +311,7 @@ function set_up_engine_and_plugins() cd /opt/escenic/ if [ ! -d engine-* ]; then - run unzip -u $download_dir/engine*.zip \ - 1>>$log 2>>$log + run unzip -u $download_dir/engine*.zip if [ -h engine ]; then run rm engine fi @@ -334,8 +331,7 @@ function set_up_engine_and_plugins() continue fi - run unzip -u $el \ - 1>>$log 2>>$log + run unzip -u $el done ece_software_setup_completed=1 @@ -350,33 +346,11 @@ function set_up_ece_scripts() (cd ece-scripts git pull 1>>$log 2>>$log) else - run git clone $git_source \ - 1>>$log 2>>$log + run git clone $git_source fi run cp -r ece-scripts/usr/* /usr/ run cp -r ece-scripts/etc/* /etc/ - - # no need to add init.d scripts to the runlevel(s) for these - # profiles - if [ $install_profile_number -eq $PROFILE_WIDGET_FRAMEWORK -o \ - $install_profile_number -eq $PROFILE_SEARCH_SERVER -o \ - $install_profile_number -eq $PROFILE_DB_SERVER -o \ - $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then - return - fi - - if [ $on_debian_or_derivative ]; then - print "Adding the ece init.d script to the default run levels..." - run update-rc.d ece defaults \ - 1>>$log 2>>$log - else - print "You remember to add /etc/intit.d/ece to the desired runlevels" - # TODO add init.d to the default runlevels, for other - # distributions too: - # - RedHat/chekcconfig - # - Gentoo: rc-update add ece default - fi } function set_up_ecedb() @@ -892,13 +866,11 @@ function set_correct_permissions() done if [ -d "$tomcat_base" ]; then - run chown -R ${ece_user}:${ece_group} $tomcat_base \ - 1>>$log 2>>$log + run chown -R ${ece_user}:${ece_group} $tomcat_base fi if [ $(ls /tmp | grep ^ece | wc -l) -gt 0 ]; then - run chown -R ${ece_user}:${ece_group} /tmp/ece-* \ - 1>>$log 2>>$log + run chown -R ${ece_user}:${ece_group} /tmp/ece-* fi } @@ -957,8 +929,7 @@ function create_publication_in_db() -F "type=webapp" \ -F "resourceFile=@${1}" \ --cookie JSESSIONID="$cookie" \ - "${ece_admin_uri}/do/publication/resource" \ - 1>>$log 2>>$log + "${ece_admin_uri}/do/publication/resource" run curl --silent \ -F "name=${publication_name}" \ @@ -966,8 +937,7 @@ function create_publication_in_db() -F "adminPassword=${publication_name}" \ -F "adminPasswordConfirm=${publication_name}" \ --cookie JSESSIONID="$cookie" \ - "${ece_admin_uri}/do/publication/insert" \ - 1>>$log 2>>$log + "${ece_admin_uri}/do/publication/insert" } function create_publication_definition_and_war() @@ -1000,8 +970,7 @@ EOF if [ -d /opt/escenic/widget-framework-core-* ]; then print "Basing ${publication_name}.war on the Widget Framework Demo ..." run cd /opt/escenic/widget-framework-core-*/publications/demo-core - run mvn package \ - 1>>$log 2>>$log + run mvn package run cp target/demo-core-*.war ${publication_war} else print "Basing your ${publication_name}.war on ECE/demo-clean ..." @@ -1069,8 +1038,7 @@ function set_up_user_enviornment() if [ $(grep bash_completion.d/ece /home/$ece_user/.bashrc 2>/dev/null | \ wc -l) -lt 1 ]; then - run echo ". /etc/bash_completion.d/ece" \ - >> /home/$ece_user/.bashrc + run echo ". /etc/bash_completion.d/ece" fi } @@ -1104,10 +1072,15 @@ function un_install_ece() /usr/bin/ece \ /var/log/escenic/ \ /etc/apt/sources.list.d/escenic.list \ -# $HOME/.m2 \ /var/run/ece-install.pid +# $HOME/.m2 \ + + # remove the Varnish key + apt-key remove C4DEFFEB + # remove the Percona key + apt-key remove CD2EFD2A - apt-get --yes --purge remove \ + for el in \ ant \ ant-contrib \ ant-optional \ @@ -1117,7 +1090,10 @@ function un_install_ece() munin* \ percona* \ varnish \ - sun-java6-jdk + sun-java6-jdk \ + ; do + apt-get --yes --purge remove $el + done apt-get clean } @@ -1364,10 +1340,10 @@ function add_apt_source() escenic_sources=/etc/apt/sources.list.d/escenic.list if [ ! -e $escenic_sources ]; then echo "$@" >> $escenic_sources - run apt-get update 1>>$log 2>>$log + run apt-get update elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then echo "$@" >> $escenic_sources - run apt-get update 1>>$log 2>>$log + run apt-get update fi } @@ -1376,8 +1352,12 @@ function install_cache_server() print "Installing a caching server on $HOSTNAME ..." if [ $on_debian_or_derivative -eq 1 ]; then - run curl http://repo.varnish-cache.org/debian/GPG-key.txt 2>>$log | \ - apt-key add - 1>>$log 2>>$log + curl --silent \ + http://repo.varnish-cache.org/debian/GPG-key.txt \ + 2>> $log | \ + apt-key add - \ + 1>>$log 2>>$log + run apt-get update fi if [ $on_debian -eq 1 ]; then @@ -1447,13 +1427,14 @@ function install_database_server() print "Installing the Percona database ..." if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then - run gpg --keyserver hkp://keys.gnupg.net \ + gpg --keyserver hkp://keys.gnupg.net \ -a \ --export 1C4CBDCDCD2EFD2A | \ apt-key add - \ 1>>$log 2>>$log - run gpg -a --export CD2EFD2A | apt-key add - \ + gpg -a --export CD2EFD2A | apt-key add - \ 1>>$log 2>>$log + run apt-get update fi add_apt_source "deb http://repo.percona.com/apt ${code_name} main" packages="percona-server-server percona-server-client" @@ -1800,7 +1781,7 @@ EOF assert_pre_prequesite mvn wf_maven_dir=$(echo /opt/escenic/widget-framework-core-*/maven) cd $wf_maven_dir - print "Installing Widget Framework into your local Maven repository ..." + print "Installing Widget Framework into your Maven repository ..." export JAVA_HOME=$java_home mvn install \ 1>>$log 2>>$log @@ -1942,15 +1923,13 @@ function install_munin_node() # install the escenic_jstat munin plugin file=/usr/share/munin/plugins/escenic_jstat_ - wget https://github.com/mogsie/escenic-munin/raw/master/escenic_jstat_ \ - -O $file \ - 1>>$log 2>>$log + run wget \ + --continue \ + --inet4-only \ + https://github.com/mogsie/escenic-munin/raw/master/escenic_jstat_ \ + -O $file run chmod 755 $file - # TODO hack to get escenic_jstat to work with the new ece scripts. - cd $(dirname $file) - run patch -p1 < $download_dir/escenic_jstat_.diff - escenic_jstat_modules="_gc _gcoverhead _heap _uptime" for el in /etc/escenic/ece-*.conf; do current_instance=$(basename ${el} | \ @@ -1984,8 +1963,33 @@ function install_munin_node() add_next_step "- A Munin node has been installed on $HOSTNAME" } +function add_server_to_runlevels() +{ + # no need to add init.d scripts to the runlevel(s) for these + # profiles + if [ $install_profile_number -eq $PROFILE_WIDGET_FRAMEWORK -o \ + $install_profile_number -eq $PROFILE_SEARCH_SERVER -o \ + $install_profile_number -eq $PROFILE_DB_SERVER -o \ + $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then + return + fi + + if [ $on_debian_or_derivative ]; then + print "Adding the ece init.d script to the default run levels ..." + run update-rc.d ece defaults + else + add_next_step "- Remember to add /etc/intit.d/ece to the desired" + add_next_step " run levels." + # TODO add init.d to the default runlevels, for other + # distributions too: + # - RedHat/chekcconfig + # - Gentoo: rc-update add ece default + fi +} + function common_post_install() { + add_server_to_runlevels install_munin_node print_status_and_next_steps rm $pid_file @@ -2010,15 +2014,12 @@ if [ $fai_enabled -eq 1 ]; then if [ $(get_boolean_conf_value fai_all_install) -eq 1 ]; then install_profile_number=$PROFILE_ALL_IN_ONE install_all_in_one_environment - common_post_install elif [ $(get_boolean_conf_value fai_editor_install) -eq 1 ]; then install_profile_number=$PROFILE_EDITORIAL_SERVER install_editorial_server - common_post_install elif [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then install_profile_number=$PROFILE_DB_SERVER install_database_server - common_post_install elif [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then install_profile_number=$PROFILE_CACHE_SERVER install_cache_server From 3f8806735c5209c427b60cf9257e85f6caa0f6b6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Sep 2011 15:29:50 +0800 Subject: [PATCH 0039/2585] - added information about the web server and Munin --- usr/share/doc/escenic/ece-install-guide.org | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 12b6f5be..d8e572a3 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -120,6 +120,14 @@ Select 1-7 and press ENTER 8 - Install Widget Framework. This will also set up a WF based publication for you (repo.escenic.com user/pass required). #+END_SRC +** Common for All Server Profiles +The script will install a Munin node on the host. It will also +install the Escenic specific Munin plugins on hosts where there are +any Escenic server components. + +TBD: there is currently an issue with the Escenic plugins not working +properly with the new ece/ece-install scripts. We're working on this. + ** Common for Editorial, Presentation and Search Instances *** Sun Java 6 *** Apache Tomcat application server @@ -200,6 +208,7 @@ If ece-install is run on a different platform, the admin must install Varnish 3.x prior to running ece-install. The script will configure Varnish for a typical Escenic site: +- it will set up the cache server on port 80 - will set up an access control lists of IPs which may access the privileged web applications such as /escenic-admin, /escenic and /webservice. @@ -215,13 +224,18 @@ The script will configure Varnish for a typical Escenic site: different backend servers that will serve the web site. - will set up configuration to strip away cookies from static resources, such as CSS, JS and pictures. +- will install the nginx web server for serving static content and + will configure Varnish accordingly. This will be very useful for + Adactus servers wanting to pull content from your ECEs. +- will install a Munin gatherer and make its interface available + when accesssing the web server directly (port 81) TBD: - If run on a Linux platform, the script will tweak the kernel parameters for optimal TCP handling for a web facing server. -- Will install the nginx web server for serving static content and - will configure Varnish accordingly. This will be very useful for - Adactus servers wanting to pull content from your ECEs. +- let the /munin run through on port 80, requiring the connecting IPs + to be in the staff network ACL, defined in the Varnish + configuration. ** 8 - Install Widget Framework You'll need a user name and password for accessing the From c9a8bc45f742d2f51e7201062f447047e5a9e316 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Sep 2011 15:30:19 +0800 Subject: [PATCH 0040/2585] - updated the HTML and TXT --- usr/share/doc/escenic/ece-install-guide.html | 226 ++++++++----------- usr/share/doc/escenic/ece-install-guide.txt | 154 +++++-------- 2 files changed, 141 insertions(+), 239 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index d4ad2eca..7b7ba0cf 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -227,24 +227,25 @@

    Table of Contents

  • 4 Running the ece-install Script
  • 5 Available Server Profiles
  • 6 Full Automatic Install (FAI)
  • @@ -508,23 +509,39 @@

    5 Available Server Profiles
    -

    5.1 Common for Editorial, Presentation and Search Instances

    +

    5.1 Common for All Server Profiles

    +

    The script will install a Munin node on the host. It will also +install the Escenic specific Munin plugins on hosts where there are +any Escenic server components. +

    +

    +TBD: there is currently an issue with the Escenic plugins not working +properly with the new ece/ece-install scripts. We're working on this. +

    +
    -
    -

    5.1.1 Sun Java 6

    -
    +
    +

    5.2 Common for Editorial, Presentation and Search Instances

    +
    + + +
    + +
    +

    5.2.1 Sun Java 6

    +
    -
    -

    5.1.2 Apache Tomcat application server

    -
    +
    +

    5.2.2 Apache Tomcat application server

    +
    • Compression of textual content (previously, this was typically set @@ -544,9 +561,9 @@

      5.1.2 Apache Tomcat app

    -
    -

    5.1.3 Basic Escenic Nursery configuration

    -
    +
    +

    5.2.3 Basic Escenic Nursery configuration

    +

    The basic Nursery configuration is taken care of for you, including RMI, database, search and application server/URIs. @@ -554,17 +571,17 @@

    5.1.3 Basic Escenic Nur

    -
    -

    5.1.4 APR, native library for optimal Tomcat I/O performance

    -
    +
    +

    5.2.4 APR, native library for optimal Tomcat I/O performance

    +
    -
    -

    5.1.5 Escenic Assembly environment

    -
    +
    +

    5.2.5 Escenic Assembly environment

    +

    The reason why ece-install sets this up on each host, is to make the installation process as smooth as possible. The assembly environment @@ -574,18 +591,18 @@

    5.1.5 Escenic Assembly

    -
    -

    5.1.6 Database driver

    -
    +
    +

    5.2.6 Database driver

    +
    -
    -

    5.1.7 Compression of content

    -
    +
    +

    5.2.7 Compression of content

    +

    This was was previously accomplished by having a web server in fron t of the application server (or cache server if you used ESI). A typical @@ -600,9 +617,9 @@

    5.1.7 Compression of co

    -
    -

    5.2 1 - Full Stack on One Host

    -
    +
    +

    5.3 1 - Full Stack on One Host

    +

    This profile is suitable for developers and admins wanting to set up a test environment. It will install the full stack including caching @@ -618,9 +635,9 @@

    5.2 1 - Full Stack on One

    -
    -

    5.3 2 - Editorial Server (Publication Server)

    -
    +
    +

    5.4 2 - Editorial Server (Publication Server)

    +
    @@ -628,9 +645,9 @@

    5.3 2 - Editorial Server

    -
    -

    5.4 3 - Presentation Server

    -
    +
    +

    5.5 3 - Presentation Server

    +

    This will set up a typical presentation server:

      @@ -649,9 +666,9 @@

      5.4 3 - Presentation Serv

    -
    -

    5.5 4 - Database Server

    -
    +
    +

    5.6 4 - Database Server

    +

    If ece-install is run on a support version of Debian or Ubuntu, this will install the excellent Percona distribution of MySQL with their @@ -686,9 +703,9 @@

    5.5 4 - Database Server <

    -
    -

    5.6 5 - Cache Server

    -
    +
    +

    5.7 5 - Cache Server

    +

    If ece-install is run on a support version of Debian or Ubuntu, it will install the latest Varnish 3.0 caching server from the Varnish @@ -701,6 +718,8 @@

    5.6 5 - Cache Server

    The script will configure Varnish for a typical Escenic site:

      +
    • it will set up the cache server on port 80 +
    • will set up an access control lists of IPs which may access the privileged web applications such as /escenic-admin, /escenic and /webservice. @@ -721,6 +740,13 @@

      5.6 5 - Cache Server

      will set up configuration to strip away cookies from static resources, such as CSS, JS and pictures.
    • +
    • will install the nginx web server for serving static content and + will configure Varnish accordingly. This will be very useful for + Adactus servers wanting to pull content from your ECEs. +
    • +
    • will install a Munin gatherer and make its interface available + when accesssing the web server directly (port 81) +
    @@ -730,9 +756,9 @@

    5.6 5 - Cache Server

    If run on a Linux platform, the script will tweak the kernel parameters for optimal TCP handling for a web facing server.

  • -
  • Will install the nginx web server for serving static content and - will configure Varnish accordingly. This will be very useful for - Adactus servers wanting to pull content from your ECEs. +
  • let the /munin run through on port 80, requiring the connecting IPs + to be in the staff network ACL, defined in the Varnish + configuration.
  • @@ -741,9 +767,9 @@

    5.6 5 - Cache Server

    -
    -

    5.7 8 - Install Widget Framework

    -
    +
    +

    5.8 8 - Install Widget Framework

    +

    You'll need a user name and password for accessing the repo.escenic.com Maven repository. You should get these credentials @@ -767,9 +793,9 @@

    5.7 8 - Install Widget Fr

    -
    -

    5.8 9 - Create Publication

    -
    +
    +

    5.9 9 - Create Publication

    +

    This profile will create a publication for you, only asking you the name of the publication and which ECE instance to use for its @@ -1049,87 +1075,15 @@

    12 Example Output FAI

    The following output is produced from ece-install: +CANNOT INCLUDE FILE /tmp/ece-install.out

    - - -
    root@ubiquitous-7:~# bash ece-install
    -[ece-install-0] Full Automatic Install (FAI) enabled.
    -[ece-install-0] All user input will be read from /root/.ece-install.conf
    -[ece-install-0] I'm logging to /var/log/ece-install.log
    -[ece-install-1] Stopping conflicting processes ...
    -[ece-install-1] Installing 3rd party packages needed by ece-install ...
    -[ece-install-2] Setting up the ece UNIX scripts ...
    -[ece-install-5] Adding the ece init.d script to the default run levels...
    -[ece-install-5] Setting up the doesnt user's UNIX environment ...
    -[ece-install-5] Installing an all-in-one environment on ubiquitous-7 ...
    -[ece-install-5] Installing the database server on ubiquitous-7 ...
    -[ece-install-5] Installing the Percona database ...
    -[ece-install-65] Downloading Escenic software from technet.escenic.com ...
    -[ece-install-66] Setting up the Escenic Content Engine & its plugins ...
    -[ece-install-76] Setting up the ECE database schema ...
    -[ece-install-90] Installing 3rd party packages needed by ECE ...
    -[ece-install-106] Setting up the Assembly Tool ...
    -[ece-install-110] Setting up the basic Nursery configuration ...
    -[ece-install-110] Setting up the application server ...
    -[ece-install-116] Setting up proper log4j & Java logging configuration ...
    -[ece-install-116] Setting correct permissions on ECE related directories ...
    -[ece-install-116] Assembling, deploying & starting dev1 ...
    -[ece-install-183] Installing a caching server on ubiquitous-7 ...
    -[ece-install-215] Setting up Varnish to match your environment ...
    -[ece-install-220] Setting up solr ...
    -[ece-install-221] Creating a Maven settings file: /root/.m2/settings.xml ...
    -[ece-install-221] Downloading Widget Framework from technet.escenic.com ...
    -[ece-install-259] Installing Widget Framework into your local Maven repository ...
    -[ece-install-318] Getting ready to create a new publiation ...
    -[ece-install-318] Setting up the mypub publication ...
    -[ece-install-318] Basing mypub.war on the Widget Framework Demo ...
    -[ece-install-344] Creating publication mypub using dev1 ...
    -[ece-install-382] Assembling, deploying & starting dev1 ...
    -[ece-install-503] - The installation is now complete! It took 0d 0h 8m 22s
    -[ece-install-90] - DB is now set up on ubiquitous-7:3306
    -[ece-install-181] - New ECE instance dev1 installed.
    -[ece-install-181] - Its admin interface is available at
    -[ece-install-181]   http://ubiquitous-7:8080/escenic-admin
    -[ece-install-181] - You can view all installed plugin & engine versions with:
    -[ece-install-181]   ece -i dev1 versions
    -[ece-install-182] - type 'ece help' to see all the options of this script
    -[ece-install-182]   which allows you to control all the Escenic components.
    -[ece-install-182]   Also, see /usr/share/doc/escenic/ece-guide.txt for
    -[ece-install-182]   addtional information on the ece script.
    -[ece-install-183] - Verify that /etc/default/ece lists all the instances 
    -[ece-install-183]   on ubiquitous-7 you wish to start at boot time.
    -[ece-install-220] - Cache server is up and running at http://ubiquitous-7:80/
    -[ece-install-317] - Widget Framework has been installed into your local
    -[ece-install-318]   Maven repository and its ECE components have been installed
    -[ece-install-502] - a new publication mypub has been created
    -[ece-install-503] - See /usr/share/doc/escenic/ece-install-guide.txt for
    -[ece-install-503]   a full overview of files and directories set up by
    -[ece-install-504]   ece-install
    -[ece-install-504] - type 'ece help' to see all the options of this script
    -[ece-install-504]   which allows you to control all the Escenic components.
    -[ece-install-504]   Also, see /usr/share/doc/escenic/ece-guide.txt for
    -[ece-install-505]   further information on the ece script.
    -[ece-install-505] - For further reading, in-depth guides on installing, 
    -[ece-install-505]   configuring and developing Escenic web sites can be 
    -[ece-install-505]   fount at http://documentation.vizrt.com/ece-5.3.html
    -[ece-install-506]
    -[ece-install-506] Enjoy your time with Escenic Content Engine!
    -[ece-install-506]
    -[ece-install-506] -Vizrt Online
    -
    -
    -
    - - - -

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-09-12 19:02:48 CST + Date: 2011-09-15 15:29:55 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index ce9cb86a..a4a85655 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-09-12 19:02:40 CST +Date: 2011-09-15 15:29:53 CST Table of Contents @@ -14,21 +14,22 @@ Table of Contents 3 Downloading the ece-install Script 4 Running the ece-install Script 5 Available Server Profiles - 5.1 Common for Editorial, Presentation and Search Instances - 5.1.1 Sun Java 6 - 5.1.2 Apache Tomcat application server - 5.1.3 Basic Escenic Nursery configuration - 5.1.4 APR, native library for optimal Tomcat I/O performance - 5.1.5 Escenic Assembly environment - 5.1.6 Database driver - 5.1.7 Compression of content - 5.2 1 - Full Stack on One Host - 5.3 2 - Editorial Server (Publication Server) - 5.4 3 - Presentation Server - 5.5 4 - Database Server - 5.6 5 - Cache Server - 5.7 8 - Install Widget Framework - 5.8 9 - Create Publication + 5.1 Common for All Server Profiles + 5.2 Common for Editorial, Presentation and Search Instances + 5.2.1 Sun Java 6 + 5.2.2 Apache Tomcat application server + 5.2.3 Basic Escenic Nursery configuration + 5.2.4 APR, native library for optimal Tomcat I/O performance + 5.2.5 Escenic Assembly environment + 5.2.6 Database driver + 5.2.7 Compression of content + 5.3 1 - Full Stack on One Host + 5.4 2 - Editorial Server (Publication Server) + 5.5 3 - Presentation Server + 5.6 4 - Database Server + 5.7 5 - Cache Server + 5.8 8 - Install Widget Framework + 5.9 9 - Create Publication 6 Full Automatic Install (FAI) 7 Running More Than One Installation Process 8 Re-running ece-install (and How To Speed It Up) @@ -199,13 +200,22 @@ want to install: -5.1 Common for Editorial, Presentation and Search Instances +5.1 Common for All Server Profiles +=================================== +The script will install a Munin node on the host. It will also +install the Escenic specific Munin plugins on hosts where there are +any Escenic server components. + +TBD: there is currently an issue with the Escenic plugins not working +properly with the new ece/ece-install scripts. We're working on this. + +5.2 Common for Editorial, Presentation and Search Instances ============================================================ -5.1.1 Sun Java 6 +5.2.1 Sun Java 6 ~~~~~~~~~~~~~~~~~ -5.1.2 Apache Tomcat application server +5.2.2 Apache Tomcat application server ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Compression of textual content (previously, this was typically set up with Apache and its mod\_deflate module). @@ -214,24 +224,24 @@ want to install: - routing information in the cookies - application server access log -5.1.3 Basic Escenic Nursery configuration +5.2.3 Basic Escenic Nursery configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The basic Nursery configuration is taken care of for you, including RMI, database, search and application server/URIs. -5.1.4 APR, native library for optimal Tomcat I/O performance +5.2.4 APR, native library for optimal Tomcat I/O performance ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -5.1.5 Escenic Assembly environment +5.2.5 Escenic Assembly environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The reason why ece-install sets this up on each host, is to make the installation process as smooth as possible. The assembly environment may be removed after the installation if you want to. -5.1.6 Database driver +5.2.6 Database driver ~~~~~~~~~~~~~~~~~~~~~~ -5.1.7 Compression of content +5.2.7 Compression of content ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This was was previously accomplished by having a web server in fron t of the application server (or cache server if you used ESI). A typical @@ -241,7 +251,7 @@ compressed content (and many other things that we before needed Apache for). Thus, we'll let the application server do the compression for us now. -5.2 1 - Full Stack on One Host +5.3 1 - Full Stack on One Host =============================== This profile is suitable for developers and admins wanting to set up a test environment. It will install the full stack including caching @@ -252,11 +262,11 @@ For further details on each of the different server components, see the different profile descriptions bellow. -5.3 2 - Editorial Server (Publication Server) +5.4 2 - Editorial Server (Publication Server) ============================================== -5.4 3 - Presentation Server +5.5 3 - Presentation Server ============================ This will set up a typical presentation server: - Memcached, distributed memory cache @@ -265,7 +275,7 @@ This will set up a typical presentation server: - Deployment setup to only deploy escenic-admin and the publication(s). -5.5 4 - Database Server +5.6 4 - Database Server ======================== If ece-install is run on a support version of Debian or Ubuntu, this will install the excellent Percona distribution of MySQL with their @@ -290,7 +300,7 @@ The script will fail by itself if the DB already exists: -5.6 5 - Cache Server +5.7 5 - Cache Server ===================== If ece-install is run on a support version of Debian or Ubuntu, it will install the latest Varnish 3.0 caching server from the Varnish @@ -300,6 +310,7 @@ If ece-install is run on a different platform, the admin must install Varnish 3.x prior to running ece-install. The script will configure Varnish for a typical Escenic site: +- it will set up the cache server on port 80 - will set up an access control lists of IPs which may access the privileged web applications such as /escenic-admin, /escenic and /webservice. @@ -315,15 +326,20 @@ The script will configure Varnish for a typical Escenic site: different backend servers that will serve the web site. - will set up configuration to strip away cookies from static resources, such as CSS, JS and pictures. +- will install the nginx web server for serving static content and + will configure Varnish accordingly. This will be very useful for + Adactus servers wanting to pull content from your ECEs. +- will install a Munin gatherer and make its interface available + when accesssing the web server directly (port 81) TBD: - If run on a Linux platform, the script will tweak the kernel parameters for optimal TCP handling for a web facing server. -- Will install the nginx web server for serving static content and - will configure Varnish accordingly. This will be very useful for - Adactus servers wanting to pull content from your ECEs. +- let the /munin run through on port 80, requiring the connecting IPs + to be in the staff network ACL, defined in the Varnish + configuration. -5.7 8 - Install Widget Framework +5.8 8 - Install Widget Framework ================================= You'll need a user name and password for accessing the repo.escenic.com Maven repository. You should get these credentials @@ -339,7 +355,7 @@ complain: -5.8 9 - Create Publication +5.9 9 - Create Publication =========================== This profile will create a publication for you, only asking you the name of the publication and which ECE instance to use for its @@ -526,73 +542,5 @@ file: The following output is produced from ece-install: - - - root@ubiquitous-7:~# bash ece-install - [ece-install-0] Full Automatic Install (FAI) enabled. - [ece-install-0] All user input will be read from /root/.ece-install.conf - [ece-install-0] I'm logging to /var/log/ece-install.log - [ece-install-1] Stopping conflicting processes ... - [ece-install-1] Installing 3rd party packages needed by ece-install ... - [ece-install-2] Setting up the ece UNIX scripts ... - [ece-install-5] Adding the ece init.d script to the default run levels... - [ece-install-5] Setting up the doesnt user's UNIX environment ... - [ece-install-5] Installing an all-in-one environment on ubiquitous-7 ... - [ece-install-5] Installing the database server on ubiquitous-7 ... - [ece-install-5] Installing the Percona database ... - [ece-install-65] Downloading Escenic software from technet.escenic.com ... - [ece-install-66] Setting up the Escenic Content Engine & its plugins ... - [ece-install-76] Setting up the ECE database schema ... - [ece-install-90] Installing 3rd party packages needed by ECE ... - [ece-install-106] Setting up the Assembly Tool ... - [ece-install-110] Setting up the basic Nursery configuration ... - [ece-install-110] Setting up the application server ... - [ece-install-116] Setting up proper log4j & Java logging configuration ... - [ece-install-116] Setting correct permissions on ECE related directories ... - [ece-install-116] Assembling, deploying & starting dev1 ... - [ece-install-183] Installing a caching server on ubiquitous-7 ... - [ece-install-215] Setting up Varnish to match your environment ... - [ece-install-220] Setting up solr ... - [ece-install-221] Creating a Maven settings file: /root/.m2/settings.xml ... - [ece-install-221] Downloading Widget Framework from technet.escenic.com ... - [ece-install-259] Installing Widget Framework into your local Maven repository ... - [ece-install-318] Getting ready to create a new publiation ... - [ece-install-318] Setting up the mypub publication ... - [ece-install-318] Basing mypub.war on the Widget Framework Demo ... - [ece-install-344] Creating publication mypub using dev1 ... - [ece-install-382] Assembling, deploying & starting dev1 ... - [ece-install-503] - The installation is now complete! It took 0d 0h 8m 22s - [ece-install-90] - DB is now set up on ubiquitous-7:3306 - [ece-install-181] - New ECE instance dev1 installed. - [ece-install-181] - Its admin interface is available at - [ece-install-181] http://ubiquitous-7:8080/escenic-admin - [ece-install-181] - You can view all installed plugin & engine versions with: - [ece-install-181] ece -i dev1 versions - [ece-install-182] - type 'ece help' to see all the options of this script - [ece-install-182] which allows you to control all the Escenic components. - [ece-install-182] Also, see /usr/share/doc/escenic/ece-guide.txt for - [ece-install-182] addtional information on the ece script. - [ece-install-183] - Verify that /etc/default/ece lists all the instances - [ece-install-183] on ubiquitous-7 you wish to start at boot time. - [ece-install-220] - Cache server is up and running at http://ubiquitous-7:80/ - [ece-install-317] - Widget Framework has been installed into your local - [ece-install-318] Maven repository and its ECE components have been installed - [ece-install-502] - a new publication mypub has been created - [ece-install-503] - See /usr/share/doc/escenic/ece-install-guide.txt for - [ece-install-503] a full overview of files and directories set up by - [ece-install-504] ece-install - [ece-install-504] - type 'ece help' to see all the options of this script - [ece-install-504] which allows you to control all the Escenic components. - [ece-install-504] Also, see /usr/share/doc/escenic/ece-guide.txt for - [ece-install-505] further information on the ece script. - [ece-install-505] - For further reading, in-depth guides on installing, - [ece-install-505] configuring and developing Escenic web sites can be - [ece-install-505] fount at http://documentation.vizrt.com/ece-5.3.html - [ece-install-506] - [ece-install-506] Enjoy your time with Escenic Content Engine! - [ece-install-506] - [ece-install-506] -Vizrt Online - - - +CANNOT INCLUDE FILE /tmp/ece-install.out From a95beaf3c97125f4ee2b617840bddc7da4ca145b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Sep 2011 18:40:08 +0800 Subject: [PATCH 0041/2585] - updated start screen example --- usr/share/doc/escenic/ece-install-guide.org | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index d8e572a3..499e6b59 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -107,18 +107,24 @@ want to install: #+BEGIN_SRC sh Hi, which server profile do you wish to install? -Select 1-7 and press ENTER - - 1 - The full stack on one host. Suitable for development and - test environments (cache, ECE, assembly host & database). - 2 - Editorial (publication) server (ECE, assembly host). +Select 1-9 and press ENTER + + 1 - All in one, the full stack on one host. + Suitable for development and test environments. + It will install caching server, ECE, assembly host, database & + Widget Framework as well as set up a new publication. + 2 - Editorial (publication) server. + This will install ECE and an assembly host. 3 - Presentation server (ECE, memcached). 4 - Database server. 5 - Cache server (cache & web server). 6 - RMI hub. 7 - Standalone search instance (solr + indexer-webapp). - 8 - Install Widget Framework. This will also set up a WF based - publication for you (repo.escenic.com user/pass required). + 8 - Install the Widget Framework. + (You need user/pass for repo.escenic.com for this). + 9 - Create a new publication. + This profile will create new publication based on WF + (if available) or ECE/clean-demo. #+END_SRC ** Common for All Server Profiles The script will install a Munin node on the host. It will also From 60ca5fbc19b07359c706de63ed7a262fe6595795 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Sep 2011 13:37:33 +0800 Subject: [PATCH 0042/2585] - added get_base_dir_from_bundle - this method is used to extract exactly the bundles downloaded instead of using wildcards. This solves the problem when you already have downloaded the same artifacts, but with different versions. E.g. when you have a previous version of Tomcat or Content Engine installed. - now only added the varnish repository if there's an APT repository for the version of the OS you're running. --- usr/sbin/ece-install | 58 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c844233f..691f9d95 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -35,6 +35,7 @@ git_source=git://github.com/skybert/ece-scripts.git appserver_port=8080 on_debian_or_derivative=0 on_debian=0 +on_ubuntu=0 # because the all in one profile will run database, search and app # server profiles, all of which needs downloading and setting up the @@ -310,15 +311,21 @@ function set_up_engine_and_plugins() make_dir /opt/escenic cd /opt/escenic/ - if [ ! -d engine-* ]; then - run unzip -u $download_dir/engine*.zip + for el in $technet_download_list; do + if [ $(basename $el | grep ^engine-[0-9] | wc -l) -gt 0 ]; then + engine_dir=$(get_base_dir_from_bundle $el) + fi + done + + if [ ! -d ${engine_dir} ]; then + run unzip -u $download_dir/${engine_dir}.zip if [ -h engine ]; then run rm engine fi - run ln -s engine-* engine + run ln -s ${engine_dir} engine else - debug "engine- is already there, skipping to next step." + debug "${engine_dir} is already there, skipping to next step." fi # we now extract all the plugins. We extract them in /opt/escenic @@ -594,6 +601,16 @@ function install_ece_third_party_packages done } +function get_base_dir_from_bundle() +{ + file_name=$(basename $1) + for el in .tar.gz .tar.bz2 .zip; do + file_name=${file_name%$el} + done + + echo $file_name +} + function set_up_app_server { print "Setting up the application server ..." @@ -675,15 +692,20 @@ function set_up_app_server run cd $download_dir run wget --continue --inet4-only $tomcat_download + tomcat_dir=$(get_base_dir_from_bundle $tomcat_download) run cd /opt/ - run tar xzf $download_dir/apache-tomcat*.tar.gz - run ln -sf apache-tomcat* tomcat + run tar xzf $download_dir/${tomcat_dir}.tar.gz + + if [ -e tomcat ]; then + run rm tomcat + fi + run ln -sf ${tomcat_dir} tomcat tomcat_base=/opt/tomcat-${instance_name} make_dir $tomcat_base - run cp -r /opt/apache-tomcat*/conf/ $tomcat_base + run cp -r /opt/${tomcat_dir}/ $tomcat_base for el in bin escenic/lib lib work logs temp webapps; do make_dir $tomcat_base/$el done @@ -1359,10 +1381,21 @@ function install_cache_server() 1>>$log 2>>$log run apt-get update fi + + code_name=$(lsb_release -s -c) + + supported_code_name=0 + # list taken from http://repo.varnish-cache.org/debian/dists/ + supported_list="lenny squeeze hardy lucid" + for el in $supported_list; do + if [ $code_name = $el ]; then + supported_code_name=1 + fi + done - if [ $on_debian -eq 1 ]; then + if [ $supported_code_name -eq 1 -a $on_debian -eq 1 ]; then add_apt_source "deb http://repo.varnish-cache.org/debian/ $(lsb_release -s -c) varnish-3.0" - elif [ $on_ubuntu -eq 1 ]; then + elif [ $supported_code_name -eq 1 -a $on_ubuntu -eq 1 ]; then add_apt_source "deb http://repo.varnish-cache.org/ubuntu/ $(lsb_release -s -c) varnish-3.0" fi @@ -1440,10 +1473,9 @@ function install_database_server() packages="percona-server-server percona-server-client" install_packages_if_missing $packages else - print -n "The Percona APT repsository " - print "doesn't have packages for your Debian (or derivative) " - print "version with code name $code_name. I will use vanilla " - print "MySQL instead." + print "The Percona APT repsository fdoesn't have packages for your" + print "Debian (or derivative) version with code name $code_name. " + print "I will use vanilla MySQL instead." packages="mysql-server mysql-client libmysql-java" install_packages_if_missing $packages From 42baeffe36a2abf6aef485bbe5bc434b9d785db4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Sep 2011 13:48:59 +0800 Subject: [PATCH 0043/2585] - speling - made one output entry fit on 80 character terminals --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 691f9d95..8491f644 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1473,7 +1473,7 @@ function install_database_server() packages="percona-server-server percona-server-client" install_packages_if_missing $packages else - print "The Percona APT repsository fdoesn't have packages for your" + print "The Percona APT repsository doesn't have packages for your" print "Debian (or derivative) version with code name $code_name. " print "I will use vanilla MySQL instead." @@ -1754,7 +1754,7 @@ function install_widget_framework() wf_password=$(get_conf_value wf_password) if [ -z "$wf_user" -o -z "$wf_password" ]; then - print "Be sure to set wf_user and wf_password in $conf_file" + print "Missing wf_user and wf_password in $conf_file" print "If you don't have these, please contact support@escenic.com" exit 1 fi From 8577a827b8922c9a48760f1b6cfc9ab92a074acf Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Sep 2011 18:25:26 +0800 Subject: [PATCH 0044/2585] - added check for the callee being root - chown now runs with the --changes options, only reporting when it actually changes something. --- etc/init.d/ece | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/etc/init.d/ece b/etc/init.d/ece index 1ee0751e..d5976036 100755 --- a/etc/init.d/ece +++ b/etc/init.d/ece @@ -25,6 +25,11 @@ # would be: update-rc.d ece defaults ####################################################################### +if [ $(whoami) != "root" ]; then + echo "Sorry, you must be root for running $0" + exit 1 +fi + ####################################################################### # Default values ####################################################################### @@ -69,7 +74,7 @@ function ensure_dirs_are_ok() if [ ! -d $el ]; then mkdir -p $el fi - chown -R $ece_unix_user:$ece_unix_group $el + chown --changes -R $ece_unix_user:$ece_unix_group $el done } From c8f4373cbfc0a9224a0f6c247914737d6b26ca90 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Sep 2011 18:51:00 +0800 Subject: [PATCH 0045/2585] - added static backend to varnish setup - the Munin Gatherer can now be accessed on the public host /munin, e.g. http://mysite.com/munin - /munin added to the privileged list of URIs in the Varnish configuration. --- usr/sbin/ece-install | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8491f644..6fc6a5a6 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1164,6 +1164,11 @@ acl adactus { "203.33.232.216"; } +/* Our web server for serving static content */ +backend static { + .host = "localhost"; + .port = "81"; +} EOF for el in $backend_servers; do appserver_id=$(echo $el | cut -d':' -f1 | sed 's/-/_/g') @@ -1203,6 +1208,7 @@ sub vcl_recv { if (!client.ip ~ staff && (req.url ~ "^/escenic" || req.url ~ "^/studio" || + req.url ~ "^/munin" || req.url ~ "^/webservice" || req.url ~ "^/escenic-admin")) { error 405 "Not allowed."; @@ -1214,7 +1220,13 @@ sub vcl_recv { error 405 "Not allowed."; } - set req.backend = webdirector; + if (req.url ~ "^/munin") { + set req.url = regsub(req.url, "^/munin", "/"); + set req.backend = static; + } + else { + set req.backend = webdirector; + } if (req.url ~ "\.(png|gif|jpg|css|js)$" || req.url == "/favicon.ico") { remove req.http.Cookie; @@ -1986,6 +1998,17 @@ function install_munin_node() make_ln $el done done + + # TODO in which version(s) of munin is this directory called + # client-conf.d? + file=/etc/munin/plugin-conf.d/munin-node + if [ -e $file ]; then + cat >> $file < Date: Fri, 16 Sep 2011 19:06:29 +0800 Subject: [PATCH 0046/2585] - removed the munin gatherer from the cache server profile (it's still in the install all profile). - added missing 'fi', argh! - added munin gatherer profile, it's not fully integrated everywhere, but the basic gatherer should get installed. TODO: make sure VCL is sane - support for fai_munin_gather_host --- usr/sbin/ece-install | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 6fc6a5a6..2e9af2ef 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -116,6 +116,7 @@ PROFILE_RMI_HUB=6 PROFILE_SEARCH_SERVER=7 PROFILE_WIDGET_FRAMEWORK=8 PROFILE_CREATE_PUBLICATION=9 +PROFILE_MUNIN_GATHERER=10 # Because of issue VF-3559, we also create the default family and host # directories. @@ -1275,7 +1276,7 @@ function read_user_input() { echo "Hi, which server profile do you wish to install?" echo "" - echo "Select 1-9 and press ENTER" + echo "Select 1-10 and press ENTER" echo "" echo " $PROFILE_ALL_IN_ONE - All in one, the full stack on one host. " echo " Suitable for development and test environments." @@ -1296,6 +1297,7 @@ function read_user_input() echo " $PROFILE_CREATE_PUBLICATION - Create a new publication." echo " This profile will create new publication based on WF" echo " (if available) or ECE/clean-demo." + echo -n " $PROFILE_MUNIN_GATHERER - The Munin gatherer." echo "" echo -n "Your choice [1]> " read install_profile_number @@ -2004,11 +2006,11 @@ function install_munin_node() file=/etc/munin/plugin-conf.d/munin-node if [ -e $file ]; then cat >> $file < Date: Mon, 19 Sep 2011 19:45:20 +0800 Subject: [PATCH 0047/2585] - fixed bug introduced in 60ca5fbc19b07359c706de63ed7a262fe6595795, instead of just copying the conf directory of tomcat_home to _base, the entire tomcat_home was copied into _base. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 2e9af2ef..e525bd45 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -706,7 +706,7 @@ function set_up_app_server tomcat_base=/opt/tomcat-${instance_name} make_dir $tomcat_base - run cp -r /opt/${tomcat_dir}/ $tomcat_base + run cp -r /opt/${tomcat_dir}/conf $tomcat_base for el in bin escenic/lib lib work logs temp webapps; do make_dir $tomcat_base/$el done From 86f6497fdbe0aa2f1e04630e71629cc600384d16 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Sep 2011 12:08:51 +0800 Subject: [PATCH 0048/2585] - sometimes, the GPG key ring doesn't work when running as root, this should hopefully fix it (problem: installing Percona) --- usr/sbin/ece-install | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index e525bd45..a4cfdea7 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1474,13 +1474,13 @@ function install_database_server() print "Installing the Percona database ..." if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then - gpg --keyserver hkp://keys.gnupg.net \ + run gpg --keyserver hkp://keys.gnupg.net \ + --recv-keys 1C4CBDCDCD2EFD2A + gpg --armor \ -a \ --export 1C4CBDCDCD2EFD2A | \ apt-key add - \ 1>>$log 2>>$log - gpg -a --export CD2EFD2A | apt-key add - \ - 1>>$log 2>>$log run apt-get update fi add_apt_source "deb http://repo.percona.com/apt ${code_name} main" From 92713e818222932cd98550332a7ab00716a98d30 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Sep 2011 13:14:53 +0800 Subject: [PATCH 0049/2585] - added create_user_and_group_if_not_present, called from common_pre_install - user and groups no longer created in set_correct_permissions (it's too late!) --- usr/sbin/ece-install | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a4cfdea7..178227d7 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -865,22 +865,28 @@ EOF } +function create_user_and_group_if_not_present() +{ + if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then + print "Creating UNIX user $ece_user ..." + # TODO add support for useradd + run adduser $ece_user \ + --disabled-password \ + --gecos "Escenic-user,Room,Work,Home,Other" + add_next_step "- Set a password for the new $ece_user (passwd $ece_user)" + fi + + if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then + print "Creating UNIX group $ece_group ..." + run addgroup $ece_group + fi + } + # last, give the control back to the ECE user & group function set_correct_permissions() { print "Setting correct permissions on ECE related directories ..." - if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then - # TODO add support for useradd - run adduser $ece_user \ - --disabled-password \ - --gecos "Escenic-user,Room,Work,Home,Other" - add_next_step "- Set a password for the new $ece_user (passwd $ece_user)" - fi - if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then - run addgroup $ece_group - fi - for el in $engine_dir_list; do if [ -d $el ]; then run chown -R ${ece_user}:${ece_group} $el \ @@ -1360,6 +1366,7 @@ function common_pre_install() stop_conflicting_processes install_common_os_packages set_up_ece_scripts + create_user_and_group_if_not_present set_up_user_enviornment } From 850df5a952e23970dd07c39db1b6a69a4dfbcbd6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Sep 2011 15:35:30 +0800 Subject: [PATCH 0050/2585] - added more help on setting the ECE user's password. --- usr/sbin/ece-install | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 178227d7..8754875b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -873,7 +873,8 @@ function create_user_and_group_if_not_present() run adduser $ece_user \ --disabled-password \ --gecos "Escenic-user,Room,Work,Home,Other" - add_next_step "- Set a password for the new $ece_user (passwd $ece_user)" + add_next_step "- Set a password for the new UNIX user $ece_user" + add_next_step " You can do this by running [passwd $ece_user]" fi if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then From 8df58773cecb9fd08049ed0b9bedfabca10efcf4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Sep 2011 16:33:07 +0800 Subject: [PATCH 0051/2585] - added new command to ece, loglist which will produce output like: ece -i dev1 loglist [ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out [ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log [ece#engine-dev1] log4j log: /var/log/escenic/messages - added -q/--quiet option to ece, will suppress all user information from ece. This is also useful for the loglist command as it will then only output the paths without any prefix or information. - removed pager from "ece help" --- usr/share/doc/escenic/ece-guide.org | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index 0b50e445..c5b63f48 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -82,6 +82,38 @@ The following commands are available: *) only applicable if type is 'engine' #+END_SRC +* Tailing log files +The script has a number of options to let you keep track of what's +going on on your system. + +** Tailing the logs +There are three different logs, which ece will let you tail. When +tailing the log file, ece will also first print out where the file is +located on the file system so you can open it later for further +inspection. + +#+BEGIN_SRC sh +$ ece -i dev1 outlog +[ece#engine-dev1] Tailing the system out log /var/log/escenic/engine-dev1.out +#+END_SRC + +** Getting an overview of the log files +The ece script will be more than happy to tell you about the log files +on your system that are of interest to your instance: +#+BEGIN_SRC sh +$ ece -i dev1 loglist +[ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out +[ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log +[ece#engine-dev1] log4j log: /var/log/escenic/messages +#+END_SRC + +If you wish to pass the log files onto another pipe, for instance to +grep all the log files for a certain exception or error message, you +will find the -q parameter useful: +#+BEGIN_SRC sh +$ ece -i dev1 -q loglist | xargs grep IllegalArgumentException +#+END_SRC + * Installation The ece script and ece.conf may be used on any Unix like system that has a fairly recent version of BASH installed. From eb2116714aff7146e5442ad917e6405fb815434c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Sep 2011 16:33:46 +0800 Subject: [PATCH 0052/2585] - added new command to ece, loglist which will produce output like: $ ece -i dev1 loglist [ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out [ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log [ece#engine-dev1] log4j log: /var/log/escenic/messages - added -q/--quiet option to ece, will suppress all user information from ece. This is also useful for the loglist command as it will then only output the paths without any prefix or information. - removed pager from "ece help" --- usr/bin/ece | 77 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 22 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 09f6a536..02fec993 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -12,6 +12,7 @@ ###################################################################### verbose=0 force_ipv4=0 +quiet=0 ###################################################################### # Dear user, don't touch anyting in this script, especially below this @@ -88,7 +89,9 @@ function debug() function print() { - echo "$id" $@ + if [ $quiet -eq 0 ]; then + echo "$id" $@ + fi } function log() @@ -729,15 +732,22 @@ function assemble() debug $assemblytool_home/dist/engine.ear "is now ready" } -function tail_messages_log() +function get_log4j_log() { for el in $log_file_list; do if [ -r $el ]; then - print "tailing $el" - tail -f $el - break + echo $el fi done + + echo "" +} + +function tail_messages_log() +{ + log4j_log=$(get_log4j_log) + print "tailing $log4j_log" + tail -f $log4j_log } function tail_out_log() @@ -751,13 +761,8 @@ function tail_out_log() tail -f $tail_list } -function tail_app_log() +function get_app_log() { - if [ $type = "rmi-hub" ]; then - print "There is no application server log for $type" - exit 1 - fi - if [ $appserver = "tomcat" ]; then log_file=$tomcat_base/logs/localhost.`date +%F`.log elif [ $appserver = "resin" -a -e $resin_home/log/jvm-default.log ]; then @@ -768,9 +773,20 @@ function tail_app_log() print "tail_app_log()" exit 1 fi + + echo $log_file +} + + +function tail_app_log() +{ + if [ $type = "rmi-hub" ]; then + print "There is no application server log for $type" + exit 1 + fi print "Tailing the application server log $log_file" - tail -f $log_file + tail -f $(get_app_log) } function make_thread_dump() @@ -803,6 +819,10 @@ function set_type_command_and_instance() verbose=1 continue fi + if [ $el = "-q" -o $el = "--quiet" ]; then + quiet=1 + continue + fi if [ $el = "--help" ]; then command=help continue @@ -878,7 +898,7 @@ function clean_up() tmp_dir_prefix="`basename $0`-" if [ `ls $tmp_dir | grep $tmp_dir_prefix | wc -l` -gt 0 ]; then print "Cleaning up generated files in $tmp_dir ..." - rm -rf $tmp_dir/$tmp_dir_prefix-[0-9]* \ + rm -rf $tmp_dir/$tmp_dir_prefix[0-9]* \ 1>>$log_file \ 2>>$log_file fi @@ -1015,6 +1035,19 @@ function update_publication_resources() } +function show_all_log_paths() +{ + if [ $quiet -eq 0 ]; then + print "System out log: "$log_file + print "App server log: "$(get_app_log) + print "log4j log: "$(get_log4j_log) + else + echo $log_file + echo $(get_app_log) + echo $(get_log4j_log) + fi +} + set_type_command_and_instance $@ set_id set_common_settings @@ -1046,6 +1079,8 @@ function print_help() echo " Used for updating publication resources." echo " Must be one of: content-type, feature, layout, layout-group" echo " image-version, menu" + echo " -q --quiet" + echo " Makes $(basename $0) as quiet as possible." echo "" echo " -v --verbose" echo " Prints out debug statements, useful for debugging." @@ -1057,7 +1092,8 @@ function print_help() echo " deploy deploys the assembled EAR *)" echo " help prints this help screen" echo " kill uses force to stop the type" - echo " log the type's Log4J log" + echo " log the type's log4j log **)" + echo " loglist list all the log paths" echo " outlog the $id script log (system out log)" echo " restart restarts the type" echo " start starts the type" @@ -1068,6 +1104,7 @@ function print_help() echo " versions lists the ECE component versions" echo "" echo "*) only applicable if type is 'engine'" + echo "**) not applicable if type is 'rmi-hub'" } for el in $command; do @@ -1087,6 +1124,9 @@ for el in $command; do log) tail_messages_log ;; + loglist) + show_all_log_paths + ;; outlog) tail_out_log ;; @@ -1115,14 +1155,7 @@ for el in $command; do update_publication_resources ;; help) - if [ -x $(which less) ]; then - print_help | less - elif [ -x $(which more) ]; then - print_help | more - else - print_help - fi - + print_help ;; *) print "Invalid command: '$el' :-(" From d3432d420225d0b72493034b23320bf2ec1ae98a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Sep 2011 16:35:52 +0800 Subject: [PATCH 0053/2585] - added loglist to command completions --- etc/bash_completion.d/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index f22da95f..cdaec16d 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -4,7 +4,7 @@ _ece_commands() local cur=${COMP_WORDS[COMP_CWORD]} local prev=${COMP_WORDS[COMP_CWORD-1]} - commands="applog assemble clean deploy help kill log outlog restart start status stop threaddump update versions" + commands="applog assemble clean deploy help kill log loglist outlog restart start status stop threaddump update versions" resources="content-type feature layout layout-group image-version menu" types="engine search analysis rmi-hub" From 7bac9bddbd8d99bb232878be9d0cd2a6796e1bdc Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 21 Sep 2011 12:55:27 +0800 Subject: [PATCH 0054/2585] - ece nows use different options to ps to determine the type PID, depending on running on Linux (auxww) or other Unix systems (-ef). Using auxww on Linux systems make the script also return correctly when running in non-optimal shell environments, which default wise would cut off the process list enough for the ece to fail finding the running ECE processes. --- usr/bin/ece | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 02fec993..726e4f68 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -224,6 +224,12 @@ function set_type_settings() function set_type_pid() { + if [ $(uname) = "Linux" ]; then + ps_options="auxww" + else + ps_options="-ef" + fi + # Get a hold of the PID of the process. Although we've got the PID # file, we stil get the PID from the OS here as we used it for # sanity checking later on, see stop_type(). @@ -231,11 +237,11 @@ function set_type_pid() # we need to be able to differentiate between an ECE instance # and an rmi hub, for this we use java.security.policy which # the hub doesn't have. - type_pid=`ps -ef | grep -v java.security.policy | \ + type_pid=`ps $ps_options | grep -v java.security.policy | \ awk "/Djava.rmi.server.hostname=$ece_server_hostname / && \ !/awk/"' {print $2}'` else - type_pid=`ps -ef | awk "/Descenic.server=$ece_server / && !/awk/"' {print $2}'` + type_pid=`ps $ps_options | awk "/Descenic.server=$ece_server / && !/awk/"' {print $2}'` fi debug "type_pid is now=$type_pid" From 798b7eb5b1e378794805641c829eff4d7dd05188 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 21 Sep 2011 13:56:13 +0800 Subject: [PATCH 0055/2585] - tail_app_log now prints the correct log file name --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index 726e4f68..cfdd68e0 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -791,7 +791,7 @@ function tail_app_log() exit 1 fi - print "Tailing the application server log $log_file" + print "Tailing the application server log $(get_app_log)" tail -f $(get_app_log) } From 98be4ae9b6071da825bd2064cbc23de8f123a73a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 21 Sep 2011 14:53:16 +0800 Subject: [PATCH 0056/2585] - it's important to append (and not pre-pend) the ECE libraries so that things put it in the standard loader behaves as expected, such as log4j configuration put in ${tomcat.base}/lib. This change in set_up_app_server, made the app server pick up the log4j configuration in tomcat/lib properly. --- usr/sbin/ece-install | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8754875b..1fbe65d7 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -612,6 +612,19 @@ function get_base_dir_from_bundle() echo $file_name } +# Returns an escaped string useful for sed and other BASH commands. +# +# $1 : the string +function get_escaped_bash_string() +{ + result=$(echo $1 | \ + sed -e 's/\$/\\$/g' \ + -e 's/\*/\\*/g' \ + -e 's#/#\\/#g' \ + -e 's/\./\\./g') + echo $result +} + function set_up_app_server { print "Setting up the application server ..." @@ -717,9 +730,15 @@ function set_up_app_server run cd $tomcat_base/lib make_ln $jdbc_driver + # it's important to append (and not pre-pend) the ECE libraries so + # that things put it in the standard loader behaves as expected, + # such as log4j configuration put in ${tomcat.base}/lib. file=$tomcat_base/conf/catalina.properties - sed -i 's/common.loader=/common.loader=\$\{catalina.base\}\/escenic\/lib\/\*\.jar\,/g' \ - $file \ + common_loader=$(grep ^common.loader $file) + escenic_loader=",$\{catalina.base\}/escenic/lib/*.jar" + old=$(get_escaped_bash_string ${common_loader}) + new=${escaped_common_loader}$(get_escaped_bash_string ${escenic_loader}) + sed -i "s#${old}#${new}#g" $file \ 1>>$log 2>>$log exit_on_error "sed on $file" From 988d98a443e409832855e357e32ec4d8827429cd Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 22 Sep 2011 11:09:03 +0800 Subject: [PATCH 0057/2585] - removed TBD: we no longer have an issue with the Escenic Munin plugin --- usr/share/doc/escenic/ece-install-guide.org | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 499e6b59..d992d898 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -131,9 +131,6 @@ The script will install a Munin node on the host. It will also install the Escenic specific Munin plugins on hosts where there are any Escenic server components. -TBD: there is currently an issue with the Escenic plugins not working -properly with the new ece/ece-install scripts. We're working on this. - ** Common for Editorial, Presentation and Search Instances *** Sun Java 6 *** Apache Tomcat application server @@ -421,6 +418,9 @@ fai_enabled=1 fai_all_install=1 #+END_SRC +All the different components described in this guide are installed on +the host that runs the install script. + The following output is produced from ece-install: -#+INCLUDE: "/tmp/ece-install.out" src text +#+INCLUDE: "~/tmp/ece-install.out" src text From 4a3abf57f8a669e0f4312b9f901593f8aa7813eb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 22 Sep 2011 11:19:21 +0800 Subject: [PATCH 0058/2585] - added section on the Monitoring Server profile --- usr/share/doc/escenic/ece-install-guide.org | 28 ++++++++++++--------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index d992d898..1649637a 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -160,7 +160,7 @@ compressed content (and many other things that we before needed Apache for). Thus, we'll let the application server do the compression for us now. -** 1 - Full Stack on One Host +** Profile - Full Stack on One Host This profile is suitable for developers and admins wanting to set up a test environment. It will install the full stack including caching server, application server, ECE, assembly host, database, Widget @@ -170,18 +170,17 @@ For further details on each of the different server components, see the different profile descriptions bellow. -** 2 - Editorial Server (Publication Server) +** Profile - Editorial Server (Publication Server) +Will set up a typical editorial server (often referred to as the +publication server in Escenic literature). - -** 3 - Presentation Server +** Profile - Presentation Server This will set up a typical presentation server: - Memcached, distributed memory cache -- Escenic Assembly environment (which may be removed after the - installation). - Deployment setup to only deploy escenic-admin and the publication(s). -** 4 - Database Server +** Profile - Database Server If ece-install is run on a support version of Debian or Ubuntu, this will install the excellent Percona distribution of MySQL with their set of high performance patches. @@ -202,7 +201,7 @@ ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists [ece-install] running tables FAILED, exiting :-( #+END_SRC -** 5 - Cache Server +** Profile - Cache Server If ece-install is run on a support version of Debian or Ubuntu, it will install the latest Varnish 3.0 caching server from the Varnish APT repository. @@ -230,8 +229,6 @@ The script will configure Varnish for a typical Escenic site: - will install the nginx web server for serving static content and will configure Varnish accordingly. This will be very useful for Adactus servers wanting to pull content from your ECEs. -- will install a Munin gatherer and make its interface available - when accesssing the web server directly (port 81) TBD: - If run on a Linux platform, the script will tweak the kernel @@ -240,7 +237,7 @@ TBD: to be in the staff network ACL, defined in the Varnish configuration. -** 8 - Install Widget Framework +** Profile - Install Widget Framework You'll need a user name and password for accessing the repo.escenic.com Maven repository. You should get these credentials when you bought Widget Framework. If you for some reason do not have @@ -252,7 +249,7 @@ complain: [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf [ece-install] If you don't have these, please contact support@escenic.com #+END_SRC -** 9 - Create Publication +** Profile - Create Publication This profile will create a publication for you, only asking you the name of the publication and which ECE instance to use for its creation. @@ -260,6 +257,13 @@ creation. This installation profile will base the publication on the Widget Framework if its present on the system, if not, ECE's clean demo WAR is used as a basis. +** Profile - Monitoring Server +This will install a Munin gatherer and make its interface available +when accesssing the web server directly (port 81). + +TBD: This profile will also install the Nagios interface for +monitoring the different nodes. + * Full Automatic Install (FAI) The ece-install script has support for doing a full automatic install (FAI). From d3fe20e297c1858abbc7a5666d016732bd0514ed Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 22 Sep 2011 11:21:04 +0800 Subject: [PATCH 0059/2585] - renamed munin gatherer profile to Monitoring server. --- usr/sbin/ece-install | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 1fbe65d7..0badc2b1 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -116,7 +116,7 @@ PROFILE_RMI_HUB=6 PROFILE_SEARCH_SERVER=7 PROFILE_WIDGET_FRAMEWORK=8 PROFILE_CREATE_PUBLICATION=9 -PROFILE_MUNIN_GATHERER=10 +PROFILE_MONITORING_SERVER=10 # Because of issue VF-3559, we also create the default family and host # directories. @@ -1323,7 +1323,8 @@ function read_user_input() echo " $PROFILE_CREATE_PUBLICATION - Create a new publication." echo " This profile will create new publication based on WF" echo " (if available) or ECE/clean-demo." - echo -n " $PROFILE_MUNIN_GATHERER - The Munin gatherer." + echo -n " $PROFILE_MONITORING_SERVER - A monitoring server." + echo " This will install the Munin gatherer" echo "" echo -n "Your choice [1]> " read install_profile_number @@ -2121,7 +2122,7 @@ if [ $fai_enabled -eq 1 ]; then install_profile_number=$PROFILE_CREATE_PUBLICATION create_publication elif [ $(get_boolean_conf_value fai_munin_gatherer_create) -eq 1 ]; then - install_profile_number=$PROFILE_MUNIN_GATHERER + install_profile_number=$PROFILE_MONITORING_SERVER install_munin_gatherer fi @@ -2159,7 +2160,7 @@ else $PROFILE_CREATE_PUBLICATION) create_publication ;; - $PROFILE_MUNIN_GATHERER) + $PROFILE_MONITORING_SERVER) install_munin_gatherer ;; default) From f71d30ab736650c9d2ad6a60b34ce4ee1aa7fc5f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 26 Sep 2011 12:49:51 +0800 Subject: [PATCH 0060/2585] - added support for updating the VCE security resource --- usr/bin/ece | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index cfdd68e0..fe589ab1 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1023,11 +1023,12 @@ function update_publication_resources() menu) url=${url}/${publication}/escenic/plugin/menu ;; - + security) + url=${url}/${publication}/escenic/plugin/community/security + ;; *) print "Invalid resource: $(basename $resource) :-(" exit 1 - esac print "Updating the $(basename $resource) resource for the $publication" \ From 97ad92c4a68607c8945de298bea0f5d3a1dcdf68 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 27 Sep 2011 20:20:44 +0800 Subject: [PATCH 0061/2585] - corrected log4j log file name, now in accordance with the log files created by ece-install - fixed a wee error which would list all log4j files if there were more than one match. --- usr/bin/ece | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index fe589ab1..2a34e817 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -186,6 +186,7 @@ function set_common_settings() fi log_file_list=" + $log_dir/${HOSTNAME}-${instance}-messages $log_dir/messages $log_dir/Escenic-error.log " @@ -743,6 +744,7 @@ function get_log4j_log() for el in $log_file_list; do if [ -r $el ]; then echo $el + return fi done From 04a8b966831922079a58bee26739c3e01751300f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 27 Sep 2011 20:21:50 +0800 Subject: [PATCH 0062/2585] - added note and hints on adding community widget WF support to ece-install --- usr/sbin/ece-install | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0badc2b1..27e2c2a2 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1025,7 +1025,15 @@ EOF print "Basing your ${publication_name}.war on ECE/demo-clean ..." run cp /opt/escenic/engine/contrib/wars/demo-clean.war ${publication_war} fi - + + # TODO add support for the community widgets + # + # 1 - add the core widgets to the + # publications/demo-community/pom.xml + # + # 2 - add the definition for core + # widgets in: + # publications/demo-community/src/main/webapp/META-INF/escenic/publication-resources/escenic/content-type } function check_for_required_downloads() From feb604c530d1a1a459e2192e22b720568fd1957a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 27 Sep 2011 20:27:49 +0800 Subject: [PATCH 0063/2585] - updated the user guide on the log file goodness --- usr/share/doc/escenic/ece-guide.org | 32 ++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index c5b63f48..0a2200e3 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -86,7 +86,17 @@ The following commands are available: The script has a number of options to let you keep track of what's going on on your system. -** Tailing the logs +** Getting an overview of the log files +The ece script will be more than happy to tell you about the log files +on your system that are of interest to your instance: +#+BEGIN_SRC sh +$ ece -i dev1 loglist +[ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out +[ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log +[ece#engine-dev1] log4j log: /var/log/escenic/messages +#+END_SRC + +** Commands for tailing the log files There are three different logs, which ece will let you tail. When tailing the log file, ece will also first print out where the file is located on the file system so you can open it later for further @@ -97,16 +107,12 @@ $ ece -i dev1 outlog [ece#engine-dev1] Tailing the system out log /var/log/escenic/engine-dev1.out #+END_SRC -** Getting an overview of the log files -The ece script will be more than happy to tell you about the log files -on your system that are of interest to your instance: -#+BEGIN_SRC sh -$ ece -i dev1 loglist -[ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out -[ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log -[ece#engine-dev1] log4j log: /var/log/escenic/messages -#+END_SRC +The available log file commands are: +- outlog :: the system out log +- applog :: the app server log +- log :: the log4j log +** Piping the log files to another command If you wish to pass the log files onto another pipe, for instance to grep all the log files for a certain exception or error message, you will find the -q parameter useful: @@ -114,6 +120,12 @@ will find the -q parameter useful: $ ece -i dev1 -q loglist | xargs grep IllegalArgumentException #+END_SRC +This approach is also useful if you wish to tail all the log files at +once: +#+BEGIN_SRC sh +$ ece -i dev1 -q loglist | xargs tail -f +#+END_SRC + * Installation The ece script and ece.conf may be used on any Unix like system that has a fairly recent version of BASH installed. From da60c0638b8c4f77245508a544b190ba609ed2cb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 27 Sep 2011 20:28:57 +0800 Subject: [PATCH 0064/2585] - updated HTML and TXT guides --- usr/share/doc/escenic/ece-guide.html | 188 ++++++++++++++++++++------- usr/share/doc/escenic/ece-guide.txt | 78 ++++++++--- 2 files changed, 202 insertions(+), 64 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.html b/usr/share/doc/escenic/ece-guide.html index c0903d8c..aadeb8b0 100644 --- a/usr/share/doc/escenic/ece-guide.html +++ b/usr/share/doc/escenic/ece-guide.html @@ -7,7 +7,7 @@ Guide for the /usr/bin/ece Script - + @@ -44,6 +44,12 @@ dt { font-weight: bold; } div.figure { padding: 0.5em; } div.figure p { text-align: center; } + div.inlinetask { + padding:10px; + border:2px solid gray; + margin:10px; + background: #ffffcc; + } textarea { overflow-x: auto; } .linenr { font-size:smaller } .code-highlighted {background-color:#ffff00;} @@ -161,8 +167,8 @@ -
    +
    -

    Guide for the /usr/bin/ece Script

    + +
    + + @@ -251,15 +269,12 @@

    1.1 Completing ece comman

    -
    $ ece TAB TAB
    +
    $ ece TAB TAB
     applog      deploy      log         start       threaddump  
     assemble    help        outlog      status      update      
     clean       kill        restart     stop        versions 
    -
     
    - -

    The commands are all described in detail in "ece help"

    @@ -274,14 +289,11 @@

    1.2 Completing types of s -
    $ ece -t TAB TAB
    +
    $ ece -t TAB TAB
     analysis  engine    rmi-hub   search 
    -
     
    - -

    @@ -299,14 +311,11 @@

    1.3 Completing the public

    -
    $ ece -p mypub -r TAB TAB
    +
    $ ece -p mypub -r TAB TAB
     content-type   image-version  layout-group   
     feature        layout         menu
    -
     
    - -

    @@ -319,7 +328,7 @@

    2 Getting an overview of al -
    $ ece help
    +
    $ ece help
     Usage: /home/torstein/bin/ece [-t <type>] [-i <instance>] <command>
     
     DESCRIPTION
    @@ -330,7 +339,7 @@ 

    2 Getting an overview of al search - A standalone search indexer and solr instance rmi-hub - The RMI hub responsible for the internal communication between the ECE instances. - analysis - The Escenic Analysis Engine also knows as 'Stats' + analysis - The Escenic Analysis Engine also knows as 'Stats' -i --instance <instance> The type instance, such as editor1, web1 or search1 @@ -347,13 +356,13 @@

    2 Getting an overview of al Prints out debug statements, useful for debugging. The following commands are available: - applog the type's app server log - assemble runs the Assembly Tool *) - clean removes temporary files created by /home/torstein/bin/ece *) - deploy deploys the assembled EAR *) - help prints this help screen - kill uses force to stop the type - log the type's Log4J log + applog the type's app server log + assemble runs the Assembly Tool *) + clean removes temporary files created by /home/torstein/bin/ece *) + deploy deploys the assembled EAR *) + help prints this help screen + kill uses force to stop the type + log the type's Log4J log outlog the [ece#engine] script log (system out log) restart restarts the type start starts the type @@ -363,20 +372,109 @@

    2 Getting an overview of al update update publication resources versions lists the ECE component versions -*) only applicable if type is 'engine' +*) only applicable if type is 'engine' +

    + + +

    + + + +
    +

    3 Tailing log files

    +
    + +

    The script has a number of options to let you keep track of what's +going on on your system. +

    + +
    + +
    +

    3.1 Getting an overview of the log files

    +
    + +

    The ece script will be more than happy to tell you about the log files +on your system that are of interest to your instance: +

    + +
    $ ece -i dev1 loglist
    +[ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out
    +[ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log
    +[ece#engine-dev1] log4j log: /var/log/escenic/messages
     
    +
    + +
    + +
    +

    3.2 Commands for tailing the log files

    +
    + +

    There are three different logs, which ece will let you tail. When +tailing the log file, ece will also first print out where the file is +located on the file system so you can open it later for further +inspection. +

    + + + +
    $ ece -i dev1 outlog
    +[ece#engine-dev1] Tailing the system out log /var/log/escenic/engine-dev1.out
    +
    + + +

    +The available log file commands are: +

    +
    outlog
    the system out log +
    +
    applog
    the app server log +
    +
    log
    the log4j log +
    +
    -
    -

    3 Installation

    -
    +
    +

    3.3 Piping the log files to another command

    +
    + +

    If you wish to pass the log files onto another pipe, for instance to +grep all the log files for a certain exception or error message, you +will find the -q parameter useful: +

    + + +
    $ ece -i dev1 -q loglist | xargs grep IllegalArgumentException
    +
    + + +

    +This approach is also useful if you wish to tail all the log files at +once: +

    + + +
    $ ece -i dev1 -q loglist | xargs tail -f
    +
    + + +
    +
    + +
    + +
    +

    4 Installation

    +

    The ece script and ece.conf may be used on any Unix like system that has a fairly recent version of BASH installed. @@ -384,9 +482,9 @@

    3 Installation

    -
    -

    3.1 Overview of File Paths Used by the ece script

    -
    +
    +

    4.1 Overview of File Paths Used by the ece script

    +

    These are recommended files and locations for using the ece script:

    @@ -438,13 +536,10 @@

    3.1 Overview of File Path

    -
    ECE_CONF_LOCATIONS 
    -
    +
    ECE_CONF_LOCATIONS 
     
    - -

    The reason for having so many options is because various Escenic consultants, partners and customers have requested these locations to @@ -456,23 +551,20 @@

    3.1 Overview of File Path

    -
    -

    4 Running the ece script

    -
    +
    +

    5 Running the ece script

    +

    You must be normal user to run the ece script, otherwise it will complain:

    -
    [ece#engine] Sorry, you cannot be root when running ece
    +
    [ece#engine] Sorry, you cannot be root when running ece
     [ece#engine] The root user can only use /etc/init.d/ece
    -
     
    - -

    As it mentions, the root user may use the init.d script and the accompanying /etc/default/ece to command the different ECE, EAE and @@ -480,12 +572,14 @@

    4 Running the ece script

    +
    +

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-09-12 18:14:44 CST -

    + Date: 2011-09-27 20:28:00 CST +

    diff --git a/usr/share/doc/escenic/ece-guide.txt b/usr/share/doc/escenic/ece-guide.txt index d3ee7c8d..4b053049 100644 --- a/usr/share/doc/escenic/ece-guide.txt +++ b/usr/share/doc/escenic/ece-guide.txt @@ -2,7 +2,7 @@ ================================= Author: Torstein Krause Johansen -Date: 2011-09-12 18:14:41 CST +Date: 2011-09-27 20:27:57 CST Table of Contents @@ -12,9 +12,13 @@ Table of Contents 1.2 Completing types of servers the ece scripts can operate on 1.3 Completing the publication resources available 2 Getting an overview of all available options -3 Installation - 3.1 Overview of File Paths Used by the ece script -4 Running the ece script +3 Tailing log files + 3.1 Getting an overview of the log files + 3.2 Commands for tailing the log files + 3.3 Piping the log files to another command +4 Installation + 4.1 Overview of File Paths Used by the ece script +5 Running the ece script 1 TAB completion @@ -34,8 +38,6 @@ to remember all the different options and their correct wording: assemble help outlog status update clean kill restart stop versions - - The commands are all described in detail in "ece help" 1.2 Completing types of servers the ece scripts can operate on @@ -46,8 +48,6 @@ The commands are all described in detail in "ece help" analysis engine rmi-hub search - - 1.3 Completing the publication resources available =================================================== The ece script can update the publication resources for a given @@ -61,8 +61,6 @@ list and complete the available resource names: content-type image-version layout-group feature layout menu - - 2 Getting an overview of all available options ----------------------------------------------- @@ -114,14 +112,64 @@ list and complete the available resource names: *) only applicable if type is 'engine' +3 Tailing log files +-------------------- +The script has a number of options to let you keep track of what's +going on on your system. + +3.1 Getting an overview of the log files +========================================= +The ece script will be more than happy to tell you about the log files +on your system that are of interest to your instance: + + + $ ece -i dev1 loglist + [ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out + [ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log + [ece#engine-dev1] log4j log: /var/log/escenic/messages + + +3.2 Commands for tailing the log files +======================================= +There are three different logs, which ece will let you tail. When +tailing the log file, ece will also first print out where the file is +located on the file system so you can open it later for further +inspection. + + + + $ ece -i dev1 outlog + [ece#engine-dev1] Tailing the system out log /var/log/escenic/engine-dev1.out -3 Installation +The available log file commands are: +outlog: the system out log +applog: the app server log +log: the log4j log + +3.3 Piping the log files to another command +============================================ +If you wish to pass the log files onto another pipe, for instance to +grep all the log files for a certain exception or error message, you +will find the -q parameter useful: + + + $ ece -i dev1 -q loglist | xargs grep IllegalArgumentException + + +This approach is also useful if you wish to tail all the log files at +once: + + + $ ece -i dev1 -q loglist | xargs tail -f + + +4 Installation --------------- The ece script and ece.conf may be used on any Unix like system that has a fairly recent version of BASH installed. -3.1 Overview of File Paths Used by the ece script +4.1 Overview of File Paths Used by the ece script ================================================== These are recommended files and locations for using the ece script: @@ -155,14 +203,12 @@ this environment variable in your .bashrc or similar: ECE_CONF_LOCATIONS - - The reason for having so many options is because various Escenic consultants, partners and customers have requested these locations to fit their systems. As you can see, fitting everyone's fancy adds up over time :-) -4 Running the ece script +5 Running the ece script ------------------------- You must be normal user to run the ece script, otherwise it will complain: @@ -172,8 +218,6 @@ complain: [ece#engine] The root user can only use /etc/init.d/ece - - As it mentions, the root user may use the init.d script and the accompanying /etc/default/ece to command the different ECE, EAE and RMI hub instances on your system. From 7c877d4b471adb6fb1cfa251b7c627326b4c8a87 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 29 Sep 2011 13:11:59 +0800 Subject: [PATCH 0065/2585] - added backup command which will run as follows: $ ece -i dev1 backup [ece#engine-dev1] Cleaning up generated files in /opt/escenic/assemblytool ... [ece#engine-dev1] Backing up the dev1 instance of engine on quanah ... [ece#engine-dev1] Database dumped: /var/backups/escenic/ece5db-2011-09-29.sql.gz [ece#engine-dev1] Creating snapshot ... (this will take a while) [ece#engine-dev1] Backup ready: /tmp/engine-dev1-backup-2011-09-29.tar.gz [ece#engine-dev1] The backup arhcive includes: [ece#engine-dev1] - Database snapshot [ece#engine-dev1] - All Solr & Escenic data files from /var/lib/escenic [ece#engine-dev1] - All app servers in /opt [ece#engine-dev1] - All Escenic binaries & publication templates in /opt/escenic [ece#engine-dev1] - All Escenic configuration in /etc/escenic and /etc/default/ece [ece#engine-dev1] - All Escenic bootstrap scripts from /etc/init.d [ece#engine-dev1] Enjoy! - the DB settings are read from the instance application server configuration. Currently, only Tomcat is supported. --- usr/bin/ece | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 2a34e817..8813b373 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -524,6 +524,107 @@ function deploy() esac } +function backup_type() +{ + message="Backing up the $instance instance of $type on $HOSTNAME ..." + print $message + log $message + + if [ $type = "rmi-hub" ]; then + ensure_that_required_fields_are_set $hub_required_fields + elif [ $type = "search" ]; then + # TODO trim & tun the default parameters for the search + # instance. + ensure_that_required_fields_are_set $engine_required_fields + elif [ $type = "engine" ]; then + ensure_that_required_fields_are_set $engine_required_fields + elif [ $type = "analysis" ]; then + ensure_that_required_fields_are_set $analysis_required_fields + fi + + # TODO add support for backup_dir in ece.conf + if [ -z "$backup_dir" ]; then + backup_dir=/var/backups/escenic + fi + if [ ! -w $backup_dir ]; then + print $backup_dir "must exist & be write-able for $USER" + exit 1 + fi + + if [ $appserver = "tomcat" ]; then + connect_string=$(find $tomcat_base/conf | xargs \ + grep jdbc | \ + grep url | \ + head -1 | \ + cut -d'"' -f2) + db_port=$(echo $connect_string | cut -d':' -f4 | cut -d'/' -f1) + db=$(echo $connect_string | cut -d'/' -f4 | cut -d'?' -f1) + db_host=$(echo $connect_string | cut -d'/' -f3- | cut -d':' -f1) + + db_user=$(find $tomcat_base/conf -name "*.xml" | \ + grep -v tomcat-users.xml | xargs \ + grep username | \ + grep -v \ $db_backup_file + + print "Database dumped: $db_backup_file" + fi + + archive_dir=/tmp + archive_file=/tmp/${type}-${instance}-backup-$(date --iso).tar.gz + possible_backup=" +/etc/escenic +/opt/escenic +/etc/init.d/ece +/etc/default/ece +/etc/inti.d/rmi-hub +/var/lib/escenic +" + actual_backup="" + for el in $possible_backup; do + if [ -r $el ]; then + actual_backup="$el $actual_backup" + fi + done + + if [ $appserver = "tomcat" ]; then + actual_backup="$tomcat_home $tomcat_base $actual_backup" + fi + + print "Creating snapshot ... (this will take a while)" + tar czf $archive_file \ + $actual_backup \ + $db_backup_file \ + 1>>$log_file \ + 2>>$log_file + + message="Backup ready: $archive_file" + print $message + log $message + print "The backup arhcive includes:" + print "- Database snapshot" + print "- All Solr & Escenic data files from /var/lib/escenic" + print "- All app servers in /opt" + print "- All Escenic binaries & publication templates in /opt/escenic" + print "- All Escenic configuration in /etc/escenic and /etc/default/ece" + print "- All Escenic bootstrap scripts from /etc/init.d" + print "Enjoy!" + } + function start_type() { @@ -1097,6 +1198,7 @@ function print_help() echo "The following commands are available:" echo " applog the type's app server log" echo " assemble runs the Assembly Tool *)" + echo " backup backs up an entire instance (takes time)" echo " clean removes temporary files created by $0 *)" echo " deploy deploys the assembled EAR *)" echo " help prints this help screen" @@ -1163,6 +1265,10 @@ for el in $command; do update) update_publication_resources ;; + backup) + clean_up + backup_type + ;; help) print_help ;; From cc3e6f513898d1e9826f5b939b94df8ef1a2300c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 29 Sep 2011 13:12:17 +0800 Subject: [PATCH 0066/2585] - added the backup command --- etc/bash_completion.d/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index cdaec16d..6d816e29 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -4,7 +4,7 @@ _ece_commands() local cur=${COMP_WORDS[COMP_CWORD]} local prev=${COMP_WORDS[COMP_CWORD-1]} - commands="applog assemble clean deploy help kill log loglist outlog restart start status stop threaddump update versions" + commands="applog assemble backup clean deploy help kill log loglist outlog restart start status stop threaddump update versions" resources="content-type feature layout layout-group image-version menu" types="engine search analysis rmi-hub" From 67313ca6dbd18ceeb6a0a21b9d7f092d2bfe96be Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 29 Sep 2011 13:12:34 +0800 Subject: [PATCH 0067/2585] - added backup_dir where ece backup will put its files --- etc/escenic/ece.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/escenic/ece.conf b/etc/escenic/ece.conf index 2f3ad3bf..9b1e771e 100644 --- a/etc/escenic/ece.conf +++ b/etc/escenic/ece.conf @@ -126,3 +126,9 @@ is_production=1 ##################################################################### enable_heap_dump=1 heap_dump_dir=/var/crash/escenic + +##################################################################### +# Where 'ece -i backup' will put its snapshot +# backups. Default is /var/backups/escenic. +##################################################################### +backup_dir=/var/backups/escenic From c29b77587015c7263ef4cbfe01e6afa1e1386526 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 29 Sep 2011 13:16:59 +0800 Subject: [PATCH 0068/2585] - added some hardening to ece backup if the ece.conf isn't sane or it's running with a non existent instance --- usr/bin/ece | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 8813b373..306712e8 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -552,6 +552,12 @@ function backup_type() fi if [ $appserver = "tomcat" ]; then + if [ ! -d $tomcat_base/conf ]; then + print $tomcat_base/conf "doesn't exist :-(" + print "check your ece.conf the $instance instance of type $type" + exit 1 + fi + connect_string=$(find $tomcat_base/conf | xargs \ grep jdbc | \ grep url | \ From 259db6e0677bed9a46d66fd7d252b4719fed43cf Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 7 Oct 2011 15:03:12 +0800 Subject: [PATCH 0069/2585] - the backup archive is now put in /var/backups/escenic (where the SQL backup is as well). --- usr/bin/ece | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 306712e8..f085f83f 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -590,8 +590,7 @@ function backup_type() print "Database dumped: $db_backup_file" fi - archive_dir=/tmp - archive_file=/tmp/${type}-${instance}-backup-$(date --iso).tar.gz + archive_file=$backup_dir/${type}-${instance}-backup-$(date --iso).tar.gz possible_backup=" /etc/escenic /opt/escenic From abd475e1362e118d88eb15d83c9963f58a30378f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 7 Oct 2011 15:21:52 +0800 Subject: [PATCH 0070/2585] - moved setting of port from set_common_settings to its own method, set_type_port, which is then called from set_type_settings (which is more logical). --- usr/bin/ece | 54 ++++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index f085f83f..6a7f2000 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -190,30 +190,6 @@ function set_common_settings() $log_dir/messages $log_dir/Escenic-error.log " - - host=localhost - if [ $appserver != "tomcat" ]; then - debug "Only tomcat is supported for reading host/port right now, " - debug "trying to make an educated guess" - port=8080 - else - if [ -r $tomcat_base/logs/catalina.out ]; then - # for tomcat6-user packaged tomcats (and perhaps others) - out_log=$tomcat_base/logs/catalina.out - else - out_log=$log_file - fi - - if [ -r $out_log ]; then - port=$(grep "INFO: Starting Coyote HTTP/1.1 on http-" \ - $out_log | \ - tail -1 | \ - cut -d'-' -f2 - ) - fi - - fi - } function set_type_settings() @@ -248,6 +224,33 @@ function set_type_pid() debug "type_pid is now=$type_pid" } +function set_type_port() +{ + host=localhost + if [ $appserver != "tomcat" ]; then + debug "Only tomcat is supported for reading host/port right now, " + debug "trying to make an educated guess" + port=8080 + else + if [ -r $tomcat_base/logs/catalina.out ]; then + # for tomcat6-user packaged tomcats (and perhaps others) + out_log=$tomcat_base/logs/catalina.out + else + out_log=$log_file + fi + + if [ -r $out_log ]; then + port=$(grep "INFO: Starting Coyote HTTP/1.1 on http-" \ + $out_log | \ + tail -1 | \ + cut -d'-' -f2 + ) + fi + fi + + debug "set_type_port=$port" +} + function set_instance_settings() { # optional: possible to have instance specific conf files, @@ -277,6 +280,8 @@ function set_instance_settings() fi set_type_pid + + set_type_port # if type is rmi-hub, we don't' need more configuration. if [ $type = "rmi-hub" ]; then @@ -385,7 +390,6 @@ function set_instance_settings() debug ECE_HOME=$ECE_HOME debug JAVA_HOME=$JAVA_HOME debug JAVA_OPTS=$JAVA_OPTS - } function sanity_check() From ba67153709412b5bfbab140898e2ad521ca66177 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 7 Oct 2011 15:33:40 +0800 Subject: [PATCH 0071/2585] - fixed debian_or_derivative test bug in add_server_to_runlevels and install_packages_if_missing. The first one would lead to update-rc.d was running for all distributions. Thanks to adarshdeshratnam for getting me on to this. --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 27e2c2a2..ca419aa2 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -200,7 +200,7 @@ function download_escenic_components() # $1 : space separated string of package names install_packages_if_missing() { - if [ $on_debian_or_derivative ]; then + if [ $on_debian_or_derivative -eq 1 ]; then some_are_missing=0 for el in $@; do # we don't need to grep away "No packages found matching @@ -2067,7 +2067,7 @@ function add_server_to_runlevels() return fi - if [ $on_debian_or_derivative ]; then + if [ $on_debian_or_derivative -eq 1 ]; then print "Adding the ece init.d script to the default run levels ..." run update-rc.d ece defaults else From cd82cf37a22bd7a7995761741f7387474de533c0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 7 Oct 2011 16:48:25 +0800 Subject: [PATCH 0072/2585] - added section on assembly when upgrading ECE and/or plugins. --- usr/share/doc/escenic/ece-guide.org | 43 ++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index 0a2200e3..80ba055b 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -126,6 +126,33 @@ once: $ ece -i dev1 -q loglist | xargs tail -f #+END_SRC +* Assembling a new EAR +The script will quite happily assemble your EAR file. Please note that +you need to run +#+BEGIN_SRC sh +$ ece clean assemble +#+END_SRC +whenever you've upgraded either ECE or one of its plugins. The script +will try to take care of you though and warn you of such duplication: +#+BEGIN_SRC sh +[ece#engine] Multiple versions of ECE and/or 3rd party libraries found. +[ece#engine] Remember, you need to run 'ece clean assemble' when +[ece#engine] upgrading either ECE or one of the plugins. +[ece#engine] Press ENTER to run 'ece clean assemble' +#+END_SRC +The script will then re-run the assembly by itself. + +If you're just re-running the assembly after just adding a new plugin +or want to re-build your publications after template changes, you can +omit the clean command and just run: +#+BEGIN_SRC sh +$ ece assembly +#+END_SRC + +The EAR produced is now ready to be deployed with +#+BEGIN_SRC sh +$ ece [-i ] deploy +#+END_SRC * Installation The ece script and ece.conf may be used on any Unix like system that has a fairly recent version of BASH installed. @@ -133,13 +160,15 @@ has a fairly recent version of BASH installed. ** Overview of File Paths Used by the ece script These are recommended files and locations for using the ece script: -|----------------------------------+-----------------------------| -| Path | Explanation | -|----------------------------------+-----------------------------| -| /usr/bin/ece | The script itself | -| /etc/escenic/ece.conf | The main configuration file | -| /etc/escenic/ece-.conf | Instance specific settings | -|----------------------------------+-----------------------------| +|----------------------------------+----------------------------------| +| Path | Explanation | +|----------------------------------+----------------------------------| +| /usr/bin/ece | The script itself | +| /etc/escenic/ece.conf | The main configuration file | +| /etc/escenic/ece-.conf | Instance specific settings | +| /var/cache/escenic | Directory of assembled EAR files | +| /tmp | Directory for temporary files | +|----------------------------------+----------------------------------| As you can see in ece.conf, there are a number of default locations dealing with log files, pid files, crash files as well as application From e1796839a5e0e84484258f56ca1c40bb3ec10e4f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 7 Oct 2011 16:48:35 +0800 Subject: [PATCH 0073/2585] - updated --- usr/share/doc/escenic/ece-guide.html | 81 +++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.html b/usr/share/doc/escenic/ece-guide.html index aadeb8b0..ac540c65 100644 --- a/usr/share/doc/escenic/ece-guide.html +++ b/usr/share/doc/escenic/ece-guide.html @@ -1,13 +1,13 @@ - + Guide for the /usr/bin/ece Script - + - + @@ -239,12 +239,13 @@

    Table of Contents

  • 3.3 Piping the log files to another command
  • -
  • 4 Installation +
  • 4 Assembling a new EAR
  • +
  • 5 Installation
  • -
  • 5 Running the ece script
  • +
  • 6 Running the ece script
  • @@ -473,18 +474,68 @@

    3.3 Piping the log files

    -

    4 Installation

    +

    4 Assembling a new EAR

    +

    The script will quite happily assemble your EAR file. Please note that +you need to run +

    + + +
    $ ece clean assemble
    +
    + +

    +whenever you've upgraded either ECE or one of its plugins. The script +will try to take care of you though and warn you of such duplication: +

    + + +
    [ece#engine] Multiple versions of ECE and/or 3rd party libraries found.
    +[ece#engine] Remember, you need to run 'ece clean assemble' when
    +[ece#engine] upgrading either ECE or one of the plugins.
    +[ece#engine] Press ENTER to run 'ece clean assemble'
    +
    + +

    +The script will then re-run the assembly by itself. +

    +

    +If you're just re-running the assembly after just adding a new plugin +or want to re-build your publications after template changes, you can +omit the clean command and just run: +

    + + +
    $ ece assembly
    +
    + + +

    +The EAR produced is now ready to be deployed with +

    + + +
    $ ece [-i <instance>] deploy
    +
    + +
    + +
    + +
    +

    5 Installation

    +
    +

    The ece script and ece.conf may be used on any Unix like system that has a fairly recent version of BASH installed.

    -
    -

    4.1 Overview of File Paths Used by the ece script

    -
    +
    +

    5.1 Overview of File Paths Used by the ece script

    +

    These are recommended files and locations for using the ece script:

    @@ -499,6 +550,8 @@

    4.1 Overview of File Path /usr/bin/eceThe script itself /etc/escenic/ece.confThe main configuration file /etc/escenic/ece-<instance>.confInstance specific settings +/var/cache/escenicDirectory of assembled EAR files +/tmpDirectory for temporary files @@ -551,9 +604,9 @@

    4.1 Overview of File Path

    -
    -

    5 Running the ece script

    -
    +
    +

    6 Running the ece script

    +

    You must be normal user to run the ece script, otherwise it will complain: @@ -578,7 +631,7 @@

    5 Running the ece script

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-09-27 20:28:00 CST + Date: 2011-10-07 16:47:40 CST

    From aef01987d0e17da65d263b722cc37bb6615a47b6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 7 Oct 2011 16:48:42 +0800 Subject: [PATCH 0074/2585] - updated --- usr/share/doc/escenic/ece-guide.txt | 59 +++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.txt b/usr/share/doc/escenic/ece-guide.txt index 4b053049..8d7f9e5a 100644 --- a/usr/share/doc/escenic/ece-guide.txt +++ b/usr/share/doc/escenic/ece-guide.txt @@ -2,7 +2,7 @@ ================================= Author: Torstein Krause Johansen -Date: 2011-09-27 20:27:57 CST +Date: 2011-10-07 16:47:42 CST Table of Contents @@ -16,9 +16,10 @@ Table of Contents 3.1 Getting an overview of the log files 3.2 Commands for tailing the log files 3.3 Piping the log files to another command -4 Installation - 4.1 Overview of File Paths Used by the ece script -5 Running the ece script +4 Assembling a new EAR +5 Installation + 5.1 Overview of File Paths Used by the ece script +6 Running the ece script 1 TAB completion @@ -164,20 +165,54 @@ once: $ ece -i dev1 -q loglist | xargs tail -f -4 Installation +4 Assembling a new EAR +----------------------- +The script will quite happily assemble your EAR file. Please note that +you need to run + + + $ ece clean assemble + +whenever you've upgraded either ECE or one of its plugins. The script +will try to take care of you though and warn you of such duplication: + + + [ece#engine] Multiple versions of ECE and/or 3rd party libraries found. + [ece#engine] Remember, you need to run 'ece clean assemble' when + [ece#engine] upgrading either ECE or one of the plugins. + [ece#engine] Press ENTER to run 'ece clean assemble' + +The script will then re-run the assembly by itself. + +If you're just re-running the assembly after just adding a new plugin +or want to re-build your publications after template changes, you can +omit the clean command and just run: + + + $ ece assembly + + +The EAR produced is now ready to be deployed with + + + $ ece [-i ] deploy + +5 Installation --------------- The ece script and ece.conf may be used on any Unix like system that has a fairly recent version of BASH installed. -4.1 Overview of File Paths Used by the ece script +5.1 Overview of File Paths Used by the ece script ================================================== These are recommended files and locations for using the ece script: - Path Explanation - ----------------------------------+----------------------------- - /usr/bin/ece The script itself - /etc/escenic/ece.conf The main configuration file - /etc/escenic/ece-.conf Instance specific settings + Path Explanation + ----------------------------------+---------------------------------- + /usr/bin/ece The script itself + /etc/escenic/ece.conf The main configuration file + /etc/escenic/ece-.conf Instance specific settings + /var/cache/escenic Directory of assembled EAR files + /tmp Directory for temporary files As you can see in ece.conf, there are a number of default locations dealing with log files, pid files, crash files as well as application @@ -208,7 +243,7 @@ consultants, partners and customers have requested these locations to fit their systems. As you can see, fitting everyone's fancy adds up over time :-) -5 Running the ece script +6 Running the ece script ------------------------- You must be normal user to run the ece script, otherwise it will complain: From 885e2232cc6d5e24f2d8f4273b3bb922a576e905 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 7 Oct 2011 16:52:45 +0800 Subject: [PATCH 0075/2585] - 'ece assemble' will now tell you if the generated EAR file contains duplicates libraries and if yes, offer you to re-run the assembly after cleaning up: [ece#engine] Multiple versions of ECE and/or 3rd party libraries found. [ece#engine] Remember, you need to run 'ece clean assemble' when [ece#engine] upgrading either ECE or one of the plugins. [ece#engine] Press ENTER to run 'ece clean assemble' --- usr/bin/ece | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 6a7f2000..c0f599ca 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -840,6 +840,33 @@ function assemble() 2>>$log_file exit_on_error "$message" + # test to see if the new versions of ECE & plugins are upgrades of + # the previous ones + duplicates_found=0 + known_unharmful_duplicates="activation- $'\n' ehcache-$'\n' stax-api-$'\n'" + + cd $assemblytool_home/dist/.work/ear/lib + for el in *.jar; do + jar=$(basename $(echo $el | sed -e 's/[0-9]//g') .jar | sed 's/\.//g') + if [ $(echo $known_unharmful_duplicates | grep $jar | wc -l) -gt 0 ]; then + continue + fi + if [ $(\ls *.jar | grep "$jar[0-9]" | wc -l) -gt 1 ]; then + duplicates_found=1 + debug "More than one version of $jar" + fi + done + + if [ $duplicates_found -eq 1 ]; then + print "Multiple versions of ECE and/or 3rd party libraries found." + print "Remember, you need to run '$(basename $0) clean assemble' when " + print "upgrading either ECE or one of the plugins." + print "Press ENTER to run '$(basename $0) clean assemble'" + read ok + clean_up + assemble + fi + mkdir -p $ear_cache_dir/ exit_on_error "creating $ear_cache_dir" From a494c0a0812e1478ba2887d2fcd8ab1d6ba93217 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 10 Oct 2011 13:01:54 +0800 Subject: [PATCH 0076/2585] - added check to sanity_check: a check if the user is using the default instance when he/she really wants to start one of the instances installed on the system. - new method get_instance_list, makes an educated guess of the instances installed. Works only if the system is installed with recommended paths. This change produces output like this on systems that have one or more instances installed: $ ece status [ece#engine] You must specify the instance with -i [ece#engine] e.g. ece -i dev1 status [ece#engine] Instances available on quanah: dev1 --- usr/bin/ece | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index c0f599ca..63b9d13d 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -155,6 +155,20 @@ function read_conf_file() done } +# If the system is installed using the recommended paths, the method +# will return a list of the instances configured in +# /etc/escenic/engine/instance +function get_instance_list() +{ + instance_list="" + + if [ -r /etc/escenic/engine/instance ]; then + instance_list=$(ls /etc/escenic/engine/instance) + fi + + echo $instance_list +} + function set_common_settings() { # This behaviour can be overridden by specifying a list of @@ -407,6 +421,19 @@ function sanity_check() print "You must specificy a command, see 'ece help'." exit 1 fi + + # checks if the user is using the default instance when he/she + # really wants to start one of the instances installed on the + # system. + if [ $instance = "default" -a $type = "engine" ]; then + instance_list=$(get_instance_list) + if [ -n "$instance_list" ]; then + print "You must specify the instance with -i " + print "e.g. ece -i $(echo $instance_list | cut -d' ' -f1) $command" + print "Instances available on $HOSTNAME: $instance_list" + exit 1 + fi + fi } function deploy() From 1d65314ebc6fa2b967c33b0817da5e1d87459257 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 10 Oct 2011 13:11:57 +0800 Subject: [PATCH 0077/2585] - renamed git_source -> ece_scripts_git_source --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index ca419aa2..9115ca2e 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -29,7 +29,7 @@ download_dir=/tmp/ece-downloads log=/var/log/$(basename $0).log conf_file=$HOME/.ece-install.conf common_nursery_dir=/etc/escenic/engine/common -git_source=git://github.com/skybert/ece-scripts.git +ece_scripts_git_source=git://github.com/skybert/ece-scripts.git # globals will be set to correct values in run-time. appserver_port=8080 @@ -354,7 +354,7 @@ function set_up_ece_scripts() (cd ece-scripts git pull 1>>$log 2>>$log) else - run git clone $git_source + run git clone $ece_scripts_git_source fi run cp -r ece-scripts/usr/* /usr/ From d9bcd13f1134af8d0d58096886e2e33ca935fc4e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 10 Oct 2011 14:40:15 +0800 Subject: [PATCH 0078/2585] - New logic to: ece assemble If multiple versions of dependencies are detected, ece will try to sort it out for the user without any interaction, as follows: $ ece assemble -i dev1 [ece#engine-dev1] Assembling your EAR file ... [ece#engine-dev1] Multiple versions of ECE and/or 3rd party libraries found. [ece#engine-dev1] Remember, you need to run 'ece clean assemble' when [ece#engine-dev1] upgrading either ECE or one of the plugins. [ece#engine-dev1] I will now clean it up for you and re-run the assembly. [ece#engine-dev1] Cleaning up generated files in /opt/escenic/assemblytool ... [ece#engine-dev1] Assembling your EAR file ... The change is in the assemble() function and will work regardless of the commands issued to the ece script. For the user, this means that this will work when assemble is just one of a line of commands passed to ece, e.g.: ece assemble stop deploy start --- usr/bin/ece | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 63b9d13d..53208a67 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -888,8 +888,7 @@ function assemble() print "Multiple versions of ECE and/or 3rd party libraries found." print "Remember, you need to run '$(basename $0) clean assemble' when " print "upgrading either ECE or one of the plugins." - print "Press ENTER to run '$(basename $0) clean assemble'" - read ok + print "I will now clean it up for you and re-run the assembly." clean_up assemble fi From 87a744667a2880f7eabcafd71affa41172816f14 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 10 Oct 2011 14:57:49 +0800 Subject: [PATCH 0079/2585] - updated documentation on assembly - added section describing the difference between the default instance and the name instances. --- usr/share/doc/escenic/ece-guide.org | 37 ++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index 80ba055b..aa1e8a37 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -130,7 +130,7 @@ $ ece -i dev1 -q loglist | xargs tail -f The script will quite happily assemble your EAR file. Please note that you need to run #+BEGIN_SRC sh -$ ece clean assemble +$ ece -i clean assemble #+END_SRC whenever you've upgraded either ECE or one of its plugins. The script will try to take care of you though and warn you of such duplication: @@ -138,9 +138,10 @@ will try to take care of you though and warn you of such duplication: [ece#engine] Multiple versions of ECE and/or 3rd party libraries found. [ece#engine] Remember, you need to run 'ece clean assemble' when [ece#engine] upgrading either ECE or one of the plugins. -[ece#engine] Press ENTER to run 'ece clean assemble' +[ece#engine] I will now clean it up for you and re-run the assembly. +[ece#engine] Cleaning up generated files in /opt/escenic/assemblytool ... +[ece#engine] Assembling your EAR file ... #+END_SRC -The script will then re-run the assembly by itself. If you're just re-running the assembly after just adding a new plugin or want to re-build your publications after template changes, you can @@ -199,6 +200,7 @@ fit their systems. As you can see, fitting everyone's fancy adds up over time :-) * Running the ece script +** You must run it as a non-privileged user You must be normal user to run the ece script, otherwise it will complain: #+BEGIN_SRC sh @@ -209,4 +211,33 @@ complain: As it mentions, the root user may use the init.d script and the accompanying /etc/default/ece to command the different ECE, EAE and RMI hub instances on your system. +** Specifying instance +*** Specifying the instance +The script is made for being easy to use with multiple instances on +the same host. You specify the instance you want to operate on using the +-i parameter. E.g. to assemble and deploy the editor1 instance, you'd +do: +#+BEGIN_SRC sh +$ ece -i editor1 assemble deploy restart +#+END_SRC + +*** Using the default instance +However, on development systems and perhaps even test systems, you +probably want to just use one instance on your host, though, and this +is where the "default" instance comes in. + +To use the default instance, you must only have one instance installed +on your system. More specifically, there must no instance specific +configuration files in +#+BEGIN_SRC sh +/etc/escenic/engine/instance +#+END_SRC +and that ece.conf, typically located in: +#+BEGIN_SRC sh +/etc/escenic/ece.conf +#+END_SRC +must contain all the settings required to run the instance. + +If you've ticked all these boxes, you can issue all ece commands +without the -i parameter. From 1cf5f953523ea790b5a5e3cfbabbff8ad908c693 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 10 Oct 2011 18:12:25 +0800 Subject: [PATCH 0080/2585] - fixed bug in set_up_app_server where the a BASH escaped variable never was set (probably deleted by me coding one hour too late one day), leading to Catalina's loader lacking the essential/core Tomcat libraries. - about 50% done with the code handling recovery (from backup) - added user interaction and FAI configuration reading regarding the Munin gatherer, the Munin gatherer profile is the one which lacks the most right now, though. --- usr/sbin/ece-install | 93 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9115ca2e..0a43013f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -117,6 +117,7 @@ PROFILE_SEARCH_SERVER=7 PROFILE_WIDGET_FRAMEWORK=8 PROFILE_CREATE_PUBLICATION=9 PROFILE_MONITORING_SERVER=10 +PROFILE_RESTORE_FROM_BACKUP=11 # Because of issue VF-3559, we also create the default family and host # directories. @@ -735,6 +736,7 @@ function set_up_app_server # such as log4j configuration put in ${tomcat.base}/lib. file=$tomcat_base/conf/catalina.properties common_loader=$(grep ^common.loader $file) + escaped_common_loader=$(get_escaped_bash_string ${common_loader}) escenic_loader=",$\{catalina.base\}/escenic/lib/*.jar" old=$(get_escaped_bash_string ${common_loader}) new=${escaped_common_loader}$(get_escaped_bash_string ${escenic_loader}) @@ -1984,6 +1986,20 @@ function install_munin_gatherer() fi # TODO ask user/read fai_munin_node_list + if [ $fai_enabled -eq 0 ]; then + print "Which nodes shall this Munin monitor gather from?" + print "Separate your hosts with a space, e.g.: 'editor01 db01 web01'" + echo -n "Your choice> " + read user_munin_nodes + + if [ -n "$user_munin_nodes" ]; then + node_list=$user_munin_nodes + fi + else + node_list=$(get_conf_value fai_munin_node_list) + fi + + # TODO has a hard dependency on install_web_server, should perhaps # be called from this one? @@ -2080,6 +2096,79 @@ function add_server_to_runlevels() fi } +function restore_from_backup() +{ + # locals + restore_db=0 + restore_binaries=0 + restore_data_files=0 + restore_conf=0 + backup_file="" + + if [ $fai_enabled -eq 1 -a $fai_restore_from_backup -eq 1 ]; then + backup_file=${fai_restore_from_file} + + if [ $fai_restore_db -eq 1 -o $fai_restore_all -eq 1 ]; then + restore_db=1 + fi + if [ $fai_restore_data_files -eq 1 -o $fai_restore_all -eq 1 ]; then + restore_data_files=1 + fi + if [ $fai_restore_software_binaries -eq 1 -o $fai_restore_all -eq 1 ]; then + restore_binaries=1 + fi + if [ $fai_restore_configuration -eq 1 -o $fai_restore_all -eq 1 ]; then + restore_conf=1 + fi + else + print "From which dataset do you wish to backup?" + # TODO + fi + + if [ ! -r $backup_file ]; then + print $backup_file "either doesn't exist or cannot be read." + print "I cannot restore from it :-(" + exit 1 + fi + + dir=$(mktemp -d) + + if [ $restore_db -eq 1 ]; then + install_database_server + run cd $dir + run tar xzf $backup_file --wildcards var/backups/escenic/*.sql.gz + # picking the DB backup file to restore + sql_file=$(ls -tra var/backups/escenic/*.sql.gz | tail -1) + print "Selected the most recent database dump " $sql_file + run mysql < $sql_file + print "Database restored on $HOSTNAME from $sql_file" + fi + + if [ $restore_data_files -eq 1 ]; then + run cd $dir + run tar xzf $backup_file var/lib/escenic + run mv var/lib/escenic /var/lib + print "Restored data files on $HOSTNAME from $backup_file" + print "Check /var/lib/escenic to to see they're all there" + fi + + if [ $restore_conf -eq 1 ]; then + run cd $dir + run tar xzf $backup_file etc + run mv etc / + print "Restored configuration files on $HOSTNAME from $backup_file" + print "Check /etc to see that they're all there". + fi + + if [ $restore_binaries -eq 1 ]; then + run cd $dir + run tar xzf $backup_file opt + run mv opt / + print "Restored software binaries on $HOSTNAME from $backup_file" + print "Check /opt to see that they're all there". + fi +} + function common_post_install() { add_server_to_runlevels @@ -2132,6 +2221,9 @@ if [ $fai_enabled -eq 1 ]; then elif [ $(get_boolean_conf_value fai_munin_gatherer_create) -eq 1 ]; then install_profile_number=$PROFILE_MONITORING_SERVER install_munin_gatherer + elif [ $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then + install_profile_number=$PROFILE_RESTORE_FROM_BACKUP + restore_from_backup fi common_post_install @@ -2145,7 +2237,6 @@ else $PROFILE_CACHE_SERVER) install_cache_server install_web_server - install_munin_gatherer ;; $PROFILE_DB_SERVER) install_database_server From a9c6942a66af526bf6dbe8a1524c27b258f89ee7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 10 Oct 2011 19:27:34 +0800 Subject: [PATCH 0081/2585] - added missing quote in create_schema(), goodness knows how long this bug has been there. --- usr/sbin/drop-and-create-ecedb | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/sbin/drop-and-create-ecedb b/usr/sbin/drop-and-create-ecedb index 9eb9c574..5c0d13e6 100644 --- a/usr/sbin/drop-and-create-ecedb +++ b/usr/sbin/drop-and-create-ecedb @@ -108,6 +108,7 @@ create database $db_schema character set utf8 collate utf8_general_ci; grant all on $db_schema.* to $db_user@'%' identified by '$db_password'; grant all on $db_schema.* to $db_user@'localhost' identified by '$db_password'; EOF + exit_on_the_first_error "create db" else sqlplus /nolog << EOF connect /as sysdba; From 5435df7fc5ef7be0bd0c61d8933b519e93379aec Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 15:41:44 +0800 Subject: [PATCH 0082/2585] - recovery from backup tarball made with "ece backup" now works in FAI mode! Given these three settings in .ece-install.conf: fai_enabled=1 fai_restore_from_backup=1 fai_restore_all=1 fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz ece-install will do the following (standard output from common ece-install tasks & user feedback omitted): [ece-install-8] Restoring the database contents on ubiquitous ... [ece-install-21] Selecting the most recent database dump ece5db-2011-10-10.sql.gz [ece-install-22] Successfully restored DB on ubiquitous from ece5db-2011-10-10.sql.gz [ece-install-23] Restoring the Solr & ECE data files on ubiquitous ... [ece-install-35] Successfully restored Solr & ECE data files on ubiquitous [ece-install-35] from backup: engine-dev1-backup-2011-10-10.tar.gz [ece-install-35] Check /var/lib/escenic to verify they're all there [ece-install-36] Restoring the ECE configuration files on ubiquitous ... [ece-install-49] Successfully restored ECE configuration files on ubiquitous [ece-install-49] from backup: engine-dev1-backup-2011-10-10.tar.gz [ece-install-49] Check /etc to verify that they're all there. [ece-install-49] Restoring the Escenic & Tomcat binaries on ubiquitous ... [ece-install-69] Successfully restored Escenic & Tomcat binaries on ubiquitous [ece-install-69] from backup: engine-dev1-backup-2011-10-10.tar.gz [ece-install-69] Check /opt to verify that they're all there. The same tarball can be used on multiple hosts, choosing to only backup parts of the contents by replacing fai_restore_all=1 with one of: fai_restore_db=1 fai_restore_data_files=1 fai_restore_configuration=1 fai_restore_software_binaries=1 --- usr/sbin/ece-install | 102 ++++++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 30 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0a43013f..5c1bc03d 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -388,7 +388,6 @@ function set_up_ecedb() fi done - source /usr/sbin/drop-and-create-ecedb # TODO in future, it'll be possible to override db_user, # db_password and db_schema here by reading these via # get_conf_value @@ -1335,6 +1334,8 @@ function read_user_input() echo " (if available) or ECE/clean-demo." echo -n " $PROFILE_MONITORING_SERVER - A monitoring server." echo " This will install the Munin gatherer" + echo -n " $PROFILE_RESTORE_FROM_BACKUP - Restore from a backup." + echo " Allows you to restore parts of or an entire system." echo "" echo -n "Your choice [1]> " read install_profile_number @@ -1487,10 +1488,14 @@ function add_next_step() fi } +# $1 : optional parameter, binaries_only. If passed, $1=binaries_only, +# the ECE DB schema is not set up. function install_database_server() { print "Installing the database server on $HOSTNAME ..." + source /usr/sbin/drop-and-create-ecedb + if [ $on_debian_or_derivative -eq 1 ]; then code_name=$(lsb_release -s -c) @@ -1535,10 +1540,12 @@ function install_database_server() fi assert_pre_prequesite mysql - - download_escenic_components - set_up_engine_and_plugins - set_up_ecedb + + if [ -z "$1" ]; then + download_escenic_components + set_up_engine_and_plugins + set_up_ecedb + fi } # $1 is the default instance name, the calee is responsible for @@ -2028,6 +2035,13 @@ function install_munin_node() -O $file run chmod 755 $file + if [ ! -e /etc/escenic/ece-*.conf -o -z "$type" -o -z "$instance" ]; then + print "No ECE instances found on $HOSTNAME, so I'm not adding" + print "additional Munin configuration" + add_next_step "- A Munin node has been installed on $HOSTNAME" + return + fi + escenic_jstat_modules="_gc _gcoverhead _heap _uptime" for el in /etc/escenic/ece-*.conf; do current_instance=$(basename ${el} | \ @@ -2105,28 +2119,40 @@ function restore_from_backup() restore_conf=0 backup_file="" - if [ $fai_enabled -eq 1 -a $fai_restore_from_backup -eq 1 ]; then + if [ $(get_boolean_conf_value fai_enabled) -eq 1 -a \ + $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then backup_file=${fai_restore_from_file} + + if [ -z "$backup_file" ]; then + print "You must specifify fai_restore_from_file" + exit 1 + fi - if [ $fai_restore_db -eq 1 -o $fai_restore_all -eq 1 ]; then + if [ $(get_boolean_conf_value fai_restore_db) -eq 1 -o \ + $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then restore_db=1 fi - if [ $fai_restore_data_files -eq 1 -o $fai_restore_all -eq 1 ]; then + + if [ $(get_boolean_conf_value fai_restore_data_files) -eq 1 -o \ + $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then restore_data_files=1 fi - if [ $fai_restore_software_binaries -eq 1 -o $fai_restore_all -eq 1 ]; then + + if [ $(get_boolean_conf_value fai_restore_software_binaries) -eq 1 -o \ + $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then restore_binaries=1 fi - if [ $fai_restore_configuration -eq 1 -o $fai_restore_all -eq 1 ]; then + if [ $(get_boolean_conf_value fai_restore_configuration) -eq 1 -o \ + $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then restore_conf=1 fi - else + elif [ $(get_boolean_conf_value fai_enabled) -eq 0 ]; then print "From which dataset do you wish to backup?" # TODO fi - if [ ! -r $backup_file ]; then - print $backup_file "either doesn't exist or cannot be read." + if [ ! -r "$backup_file" ]; then + print "$backup_file either doesn't exist or cannot be read." print "I cannot restore from it :-(" exit 1 fi @@ -2134,38 +2160,51 @@ function restore_from_backup() dir=$(mktemp -d) if [ $restore_db -eq 1 ]; then - install_database_server + install_database_server "binaries_only" + print "Restoring the database contents on $HOSTNAME ..." run cd $dir run tar xzf $backup_file --wildcards var/backups/escenic/*.sql.gz # picking the DB backup file to restore sql_file=$(ls -tra var/backups/escenic/*.sql.gz | tail -1) - print "Selected the most recent database dump " $sql_file - run mysql < $sql_file - print "Database restored on $HOSTNAME from $sql_file" + print "Selecting the most recent database dump " $(basename $sql_file) + + # methods in drop-and-create-ecedb to set up the database + # schema & user + pre_install_new_ecedb + create_schema + + # db_schema is defined in drop-and-create-ecedb + gunzip < $sql_file | mysql $db_schema + exit_on_error "restoring from $sql_file" + + print "Successfully restored DB on $HOSTNAME from" $(basename $sql_file) fi if [ $restore_data_files -eq 1 ]; then + print "Restoring the Solr & ECE data files on $HOSTNAME ..." run cd $dir - run tar xzf $backup_file var/lib/escenic - run mv var/lib/escenic /var/lib - print "Restored data files on $HOSTNAME from $backup_file" - print "Check /var/lib/escenic to to see they're all there" + run tar -C / -xzf $backup_file var/lib/escenic + print "Successfully restored Solr & ECE data files on $HOSTNAME" + print "from backup: $(basename $backup_file)" + print "Check /var/lib/escenic to verify they're all there" fi if [ $restore_conf -eq 1 ]; then + print "Restoring the ECE configuration files on $HOSTNAME ..." run cd $dir - run tar xzf $backup_file etc - run mv etc / - print "Restored configuration files on $HOSTNAME from $backup_file" - print "Check /etc to see that they're all there". + run tar -C / -xzf $backup_file etc + print "Successfully restored ECE configuration files on $HOSTNAME" + print "from backup: $(basename $backup_file)" + print "Check /etc to verify that they're all there". fi if [ $restore_binaries -eq 1 ]; then + print "Restoring the Escenic & Tomcat binaries on $HOSTNAME ..." run cd $dir - run tar xzf $backup_file opt - run mv opt / - print "Restored software binaries on $HOSTNAME from $backup_file" - print "Check /opt to see that they're all there". + run tar -C / -xzf $backup_file opt + print "Successfully restored Escenic & Tomcat binaries on $HOSTNAME" + print "from backup: $(basename $backup_file)" + print "Check /opt to verify that they're all there". fi } @@ -2262,8 +2301,11 @@ else $PROFILE_MONITORING_SERVER) install_munin_gatherer ;; + $PROFILE_RESTORE_FROM_BACKUP) + restore_from_backup + ;; default) - print "You must select 1-7" + print "You must select 1-11" exit 1 ;; esac From f0ee508e4432563bb7ae23e3d4523a01b144480d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 18:38:06 +0800 Subject: [PATCH 0083/2585] - updated ece-install guide with a big section on (backup &) restore as well as giving more examples on how to use FAI. --- usr/share/doc/escenic/ece-install-guide.org | 90 ++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 1649637a..8937bb2c 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -266,8 +266,13 @@ monitoring the different nodes. * Full Automatic Install (FAI) The ece-install script has support for doing a full automatic install -(FAI). +(FAI). You can only install one profile at a time. The profiles are +the parameters with "install" in their name, such as: +#+BEGIN_SRC sh +fai_editor_install +#+END_SRC +** Overview of All FAI Parameters The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user: @@ -302,6 +307,8 @@ $HOME/.ece-install.conf file of the root user: As you've probably have guessed, 0 means "false" and 1 means "true" :-) +** Examples +*** Installing an Editorial Server & Create a Publication To automatically install an editorial server and create a publication called "jollygood", the minimal configuration in .ece-install.conf would be: @@ -313,6 +320,29 @@ fai_publication_create=1 fai_publication_name=jollygood #+END_SRC +*** Installing Two Presentation Servers On a Single Host +If you wish to only install two presentation servers called "web1" and +"web2" on your host, you will first run ece-install with: +#+BEGIN_SRC sh +fai_enabled=1 +fai_presentation_install=1 +fai_presentation_name=web1 +#+END_SRC + +And then re-issue ece-install with the following configuration: +#+BEGIN_SRC sh +fai_enabled=1 +fai_presentation_install=1 +fai_presentation_name=web2 +fai_presentation_port=8081 +fai_presentation_shutdown=8105 +#+END_SRC +Notice that this configuration has some extra options since the +previous run of ece-install could run with the default settings, +whereas the second one needs to override these. + + +** Verifying That the Script Is Running In FAI Mode When FAI is enabled, ece-install will report: #+BEGIN_SRC sh [ece-install] Full Automatic Install (FAI) enabled. @@ -428,3 +458,61 @@ the host that runs the install script. The following output is produced from ece-install: #+INCLUDE: "~/tmp/ece-install.out" src text +* Restoring from backup +ece-install can restore from a backup made by the ece script: +#+BEGIN_SRC sh +$ ece -i backup +#+END_SRC + +As stated in the ece guide, this backup may contain (depending on +what's available on the host where it's run): +- database dump of the instance +- data files (pictures, video files and so on). This is often referred + to as the "multimedia archive" in Escenic literature. +- ECE, cache and web server configuration +- Escenic software + +** Location of the backup file + +*** Running interactively +The ece script will put backups in +#+BEGIN_SRC sh +/var/backups/escenic +#+END_SRC +and the ece-install script will hence expect to find them here (when +running interactively), asking you from which one you wish to restore +the system: + +#+BEGIN_SRC sh +[ece-install-5] From which dataset do you wish to backup? + 1 - engine-dev1-backup-2011-10-10.tar.gz + 2 - engine-dev1-backup-2011-10-11.tar.gz +[ece-install-5] Enter the number next to the tarball, from 1 to 2 +Your choice [1]> +#+END_SRC + +*** Running in FAI mode +If you're running in FAI mode, you specify the file to restore from +with: +#+BEGIN_SRC sh +fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz +#+END_SRC + +** Data security +You should take heed when running restore, so that you're not +restoring a system which you didn't want to change (yes, this mishap +does happen). + +The ece-install script will help you a bit on the way, but the final +responsibility always lies with you as the user. + +If you're trying to restore the DB and the ECE schema already exists, +the restore will fail: +#+BEGIN_SRC sh +[ece-install-8] Restoring the database contents on ubiquitous ... +[ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists +ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists +[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-( +[ece-install-24] See /var/log/ece-install.log for further details. +#+END_SRC From c7d8e36357cbfebdb1b58ba08b340106e909d402 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 18:38:20 +0800 Subject: [PATCH 0084/2585] - updated HTML from .org --- usr/share/doc/escenic/ece-install-guide.html | 447 +++++++++++++------ 1 file changed, 320 insertions(+), 127 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 7b7ba0cf..ef403455 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -44,6 +44,12 @@ dt { font-weight: bold; } div.figure { padding: 0.5em; } div.figure p { text-align: center; } + div.inlinetask { + padding:10px; + border:2px solid gray; + margin:10px; + background: #ffffcc; + } textarea { overflow-x: auto; } .linenr { font-size:smaller } .code-highlighted {background-color:#ffff00;} @@ -161,8 +167,8 @@ -
    +
    -

    Welcome to the /usr/sbin/ece-install Guide

    + +
    + + @@ -330,13 +364,10 @@

    3 Downloading the ece-insta

    -
    $ git clone git@github.com:skybert/ece-scripts.git
    -
    +
    $ git clone git@github.com:skybert/ece-scripts.git
     
    - -

    @@ -350,35 +381,26 @@

    4 Running the ece-install S

    -
    [ece-install] You must be root when running ece-install
    -
    +
    [ece-install] You must be root when running ece-install
     
    - -

    To become root on Ubuntu based systems and Mac OS X, do:

    -
    $ sudo su
    -
    +
    $ sudo su
     
    - -

    On all other Unix like system, do:

    -
    $ su
    -
    +
    $ su
     
    - -

    You will need access to http://technet.escenic.com and put these credentials in the root user's $HOME/.ece-install.conf, normally this @@ -387,40 +409,31 @@

    4 Running the ece-install S

    -
    [ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-(
    -
    +
    [ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-(
     
    - -

    As for the technet credentials, the script will also tell you if it cannot find what it's looking for:

    -
    [ece-install] Be sure to set technet_user and technet_password
    -[ece-install] in /root/.ece-install.conf
    -
    +
    [ece-install] Be sure to set technet_user and technet_password
    +[ece-install] in /root/.ece-install.conf
     
    - -

    The minimum .ece-install.conf file is thus:

    -
    technet_user=<user>
    -technet_password=<password>
    -
    +
    technet_user=<user>
    +technet_password=<password>
     
    - -

    If you're installing WF as well, you'll need additional credentials for this, see the Widget Framwork section below, and if you want to @@ -433,27 +446,21 @@

    4 Running the ece-install S

    -
    # bash ece-install [-v|--verbose]
    -
    +
    # bash ece-install [-v|--verbose]
     
    - -

    ece-install will try to "fail fast", exiting as soon as it detects an error during its execution:

    -
    [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-(
    +
    [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-(
     [ece-install-1] See /var/log/ece-install.log for further details.
    -
     
    - -

    As a last resort, if something goes astray with the script, and the log file cannot give you enough clues, the ultimate way of debugging @@ -461,13 +468,10 @@

    4 Running the ece-install S

    -
    # bash -x ece-install
    -
    +
    # bash -x ece-install
     
    - -

    Here you can see everything that BASH does when running the script, how the wildcard variables expand and so on. Normally, having a tail @@ -486,26 +490,29 @@

    5 Available Server Profiles

    -
    Hi, which server profile do you wish to install?
    +
    Hi, which server profile do you wish to install?
     
    -Select 1-7 and press ENTER
    +Select 1-9 and press ENTER
     
    -  1 - The full stack on one host. Suitable for development and
    -      test environments (cache, ECE, assembly host & database).
    -  2 - Editorial (publication) server (ECE, assembly host).
    -  3 - Presentation server (ECE, memcached).
    +  1 - All in one, the full stack on one host. 
    +      Suitable for development and test environments.
    +      It will install caching server, ECE, assembly host, database &
    +      Widget Framework as well as set up a new publication.
    +  2 - Editorial (publication) server.
    +      This will install ECE and an assembly host.
    +  3 - Presentation server (ECE, memcached).
       4 - Database server.
    -  5 - Cache server (cache & web server).
    +  5 - Cache server (cache & web server).
       6 - RMI hub.
    -  7 - Standalone search instance (solr + indexer-webapp).
    -  8 - Install Widget Framework. This will also set up a WF based 
    -      publication for you (repo.escenic.com user/pass required).
    -
    +  7 - Standalone search instance (solr + indexer-webapp).
    +  8 - Install the Widget Framework.
    +      (You need user/pass for repo.escenic.com for this).
    +  9 - Create a new publication.
    +      This profile will create new publication based on WF
    +      (if available) or ECE/clean-demo.
     
    - -

    @@ -516,10 +523,6 @@

    5.1 Common for All Server install the Escenic specific Munin plugins on hosts where there are any Escenic server components.

    -

    -TBD: there is currently an issue with the Escenic plugins not working -properly with the new ece/ece-install scripts. We're working on this. -

    @@ -618,7 +621,7 @@

    5.2.7 Compression of co

    -

    5.3 1 - Full Stack on One Host

    +

    5.3 Profile - Full Stack on One Host

    This profile is suitable for developers and admins wanting to set up a @@ -636,26 +639,24 @@

    5.3 1 - Full Stack on One

    -

    5.4 2 - Editorial Server (Publication Server)

    +

    5.4 Profile - Editorial Server (Publication Server)

    - - +

    Will set up a typical editorial server (often referred to as the +publication server in Escenic literature). +

    -

    5.5 3 - Presentation Server

    +

    5.5 Profile - Presentation Server

    This will set up a typical presentation server:

    • Memcached, distributed memory cache
    • -
    • Escenic Assembly environment (which may be removed after the - installation). -
    • Deployment setup to only deploy escenic-admin and the publication(s).
    • @@ -667,7 +668,7 @@

      5.5 3 - Presentation Serv

    -

    5.6 4 - Database Server

    +

    5.6 Profile - Database Server

    If ece-install is run on a support version of Debian or Ubuntu, this @@ -689,22 +690,19 @@

    5.6 4 - Database Server <

    -
    [ece-install] Setting up the ECE database schema ...
    -ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    -ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists
    -[ece-install] running tables FAILED, exiting :-(
    -
    +
    [ece-install] Setting up the ECE database schema ...
    +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    +ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists
    +[ece-install] running tables FAILED, exiting :-(
     
    - -

    -

    5.7 5 - Cache Server

    +

    5.7 Profile - Cache Server

    If ece-install is run on a support version of Debian or Ubuntu, it @@ -744,9 +742,6 @@

    5.7 5 - Cache Server

    -
  • will install a Munin gatherer and make its interface available - when accesssing the web server directly (port 81) -
  • @@ -768,7 +763,7 @@

    5.7 5 - Cache Server

    -

    5.8 8 - Install Widget Framework

    +

    5.8 Profile - Install Widget Framework

    You'll need a user name and password for accessing the @@ -782,19 +777,16 @@

    5.8 8 - Install Widget Fr

    -
    [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf
    -[ece-install] If you don't have these, please contact support@escenic.com
    -
    +
    [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf
    +[ece-install] If you don't have these, please contact support@escenic.com
     
    - -

    -

    5.9 9 - Create Publication

    +

    5.9 Profile - Create Publication

    This profile will create a publication for you, only asking you the @@ -806,6 +798,21 @@

    5.9 9 - Create Publicatio Framework if its present on the system, if not, ECE's clean demo WAR is used as a basis.

    + +
    + +
    +

    5.10 Profile - Monitoring Server

    +
    + +

    This will install a Munin gatherer and make its interface available +when accesssing the web server directly (port 81). +

    +

    +TBD: This profile will also install the Nagios interface for +monitoring the different nodes. +

    +
    @@ -815,10 +822,23 @@

    6 Full Automatic Install (F

    The ece-install script has support for doing a full automatic install -(FAI). +(FAI). You can only install one profile at a time. The profiles are +the parameters with "install" in their name, such as:

    -

    -The ece-install script understands for the following settings in the + + +

    fai_editor_install
    +
    + + + +
    + +
    +

    6.1 Overview of All FAI Parameters

    +
    + +

    The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user:

    @@ -860,37 +880,91 @@

    6 Full Automatic Install (F

    As you've probably have guessed, 0 means "false" and 1 means "true" :-)

    -

    -To automatically install an editorial server and create a publication + + + + +

    +

    6.2 Examples

    +
    + + +
    + +
    +

    6.2.1 Installing an Editorial Server & Create a Publication

    +
    + +

    To automatically install an editorial server and create a publication called "jollygood", the minimal configuration in .ece-install.conf would be:

    -
    fai_enabled=1
    -fai_editor_install=1
    -fai_publication_create=1
    -fai_publication_name=jollygood
    +
    fai_enabled=1
    +fai_editor_install=1
    +fai_publication_create=1
    +fai_publication_name=jollygood
    +
    + + +
    + +
    + +
    +

    6.2.2 Installing Two Presentation Servers On a Single Host

    +
    +

    If you wish to only install two presentation servers called "web1" and +"web2" on your host, you will first run ece-install with: +

    + + +
    fai_enabled=1
    +fai_presentation_install=1
    +fai_presentation_name=web1
     
    +

    +And then re-issue ece-install with the following configuration: +

    + +
    fai_enabled=1
    +fai_presentation_install=1
    +fai_presentation_name=web2
    +fai_presentation_port=8081
    +fai_presentation_shutdown=8105
    +

    -When FAI is enabled, ece-install will report: +Notice that this configuration has some extra options since the +previous run of ece-install could run with the default settings, +whereas the second one needs to override these.

    +
    +
    -
    [ece-install] Full Automatic Install (FAI) enabled.
    -[ece-install] All user input will be read from /root/.ece-install.conf
    +
    - +
    +

    6.3 Verifying That the Script Is Running In FAI Mode

    +
    + +

    When FAI is enabled, ece-install will report: +

    +
    [ece-install] Full Automatic Install (FAI) enabled.
    +[ece-install] All user input will be read from /root/.ece-install.conf
    +
    +
    @@ -904,16 +978,13 @@

    7 Running More Than One Ins

    -
    [ece-install] There's already one ece-install process running. If you believe
    -[ece-install] this is wrong, e.g. if a previous run of ece-install was aborted
    -[ece-install] before it completed, you may delete /var/run/ece-install.pid and
    -[ece-install] run ece-install again.
    -
    +
    [ece-install] There's already one ece-install process running. If you believe
    +[ece-install] this is wrong, e.g. if a previous run of ece-install was aborted
    +[ece-install] before it completed, you may delete /var/run/ece-install.pid and
    +[ece-install] run ece-install again.
     
    - - @@ -1060,31 +1131,153 @@

    12 Example Output FAI

    -
    technet_user=apples
    -technet_password=andoranges
    -wf_user=foo
    -wf_password=bar
    +
    technet_user=apples
    +technet_password=andoranges
    +wf_user=foo
    +wf_password=bar
    +
    +fai_enabled=1
    +fai_all_install=1
    +
    -fai_enabled=1 -fai_all_install=1 +

    +All the different components described in this guide are installed on +the host that runs the install script. +

    +

    +The following output is produced from ece-install: +CANNOT INCLUDE FILE ~/tmp/ece-install.out +

    + + + + +
    +

    13 Restoring from backup

    +
    + +

    ece-install can restore from a backup made by the ece script: +

    + + +
    $ ece -i <instance> backup
     
    +

    +As stated in the ece guide, this backup may contain (depending on +what's available on the host where it's run): +

      +
    • database dump of the instance +
    • +
    • data files (pictures, video files and so on). This is often referred + to as the "multimedia archive" in Escenic literature. +
    • +
    • ECE, cache and web server configuration +
    • +
    • Escenic software +
    • +
    + + + +
    +
    +

    13.1 Location of the backup file

    +
    + + + +
    + +
    +

    13.1.1 Running interactively

    +
    + +

    The ece script will put backups in +

    + + +
    /var/backups/escenic
    +

    -The following output is produced from ece-install: -CANNOT INCLUDE FILE /tmp/ece-install.out +and the ece-install script will hence expect to find them here (when +running interactively), asking you from which one you wish to restore +the system: +

    + + + +
    [ece-install-5] From which dataset do you wish to backup?
    +    1 - engine-dev1-backup-2011-10-10.tar.gz
    +    2 - engine-dev1-backup-2011-10-11.tar.gz
    +[ece-install-5] Enter the number next to the tarball, from 1 to 2
    +Your choice [1]>
    +
    + + +
    + +
    + +
    +

    13.1.2 Running in FAI mode

    +
    + +

    If you're running in FAI mode, you specify the file to restore from +with: +

    + + +
    fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz
    +
    + + +
    +
    + +
    + +
    +

    13.2 Data security

    +
    + +

    You should take heed when running restore, so that you're not +restoring a system which you didn't want to change (yes, this mishap +does happen). +

    +

    +The ece-install script will help you a bit on the way, but the final +responsibility always lies with you as the user. +

    +

    +If you're trying to restore the DB and the ECE schema already exists, +the restore will fail:

    + + +
    [ece-install-8] Restoring the database contents on ubiquitous ...
    +[ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz
    +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    +ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists
    +[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-(
    +[ece-install-24] See /var/log/ece-install.log for further details.
    +
    + +
    +
    +

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-09-15 15:29:55 CST -

    + Date: 2011-10-11 18:35:36 CST +

    From eb6bfcf2830c454746d07624a32e85613b189a5a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 18:38:37 +0800 Subject: [PATCH 0085/2585] - updated .txt from .org --- usr/share/doc/escenic/ece-install-guide.txt | 233 ++++++++++++++------ 1 file changed, 164 insertions(+), 69 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index a4a85655..06d7190e 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-09-15 15:29:53 CST +Date: 2011-10-11 18:38:25 CST Table of Contents @@ -23,14 +23,20 @@ Table of Contents 5.2.5 Escenic Assembly environment 5.2.6 Database driver 5.2.7 Compression of content - 5.3 1 - Full Stack on One Host - 5.4 2 - Editorial Server (Publication Server) - 5.5 3 - Presentation Server - 5.6 4 - Database Server - 5.7 5 - Cache Server - 5.8 8 - Install Widget Framework - 5.9 9 - Create Publication + 5.3 Profile - Full Stack on One Host + 5.4 Profile - Editorial Server (Publication Server) + 5.5 Profile - Presentation Server + 5.6 Profile - Database Server + 5.7 Profile - Cache Server + 5.8 Profile - Install Widget Framework + 5.9 Profile - Create Publication + 5.10 Profile - Monitoring Server 6 Full Automatic Install (FAI) + 6.1 Overview of All FAI Parameters + 6.2 Examples + 6.2.1 Installing an Editorial Server & Create a Publication + 6.2.2 Installing Two Presentation Servers On a Single Host + 6.3 Verifying That the Script Is Running In FAI Mode 7 Running More Than One Installation Process 8 Re-running ece-install (and How To Speed It Up) 9 Overview of File Paths Used by the ece-install script @@ -38,6 +44,11 @@ Table of Contents 10.1 /etc/esecenic is shared 11 Uninstalling Everything That the ece-install Set Up 12 Example Output FAI +13 Restoring from backup + 13.1 Location of the backup file + 13.1.1 Running interactively + 13.1.2 Running in FAI mode + 13.2 Data security 1 Supported Operating Systems @@ -82,8 +93,6 @@ or be downloaded together with the other ece-scripts using git: $ git clone git@github.com:skybert/ece-scripts.git - - 4 Running the ece-install Script --------------------------------- You must run the script as the root user. If you start the script as @@ -92,23 +101,17 @@ a regular user, it will complain: [ece-install] You must be root when running ece-install - - To become root on Ubuntu based systems and Mac OS X, do: $ sudo su - - On all other Unix like system, do: $ su - - You will need access to [http://technet.escenic.com] and put these credentials in the root user's $HOME/.ece-install.conf, normally this means /root/.ece-install.conf, the script will also tell you this if @@ -118,8 +121,6 @@ you forget to provide such a configuration file: [ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-( - - As for the technet credentials, the script will also tell you if it cannot find what it's looking for: @@ -128,8 +129,6 @@ cannot find what it's looking for: [ece-install] in /root/.ece-install.conf - - The minimum .ece-install.conf file is thus: @@ -137,8 +136,6 @@ The minimum .ece-install.conf file is thus: technet_password= - - If you're installing WF as well, you'll need additional credentials for this, see the Widget Framwork section below, and if you want to runn a fully automatic install without any user interaction, you'll @@ -151,8 +148,6 @@ With the configuration file in place, you're now ready to run it: # bash ece-install [-v|--verbose] - - ece-install will try to "fail fast", exiting as soon as it detects an error during its execution: @@ -161,8 +156,6 @@ error during its execution: [ece-install-1] See /var/log/ece-install.log for further details. - - As a last resort, if something goes astray with the script, and the log file cannot give you enough clues, the ultimate way of debugging this is to run the BASH interpreter with the -x flag: @@ -171,8 +164,6 @@ this is to run the BASH interpreter with the -x flag: # bash -x ece-install - - Here you can see everything that BASH does when running the script, how the wildcard variables expand and so on. Normally, having a tail on /var/log/ece-install.log should be sufficient, though. @@ -185,20 +176,24 @@ want to install: Hi, which server profile do you wish to install? - Select 1-7 and press ENTER + Select 1-9 and press ENTER - 1 - The full stack on one host. Suitable for development and - test environments (cache, ECE, assembly host & database). - 2 - Editorial (publication) server (ECE, assembly host). + 1 - All in one, the full stack on one host. + Suitable for development and test environments. + It will install caching server, ECE, assembly host, database & + Widget Framework as well as set up a new publication. + 2 - Editorial (publication) server. + This will install ECE and an assembly host. 3 - Presentation server (ECE, memcached). 4 - Database server. 5 - Cache server (cache & web server). 6 - RMI hub. 7 - Standalone search instance (solr + indexer-webapp). - 8 - Install Widget Framework. This will also set up a WF based - publication for you (repo.escenic.com user/pass required). - - + 8 - Install the Widget Framework. + (You need user/pass for repo.escenic.com for this). + 9 - Create a new publication. + This profile will create new publication based on WF + (if available) or ECE/clean-demo. 5.1 Common for All Server Profiles =================================== @@ -206,9 +201,6 @@ The script will install a Munin node on the host. It will also install the Escenic specific Munin plugins on hosts where there are any Escenic server components. -TBD: there is currently an issue with the Escenic plugins not working -properly with the new ece/ece-install scripts. We're working on this. - 5.2 Common for Editorial, Presentation and Search Instances ============================================================ @@ -251,8 +243,8 @@ compressed content (and many other things that we before needed Apache for). Thus, we'll let the application server do the compression for us now. -5.3 1 - Full Stack on One Host -=============================== +5.3 Profile - Full Stack on One Host +===================================== This profile is suitable for developers and admins wanting to set up a test environment. It will install the full stack including caching server, application server, ECE, assembly host, database, Widget @@ -262,21 +254,20 @@ For further details on each of the different server components, see the different profile descriptions bellow. -5.4 2 - Editorial Server (Publication Server) -============================================== +5.4 Profile - Editorial Server (Publication Server) +==================================================== +Will set up a typical editorial server (often referred to as the +publication server in Escenic literature). - -5.5 3 - Presentation Server -============================ +5.5 Profile - Presentation Server +================================== This will set up a typical presentation server: - Memcached, distributed memory cache -- Escenic Assembly environment (which may be removed after the - installation). - Deployment setup to only deploy escenic-admin and the publication(s). -5.6 4 - Database Server -======================== +5.6 Profile - Database Server +============================== If ece-install is run on a support version of Debian or Ubuntu, this will install the excellent Percona distribution of MySQL with their set of high performance patches. @@ -298,10 +289,8 @@ The script will fail by itself if the DB already exists: [ece-install] running tables FAILED, exiting :-( - - -5.7 5 - Cache Server -===================== +5.7 Profile - Cache Server +=========================== If ece-install is run on a support version of Debian or Ubuntu, it will install the latest Varnish 3.0 caching server from the Varnish APT repository. @@ -329,8 +318,6 @@ The script will configure Varnish for a typical Escenic site: - will install the nginx web server for serving static content and will configure Varnish accordingly. This will be very useful for Adactus servers wanting to pull content from your ECEs. -- will install a Munin gatherer and make its interface available - when accesssing the web server directly (port 81) TBD: - If run on a Linux platform, the script will tweak the kernel @@ -339,8 +326,8 @@ TBD: to be in the staff network ACL, defined in the Varnish configuration. -5.8 8 - Install Widget Framework -================================= +5.8 Profile - Install Widget Framework +======================================= You'll need a user name and password for accessing the repo.escenic.com Maven repository. You should get these credentials when you bought Widget Framework. If you for some reason do not have @@ -353,10 +340,8 @@ complain: [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf [ece-install] If you don't have these, please contact support@escenic.com - - -5.9 9 - Create Publication -=========================== +5.9 Profile - Create Publication +================================= This profile will create a publication for you, only asking you the name of the publication and which ECE instance to use for its creation. @@ -365,11 +350,26 @@ This installation profile will base the publication on the Widget Framework if its present on the system, if not, ECE's clean demo WAR is used as a basis. +5.10 Profile - Monitoring Server +================================= +This will install a Munin gatherer and make its interface available +when accesssing the web server directly (port 81). + +TBD: This profile will also install the Nagios interface for +monitoring the different nodes. + 6 Full Automatic Install (FAI) ------------------------------- The ece-install script has support for doing a full automatic install -(FAI). +(FAI). You can only install one profile at a time. The profiles are +the parameters with "install" in their name, such as: + + fai_editor_install + + +6.1 Overview of All FAI Parameters +=================================== The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user: @@ -402,6 +402,11 @@ $HOME/.ece-install.conf file of the root user: As you've probably have guessed, 0 means "false" and 1 means "true" :-) +6.2 Examples +============= + +6.2.1 Installing an Editorial Server & Create a Publication +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To automatically install an editorial server and create a publication called "jollygood", the minimal configuration in .ece-install.conf would be: @@ -414,8 +419,33 @@ would be: fai_publication_name=jollygood +6.2.2 Installing Two Presentation Servers On a Single Host +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If you wish to only install two presentation servers called "web1" and +"web2" on your host, you will first run ece-install with: + fai_enabled=1 + fai_presentation_install=1 + fai_presentation_name=web1 + + +And then re-issue ece-install with the following configuration: + + + fai_enabled=1 + fai_presentation_install=1 + fai_presentation_name=web2 + fai_presentation_port=8081 + fai_presentation_shutdown=8105 + +Notice that this configuration has some extra options since the +previous run of ece-install could run with the default settings, +whereas the second one needs to override these. + + +6.3 Verifying That the Script Is Running In FAI Mode +===================================================== When FAI is enabled, ece-install will report: @@ -423,8 +453,6 @@ When FAI is enabled, ece-install will report: [ece-install] All user input will be read from /root/.ece-install.conf - - 7 Running More Than One Installation Process --------------------------------------------- If the script believes there's already an ece-intall process running, @@ -437,8 +465,6 @@ it will abort: [ece-install] run ece-install again. - - 8 Re-running ece-install (and How To Speed It Up) -------------------------------------------------- Although the initial thought behind ece-install, is to run it on a @@ -539,8 +565,77 @@ file: fai_all_install=1 - +All the different components described in this guide are installed on +the host that runs the install script. The following output is produced from ece-install: -CANNOT INCLUDE FILE /tmp/ece-install.out +CANNOT INCLUDE FILE ~/tmp/ece-install.out + +13 Restoring from backup +------------------------- +ece-install can restore from a backup made by the ece script: + + + $ ece -i backup + + +As stated in the ece guide, this backup may contain (depending on +what's available on the host where it's run): +- database dump of the instance +- data files (pictures, video files and so on). This is often referred + to as the "multimedia archive" in Escenic literature. +- ECE, cache and web server configuration +- Escenic software + +13.1 Location of the backup file +================================= + +13.1.1 Running interactively +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ece script will put backups in + + + /var/backups/escenic + +and the ece-install script will hence expect to find them here (when +running interactively), asking you from which one you wish to restore +the system: + + + + [ece-install-5] From which dataset do you wish to backup? + 1 - engine-dev1-backup-2011-10-10.tar.gz + 2 - engine-dev1-backup-2011-10-11.tar.gz + [ece-install-5] Enter the number next to the tarball, from 1 to 2 + Your choice [1]> + + +13.1.2 Running in FAI mode +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If you're running in FAI mode, you specify the file to restore from +with: + + + fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz + + +13.2 Data security +=================== +You should take heed when running restore, so that you're not +restoring a system which you didn't want to change (yes, this mishap +does happen). + +The ece-install script will help you a bit on the way, but the final +responsibility always lies with you as the user. + +If you're trying to restore the DB and the ECE schema already exists, +the restore will fail: + + + [ece-install-8] Restoring the database contents on ubiquitous ... + [ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz + ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists + ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists + [ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-( + [ece-install-24] See /var/log/ece-install.log for further details. From b0055dbe05e42e0250af2553aaaa5e23349741ba Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 19:05:19 +0800 Subject: [PATCH 0086/2585] - simplified the restore section structure --- usr/share/doc/escenic/ece-install-guide.org | 29 ++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 8937bb2c..1aad0602 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -472,9 +472,7 @@ what's available on the host where it's run): - ECE, cache and web server configuration - Escenic software -** Location of the backup file - -*** Running interactively +** Running interactively The ece script will put backups in #+BEGIN_SRC sh /var/backups/escenic @@ -491,10 +489,29 @@ the system: Your choice [1]> #+END_SRC -*** Running in FAI mode -If you're running in FAI mode, you specify the file to restore from -with: +** Running in FAI mode +If you're running in FAI mode, you can choose between these settings +to decide what to restore and where to find the backup file to +restore from: + +|----------------------------------+---------+------------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+---------+------------------------------------------------| +| fai\_restore\_all | 0 | Restore everything found in the backup file | +| fai\_restore\_db | 0 | Install the DB server & restore its contents | +| fai\_restore\_data\_files | 0 | Restore the Solr & ECE data files | +| fai\_restore\_configuration | 0 | Restore the Solr & ECE configuration files | +| fai\_restore\_software\_binaries | 0 | Restore the Escenic and Apache Tomcat software | +| fai\_restore\_from\_file | "" | The .tar.gz produced by "ece -i backup" | +|----------------------------------+---------+------------------------------------------------| + +For example, to restore everything possible from a given tarball, you +need this in your .ece-install.conf: #+BEGIN_SRC sh +fai_enabled=1 + +fai_restore_from_backup=1 +fai_restore_all=1 fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz #+END_SRC From 144140b816ff7d2491b09dffc4913ca50abc517a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 19:05:26 +0800 Subject: [PATCH 0087/2585] - updated html --- usr/share/doc/escenic/ece-install-guide.html | 70 ++++++++++++-------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index ef403455..06c41428 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -284,13 +284,9 @@

    Table of Contents

  • 12 Example Output FAI
  • 13 Restoring from backup
  • @@ -1185,17 +1181,9 @@

    13 Restoring from backup <
    -

    13.1 Location of the backup file

    +

    13.1 Running interactively

    - - -
    - -
    -

    13.1.1 Running interactively

    -
    -

    The ece script will put backups in

    @@ -1223,27 +1211,53 @@

    13.1.1 Running interac

    -
    -

    13.1.2 Running in FAI mode

    -
    +
    +

    13.2 Running in FAI mode

    +
    -

    If you're running in FAI mode, you specify the file to restore from -with: +

    If you're running in FAI mode, you can choose between these settings +to decide what to restore and where to find the backup file to +restore from: +

    +

    + ++ + + + + + + + + + + + +
    ParameterDefaultDescription
    fai_restore_all0Restore everything found in the backup file
    fai_restore_db0Install the DB server & restore its contents
    fai_restore_data_files0Restore the Solr & ECE data files
    fai_restore_configuration0Restore the Solr & ECE configuration files
    fai_restore_software_binaries0Restore the Escenic and Apache Tomcat software
    fai_restore_from_file""The .tar.gz produced by "ece -i <instance> backup"
    + + +

    +For example, to restore everything possible from a given tarball, you +need this in your .ece-install.conf:

    -
    fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz
    +
    fai_enabled=1
    +
    +fai_restore_from_backup=1
    +fai_restore_all=1
    +fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz
     
    -

    -
    -

    13.2 Data security

    -
    +
    +

    13.3 Data security

    +

    You should take heed when running restore, so that you're not restoring a system which you didn't want to change (yes, this mishap @@ -1276,7 +1290,7 @@

    13.2 Data security


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-10-11 18:35:36 CST + Date: 2011-10-11 19:04:29 CST

    From 629d219e1a489aec921a368e3845025dd73ecc9e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 19:08:41 +0800 Subject: [PATCH 0088/2585] - interactively selecting the backup to use now works. --- usr/sbin/ece-install | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 5c1bc03d..d3be5432 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1332,10 +1332,10 @@ function read_user_input() echo " $PROFILE_CREATE_PUBLICATION - Create a new publication." echo " This profile will create new publication based on WF" echo " (if available) or ECE/clean-demo." - echo -n " $PROFILE_MONITORING_SERVER - A monitoring server." - echo " This will install the Munin gatherer" - echo -n " $PROFILE_RESTORE_FROM_BACKUP - Restore from a backup." - echo " Allows you to restore parts of or an entire system." + echo " $PROFILE_MONITORING_SERVER - A monitoring server." + echo " This will install the Munin gatherer." + echo " $PROFILE_RESTORE_FROM_BACKUP - Restore from backup. Allows you to" + echo " restore DB, data files, binaries & publication templates." echo "" echo -n "Your choice [1]> " read install_profile_number @@ -2118,6 +2118,7 @@ function restore_from_backup() restore_data_files=0 restore_conf=0 backup_file="" + backup_dir=/var/backups/escenic if [ $(get_boolean_conf_value fai_enabled) -eq 1 -a \ $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then @@ -2148,7 +2149,30 @@ function restore_from_backup() fi elif [ $(get_boolean_conf_value fai_enabled) -eq 0 ]; then print "From which dataset do you wish to backup?" - # TODO + if [ ! -d $backup_dir ]; then + print "Backup directory $backup_dir doesn't exist or insn't readable" + exit 1 + fi + + i=0 + tarball_array=($(ls $backup_dir/*.tar.gz)) + + for (( i = 0; i <${#tarball_array[@]}; i++ )); do + echo " " $(( ${i} + 1 )) "-" $(basename ${tarball_array[$i]}) + done + + print "Enter the number next to the tarball, from 1 to $i" + echo -n "Your choice [1]> " + read user_tarball + + if [ -z "$user_tarball" ]; then + user_tarball=1 + fi + + backup_file=${tarball_array[$(( ${user_tarball} - 1 ))]} + + print "Which part of the system do you wish to restore?" + fi if [ ! -r "$backup_file" ]; then From c13cba0ae972b1f89d8803e53dba2229b5a02e11 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 11 Oct 2011 19:08:54 +0800 Subject: [PATCH 0089/2585] - updated --- usr/share/doc/escenic/ece-guide.html | 97 +++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 8 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.html b/usr/share/doc/escenic/ece-guide.html index ac540c65..57839ec2 100644 --- a/usr/share/doc/escenic/ece-guide.html +++ b/usr/share/doc/escenic/ece-guide.html @@ -7,7 +7,7 @@ Guide for the /usr/bin/ece Script - + @@ -245,7 +245,17 @@

    Table of Contents

  • 5.1 Overview of File Paths Used by the ece script
  • -
  • 6 Running the ece script
  • +
  • 6 Running the ece script + +
  • @@ -482,7 +492,7 @@

    4 Assembling a new EAR

    -
    $ ece clean assemble
    +
    $ ece -i clean assemble
     

    @@ -494,12 +504,12 @@

    4 Assembling a new EAR

    [ece#engine] Multiple versions of ECE and/or 3rd party libraries found. [ece#engine] Remember, you need to run 'ece clean assemble' when [ece#engine] upgrading either ECE or one of the plugins. -[ece#engine] Press ENTER to run 'ece clean assemble' +[ece#engine] I will now clean it up for you and re-run the assembly. +[ece#engine] Cleaning up generated files in /opt/escenic/assemblytool ... +[ece#engine] Assembling your EAR file ...
    -

    -The script will then re-run the assembly by itself. -

    +

    If you're just re-running the assembly after just adding a new plugin or want to re-build your publications after template changes, you can @@ -608,6 +618,13 @@

    5.1 Overview of File Path

    6 Running the ece script

    + +
    + +
    +

    6.1 You must run it as a non-privileged user

    +
    +

    You must be normal user to run the ece script, otherwise it will complain:

    @@ -622,7 +639,71 @@

    6 Running the ece script

    + +
    + +
    +

    6.2 Specifying instance

    +
    + + + +
    + +
    +

    6.2.1 Specifying the instance

    +
    + +

    The script is made for being easy to use with multiple instances on +the same host. You specify the instance you want to operate on using the +-i parameter. E.g. to assemble and deploy the editor1 instance, you'd +do:

    + + +
    $ ece -i editor1 assemble deploy restart
    +
    + + +
    + +
    + +
    +

    6.2.2 Using the default instance

    +
    + +

    However, on development systems and perhaps even test systems, you +probably want to just use one instance on your host, though, and this +is where the "default" instance comes in. +

    +

    +To use the default instance, you must only have one instance installed +on your system. More specifically, there must no instance specific +configuration files in +

    + + +
    /etc/escenic/engine/instance
    +
    + +

    +and that ece.conf, typically located in: +

    + + +
    /etc/escenic/ece.conf
    +
    + +

    +must contain all the settings required to run the instance. +

    +

    +If you've ticked all these boxes, you can issue all ece commands +without the -i parameter. +

    +
    @@ -631,7 +712,7 @@

    6 Running the ece script

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-10-07 16:47:40 CST + Date: 2011-10-10 14:56:44 CST

    From eabb765238957122bd074808e01915a9da6347c2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 12 Oct 2011 16:05:14 +0800 Subject: [PATCH 0090/2585] - first version of the interactive restore routine - fixed bug when running in FAI mode and not having selected any installation profile in .ece-install.conf --- usr/sbin/ece-install | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index d3be5432..4f177ae7 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2154,13 +2154,12 @@ function restore_from_backup() exit 1 fi - i=0 - tarball_array=($(ls $backup_dir/*.tar.gz)) + tarball_array=($(ls -t $backup_dir/*.tar.gz)) for (( i = 0; i <${#tarball_array[@]}; i++ )); do echo " " $(( ${i} + 1 )) "-" $(basename ${tarball_array[$i]}) done - + print "Enter the number next to the tarball, from 1 to $i" echo -n "Your choice [1]> " read user_tarball @@ -2172,7 +2171,33 @@ function restore_from_backup() backup_file=${tarball_array[$(( ${user_tarball} - 1 ))]} print "Which part of the system do you wish to restore?" + restore_profiles=( + "The database" + "The Solr and ECE data files (multimedia archive)" + "The ECE configuration files" + "The Escenic and Tomcat software binaries + publication templates" + ) + for (( i = 0; i <${#restore_profiles[@]}; i++ )); do + echo " " $(( ${i} + 1 )) "-" ${restore_profiles[$i]} + done + print "Enter the number next to the tarball, from 1 to $i" + echo -n "Your choice [1]> " + read user_restore_profile + + if [ -z "$user_restore_profile" ]; then + user_restore_profile=1 + fi + + if [ $user_restore_profile -eq 1 ]; then + restore_db=1 + elif [ $user_restore_profile -eq 2 ]; then + restore_data_files=1 + elif [ $user_restore_profile -eq 3 ]; then + restore_conf=1 + elif [ $user_restore_profile -eq 4 ]; then + restore_binaries=1 + fi fi if [ ! -r "$backup_file" ]; then @@ -2287,6 +2312,10 @@ if [ $fai_enabled -eq 1 ]; then elif [ $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then install_profile_number=$PROFILE_RESTORE_FROM_BACKUP restore_from_backup + else + print "No install profile selected, be sure to have one of the " + print "fai__install=1 in your $conf_file" + exit 1 fi common_post_install From 18c615725451f760a6516753a399079344443e1b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 12 Oct 2011 17:25:25 +0800 Subject: [PATCH 0091/2585] - added /var/backups/escenic to dir_list, so the init.d script takes care of creating it and setting the correct permissions on it. --- etc/init.d/ece | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/init.d/ece b/etc/init.d/ece index d5976036..f4cce807 100755 --- a/etc/init.d/ece +++ b/etc/init.d/ece @@ -35,6 +35,7 @@ fi ####################################################################### dir_list=" /var/cache/escenic +/var/backups/escenic /var/crash/escenic /var/log/escenic /var/run/escenic From 90cf6a229c531903bd36761808fd5dc9a197d881 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 12 Oct 2011 17:44:43 +0800 Subject: [PATCH 0092/2585] - fixed bug in "ece clean" if /opt/escenic/assemblytool doesn't exist - added run() from ece-install, this simplified the code for fixing - the above bug and will further simplify (and harden) the rest of the script once it's applied. --- usr/bin/ece | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 53208a67..e1fd1e0a 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -77,6 +77,12 @@ function exit_on_error() fi } +function run() +{ + $@ 1>>$log_file 2>>$log_file + exit_on_error $@ +} + #################################################################### # debug/verbose method #################################################################### @@ -656,8 +662,8 @@ function backup_type() print "- All Solr & Escenic data files from /var/lib/escenic" print "- All app servers in /opt" print "- All Escenic binaries & publication templates in /opt/escenic" - print "- All Escenic configuration in /etc/escenic and /etc/default/ece" - print "- All Escenic bootstrap scripts from /etc/init.d" + print "- All configuration in /etc/escenic and /etc/default/ece" + print "- All bootstrap scripts from /etc/init.d" print "Enjoy!" } @@ -1060,10 +1066,8 @@ function clean_up() { if [ $type = "engine" ]; then print "Cleaning up generated files in $assemblytool_home ..." - cd $assemblytool_home - ant clean \ - 1>>$log_file \ - 2>>$log_file + run cd $assemblytool_home + run ant clean fi tmp_dir_prefix="`basename $0`-" From 22419535017e7aaa2d94b79f2d519cdeffe427b2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 12 Oct 2011 18:58:49 +0800 Subject: [PATCH 0093/2585] - restore of both DB and data files seem to work reliably! --- usr/sbin/ece-install | 95 +++++++++++++++++++++++++++----------------- 1 file changed, 58 insertions(+), 37 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 4f177ae7..12daa801 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1,15 +1,21 @@ #! /usr/bin/env bash -# The goal of this script, is to have an ECE up and running within -# five minutes. The setup the user is getting is suitable for a -# production environment, except for the fact that with this script, -# all ECE components are set up on the same host. +# The goal of this script is install a complete production environment +# for web sites using on Escenic Content Engine as their CMS. The +# script is equally well suited for installing development, testing +# and staging environments, in addition to recover from backups create +# with the "ece -i backup" command. +# +# Always check for the latest version at +# http://github.com/skybert/ece-scripts -# by tkj@vizrt.com +# echo comments and suggestions > tkj@vizrt.com +# echo frustrations > /dev/null ##################################################################### -# User definable variables (the defaults are fine in most cases). Any -# of these may be overridden in the .ece-isntall.conf file +# User definable variables (the defaults are fine in most +# cases). These are the most likely variables you want to change and +# they can all be set in the .ece-isntall.conf file ##################################################################### ece_user=escenic ece_group=escenic @@ -20,7 +26,7 @@ tomcat_download=http://apache.stu.edu.tw/tomcat/tomcat-6/v6.0.33/bin/apache-tomc # The script will install the sun-java6-jdk package on Debian based # systems and this is the path of the JAVA home with this package. If # you're using a different system or have other preferences, change -# java_home here. +# java_home. java_home=/usr/lib/jvm/java-6-sun ##################################################################### @@ -168,9 +174,6 @@ function set_up_engine_directories() done } -# TODO download documentation to -# /usr/share/doc/escenic/content-engine-/ - function download_escenic_components() { if [ $ece_software_setup_completed -eq 1 ]; then @@ -194,7 +197,7 @@ function download_escenic_components() } -# will intall the passed packages if these are not installed from +# will install the passed packages if these are not installed from # before. # # parameters: @@ -938,17 +941,22 @@ function print_status_and_next_steps() s="- The installation is now complete!" print $s" It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s - add_next_step "- See /usr/share/doc/escenic/ece-install-guide.txt for" - add_next_step " a full overview of files and directories set up by" - add_next_step " $(basename $0)" - add_next_step "- type 'ece help' to see all the options of this script" - add_next_step " which allows you to control all the Escenic components." - add_next_step " Also, see /usr/share/doc/escenic/ece-guide.txt for" - add_next_step " further information on the ece script." - add_next_step "- For further reading, in-depth guides on installing, " - add_next_step " configuring and developing Escenic web sites can be " - add_next_step " fount at http://documentation.vizrt.com/ece-5.3.html" + if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP -a \ + $install_profile_number -ne $PROFILE_CACHE_SERVER -a \ + $install_profile_number -ne $PROFILE_WIDGET_FRAMEWORK ]; then + add_next_step "- See /usr/share/doc/escenic/ece-install-guide.txt for" + add_next_step " a full overview of files and directories set up by" + add_next_step " $(basename $0)" + add_next_step "- type 'ece help' to see all the options of this script" + add_next_step " which allows you to control all the ECE components." + add_next_step " Also, see /usr/share/doc/escenic/ece-guide.txt for" + add_next_step " further information on the ece script." + add_next_step "- For further reading, in-depth guides on installing, " + add_next_step " configuring and developing Escenic web sites can be " + add_next_step " fount at http://documentation.vizrt.com/ece-5.3.html" + fi + echo "$next_steps" @@ -1160,7 +1168,9 @@ function stop_conflicting_processes() { print "Stopping conflicting processes ..." # TODO this is dirty - killall java 1>>$log 2>>$log + if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then + killall java 1>>$log 2>>$log + fi } function set_up_varnish() @@ -2035,7 +2045,7 @@ function install_munin_node() -O $file run chmod 755 $file - if [ ! -e /etc/escenic/ece-*.conf -o -z "$type" -o -z "$instance" ]; then + if [ ! -e /etc/escenic -o ! -e /etc/escenic/ece-*.conf ]; then print "No ECE instances found on $HOSTNAME, so I'm not adding" print "additional Munin configuration" add_next_step "- A Munin node has been installed on $HOSTNAME" @@ -2215,7 +2225,7 @@ function restore_from_backup() run tar xzf $backup_file --wildcards var/backups/escenic/*.sql.gz # picking the DB backup file to restore sql_file=$(ls -tra var/backups/escenic/*.sql.gz | tail -1) - print "Selecting the most recent database dump " $(basename $sql_file) + print "Selecting the most recent database dump $(basename $sql_file)" # methods in drop-and-create-ecedb to set up the database # schema & user @@ -2226,41 +2236,52 @@ function restore_from_backup() gunzip < $sql_file | mysql $db_schema exit_on_error "restoring from $sql_file" - print "Successfully restored DB on $HOSTNAME from" $(basename $sql_file) + add_next_step "Successfully restored DB from $(basename $sql_file)" fi if [ $restore_data_files -eq 1 ]; then print "Restoring the Solr & ECE data files on $HOSTNAME ..." run cd $dir run tar -C / -xzf $backup_file var/lib/escenic - print "Successfully restored Solr & ECE data files on $HOSTNAME" - print "from backup: $(basename $backup_file)" - print "Check /var/lib/escenic to verify they're all there" + add_next_step "Successfully restored Solr & ECE data files" + add_next_step "from backup: $(basename $backup_file)" + add_next_step "Check /var/lib/escenic to verify they're all there" fi if [ $restore_conf -eq 1 ]; then print "Restoring the ECE configuration files on $HOSTNAME ..." run cd $dir run tar -C / -xzf $backup_file etc - print "Successfully restored ECE configuration files on $HOSTNAME" - print "from backup: $(basename $backup_file)" - print "Check /etc to verify that they're all there". + add_next_step "Successfully restored ECE configuration files" + add_next_step "from backup: $(basename $backup_file)" + add_next_step "Check /etc to verify that they're all there". fi if [ $restore_binaries -eq 1 ]; then print "Restoring the Escenic & Tomcat binaries on $HOSTNAME ..." run cd $dir run tar -C / -xzf $backup_file opt - print "Successfully restored Escenic & Tomcat binaries on $HOSTNAME" - print "from backup: $(basename $backup_file)" - print "Check /opt to verify that they're all there". + add_next_step "Successfully restored Escenic & Tomcat binaries" + add_next_step "from backup: $(basename $backup_file)" + add_next_step "Check /opt to verify that they're all there". fi } function common_post_install() { - add_server_to_runlevels - install_munin_node + if [ $install_profile_number -eq $PROFILE_ALL_IN_ONE -o \ + $install_profile_number -eq $PROFILE_PRESENTATION_SERVER -o \ + $install_profile_number -eq $PROFILE_RMI_HUB -o \ + $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + add_server_to_runlevels + fi + + if [ $install_profile_number -ne $PROFILE_WIDGET_FRAMEWORK -a \ + $install_profile_number -ne $PROFILE_CREATE_PUBLICATION -a \ + $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then + install_munin_node + fi + print_status_and_next_steps rm $pid_file } From 9e6a0e6c13fb0b55ee401f0307ed804be4c40f03 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 13 Oct 2011 12:39:23 +0800 Subject: [PATCH 0094/2585] - I hate inconsistent feedback, fixed backup -> restore --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 12daa801..367c1906 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2158,9 +2158,9 @@ function restore_from_backup() restore_conf=1 fi elif [ $(get_boolean_conf_value fai_enabled) -eq 0 ]; then - print "From which dataset do you wish to backup?" + print "From which dataset do you wish to restore?" if [ ! -d $backup_dir ]; then - print "Backup directory $backup_dir doesn't exist or insn't readable" + print "Directory $backup_dir doesn't exist or isn't readable" exit 1 fi From e834b354fb9724a08ca4a8009ae21d4ca217646e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 13 Oct 2011 13:44:15 +0800 Subject: [PATCH 0095/2585] - cleaner listing of installation profiles, making it easier to add new profiles without forgetting to update the total number of options on the opening screen. --- usr/sbin/ece-install | 64 +++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 367c1906..6f2806e7 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -939,7 +939,7 @@ function print_status_and_next_steps() minutes=$(( seconds_left / 60 )) seconds_left=$(( seconds_left - $minutes * 60 )) - s="- The installation is now complete!" + s="The installation is now complete!" print $s" It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP -a \ @@ -948,7 +948,7 @@ function print_status_and_next_steps() add_next_step "- See /usr/share/doc/escenic/ece-install-guide.txt for" add_next_step " a full overview of files and directories set up by" add_next_step " $(basename $0)" - add_next_step "- type 'ece help' to see all the options of this script" + add_next_step "- Type 'ece help' to see all the options of this script" add_next_step " which allows you to control all the ECE components." add_next_step " Also, see /usr/share/doc/escenic/ece-guide.txt for" add_next_step " further information on the ece script." @@ -1319,41 +1319,37 @@ EOF function read_user_input() { - echo "Hi, which server profile do you wish to install?" - echo "" - echo "Select 1-10 and press ENTER" - echo "" - echo " $PROFILE_ALL_IN_ONE - All in one, the full stack on one host. " - echo " Suitable for development and test environments." - echo " It will install caching server, ECE, assembly host, database &" - echo " Widget Framework as well as set up a new publication." - echo " $PROFILE_EDITORIAL_SERVER - Editorial (publication) server." - echo " This will install ECE and an assembly host." - echo -n " $PROFILE_PRESENTATION_SERVER - Presentation server " - echo "(ECE, memcached)." - echo " $PROFILE_DB_SERVER - Database server." - echo -n " $PROFILE_CACHE_SERVER - Cache server " - echo "(cache & web server)." - echo " $PROFILE_RMI_HUB - RMI hub." - echo -n " $PROFILE_SEARCH_SERVER - Standalone search instance " - echo "(solr + indexer-webapp)." - echo " $PROFILE_WIDGET_FRAMEWORK - Install the Widget Framework." - echo " (You need user/pass for repo.escenic.com for this)." - echo " $PROFILE_CREATE_PUBLICATION - Create a new publication." - echo " This profile will create new publication based on WF" - echo " (if available) or ECE/clean-demo." - echo " $PROFILE_MONITORING_SERVER - A monitoring server." - echo " This will install the Munin gatherer." - echo " $PROFILE_RESTORE_FROM_BACKUP - Restore from backup. Allows you to" - echo " restore DB, data files, binaries & publication templates." - echo "" + installation_profiles=( + "$PROFILE_ALL_IN_ONE - All in one, full stack on one host,"\ +" suitable for dev & test environments" + "$PROFILE_EDITORIAL_SERVER - Editorial (publication) server" + "$PROFILE_PRESENTATION_SERVER - Presentation server (ECE + memcached)." + "$PROFILE_DB_SERVER - Database server" + "$PROFILE_CACHE_SERVER - Cache server (cache and web server)" + "$PROFILE_RMI_HUB - RMI hub" + "$PROFILE_SEARCH_SERVER - Search server (Solr + indexer-webapp)" + "$PROFILE_WIDGET_FRAMEWORK - Install Widget Framework." + "$PROFILE_CREATE_PUBLICATION - Create a new publication"\ +" based on WF if available, ECE/clean-demo if not" + "$PROFILE_MONITORING_SERVER - A monitoring server (web server +"\ +" Munin gatherer)" + "$PROFILE_RESTORE_FROM_BACKUP - Restore from backup"\ +" (DB, data files, binaries, conf & publications)" + ) + + echo "Hi, which server profile do you wish to install?"$'\n' + + for (( i = 0; i < ${#installation_profiles[@]}; i++ )); do + echo " " ${installation_profiles[$i]} + done + + echo $'\n'"Select 1-${#installation_profiles[@]} and press ENTER" echo -n "Your choice [1]> " read install_profile_number if [ -z "$install_profile_number" ]; then install_profile_number=$PROFILE_ALL_IN_ONE fi - } function assert_correct_runtime_environment() @@ -1492,9 +1488,11 @@ function install_cache_server() function add_next_step() { if [ -n "$next_steps" ]; then - next_steps=${next_steps}${NEW_LINE}"[$(basename $0)] "${1} + # next_steps=${next_steps}${NEW_LINE}"[$(basename $0)] "${1} + next_steps=${next_steps}${NEW_LINE}"${1}" else - next_steps="[$(basename $0)] "${1} + # next_steps="[$(basename $0)] "${1} + next_steps=${1} fi } From 1016072ddb906abc45e08ec1848ad611077aefa7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 13 Oct 2011 15:08:10 +0800 Subject: [PATCH 0096/2585] - added --exclude-binaries to the backup command. This will exclude the app server and Escenic binaries from the backup, significantly reducing the size of the backup. --- usr/bin/ece | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index e1fd1e0a..8402a3fd 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -21,6 +21,7 @@ quiet=0 type=engine instance=default command="" +backup_exclude_binaries=0 type_list=" engine @@ -638,21 +639,21 @@ function backup_type() " actual_backup="" for el in $possible_backup; do - if [ -r $el ]; then + if [ $el = "/opt/escenic" -a $backup_exclude_binaries -eq 1 ]; then + continue + elif [ -r $el ]; then actual_backup="$el $actual_backup" fi done - if [ $appserver = "tomcat" ]; then + if [ $appserver = "tomcat" -a $backup_exclude_binaries -eq 0 ]; then actual_backup="$tomcat_home $tomcat_base $actual_backup" fi print "Creating snapshot ... (this will take a while)" - tar czf $archive_file \ + run tar czf $archive_file \ $actual_backup \ - $db_backup_file \ - 1>>$log_file \ - 2>>$log_file + $db_backup_file message="Backup ready: $archive_file" print $message @@ -660,8 +661,12 @@ function backup_type() print "The backup arhcive includes:" print "- Database snapshot" print "- All Solr & Escenic data files from /var/lib/escenic" - print "- All app servers in /opt" - print "- All Escenic binaries & publication templates in /opt/escenic" + + if [ $backup_exclude_binaries -eq 0 ]; then + print "- All app servers in /opt" + print "- All Escenic binaries & publication templates in /opt/escenic" + fi + print "- All configuration in /etc/escenic and /etc/default/ece" print "- All bootstrap scripts from /etc/init.d" print "Enjoy!" @@ -1056,6 +1061,11 @@ function set_type_command_and_instance() else next_is_resource=0 fi + + if [ $el = "--exclude-binaries" ]; then + backup_exclude_binaries=1 + continue + fi # the only thing left at this point, is the command command="$command $el" From 902a242ba949d51ca03b3aff266d6b9120cecfd2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 13 Oct 2011 15:27:47 +0800 Subject: [PATCH 0097/2585] - added BASH completion for instances (-i) and publications (-p) - added BASH completion for backup's --exclude-binaries option --- etc/bash_completion.d/ece | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index 6d816e29..54d5ae2c 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -11,13 +11,28 @@ _ece_commands() # default completions is the list of commands completions=$commands - long_options="--help" - - # TODO make educated guesses on these. instances="" + dir=/etc/escenic/engine/instance + if [ -r $dir ] ; then + cd $dir + for el in *; do + instances=${el}" "$instances + done + fi + publications="" + dir=/opt/escenic/assemblytool/publications + if [ -r $dir ] ; then + cd $dir + for el in *.properties; do + publications=$(basename $el .properties)" "$publications + done + fi case "$prev" in + backup) + completions="--exclude-binaries" + ;; -t) completions=$types ;; From 13367f7ec70e8d97f4e0c416d11786fb025cd2e0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 13 Oct 2011 16:27:44 +0800 Subject: [PATCH 0098/2585] - fixed bug in "ece assemble" where assemble would enter an eternal loop of "ece clean assemble" when duplicate plugins were to be found in the plugins directory. Now, the ece script fails with a message telling the user what to do: [ece#engine-dev1] I've tried to assemble twice now and FAILED :-( [ece#engine-dev1] You probably have multiple versions of one or more plugins [ece#engine-dev1] Check your plugins directory, sort it out and try again. --- usr/bin/ece | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 8402a3fd..070f6da4 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -861,8 +861,18 @@ function restart_type() start_type } +assemble_attempts=0 + function assemble() { + if [ $assemble_attempts -gt 1 ]; then + print "I've tried to assemble twice now and FAILED :-(" + print "You probably have multiple versions of one or more plugins" + print "Check your plugins directory, sort it out and try again." + exit 1 + fi + assemble_attempts=$(( $assemble_attempts + 1 )) + if [ ! $type = "engine" ]; then print "You cannot assemble $type" exit 1 From 949b142b2bb8d83be8d89d48120c046eba816409 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 13 Oct 2011 17:05:34 +0800 Subject: [PATCH 0099/2585] - changed the way next_steps is accumulated. It's now an array where elements get added in the add_next_step method (as before). However, having an array makes it easier to display the messages in tidier manner with the ece-install- when they are printed in the report and not when they are added to the next_steps. - made the user feedback more compact. The report screen was getting too big IMO. Now it's short and sweet, people might even want to read it :-) Example report screen: [ece-install-77] The installation is now complete! It took 0d 0h 1m 17s [ece-install-77] - New ECE instance web1 installed. [ece-install-77] - Admin interface: http://quanah:8080/escenic-admin/ [ece-install-77] - View installed versions with: ece -i web1 versions [ece-install-77] - Type 'ece help' to see all the options of this script [ece-install-77] - Read its guide: /usr/share/doc/escenic/ece-guide.txt [ece-install-78] - /etc/default/ece lists all instances started at boot time [ece-install-78] - A Munin node has been installed on quanah [ece-install-78] - More info: /usr/share/doc/escenic/ece-install-guide.txt [ece-install-78] - Guide books: http://documentation.vizrt.com/ece-5.3.html [ece-install-78] Enjoy your time with Escenic Content Engine! [ece-install-78] -Vizrt Online - added a slash to the /escenic-admin url printed to the user in the installation report. Without the trailing slash, the URL cannot be copy and pasted -> success. --- usr/sbin/ece-install | 107 +++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 6f2806e7..76405e4a 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -52,7 +52,7 @@ NEW_LINE=" " # the next steps printed when the user has installed his/her # components. -next_steps="" +next_steps=() function get_seconds_since_start() { @@ -399,7 +399,7 @@ function set_up_ecedb() cd ~/ run rm -rf /opt/escenic/engine/plugins - add_next_step "- DB is now set up on $HOSTNAME:3306" + add_next_step "DB is now set up on $HOSTNAME:3306" } # returns a string of asterixes with the same lenght as the inputted @@ -896,8 +896,8 @@ function create_user_and_group_if_not_present() run adduser $ece_user \ --disabled-password \ --gecos "Escenic-user,Room,Work,Home,Other" - add_next_step "- Set a password for the new UNIX user $ece_user" - add_next_step " You can do this by running [passwd $ece_user]" + add_next_step "Set a password for the new UNIX user $ece_user "\ +"(passwd $ece_user)" fi if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then @@ -939,30 +939,25 @@ function print_status_and_next_steps() minutes=$(( seconds_left / 60 )) seconds_left=$(( seconds_left - $minutes * 60 )) - s="The installation is now complete!" + if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then + s="The installation is now complete!" + else + s="The restore is now complete!" + fi print $s" It took" ${days}d ${hours}h ${minutes}m ${seconds_left}s if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP -a \ $install_profile_number -ne $PROFILE_CACHE_SERVER -a \ $install_profile_number -ne $PROFILE_WIDGET_FRAMEWORK ]; then - add_next_step "- See /usr/share/doc/escenic/ece-install-guide.txt for" - add_next_step " a full overview of files and directories set up by" - add_next_step " $(basename $0)" - add_next_step "- Type 'ece help' to see all the options of this script" - add_next_step " which allows you to control all the ECE components." - add_next_step " Also, see /usr/share/doc/escenic/ece-guide.txt for" - add_next_step " further information on the ece script." - add_next_step "- For further reading, in-depth guides on installing, " - add_next_step " configuring and developing Escenic web sites can be " - add_next_step " fount at http://documentation.vizrt.com/ece-5.3.html" + add_next_step "More info: /usr/share/doc/escenic/ece-install-guide.txt" + add_next_step "Guide books: http://documentation.vizrt.com/ece-5.3.html" fi + for (( i = 0; i < ${#next_steps[@]}; i++ )); do + print " - " ${next_steps[$i]} + done - echo "$next_steps" - - print "" - print "Enjoy your time with Escenic Content Engine!" - print "" + print $'\n'"Enjoy your time with Escenic Content Engine!"$'\n' print "-Vizrt Online" } @@ -1480,19 +1475,20 @@ function install_cache_server() set_up_varnish $backend_servers - add_next_step "- Cache server is up and running at http://${HOSTNAME}:80/" + add_next_step "Cache server is up and running at http://${HOSTNAME}:80/" } # Parameters: # $1 : your added line function add_next_step() { + next_steps[${#next_steps[@]}]="${1}" + return + if [ -n "$next_steps" ]; then - # next_steps=${next_steps}${NEW_LINE}"[$(basename $0)] "${1} - next_steps=${next_steps}${NEW_LINE}"${1}" + next_steps=${next_steps}${NEW_LINE}"[$(basename $0)] "${1} else - # next_steps="[$(basename $0)] "${1} - next_steps=${1} + next_steps="[$(basename $0)] "${1} fi } @@ -1655,18 +1651,14 @@ EOF set_conf_file_value ece_unix_user $ece_user /etc/default/ece set_conf_file_value ece_unix_group $ece_group /etc/default/ece - admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin - add_next_step "- New ECE instance $instance_name installed." - add_next_step "- Its admin interface is available at" - add_next_step " $admin_uri" - add_next_step "- You can view all installed plugin & engine versions with:" - add_next_step " ece -i $instance_name versions" - add_next_step "- type 'ece help' to see all the options of this script" - add_next_step " which allows you to control all the Escenic components." - add_next_step " Also, see /usr/share/doc/escenic/ece-guide.txt for" - add_next_step " addtional information on the ece script." - add_next_step "- Verify that /etc/default/ece lists all the instances " - add_next_step " on $HOSTNAME you wish to start at boot time." + admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin/ + add_next_step "New ECE instance $instance_name installed." + add_next_step "Admin interface: $admin_uri" + add_next_step "View installed versions with:"\ +" ece -i $instance_name versions" + add_next_step "Type 'ece help' to see all the options of this script" + add_next_step "Read its guide: /usr/share/doc/escenic/ece-guide.txt" + add_next_step "/etc/default/ece lists all instances started at boot time" } function install_presentation_server() @@ -1737,7 +1729,7 @@ function create_publication() create_publication_in_db $publication_war assemble_deploy_and_restart_type - add_next_step "- a new publication $publication_name has been created" + add_next_step "A new publication $publication_name has been created." } function install_editorial_server() @@ -1768,11 +1760,7 @@ clientConfiguration=/neo/io/services/HubConnection pingTime=10000 EOF - add_next_step "- Restart all your instances to make the hub seem them:" - for el in /etc/escenic/engine/instance/*; do - instance_name=$(basename $el) - add_next_step " - $ ece -i ${instance_name} restart" - done + add_next_step "Restart all your instances to make the hub see them." print "Starting the RMI-hub on $HOSTNAME ..." ece_command="ece -t rmi-hub restart" @@ -1894,8 +1882,8 @@ EOF cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ - add_next_step "- Widget Framework has been installed into your local" - add_next_step " Maven repository and its ECE components have been installed" + add_next_step "Widget Framework has been installed into your local"\ +" Maven repository" } function install_search_instance() @@ -1982,9 +1970,9 @@ server { EOF /etc/init.d/nginx restart 1>>$log 2>>$log - add_next_step "- Web server is install on http://${HOSTNAME}:81/" - add_next_step " http://${HOSTNAME}:81/ gives you the munin interface" - add_next_step " http://${HOSTNAME}:81/binary gives you the Adactus endpoint" + add_next_step "Web server is install on http://${HOSTNAME}:81/" + add_next_step "http://${HOSTNAME}:81/ gives you the munin interface" + add_next_step "http://${HOSTNAME}:81/binary gives you the Adactus endpoint" } function install_munin_gatherer() @@ -2018,7 +2006,7 @@ function install_munin_gatherer() # TODO has a hard dependency on install_web_server, should perhaps # be called from this one? - add_next_step "- Munin interface available at http://$HOSTNAME:81/" + add_next_step "Munin interface is available at http://$HOSTNAME:81/" } function install_munin_node() @@ -2043,10 +2031,11 @@ function install_munin_node() -O $file run chmod 755 $file - if [ ! -e /etc/escenic -o ! -e /etc/escenic/ece-*.conf ]; then + if [ ! -e /etc/escenic -o \ + $(ls /etc/escenic/ | grep ece- | wc -l) -lt 1 ]; then print "No ECE instances found on $HOSTNAME, so I'm not adding" print "additional Munin configuration" - add_next_step "- A Munin node has been installed on $HOSTNAME" + add_next_step "A Munin node has been installed on $HOSTNAME" return fi @@ -2091,7 +2080,7 @@ EOF 1>>$log 2>>$log fi - add_next_step "- A Munin node has been installed on $HOSTNAME" + add_next_step "A Munin node has been installed on $HOSTNAME" } function add_server_to_runlevels() @@ -2109,8 +2098,8 @@ function add_server_to_runlevels() print "Adding the ece init.d script to the default run levels ..." run update-rc.d ece defaults else - add_next_step "- Remember to add /etc/intit.d/ece to the desired" - add_next_step " run levels." + add_next_step "Remember to add /etc/intit.d/ece to the desired "\ +"run levels." # TODO add init.d to the default runlevels, for other # distributions too: # - RedHat/chekcconfig @@ -2242,8 +2231,8 @@ function restore_from_backup() run cd $dir run tar -C / -xzf $backup_file var/lib/escenic add_next_step "Successfully restored Solr & ECE data files" - add_next_step "from backup: $(basename $backup_file)" - add_next_step "Check /var/lib/escenic to verify they're all there" + add_next_step "Backup file used: $(basename $backup_file)" + add_next_step "Check /var/lib/escenic to verify they're all there." fi if [ $restore_conf -eq 1 ]; then @@ -2251,8 +2240,8 @@ function restore_from_backup() run cd $dir run tar -C / -xzf $backup_file etc add_next_step "Successfully restored ECE configuration files" - add_next_step "from backup: $(basename $backup_file)" - add_next_step "Check /etc to verify that they're all there". + add_next_step "Backup file used: $(basename $backup_file)" + add_next_step "Check /etc to verify that they're all there." fi if [ $restore_binaries -eq 1 ]; then @@ -2260,7 +2249,7 @@ function restore_from_backup() run cd $dir run tar -C / -xzf $backup_file opt add_next_step "Successfully restored Escenic & Tomcat binaries" - add_next_step "from backup: $(basename $backup_file)" + add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check /opt to verify that they're all there". fi } From bb84670c1dedd58aa2c0d1d1ec4897986ec05bc8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 13 Oct 2011 18:28:15 +0800 Subject: [PATCH 0100/2585] - fixed bug in the duplicate detection test, making commons-httpclient-3.1.jar resolvers-commons-httpclient-3.2.jar produce false positives. A hat fixed it (^). --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index 070f6da4..26a6864f 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -899,7 +899,7 @@ function assemble() if [ $(echo $known_unharmful_duplicates | grep $jar | wc -l) -gt 0 ]; then continue fi - if [ $(\ls *.jar | grep "$jar[0-9]" | wc -l) -gt 1 ]; then + if [ $(\ls *.jar | grep "^${jar}[0-9]" | wc -l) -gt 1 ]; then duplicates_found=1 debug "More than one version of $jar" fi From 57ed307b8dece5804e4ee541cc4f41ac7e047127 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 14 Oct 2011 19:05:35 +0800 Subject: [PATCH 0101/2585] - on Debian systems, ece-install will now add the non-free (and contrib) pools if they don't already exist in the APT sources list. This will make it easier to install the ECE instances, which all require Sun, err, Oracle Java 6 to be installed and this is in the non-free pool. --- usr/sbin/ece-install | 75 +++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 76405e4a..9ef7f257 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -240,7 +240,7 @@ function install_common_os_packages() install_packages_if_missing $packages fi - for el in curl wget git unzip; do + for el in lsb_release curl wget git unzip; do assert_pre_prequesite $el done } @@ -579,7 +579,18 @@ function install_ece_third_party_packages if [ $on_debian_or_derivative -eq 1 ]; then if [ $on_ubuntu -eq 1 ]; then - add_apt_source "deb http://archive.canonical.com/ $(lsb_release -s -c) partner" + add_apt_source "deb http://archive.canonical.com/ $(lsb_release -s -c) partner" + elif [ $on_debian -eq 1 ]; then + code_name=$(lsb_release -s -c) + has_non_free=$(grep $(lsb_release -s -c) \ + /etc/apt/sources.list | \ + egrep -v "^#|deb-src" | \ + grep -v security | \ + grep non-free | \ + wc -l) + if [ $has_non_free -eq 0 ]; then + add_apt_source "deb http://ftp.tw.debian.org/debian/ $(lsb_release -s -c) contrib non-free" + fi fi echo "sun-java6-jdk shared/accepted-sun-dlj-v1-1 boolean true" | \ @@ -949,7 +960,7 @@ function print_status_and_next_steps() if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP -a \ $install_profile_number -ne $PROFILE_CACHE_SERVER -a \ $install_profile_number -ne $PROFILE_WIDGET_FRAMEWORK ]; then - add_next_step "More info: /usr/share/doc/escenic/ece-install-guide.txt" + add_next_step "Install info: "\ "/usr/share/doc/escenic/ece-install-guide.txt" add_next_step "Guide books: http://documentation.vizrt.com/ece-5.3.html" fi @@ -1389,6 +1400,13 @@ function common_pre_install() export DEBIAN_FRONTEND=noninteractive fi + # chicken and the egg problem, we need lsb_release to install the + # packages later on, hence as soon as we know we've got a Debian + # based platform, we install lsb-release. Also note, the + # executable, lsb_release, is in the list of required binaries in + # install_common_os_packages. + install_packages_if_missing "lsb-release" + if [ $(lsb_release -i | grep Ubuntu | wc -l) -gt 0 ]; then on_ubuntu=1 elif [ $(lsb_release -i | grep Debian | wc -l) -gt 0 ]; then @@ -2110,6 +2128,7 @@ function add_server_to_runlevels() function restore_from_backup() { # locals + restore_all=0 restore_db=0 restore_binaries=0 restore_data_files=0 @@ -2125,23 +2144,17 @@ function restore_from_backup() print "You must specifify fai_restore_from_file" exit 1 fi - - if [ $(get_boolean_conf_value fai_restore_db) -eq 1 -o \ - $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then + + if [ $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then + restore_all=1 + elif [ $(get_boolean_conf_value fai_restore_db) -eq 1 ]; then restore_db=1 - fi - - if [ $(get_boolean_conf_value fai_restore_data_files) -eq 1 -o \ - $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then + elif [ $(get_boolean_conf_value fai_restore_data_files) -eq 1 ]; then restore_data_files=1 - fi - - if [ $(get_boolean_conf_value fai_restore_software_binaries) -eq 1 -o \ - $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then + elif [ $(get_boolean_conf_value fai_restore_software_binaries) -eq 1 ] + then restore_binaries=1 - fi - if [ $(get_boolean_conf_value fai_restore_configuration) -eq 1 -o \ - $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then + elif [ $(get_boolean_conf_value fai_restore_configuration) -eq 1 ]; then restore_conf=1 fi elif [ $(get_boolean_conf_value fai_enabled) -eq 0 ]; then @@ -2151,7 +2164,12 @@ function restore_from_backup() exit 1 fi - tarball_array=($(ls -t $backup_dir/*.tar.gz)) + if [ $(ls $backup_dir | grep .tar | wc -l) -lt 1 ]; then + print "No backup files (.tar) found in $backup_dir, exiting." + exit 0 + fi + + tarball_array=($(ls -t $backup_dir/*.tar)) for (( i = 0; i <${#tarball_array[@]}; i++ )); do echo " " $(( ${i} + 1 )) "-" $(basename ${tarball_array[$i]}) @@ -2173,6 +2191,7 @@ function restore_from_backup() "The Solr and ECE data files (multimedia archive)" "The ECE configuration files" "The Escenic and Tomcat software binaries + publication templates" + "Restore everything of the above" ) for (( i = 0; i <${#restore_profiles[@]}; i++ )); do echo " " $(( ${i} + 1 )) "-" ${restore_profiles[$i]} @@ -2194,6 +2213,8 @@ function restore_from_backup() restore_conf=1 elif [ $user_restore_profile -eq 4 ]; then restore_binaries=1 + elif [ $user_restore_profile -eq 5 ]; then + restore_all=1 fi fi @@ -2205,11 +2226,11 @@ function restore_from_backup() dir=$(mktemp -d) - if [ $restore_db -eq 1 ]; then + if [ $restore_db -eq 1 -o $restore_all -eq 1 ]; then install_database_server "binaries_only" print "Restoring the database contents on $HOSTNAME ..." run cd $dir - run tar xzf $backup_file --wildcards var/backups/escenic/*.sql.gz + run tar xf $backup_file --wildcards var/backups/escenic/*.sql.gz # picking the DB backup file to restore sql_file=$(ls -tra var/backups/escenic/*.sql.gz | tail -1) print "Selecting the most recent database dump $(basename $sql_file)" @@ -2226,31 +2247,32 @@ function restore_from_backup() add_next_step "Successfully restored DB from $(basename $sql_file)" fi - if [ $restore_data_files -eq 1 ]; then + if [ $restore_data_files -eq 1 -o $restore_all -eq 1 ]; then print "Restoring the Solr & ECE data files on $HOSTNAME ..." run cd $dir - run tar -C / -xzf $backup_file var/lib/escenic + run tar -C / -xf $backup_file var/lib/escenic add_next_step "Successfully restored Solr & ECE data files" add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check /var/lib/escenic to verify they're all there." fi - if [ $restore_conf -eq 1 ]; then + if [ $restore_conf -eq 1 -o $restore_all -eq 1 ]; then print "Restoring the ECE configuration files on $HOSTNAME ..." run cd $dir - run tar -C / -xzf $backup_file etc + run tar -C / -xf $backup_file etc add_next_step "Successfully restored ECE configuration files" add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check /etc to verify that they're all there." fi - if [ $restore_binaries -eq 1 ]; then + if [ $restore_binaries -eq 1 -o $restore_all -eq 1 ]; then print "Restoring the Escenic & Tomcat binaries on $HOSTNAME ..." run cd $dir - run tar -C / -xzf $backup_file opt + run tar -C / -xf $backup_file opt add_next_step "Successfully restored Escenic & Tomcat binaries" add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check /opt to verify that they're all there". + install_ece_third_party_packages fi } @@ -2330,6 +2352,7 @@ if [ $fai_enabled -eq 1 ]; then else read_user_input common_pre_install + case $install_profile_number in $PROFILE_ALL_IN_ONE) install_all_in_one_environment From 4947c197f6938180266c9fdc2b3a0a63675f0d04 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 14 Oct 2011 22:24:33 +0800 Subject: [PATCH 0102/2585] - changed backup format to .tar instead of .tar.gz as I discovered some problems extracting such a big (~6GB) archive on two servers. Will revise this if some light can shed as to why this can be a problem. The change to uncompressed tarballs significantly speeds up the backup and recovery, so I might keep it. - added get_actual_file for sorting out members of the backup being variables holding links to the actual contents (like /opt/tomcat -> apache-tomcat...) Returns the file (can be a directory) passed to the function only if it's the actual file/directory and not a link to it. If the passed file is a link, the link target is returned instead. --- usr/bin/ece | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 26a6864f..e6963e46 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -562,6 +562,37 @@ function deploy() esac } +# Returns the file (can be a directory) passed to the function only +# if it's the actual file/directory and not a link to it. If the +# passed file is a link, the link target is returned instead. +# +# $1 - the file (which could be a link) +function get_actual_file() +{ + if [ -h ${1} ]; then + dir=$(dirname $1) + real_file=$(ls -l ${1} | awk '{print $11}') + + # Because of the test if the file we want to returns is + # absolute, we go to the root before testing. We want to + # preserve the cwd, therefore, we're doing the "cd /" in a + # subshell. + real_file=$( + cd / + if [ ! -e ${real_file} ]; then + real_file=${dir}/${real_file} + fi + + # this is the return value from the sub process + echo ${real_file} + ) + else + real_file=${1} + fi + + echo ${real_file} +} + function backup_type() { message="Backing up the $instance instance of $type on $HOSTNAME ..." @@ -628,7 +659,7 @@ function backup_type() print "Database dumped: $db_backup_file" fi - archive_file=$backup_dir/${type}-${instance}-backup-$(date --iso).tar.gz + archive_file=$backup_dir/${type}-${instance}-backup-$(date --iso).tar possible_backup=" /etc/escenic /opt/escenic @@ -647,11 +678,11 @@ function backup_type() done if [ $appserver = "tomcat" -a $backup_exclude_binaries -eq 0 ]; then - actual_backup="$tomcat_home $tomcat_base $actual_backup" + actual_backup="$(get_actual_file $tomcat_home) $tomcat_base $actual_backup" fi print "Creating snapshot ... (this will take a while)" - run tar czf $archive_file \ + run tar cf $archive_file \ $actual_backup \ $db_backup_file From 9e3d604543829e36700e381a73b146422df81079 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 14 Oct 2011 22:25:36 +0800 Subject: [PATCH 0103/2585] - simplification of feedback on DB dump selected for restore. The rationale is that I want to confuse the user as little as possible, simply stating which DB backup was used is hopefully enough. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9ef7f257..ea2ba879 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2233,7 +2233,7 @@ function restore_from_backup() run tar xf $backup_file --wildcards var/backups/escenic/*.sql.gz # picking the DB backup file to restore sql_file=$(ls -tra var/backups/escenic/*.sql.gz | tail -1) - print "Selecting the most recent database dump $(basename $sql_file)" + print "Selecting database dump: $(basename $sql_file)" # methods in drop-and-create-ecedb to set up the database # schema & user From 829dbd058ed3cef18dbe1576c2a4dbca22de52da Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 18 Oct 2011 10:53:11 +0800 Subject: [PATCH 0104/2585] - removed the is_production setting for ece{.conf}. If set to 1, it would add the "-server" parameter to the JVM options. However, the "-server" VM server class is the only one available on 64bits systems on Java 6 and since all systems these days are both 64bit and use Java 6, this setting can be omitted. Trivia: The original idea behind the is_production was to allow a number of production specific settings/switches to ece, however as the script grew, more specific options were added and a general is_production_environment/is_development_environment switch became redundant. --- etc/escenic/ece.conf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/etc/escenic/ece.conf b/etc/escenic/ece.conf index 9b1e771e..be5ded7d 100644 --- a/etc/escenic/ece.conf +++ b/etc/escenic/ece.conf @@ -116,11 +116,6 @@ remote_monitoring_port=5792 ##################################################################### force_ipv4=0 -##################################################################### -# Is this a production system? -##################################################################### -is_production=1 - ##################################################################### # Ask the JVM to create a heap dump when/if it crashes. ##################################################################### From b3788599637eb9079280f8738f26341b696a4ae1 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 18 Oct 2011 10:53:26 +0800 Subject: [PATCH 0105/2585] - removed the is_production setting for ece{.conf}. If set to 1, it would add the "-server" parameter to the JVM options. However, the "-server" VM server class is the only one available on 64bits systems on Java 6 and since all systems these days are both 64bit and use Java 6, this setting can be omitted. Trivia: The original idea behind the is_production was to allow a number of production specific settings/switches to ece, however as the script grew, more specific options were added and a general is_production_environment/is_development_environment switch became redundant. --- usr/bin/ece | 6 ------ 1 file changed, 6 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index e6963e46..5f33d14a 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -51,7 +51,6 @@ ece_security_configuration_dir enable_heap_dump enable_remote_debugging enable_remote_monitoring -is_production java_home solr_home " @@ -352,11 +351,6 @@ function set_instance_settings() ece_args=$ece_args" -XX:MaxPermSize=$max_permgen_size" fi - # settings specific to a production environment - if [ $is_production -eq 1 ]; then - ece_args=$ece_args" -server" - fi - if [ $jvm_encoding ]; then ece_args=$ece_args" -Dsun.jnu.encoding=$jvm_encoding" ece_args=$ece_args" -Dfile.encoding=$jvm_encoding" From 69e20997d802ff9e2fdde42a934cfafd4c178cd8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 19 Oct 2011 12:54:58 +0800 Subject: [PATCH 0106/2585] - ece-install/restore now reads the ece UNIX user & group from the restored /etc/default/ece - if exists - and based on that, sets the correct permissions, as well as creating the user & group if they don't exist on the system. - getting the Percona APT key has failed several times now during the last six months. Therefore, I've added logic to the install_database_server method, that it will force the installation of Percona if the key couldn't be retrieved. A warning message about this is logged. - bug fix to the test for existent tarballs - added log and print_and_log methods --- usr/sbin/ece-install | 89 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 12 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index ea2ba879..76b90734 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -42,6 +42,7 @@ appserver_port=8080 on_debian_or_derivative=0 on_debian=0 on_ubuntu=0 +force_packages=0 # because the all in one profile will run database, search and app # server profiles, all of which needs downloading and setting up the @@ -80,6 +81,17 @@ function print() echo $(get_id) $@ } +function log() +{ + echo $(get_id) $@ >> $log +} + +function print_and_log() +{ + print "$@" + log "$@" +} + function exit_on_error() { if [ $? -gt 0 ]; then @@ -218,8 +230,12 @@ install_packages_if_missing() if [ $some_are_missing -eq 0 ]; then return fi - - run apt-get install -y $@ + + if [ $force_packages -eq 1 ]; then + run apt-get install --assume-yes --force-yes $@ + else + run apt-get install --assume-yes $@ + fi fi } @@ -907,8 +923,8 @@ function create_user_and_group_if_not_present() run adduser $ece_user \ --disabled-password \ --gecos "Escenic-user,Room,Work,Home,Other" - add_next_step "Set a password for the new UNIX user $ece_user "\ -"(passwd $ece_user)" + add_next_step "I created a new UNIX user called $ece_user" + add_next_step "and you must set a password using: passwd $ece_user" fi if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then @@ -1539,18 +1555,35 @@ function install_database_server() print "Installing the Percona database ..." if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then - run gpg --keyserver hkp://keys.gnupg.net \ - --recv-keys 1C4CBDCDCD2EFD2A - gpg --armor \ - -a \ - --export 1C4CBDCDCD2EFD2A | \ - apt-key add - \ - 1>>$log 2>>$log + gpg --keyserver hkp://keys.gnupg.net \ + --recv-keys 1C4CBDCDCD2EFD2A \ + 1>>$log 2>>$log + + # There has been twice now, during six months, that + # the key cannot be retrieved from + # keys.gnupg.net. Therefore, we're checking if it + # failed and if yes, force the package installation. + if [ $? -gt 0 ]; then + s="Failed retrieving the Percona key from keys.gnupg.net" + print_and_log $s + s="Will install the Percona packages without the GPG key" + print_and_log $s + force_packages=1 + else + gpg --armor \ + -a \ + --export 1C4CBDCDCD2EFD2A | \ + apt-key add - \ + 1>>$log 2>>$log + fi + run apt-get update fi + add_apt_source "deb http://repo.percona.com/apt ${code_name} main" packages="percona-server-server percona-server-client" install_packages_if_missing $packages + force_packages=0 else print "The Percona APT repsository doesn't have packages for your" print "Debian (or derivative) version with code name $code_name. " @@ -2164,7 +2197,7 @@ function restore_from_backup() exit 1 fi - if [ $(ls $backup_dir | grep .tar | wc -l) -lt 1 ]; then + if [ $(ls $backup_dir | grep ".tar$" | wc -l) -lt 1 ]; then print "No backup files (.tar) found in $backup_dir, exiting." exit 0 fi @@ -2273,6 +2306,38 @@ function restore_from_backup() add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check /opt to verify that they're all there". install_ece_third_party_packages + set_up_engine_directories + + # doing some educated guessing on which tomcat_base/home as + # well as UNIX user/group to use for this. + file=/etc/default/ece + if [ -r $file ]; then + ece_user=$(grep ^ece_unix_user $file | \ + cut -d'=' -f2 | \ + sed -e "s/'//g" -e 's/"//g' + ) + ece_group=$(grep ^ece_unix_group $file | \ + cut -d'=' -f2 | \ + sed -e "s/'//g" -e 's/"//g' + ) + create_user_and_group_if_not_present + + if [ -d /etc/escenic -a \ + $(ls /etc/escenic/ | grep ^ece- | wc -l) -gt 0 -a \ + $(grep ^tomcat_base /etc/escenic/ece*.conf | wc -l) -gt 0 ]; then + directories=$(grep ^tomcat_base /etc/escenic/ece*.conf | \ + cut -d'=' -f2 | \ + sed -e "s/'//g" -e 's/"//g' + ) + s="Setting file permissions according to /etc/default/ece" + print_and_log $s + s="and /etc/escenic/ece*.conf" + print_and_log $s + run chown -R ${ece_user}:${ece_group} ${directories} + fi + fi + + set_correct_permissions fi } From 9c311874b8c2576f7b22919471ceeec029c03048 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 19 Oct 2011 13:17:55 +0800 Subject: [PATCH 0107/2585] - added calls to remove_pid_and_exit_in_error all places where exit 1 previously was called. This means that re-running ece-install after a (controlled) failed attempt can proceed without needing to remove the PID file first. - added remove_pid_and_exit_in_error - added hardening to the user input checking (selecting profile number to install). - added is_number method --- usr/sbin/ece-install | 69 ++++++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 76b90734..1a9ec57e 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -92,12 +92,18 @@ function print_and_log() log "$@" } +function remove_pid_and_exit_in_error() +{ + rm $pid_file + exit 1 +} + function exit_on_error() { if [ $? -gt 0 ]; then print "The command ["$@"] FAILED, exiting :-(" print "See $log for further details." - exit 1 + remove_pid_and_exit_in_error fi } @@ -107,6 +113,22 @@ function run() exit_on_error $@ } +# Returns 1 if the passed value is a number, 0 if not +# +# $1 : what you want to test +function is_number() +{ + if [ -z $1 ]; then + echo 0 + elif [ $(echo $1 | grep [a-z] | wc -l) -gt 0 ]; then + echo 0 + elif [ $(echo $1 | grep [A-Z] | wc -l) -gt 0 ]; then + echo 0 + else + echo 1 + fi +} + technet_download_list=" http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip @@ -167,14 +189,14 @@ function make_ln() run ln -s $1 $2 elif [ ! -e $1 ]; then print "Tried to make a symlink to $1, but it doesn't exist" - exit 1 + remove_pid_and_exit_in_error fi else if [ -e $1 -a ! -h $(basename $1) ]; then run ln -s $1 elif [ ! -e $1 ]; then print "Tried to make a symlink to $1, but it doesn't exist" - exit 1 + remove_pid_and_exit_in_error fi fi } @@ -1002,7 +1024,7 @@ function create_publication_in_db() if [ "$cookie" == "" ] ; then print "Unable to get a session cookie." - exit 1; + remove_pid_and_exit_in_error; fi run curl --silent \ @@ -1085,7 +1107,7 @@ function check_for_required_downloads() wc -l) \ -lt 1 ]; then print "Couldn't find $el* in $download_dir" - exit 1 + remove_pid_and_exit_in_error fi done } @@ -1372,13 +1394,18 @@ function read_user_input() if [ -z "$install_profile_number" ]; then install_profile_number=$PROFILE_ALL_IN_ONE fi + + if [ $(is_number $install_profile_number) -eq 0 ]; then + print "The profile number, $install_profile_number, is not a number" + remove_pid_and_exit_in_error + fi } function assert_correct_runtime_environment() { if [ $(whoami) != "root" ]; then print "You must be root when running $(basename $0)" - exit 1 + remove_pid_and_exit_in_error fi if [ -e $pid_file ]; then @@ -1386,7 +1413,7 @@ function assert_correct_runtime_environment() print "If you blelieve this is wrong, e.g. if a previous run of" print "$(basename $0) was aborted before it completed, you" print "may remove ${pid_file} and run $(basename $0) again." - exit 1 + remove_pid_and_exit_in_error else echo $BASHPID > $pid_file started=`stat -c %Y $pid_file` @@ -1395,7 +1422,7 @@ function assert_correct_runtime_environment() if [ ! -e "$conf_file" ]; then print $conf_file "doesn't exist." print "I cannot live without it, so I'm exiting :-(" - exit 1 + remove_pid_and_exit_in_error fi } @@ -1408,7 +1435,7 @@ function common_pre_install() if [ -z "$technet_user" -o -z "$technet_password" ]; then print "Be sure to set technet_user and technet_password " print "in $conf_file" - exit 1 + remove_pid_and_exit_in_error fi if [ -e /etc/debian_version -a -x /usr/bin/dpkg ]; then @@ -1441,7 +1468,7 @@ function assert_pre_prequesite() { if [ $(which $1 | wc -l) -lt 1 ]; then print "Please install $1 and then run $(basename $0) again." - exit 1 + remove_pid_and_exit_in_error fi } @@ -1750,7 +1777,7 @@ function create_publication() ! -e /opt/escenic/assemblytool ]; then print "Please install ECE and an assembly environment before" print "running this installation profile again." - exit 1 + remove_pid_and_exit_in_error fi print "Getting ready to create a new publiation ..." @@ -1826,7 +1853,7 @@ function get_conf_value() { if [ ! -e "$conf_file" ]; then print $conf_file "doesn't exist." - exit 1 + remove_pid_and_exit_in_error fi if [ $(grep $1 $conf_file | grep -v ^# | wc -l) -gt 0 ]; then @@ -1836,7 +1863,7 @@ function get_conf_value() # returns 0 for false, 1 for true # -# paramaters: $1 : the conf key, see get_conf_value +# parameters: $1 : the conf key, see get_conf_value function get_boolean_conf_value() { value=$(get_conf_value $1) @@ -1862,7 +1889,7 @@ function install_widget_framework() if [ -z "$wf_user" -o -z "$wf_password" ]; then print "Missing wf_user and wf_password in $conf_file" print "If you don't have these, please contact support@escenic.com" - exit 1 + remove_pid_and_exit_in_error fi print "Creating a Maven settings file: $HOME/.m2/settings.xml ..." @@ -2175,7 +2202,7 @@ function restore_from_backup() if [ -z "$backup_file" ]; then print "You must specifify fai_restore_from_file" - exit 1 + remove_pid_and_exit_in_error fi if [ $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then @@ -2194,7 +2221,7 @@ function restore_from_backup() print "From which dataset do you wish to restore?" if [ ! -d $backup_dir ]; then print "Directory $backup_dir doesn't exist or isn't readable" - exit 1 + remove_pid_and_exit_in_error fi if [ $(ls $backup_dir | grep ".tar$" | wc -l) -lt 1 ]; then @@ -2254,7 +2281,7 @@ function restore_from_backup() if [ ! -r "$backup_file" ]; then print "$backup_file either doesn't exist or cannot be read." print "I cannot restore from it :-(" - exit 1 + remove_pid_and_exit_in_error fi dir=$(mktemp -d) @@ -2410,7 +2437,7 @@ if [ $fai_enabled -eq 1 ]; then else print "No install profile selected, be sure to have one of the " print "fai__install=1 in your $conf_file" - exit 1 + remove_pid_and_exit_in_error fi common_post_install @@ -2453,9 +2480,9 @@ else $PROFILE_RESTORE_FROM_BACKUP) restore_from_backup ;; - default) - print "You must select 1-11" - exit 1 + *) + print "Invalid profile number $install_profile_number, must be 1-11" + remove_pid_and_exit_in_error ;; esac common_post_install From c4a613cbf333157e02492230d7daf2fcc46fb5e8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 20 Oct 2011 18:28:18 +0800 Subject: [PATCH 0108/2585] - the install_munin_node function called from common_post_install now prompts the user for the IP of the monitoring server (if the profile is not monitoring of course!) or, if run in FAI mode, reads the new settings: fai_monitoring_server_install fai_monitoring_server_ip If set/passed, this IP is then used in the node configuration to grant access to that nodes data (from the monitoring server). - added parameters to install_web_server so that it can be used by multiple profiles: cache server, monitoring server and both (i.e. currently three). - fixed bug in the RMI hub install profile - added method get_ip(host_name) - added method get_perl_escaped(string), needed because of Munin's dependency of Perl and some short coming in one of its default libraries. --- usr/sbin/ece-install | 145 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 14 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 1a9ec57e..13d17b89 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1240,7 +1240,7 @@ EOF w | cut -d' ' -f7 | sort | uniq | while read l; do if [ $(echo $l | wc -c) -gt 8 ]; then cat >> /etc/varnish/default.vcl < common/io/api/EventManager.properties < $common_nursery_dir/io/api/EventManager.properties < $file < $file < $file < $file <>$log 2>>$log - add_next_step "Web server is install on http://${HOSTNAME}:81/" - add_next_step "http://${HOSTNAME}:81/ gives you the munin interface" - add_next_step "http://${HOSTNAME}:81/binary gives you the Adactus endpoint" + if [ $1 -eq 0 ]; then + add_next_step "http://${HOSTNAME}:${port}/binary gives the Adactus endpoint" + elif [ $1 -eq 1 ]; then + add_next_step "http://${HOSTNAME}:${port}/ gives the Munin interface" + elif [ $2 -eq 1 ]; then + add_next_step "http://${HOSTNAME}:${port}/ gives the Munin interface" + add_next_step "http://${HOSTNAME}:${port}/binary gives the Adactus endpoint" + fi } function install_munin_gatherer() @@ -2080,17 +2124,77 @@ function install_munin_gatherer() node_list=$(get_conf_value fai_munin_node_list) fi + if [ -z "$node_list" ]; then + return + fi + + for el in $node_list; do + print "Adding ${el} to the Muning gatherer on ${HOSTNAME} ..." + file=/etc/munin/munin-conf.d/escenic.conf + cat >> $file </dev/null | \ + grep "bytes from" | \ + cut -d'(' -f2 | \ + cut -d ')' -f1) - # TODO has a hard dependency on install_web_server, should perhaps - # be called from this one? + if [ -z "$ip" ]; then + echo $1 + fi - add_next_step "Munin interface is available at http://$HOSTNAME:81/" + echo $ip +} + +# munin nodes need the IP of the munin gatherer to be escaped. Hence this function. +# $parameters: +# $1 : the IP +function get_perl_escaped() +{ + local escaped_input=$( + echo $1 | sed 's/\./\\./g' + ) + echo "^${escaped_input}$" } function install_munin_node() { print "Installing a Munin node on $HOSTNAME ..." + # the IP of the monitoring server + local default_ip=127.0.0.1 + if [ $fai_enabled -eq 1 ]; then + if [ -n "fai_monitoring_server_ip" ]; then + monitoring_server_ip=${fai_monitoring_server_ip} + fi + elif [ $install_profile_number -ne $PROFILE_MONITORING_SERVER ]; then + print "What is the IP of your monitoring server? If you don't know" + print "this, don't worry and just press ENTER" + echo -n "Your choice [${default_ip}]> " + read user_monitoring_server + + if [ -n "$user_monitoring_server" ]; then + monitoring_server_ip=$user_monitoring_server + fi + fi + + if [ -z "$monitoring_server_ip" ]; then + monitoring_server_ip=$default_ip + fi + if [ $on_debian_or_derivative -eq 1 ]; then packages="munin-node munin-plugins-extra munin-java-plugins" install_packages_if_missing $packages @@ -2100,6 +2204,17 @@ function install_munin_node() return fi + if [ -n "$monitoring_server_ip" ]; then + escaped_munin_gather_ip=$(get_perl_escaped ${monitoring_server_ip}) + file=/etc/munin/munin-node.conf + cat >> $file < Date: Thu, 20 Oct 2011 18:32:32 +0800 Subject: [PATCH 0109/2585] - updated guide to include the FAI options of the monitoring server --- usr/share/doc/escenic/ece-install-guide.org | 106 ++++++++++++-------- 1 file changed, 66 insertions(+), 40 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 1aad0602..d094fd63 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -194,7 +194,7 @@ this, the script will make a call to drop-and-create-ecedb in the same directory as the ece-intall script itself. The script will fail by itself if the DB already exists: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install] Setting up the ECE database schema ... ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists @@ -258,8 +258,8 @@ This installation profile will base the publication on the Widget Framework if its present on the system, if not, ECE's clean demo WAR is used as a basis. ** Profile - Monitoring Server -This will install a Munin gatherer and make its interface available -when accesssing the web server directly (port 81). +This will install a Munin gatherer and web server. The latter for +accessing the reports generated by the former. TBD: This profile will also install the Nagios interface for monitoring the different nodes. @@ -276,34 +276,36 @@ fai_editor_install The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user: -|---------------------------------+------------------+-----------------------------------------------| -| Parameter | Default | Description | -|---------------------------------+------------------+-----------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|---------------------------------+------------------+-----------------------------------------------| +|----------------------------------+------------------+-----------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+------------------+-----------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|----------------------------------+------------------+-----------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) @@ -473,15 +475,14 @@ what's available on the host where it's run): - Escenic software ** Running interactively -The ece script will put backups in -#+BEGIN_SRC sh -/var/backups/escenic +*** Start ece-install and choose the Option, "Restore from backup" +#+BEGIN_SRC text +# ece-install #+END_SRC -and the ece-install script will hence expect to find them here (when -running interactively), asking you from which one you wish to restore -the system: -#+BEGIN_SRC sh +*** Select Which Backup to Restore + +#+BEGIN_SRC text [ece-install-5] From which dataset do you wish to backup? 1 - engine-dev1-backup-2011-10-10.tar.gz 2 - engine-dev1-backup-2011-10-11.tar.gz @@ -489,6 +490,31 @@ the system: Your choice [1]> #+END_SRC +The ece script mentioned above will create backups in +#+BEGIN_SRC sh +/var/backups/escenic +#+END_SRC +and the ece-install script will hence expect to find them here. + +*** Choose What to Restore +#+BEGIN_SRC text +[ece-install-12] Which part of the system do you wish to restore? + 1 - The database + 2 - The Solr and ECE data files (multimedia archive) + 3 - The ECE configuration files + 4 - The Escenic and Tomcat software binaries + publication templates +Your choice [1]> 2 +#+END_SRC + +*** Sit Back and Watch ece-install Restore the Data for You +#+BEGIN_SRC text +[ece-install-13] Restoring the Solr & ECE data files on ubiquitous ... +[ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s +[ece-install] Successfully restored Solr & ECE data files +[ece-install] from backup: engine-dev1-backup-2011-10-12.tar.gz +[ece-install] Check /var/lib/escenic to verify they're all there +#+END_SRC + ** Running in FAI mode If you're running in FAI mode, you can choose between these settings to decide what to restore and where to find the backup file to @@ -507,7 +533,7 @@ restore from: For example, to restore everything possible from a given tarball, you need this in your .ece-install.conf: -#+BEGIN_SRC sh +#+BEGIN_SRC conf fai_enabled=1 fai_restore_from_backup=1 @@ -525,7 +551,7 @@ responsibility always lies with you as the user. If you're trying to restore the DB and the ECE schema already exists, the restore will fail: -#+BEGIN_SRC sh +#+BEGIN_SRC text [ece-install-8] Restoring the database contents on ubiquitous ... [ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists From 0b8e2a850ebb27aed006a473f6cd84626e8107e5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 20 Oct 2011 18:33:18 +0800 Subject: [PATCH 0110/2585] - updated HTML and TEXT versions of the guide --- usr/share/doc/escenic/ece-install-guide.html | 115 ++++++++++++--- usr/share/doc/escenic/ece-install-guide.txt | 145 +++++++++++++------ 2 files changed, 191 insertions(+), 69 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 06c41428..a87f6504 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -284,7 +284,14 @@

    Table of Contents

  • 12 Example Output FAI
  • 13 Restoring from backup @@ -686,8 +693,8 @@

    5.6 Profile - Database Se

    -
    [ece-install] Setting up the ECE database schema ...
    -ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    +
    [ece-install] Setting up the ECE database schema ...
    +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
     ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists
     [ece-install] running tables FAILED, exiting :-(
     
    @@ -801,8 +808,8 @@

    5.9 Profile - Create Publ

    5.10 Profile - Monitoring Server

    -

    This will install a Munin gatherer and make its interface available -when accesssing the web server directly (port 81). +

    This will install a Munin gatherer and web server. The latter for +accessing the reports generated by the former.

    TBD: This profile will also install the Nagios interface for @@ -858,6 +865,8 @@

    6.1 Overview of All FAI P fai_enabled0Whether or not to run ece-install in FAI mode fai_public_host_name${HOSTNAME}:8080The public address for your website fai_presentation_install0Install the presentation server profile +fai_monitoring_server_install0Install the monitoring server profile. +fai_monitoring_server_ip127.0.0.1The IP of the monitoring server. fai_presentation_nameweb1Name of the presentation server instance fai_presentation_port8080HTTP port of the presentation server instance fai_presentation_shutdown8005Shutdown port of the presentation instance @@ -1184,22 +1193,33 @@

    13 Restoring from backup <

    13.1 Running interactively

    -

    The ece script will put backups in -

    +
    -
    /var/backups/escenic
    +
    +

    13.1.1 Start ece-install and choose the Option, "Restore from backup"

    +
    + + + + +
    # ece-install
     
    -

    -and the ece-install script will hence expect to find them here (when -running interactively), asking you from which one you wish to restore -the system: -

    + +
    + +
    + +
    +

    13.1.2 Select Which Backup to Restore

    +
    -
    [ece-install-5] From which dataset do you wish to backup?
    +
    +
    +
    [ece-install-5] From which dataset do you wish to backup?
         1 - engine-dev1-backup-2011-10-10.tar.gz
         2 - engine-dev1-backup-2011-10-11.tar.gz
     [ece-install-5] Enter the number next to the tarball, from 1 to 2
    @@ -1207,6 +1227,57 @@ 

    13.1 Running interactive

    +

    +The ece script mentioned above will create backups in +

    + + +
    /var/backups/escenic
    +
    + +

    +and the ece-install script will hence expect to find them here. +

    +
    + +
    + +
    +

    13.1.3 Choose What to Restore

    +
    + + + + +
    [ece-install-12] Which part of the system do you wish to restore?
    +    1 - The database
    +    2 - The Solr and ECE data files (multimedia archive)
    +    3 - The ECE configuration files
    +    4 - The Escenic and Tomcat software binaries + publication templates
    +Your choice [1]> 2
    +
    + + +
    + +
    + +
    +

    13.1.4 Sit Back and Watch ece-install Restore the Data for You

    +
    + + + + +
    [ece-install-13] Restoring the Solr & ECE data files on ubiquitous ...
    +[ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s
    +[ece-install] Successfully restored Solr & ECE data files
    +[ece-install] from backup:  engine-dev1-backup-2011-10-12.tar.gz
    +[ece-install] Check /var/lib/escenic to verify they're all there
    +
    + + +
    @@ -1243,7 +1314,7 @@

    13.2 Running in FAI mode

    -
    fai_enabled=1
    +
    fai_enabled=1
     
     fai_restore_from_backup=1
     fai_restore_all=1
    @@ -1273,12 +1344,12 @@ 

    13.3 Data security

    -
    [ece-install-8] Restoring the database contents on ubiquitous ...
    +
    [ece-install-8] Restoring the database contents on ubiquitous ...
     [ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz
    -ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    -ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists
    -[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-(
    -[ece-install-24] See /var/log/ece-install.log for further details.
    +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    +ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists
    +[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-(
    +[ece-install-24] See /var/log/ece-install.log for further details.
     

  • @@ -1290,7 +1361,7 @@

    13.3 Data security


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-10-11 19:04:29 CST + Date: 2011-10-20 18:33:00 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index 06d7190e..2e955d8f 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-10-11 18:38:25 CST +Date: 2011-10-20 18:33:13 CST Table of Contents @@ -45,10 +45,13 @@ Table of Contents 11 Uninstalling Everything That the ece-install Set Up 12 Example Output FAI 13 Restoring from backup - 13.1 Location of the backup file - 13.1.1 Running interactively - 13.1.2 Running in FAI mode - 13.2 Data security + 13.1 Running interactively + 13.1.1 Start ece-install and choose the Option, "Restore from backup" + 13.1.2 Select Which Backup to Restore + 13.1.3 Choose What to Restore + 13.1.4 Sit Back and Watch ece-install Restore the Data for You + 13.2 Running in FAI mode + 13.3 Data security 1 Supported Operating Systems @@ -352,8 +355,8 @@ is used as a basis. 5.10 Profile - Monitoring Server ================================= -This will install a Munin gatherer and make its interface available -when accesssing the web server directly (port 81). +This will install a Munin gatherer and web server. The latter for +accessing the reports generated by the former. TBD: This profile will also install the Nagios interface for monitoring the different nodes. @@ -373,32 +376,34 @@ the parameters with "install" in their name, such as: The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user: - Parameter Default Description - ---------------------------------+------------------+----------------------------------------------- - fai\_all\_install 0 Install all components on your server. - fai\_cache\_install 0 Install cache server profile - fai\_cache\_backends ${HOSTNAME}:8080 Space separated, e.g. "app1:8080 app2:8080" - fai\_db\_install 0 Install db profile - fai\_db\_host $HOSTNAME Useful for editor & presentation profiles - fai\_db\_port 3306 Useful for editor & presentation profiles - fai\_editor\_install 0 Install the editorial profile - fai\_editor\_name editor1 Name of the editor instance - fai\_editor\_port 8080 HTTP port of the editor instance - fai\_editor\_shutdown 8005 Shutdown port of the editor instance - fai\_enabled 0 Whether or not to run ece-install in FAI mode - fai\_public\_host\_name ${HOSTNAME}:8080 The public address for your website - fai\_presentation\_install 0 Install the presentation server profile - fai\_presentation\_name web1 Name of the presentation server instance - fai\_presentation\_port 8080 HTTP port of the presentation server instance - fai\_presentation\_shutdown 8005 Shutdown port of the presentation instance - fai\_publication\_create 0 Create a new publication - fai\_publication\_name mypub Name of the publication - fai\_publication\_use\_instance dev1 Name of local instance to use for creation - fai\_rmi\_install 0 Install RMI hub profile - fai\_search\_name search1 Name of the search instance - fai\_search\_port 8080 HTTP port of the search instance - fai\_search\_shutdown 8005 Shutdown port of the search instance - fai\_wf\_install 0 Install Widget Framework profile + Parameter Default Description + ----------------------------------+------------------+----------------------------------------------- + fai\_all\_install 0 Install all components on your server. + fai\_cache\_install 0 Install cache server profile + fai\_cache\_backends ${HOSTNAME}:8080 Space separated, e.g. "app1:8080 app2:8080" + fai\_db\_install 0 Install db profile + fai\_db\_host $HOSTNAME Useful for editor & presentation profiles + fai\_db\_port 3306 Useful for editor & presentation profiles + fai\_editor\_install 0 Install the editorial profile + fai\_editor\_name editor1 Name of the editor instance + fai\_editor\_port 8080 HTTP port of the editor instance + fai\_editor\_shutdown 8005 Shutdown port of the editor instance + fai\_enabled 0 Whether or not to run ece-install in FAI mode + fai\_public\_host\_name ${HOSTNAME}:8080 The public address for your website + fai\_presentation\_install 0 Install the presentation server profile + fai\_monitoring\_server\_install 0 Install the monitoring server profile. + fai\_monitoring\_server\_ip 127.0.0.1 The IP of the monitoring server. + fai\_presentation\_name web1 Name of the presentation server instance + fai\_presentation\_port 8080 HTTP port of the presentation server instance + fai\_presentation\_shutdown 8005 Shutdown port of the presentation instance + fai\_publication\_create 0 Create a new publication + fai\_publication\_name mypub Name of the publication + fai\_publication\_use\_instance dev1 Name of local instance to use for creation + fai\_rmi\_install 0 Install RMI hub profile + fai\_search\_name search1 Name of the search instance + fai\_search\_port 8080 HTTP port of the search instance + fai\_search\_shutdown 8005 Shutdown port of the search instance + fai\_wf\_install 0 Install Widget Framework profile As you've probably have guessed, 0 means "false" and 1 means "true" :-) @@ -587,19 +592,18 @@ what's available on the host where it's run): - ECE, cache and web server configuration - Escenic software -13.1 Location of the backup file -================================= +13.1 Running interactively +=========================== -13.1.1 Running interactively -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ece script will put backups in +13.1.1 Start ece-install and choose the Option, "Restore from backup" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - /var/backups/escenic + # ece-install + -and the ece-install script will hence expect to find them here (when -running interactively), asking you from which one you wish to restore -the system: +13.1.2 Select Which Backup to Restore +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -610,16 +614,63 @@ the system: Your choice [1]> -13.1.2 Running in FAI mode -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you're running in FAI mode, you specify the file to restore from -with: +The ece script mentioned above will create backups in + + + /var/backups/escenic + +and the ece-install script will hence expect to find them here. + +13.1.3 Choose What to Restore +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + [ece-install-12] Which part of the system do you wish to restore? + 1 - The database + 2 - The Solr and ECE data files (multimedia archive) + 3 - The ECE configuration files + 4 - The Escenic and Tomcat software binaries + publication templates + Your choice [1]> 2 + +13.1.4 Sit Back and Watch ece-install Restore the Data for You +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + [ece-install-13] Restoring the Solr & ECE data files on ubiquitous ... + [ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s + [ece-install] Successfully restored Solr & ECE data files + [ece-install] from backup: engine-dev1-backup-2011-10-12.tar.gz + [ece-install] Check /var/lib/escenic to verify they're all there + + +13.2 Running in FAI mode +========================= +If you're running in FAI mode, you can choose between these settings +to decide what to restore and where to find the backup file to +restore from: + + Parameter Default Description + ----------------------------------+---------+---------------------------------------------------- + fai\_restore\_all 0 Restore everything found in the backup file + fai\_restore\_db 0 Install the DB server & restore its contents + fai\_restore\_data\_files 0 Restore the Solr & ECE data files + fai\_restore\_configuration 0 Restore the Solr & ECE configuration files + fai\_restore\_software\_binaries 0 Restore the Escenic and Apache Tomcat software + fai\_restore\_from\_file "" The .tar.gz produced by "ece -i backup" + +For example, to restore everything possible from a given tarball, you +need this in your .ece-install.conf: + + + fai_enabled=1 + + fai_restore_from_backup=1 + fai_restore_all=1 fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz -13.2 Data security +13.3 Data security =================== You should take heed when running restore, so that you're not restoring a system which you didn't want to change (yes, this mishap From 45e15abdc8c24b41ded4c50365c07349af312b00 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 20 Oct 2011 18:37:29 +0800 Subject: [PATCH 0111/2585] - moved restore profile description to where the other profile descriptions are - updated the start up screen for ece-install - updated generate .txt and .html version --- usr/share/doc/escenic/ece-install-guide.html | 459 +++++++++---------- usr/share/doc/escenic/ece-install-guide.org | 230 +++++----- usr/share/doc/escenic/ece-install-guide.txt | 276 ++++++----- 3 files changed, 475 insertions(+), 490 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index a87f6504..feb244ab 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -258,6 +258,17 @@

    Table of Contents

  • 5.8 Profile - Install Widget Framework
  • 5.9 Profile - Create Publication
  • 5.10 Profile - Monitoring Server
  • +
  • 5.11 Profile - Restoring from backup
  • +
  • 5.12 Running interactively + +
  • +
  • 5.13 Running in FAI mode
  • +
  • 5.14 Data security
  • 6 Full Automatic Install (FAI) @@ -282,20 +293,6 @@

    Table of Contents

  • 11 Uninstalling Everything That the ece-install Set Up
  • 12 Example Output FAI
  • -
  • 13 Restoring from backup - -
  • @@ -495,24 +492,20 @@

    5 Available Server Profiles
    Hi, which server profile do you wish to install?
     
    -Select 1-9 and press ENTER
    -
    -  1 - All in one, the full stack on one host. 
    -      Suitable for development and test environments.
    -      It will install caching server, ECE, assembly host, database &
    -      Widget Framework as well as set up a new publication.
    -  2 - Editorial (publication) server.
    -      This will install ECE and an assembly host.
    -  3 - Presentation server (ECE, memcached).
    -  4 - Database server.
    -  5 - Cache server (cache & web server).
    -  6 - RMI hub.
    -  7 - Standalone search instance (solr + indexer-webapp).
    -  8 - Install the Widget Framework.
    -      (You need user/pass for repo.escenic.com for this).
    -  9 - Create a new publication.
    -      This profile will create new publication based on WF
    -      (if available) or ECE/clean-demo.
    +   1 - All in one, full stack on one host, suitable for dev & test environments
    +   2 - Editorial (publication) server
    +   3 - Presentation server (ECE + memcached).
    +   4 - Database server
    +   5 - Cache server (cache and web server)
    +   6 - RMI hub
    +   7 - Search server (Solr + indexer-webapp)
    +   8 - Install Widget Framework.
    +   9 - Create a new publication based on WF if available, ECE/clean-demo if not
    +   10 - A monitoring server (web server + Munin gatherer)
    +   11 - Restore from backup (DB, data files, binaries, conf & publications)
    +
    +Select 1-11 and press ENTER
    +Your choice [1]> 
     
    @@ -815,6 +808,205 @@

    5.10 Profile - Monitorin TBD: This profile will also install the Nagios interface for monitoring the different nodes.

    +

    + +
    + +
    +

    5.11 Profile - Restoring from backup

    +
    + +

    ece-install can restore from a backup made by the ece script: +

    + + +
    $ ece -i <instance> backup
    +
    + + +

    +As stated in the ece guide, this backup may contain (depending on +what's available on the host where it's run): +

      +
    • database dump of the instance +
    • +
    • data files (pictures, video files and so on). This is often referred + to as the "multimedia archive" in Escenic literature. +
    • +
    • ECE, cache and web server configuration +
    • +
    • Escenic software +
    • +
    + + +
    + +
    + +
    +

    5.12 Running interactively

    +
    + + +
    + +
    +

    5.12.1 Start ece-install and choose the Option, "Restore from backup"

    +
    + + + + +
    # ece-install
    +
    + + +
    + +
    + +
    +

    5.12.2 Select Which Backup to Restore

    +
    + + + + + +
    [ece-install-5] From which dataset do you wish to backup?
    +    1 - engine-dev1-backup-2011-10-10.tar.gz
    +    2 - engine-dev1-backup-2011-10-11.tar.gz
    +[ece-install-5] Enter the number next to the tarball, from 1 to 2
    +Your choice [1]>
    +
    + + +

    +The ece script mentioned above will create backups in +

    + + +
    /var/backups/escenic
    +
    + +

    +and the ece-install script will hence expect to find them here. +

    +
    + +
    + +
    +

    5.12.3 Choose What to Restore

    +
    + + + + +
    [ece-install-12] Which part of the system do you wish to restore?
    +    1 - The database
    +    2 - The Solr and ECE data files (multimedia archive)
    +    3 - The ECE configuration files
    +    4 - The Escenic and Tomcat software binaries + publication templates
    +Your choice [1]> 2
    +
    + + +
    + +
    + +
    +

    5.12.4 Sit Back and Watch ece-install Restore the Data for You

    +
    + + + + +
    [ece-install-13] Restoring the Solr & ECE data files on ubiquitous ...
    +[ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s
    +[ece-install] Successfully restored Solr & ECE data files
    +[ece-install] from backup:  engine-dev1-backup-2011-10-12.tar.gz
    +[ece-install] Check /var/lib/escenic to verify they're all there
    +
    + + +
    +
    + +
    + +
    +

    5.13 Running in FAI mode

    +
    + +

    If you're running in FAI mode, you can choose between these settings +to decide what to restore and where to find the backup file to +restore from: +

    + + ++ + + + + + + + + + + + +
    ParameterDefaultDescription
    fai_restore_all0Restore everything found in the backup file
    fai_restore_db0Install the DB server & restore its contents
    fai_restore_data_files0Restore the Solr & ECE data files
    fai_restore_configuration0Restore the Solr & ECE configuration files
    fai_restore_software_binaries0Restore the Escenic and Apache Tomcat software
    fai_restore_from_file""The .tar.gz produced by "ece -i <instance> backup"
    + + +

    +For example, to restore everything possible from a given tarball, you +need this in your .ece-install.conf: +

    + + +
    fai_enabled=1
    +
    +fai_restore_from_backup=1
    +fai_restore_all=1
    +fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz
    +
    + + +
    + +
    + +
    +

    5.14 Data security

    +
    + +

    You should take heed when running restore, so that you're not +restoring a system which you didn't want to change (yes, this mishap +does happen). +

    +

    +The ece-install script will help you a bit on the way, but the final +responsibility always lies with you as the user. +

    +

    +If you're trying to restore the DB and the ECE schema already exists, +the restore will fail: +

    + + +
    [ece-install-8] Restoring the database contents on ubiquitous ...
    +[ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz
    +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    +ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists
    +[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-(
    +[ece-install-24] See /var/log/ece-install.log for further details.
    +
    +
    @@ -1155,205 +1347,6 @@

    12 Example Output FAI

    - - - -
    -

    13 Restoring from backup

    -
    - -

    ece-install can restore from a backup made by the ece script: -

    - - -
    $ ece -i <instance> backup
    -
    - - -

    -As stated in the ece guide, this backup may contain (depending on -what's available on the host where it's run): -

      -
    • database dump of the instance -
    • -
    • data files (pictures, video files and so on). This is often referred - to as the "multimedia archive" in Escenic literature. -
    • -
    • ECE, cache and web server configuration -
    • -
    • Escenic software -
    • -
    - - - -
    - -
    -

    13.1 Running interactively

    -
    - - -
    - -
    -

    13.1.1 Start ece-install and choose the Option, "Restore from backup"

    -
    - - - - -
    # ece-install
    -
    - - -
    - -
    - -
    -

    13.1.2 Select Which Backup to Restore

    -
    - - - - - -
    [ece-install-5] From which dataset do you wish to backup?
    -    1 - engine-dev1-backup-2011-10-10.tar.gz
    -    2 - engine-dev1-backup-2011-10-11.tar.gz
    -[ece-install-5] Enter the number next to the tarball, from 1 to 2
    -Your choice [1]>
    -
    - - -

    -The ece script mentioned above will create backups in -

    - - -
    /var/backups/escenic
    -
    - -

    -and the ece-install script will hence expect to find them here. -

    -
    - -
    - -
    -

    13.1.3 Choose What to Restore

    -
    - - - - -
    [ece-install-12] Which part of the system do you wish to restore?
    -    1 - The database
    -    2 - The Solr and ECE data files (multimedia archive)
    -    3 - The ECE configuration files
    -    4 - The Escenic and Tomcat software binaries + publication templates
    -Your choice [1]> 2
    -
    - - -
    - -
    - -
    -

    13.1.4 Sit Back and Watch ece-install Restore the Data for You

    -
    - - - - -
    [ece-install-13] Restoring the Solr & ECE data files on ubiquitous ...
    -[ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s
    -[ece-install] Successfully restored Solr & ECE data files
    -[ece-install] from backup:  engine-dev1-backup-2011-10-12.tar.gz
    -[ece-install] Check /var/lib/escenic to verify they're all there
    -
    - - -
    -
    - -
    - -
    -

    13.2 Running in FAI mode

    -
    - -

    If you're running in FAI mode, you can choose between these settings -to decide what to restore and where to find the backup file to -restore from: -

    - - -- - - - - - - - - - - - -
    ParameterDefaultDescription
    fai_restore_all0Restore everything found in the backup file
    fai_restore_db0Install the DB server & restore its contents
    fai_restore_data_files0Restore the Solr & ECE data files
    fai_restore_configuration0Restore the Solr & ECE configuration files
    fai_restore_software_binaries0Restore the Escenic and Apache Tomcat software
    fai_restore_from_file""The .tar.gz produced by "ece -i <instance> backup"
    - - -

    -For example, to restore everything possible from a given tarball, you -need this in your .ece-install.conf: -

    - - -
    fai_enabled=1
    -
    -fai_restore_from_backup=1
    -fai_restore_all=1
    -fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz
    -
    - - -
    - -
    - -
    -

    13.3 Data security

    -
    - -

    You should take heed when running restore, so that you're not -restoring a system which you didn't want to change (yes, this mishap -does happen). -

    -

    -The ece-install script will help you a bit on the way, but the final -responsibility always lies with you as the user. -

    -

    -If you're trying to restore the DB and the ECE schema already exists, -the restore will fail: -

    - - -
    [ece-install-8] Restoring the database contents on ubiquitous ...
    -[ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz
    -ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    -ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists
    -[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-(
    -[ece-install-24] See /var/log/ece-install.log for further details.
    -
    - -
    -
    @@ -1361,7 +1354,7 @@

    13.3 Data security


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-10-20 18:33:00 CST + Date: 2011-10-20 18:36:20 CST

    diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index d094fd63..b4d5b071 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -107,24 +107,20 @@ want to install: #+BEGIN_SRC sh Hi, which server profile do you wish to install? -Select 1-9 and press ENTER - - 1 - All in one, the full stack on one host. - Suitable for development and test environments. - It will install caching server, ECE, assembly host, database & - Widget Framework as well as set up a new publication. - 2 - Editorial (publication) server. - This will install ECE and an assembly host. - 3 - Presentation server (ECE, memcached). - 4 - Database server. - 5 - Cache server (cache & web server). - 6 - RMI hub. - 7 - Standalone search instance (solr + indexer-webapp). - 8 - Install the Widget Framework. - (You need user/pass for repo.escenic.com for this). - 9 - Create a new publication. - This profile will create new publication based on WF - (if available) or ECE/clean-demo. + 1 - All in one, full stack on one host, suitable for dev & test environments + 2 - Editorial (publication) server + 3 - Presentation server (ECE + memcached). + 4 - Database server + 5 - Cache server (cache and web server) + 6 - RMI hub + 7 - Search server (Solr + indexer-webapp) + 8 - Install Widget Framework. + 9 - Create a new publication based on WF if available, ECE/clean-demo if not + 10 - A monitoring server (web server + Munin gatherer) + 11 - Restore from backup (DB, data files, binaries, conf & publications) + +Select 1-11 and press ENTER +Your choice [1]> #+END_SRC ** Common for All Server Profiles The script will install a Munin node on the host. It will also @@ -264,6 +260,105 @@ accessing the reports generated by the former. TBD: This profile will also install the Nagios interface for monitoring the different nodes. +** Profile - Restoring from backup +ece-install can restore from a backup made by the ece script: +#+BEGIN_SRC sh +$ ece -i backup +#+END_SRC + +As stated in the ece guide, this backup may contain (depending on +what's available on the host where it's run): +- database dump of the instance +- data files (pictures, video files and so on). This is often referred + to as the "multimedia archive" in Escenic literature. +- ECE, cache and web server configuration +- Escenic software + +** Running interactively +*** Start ece-install and choose the Option, "Restore from backup" +#+BEGIN_SRC text +# ece-install +#+END_SRC + +*** Select Which Backup to Restore + +#+BEGIN_SRC text +[ece-install-5] From which dataset do you wish to backup? + 1 - engine-dev1-backup-2011-10-10.tar.gz + 2 - engine-dev1-backup-2011-10-11.tar.gz +[ece-install-5] Enter the number next to the tarball, from 1 to 2 +Your choice [1]> +#+END_SRC + +The ece script mentioned above will create backups in +#+BEGIN_SRC sh +/var/backups/escenic +#+END_SRC +and the ece-install script will hence expect to find them here. + +*** Choose What to Restore +#+BEGIN_SRC text +[ece-install-12] Which part of the system do you wish to restore? + 1 - The database + 2 - The Solr and ECE data files (multimedia archive) + 3 - The ECE configuration files + 4 - The Escenic and Tomcat software binaries + publication templates +Your choice [1]> 2 +#+END_SRC + +*** Sit Back and Watch ece-install Restore the Data for You +#+BEGIN_SRC text +[ece-install-13] Restoring the Solr & ECE data files on ubiquitous ... +[ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s +[ece-install] Successfully restored Solr & ECE data files +[ece-install] from backup: engine-dev1-backup-2011-10-12.tar.gz +[ece-install] Check /var/lib/escenic to verify they're all there +#+END_SRC + +** Running in FAI mode +If you're running in FAI mode, you can choose between these settings +to decide what to restore and where to find the backup file to +restore from: + +|----------------------------------+---------+------------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+---------+------------------------------------------------| +| fai\_restore\_all | 0 | Restore everything found in the backup file | +| fai\_restore\_db | 0 | Install the DB server & restore its contents | +| fai\_restore\_data\_files | 0 | Restore the Solr & ECE data files | +| fai\_restore\_configuration | 0 | Restore the Solr & ECE configuration files | +| fai\_restore\_software\_binaries | 0 | Restore the Escenic and Apache Tomcat software | +| fai\_restore\_from\_file | "" | The .tar.gz produced by "ece -i backup" | +|----------------------------------+---------+------------------------------------------------| + +For example, to restore everything possible from a given tarball, you +need this in your .ece-install.conf: +#+BEGIN_SRC conf +fai_enabled=1 + +fai_restore_from_backup=1 +fai_restore_all=1 +fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz +#+END_SRC + +** Data security +You should take heed when running restore, so that you're not +restoring a system which you didn't want to change (yes, this mishap +does happen). + +The ece-install script will help you a bit on the way, but the final +responsibility always lies with you as the user. + +If you're trying to restore the DB and the ECE schema already exists, +the restore will fail: +#+BEGIN_SRC text +[ece-install-8] Restoring the database contents on ubiquitous ... +[ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists +ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists +[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-( +[ece-install-24] See /var/log/ece-install.log for further details. +#+END_SRC * Full Automatic Install (FAI) The ece-install script has support for doing a full automatic install (FAI). You can only install one profile at a time. The profiles are @@ -460,102 +555,3 @@ the host that runs the install script. The following output is produced from ece-install: #+INCLUDE: "~/tmp/ece-install.out" src text -* Restoring from backup -ece-install can restore from a backup made by the ece script: -#+BEGIN_SRC sh -$ ece -i backup -#+END_SRC - -As stated in the ece guide, this backup may contain (depending on -what's available on the host where it's run): -- database dump of the instance -- data files (pictures, video files and so on). This is often referred - to as the "multimedia archive" in Escenic literature. -- ECE, cache and web server configuration -- Escenic software - -** Running interactively -*** Start ece-install and choose the Option, "Restore from backup" -#+BEGIN_SRC text -# ece-install -#+END_SRC - -*** Select Which Backup to Restore - -#+BEGIN_SRC text -[ece-install-5] From which dataset do you wish to backup? - 1 - engine-dev1-backup-2011-10-10.tar.gz - 2 - engine-dev1-backup-2011-10-11.tar.gz -[ece-install-5] Enter the number next to the tarball, from 1 to 2 -Your choice [1]> -#+END_SRC - -The ece script mentioned above will create backups in -#+BEGIN_SRC sh -/var/backups/escenic -#+END_SRC -and the ece-install script will hence expect to find them here. - -*** Choose What to Restore -#+BEGIN_SRC text -[ece-install-12] Which part of the system do you wish to restore? - 1 - The database - 2 - The Solr and ECE data files (multimedia archive) - 3 - The ECE configuration files - 4 - The Escenic and Tomcat software binaries + publication templates -Your choice [1]> 2 -#+END_SRC - -*** Sit Back and Watch ece-install Restore the Data for You -#+BEGIN_SRC text -[ece-install-13] Restoring the Solr & ECE data files on ubiquitous ... -[ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s -[ece-install] Successfully restored Solr & ECE data files -[ece-install] from backup: engine-dev1-backup-2011-10-12.tar.gz -[ece-install] Check /var/lib/escenic to verify they're all there -#+END_SRC - -** Running in FAI mode -If you're running in FAI mode, you can choose between these settings -to decide what to restore and where to find the backup file to -restore from: - -|----------------------------------+---------+------------------------------------------------| -| Parameter | Default | Description | -|----------------------------------+---------+------------------------------------------------| -| fai\_restore\_all | 0 | Restore everything found in the backup file | -| fai\_restore\_db | 0 | Install the DB server & restore its contents | -| fai\_restore\_data\_files | 0 | Restore the Solr & ECE data files | -| fai\_restore\_configuration | 0 | Restore the Solr & ECE configuration files | -| fai\_restore\_software\_binaries | 0 | Restore the Escenic and Apache Tomcat software | -| fai\_restore\_from\_file | "" | The .tar.gz produced by "ece -i backup" | -|----------------------------------+---------+------------------------------------------------| - -For example, to restore everything possible from a given tarball, you -need this in your .ece-install.conf: -#+BEGIN_SRC conf -fai_enabled=1 - -fai_restore_from_backup=1 -fai_restore_all=1 -fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz -#+END_SRC - -** Data security -You should take heed when running restore, so that you're not -restoring a system which you didn't want to change (yes, this mishap -does happen). - -The ece-install script will help you a bit on the way, but the final -responsibility always lies with you as the user. - -If you're trying to restore the DB and the ECE schema already exists, -the restore will fail: -#+BEGIN_SRC text -[ece-install-8] Restoring the database contents on ubiquitous ... -[ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz -ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists -ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists -[ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-( -[ece-install-24] See /var/log/ece-install.log for further details. -#+END_SRC diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt index 2e955d8f..e1078ee3 100644 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ b/usr/share/doc/escenic/ece-install-guide.txt @@ -2,7 +2,7 @@ ========================================== Author: Torstein Krause Johansen -Date: 2011-10-20 18:33:13 CST +Date: 2011-10-20 18:37:17 CST Table of Contents @@ -31,6 +31,14 @@ Table of Contents 5.8 Profile - Install Widget Framework 5.9 Profile - Create Publication 5.10 Profile - Monitoring Server + 5.11 Profile - Restoring from backup + 5.12 Running interactively + 5.12.1 Start ece-install and choose the Option, "Restore from backup" + 5.12.2 Select Which Backup to Restore + 5.12.3 Choose What to Restore + 5.12.4 Sit Back and Watch ece-install Restore the Data for You + 5.13 Running in FAI mode + 5.14 Data security 6 Full Automatic Install (FAI) 6.1 Overview of All FAI Parameters 6.2 Examples @@ -44,14 +52,6 @@ Table of Contents 10.1 /etc/esecenic is shared 11 Uninstalling Everything That the ece-install Set Up 12 Example Output FAI -13 Restoring from backup - 13.1 Running interactively - 13.1.1 Start ece-install and choose the Option, "Restore from backup" - 13.1.2 Select Which Backup to Restore - 13.1.3 Choose What to Restore - 13.1.4 Sit Back and Watch ece-install Restore the Data for You - 13.2 Running in FAI mode - 13.3 Data security 1 Supported Operating Systems @@ -179,24 +179,20 @@ want to install: Hi, which server profile do you wish to install? - Select 1-9 and press ENTER + 1 - All in one, full stack on one host, suitable for dev & test environments + 2 - Editorial (publication) server + 3 - Presentation server (ECE + memcached). + 4 - Database server + 5 - Cache server (cache and web server) + 6 - RMI hub + 7 - Search server (Solr + indexer-webapp) + 8 - Install Widget Framework. + 9 - Create a new publication based on WF if available, ECE/clean-demo if not + 10 - A monitoring server (web server + Munin gatherer) + 11 - Restore from backup (DB, data files, binaries, conf & publications) - 1 - All in one, the full stack on one host. - Suitable for development and test environments. - It will install caching server, ECE, assembly host, database & - Widget Framework as well as set up a new publication. - 2 - Editorial (publication) server. - This will install ECE and an assembly host. - 3 - Presentation server (ECE, memcached). - 4 - Database server. - 5 - Cache server (cache & web server). - 6 - RMI hub. - 7 - Standalone search instance (solr + indexer-webapp). - 8 - Install the Widget Framework. - (You need user/pass for repo.escenic.com for this). - 9 - Create a new publication. - This profile will create new publication based on WF - (if available) or ECE/clean-demo. + Select 1-11 and press ENTER + Your choice [1]> 5.1 Common for All Server Profiles =================================== @@ -361,6 +357,120 @@ accessing the reports generated by the former. TBD: This profile will also install the Nagios interface for monitoring the different nodes. +5.11 Profile - Restoring from backup +===================================== +ece-install can restore from a backup made by the ece script: + + + $ ece -i backup + + +As stated in the ece guide, this backup may contain (depending on +what's available on the host where it's run): +- database dump of the instance +- data files (pictures, video files and so on). This is often referred + to as the "multimedia archive" in Escenic literature. +- ECE, cache and web server configuration +- Escenic software + +5.12 Running interactively +=========================== + +5.12.1 Start ece-install and choose the Option, "Restore from backup" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + # ece-install + + +5.12.2 Select Which Backup to Restore +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + + [ece-install-5] From which dataset do you wish to backup? + 1 - engine-dev1-backup-2011-10-10.tar.gz + 2 - engine-dev1-backup-2011-10-11.tar.gz + [ece-install-5] Enter the number next to the tarball, from 1 to 2 + Your choice [1]> + + +The ece script mentioned above will create backups in + + + /var/backups/escenic + +and the ece-install script will hence expect to find them here. + +5.12.3 Choose What to Restore +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + [ece-install-12] Which part of the system do you wish to restore? + 1 - The database + 2 - The Solr and ECE data files (multimedia archive) + 3 - The ECE configuration files + 4 - The Escenic and Tomcat software binaries + publication templates + Your choice [1]> 2 + + +5.12.4 Sit Back and Watch ece-install Restore the Data for You +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + [ece-install-13] Restoring the Solr & ECE data files on ubiquitous ... + [ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s + [ece-install] Successfully restored Solr & ECE data files + [ece-install] from backup: engine-dev1-backup-2011-10-12.tar.gz + [ece-install] Check /var/lib/escenic to verify they're all there + + +5.13 Running in FAI mode +========================= +If you're running in FAI mode, you can choose between these settings +to decide what to restore and where to find the backup file to +restore from: + + Parameter Default Description + ----------------------------------+---------+---------------------------------------------------- + fai\_restore\_all 0 Restore everything found in the backup file + fai\_restore\_db 0 Install the DB server & restore its contents + fai\_restore\_data\_files 0 Restore the Solr & ECE data files + fai\_restore\_configuration 0 Restore the Solr & ECE configuration files + fai\_restore\_software\_binaries 0 Restore the Escenic and Apache Tomcat software + fai\_restore\_from\_file "" The .tar.gz produced by "ece -i backup" + +For example, to restore everything possible from a given tarball, you +need this in your .ece-install.conf: + + + fai_enabled=1 + + fai_restore_from_backup=1 + fai_restore_all=1 + fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz + + +5.14 Data security +=================== +You should take heed when running restore, so that you're not +restoring a system which you didn't want to change (yes, this mishap +does happen). + +The ece-install script will help you a bit on the way, but the final +responsibility always lies with you as the user. + +If you're trying to restore the DB and the ECE schema already exists, +the restore will fail: + + + [ece-install-8] Restoring the database contents on ubiquitous ... + [ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz + ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists + ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists + [ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-( + [ece-install-24] See /var/log/ece-install.log for further details. + 6 Full Automatic Install (FAI) ------------------------------- The ece-install script has support for doing a full automatic install @@ -576,117 +686,3 @@ the host that runs the install script. The following output is produced from ece-install: CANNOT INCLUDE FILE ~/tmp/ece-install.out -13 Restoring from backup -------------------------- -ece-install can restore from a backup made by the ece script: - - - $ ece -i backup - - -As stated in the ece guide, this backup may contain (depending on -what's available on the host where it's run): -- database dump of the instance -- data files (pictures, video files and so on). This is often referred - to as the "multimedia archive" in Escenic literature. -- ECE, cache and web server configuration -- Escenic software - -13.1 Running interactively -=========================== - -13.1.1 Start ece-install and choose the Option, "Restore from backup" -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - # ece-install - - -13.1.2 Select Which Backup to Restore -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - - [ece-install-5] From which dataset do you wish to backup? - 1 - engine-dev1-backup-2011-10-10.tar.gz - 2 - engine-dev1-backup-2011-10-11.tar.gz - [ece-install-5] Enter the number next to the tarball, from 1 to 2 - Your choice [1]> - - -The ece script mentioned above will create backups in - - - /var/backups/escenic - -and the ece-install script will hence expect to find them here. - -13.1.3 Choose What to Restore -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - [ece-install-12] Which part of the system do you wish to restore? - 1 - The database - 2 - The Solr and ECE data files (multimedia archive) - 3 - The ECE configuration files - 4 - The Escenic and Tomcat software binaries + publication templates - Your choice [1]> 2 - - -13.1.4 Sit Back and Watch ece-install Restore the Data for You -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - [ece-install-13] Restoring the Solr & ECE data files on ubiquitous ... - [ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s - [ece-install] Successfully restored Solr & ECE data files - [ece-install] from backup: engine-dev1-backup-2011-10-12.tar.gz - [ece-install] Check /var/lib/escenic to verify they're all there - - -13.2 Running in FAI mode -========================= -If you're running in FAI mode, you can choose between these settings -to decide what to restore and where to find the backup file to -restore from: - - Parameter Default Description - ----------------------------------+---------+---------------------------------------------------- - fai\_restore\_all 0 Restore everything found in the backup file - fai\_restore\_db 0 Install the DB server & restore its contents - fai\_restore\_data\_files 0 Restore the Solr & ECE data files - fai\_restore\_configuration 0 Restore the Solr & ECE configuration files - fai\_restore\_software\_binaries 0 Restore the Escenic and Apache Tomcat software - fai\_restore\_from\_file "" The .tar.gz produced by "ece -i backup" - -For example, to restore everything possible from a given tarball, you -need this in your .ece-install.conf: - - - fai_enabled=1 - - fai_restore_from_backup=1 - fai_restore_all=1 - fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz - - -13.3 Data security -=================== -You should take heed when running restore, so that you're not -restoring a system which you didn't want to change (yes, this mishap -does happen). - -The ece-install script will help you a bit on the way, but the final -responsibility always lies with you as the user. - -If you're trying to restore the DB and the ECE schema already exists, -the restore will fail: - - - [ece-install-8] Restoring the database contents on ubiquitous ... - [ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz - ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists - ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists - [ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-( - [ece-install-24] See /var/log/ece-install.log for further details. - From 7f085aba448e023dd4fcda24f2b4473d430d01b2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 25 Oct 2011 15:43:06 +0800 Subject: [PATCH 0112/2585] - small change to the log message so that failed "ece assemble"-s are not confusing. --- usr/bin/ece | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 5f33d14a..8ad8e684 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -903,9 +903,9 @@ function assemble() exit 1 fi - message="Assembling your EAR file ..." - print $message - log $message >> $log_file + message="Assembling your EAR file" + print $message ".." + log $message "..." >> $log_file cd $assemblytool_home && \ ant -q ear -DskipRedundancyCheck=true \ From 5db9b7a3df7d1ba0ebb18930ae7ca6df5a693017 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 26 Oct 2011 12:53:53 +0800 Subject: [PATCH 0113/2585] - if help is one of the commands, the instance test is omitted. This means you can do "ece help" or indeed "ece help" without specifying the instance. --- usr/bin/ece | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index 8ad8e684..d1c51004 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -426,7 +426,9 @@ function sanity_check() # checks if the user is using the default instance when he/she # really wants to start one of the instances installed on the # system. - if [ $instance = "default" -a $type = "engine" ]; then + if [ $instance = "default" -a \ + $type = "engine" -a \ + $(echo $command | grep help | wc -l) -eq 0 ]; then instance_list=$(get_instance_list) if [ -n "$instance_list" ]; then print "You must specify the instance with -i " From 206bb9553b77857530099be23cc38ccc04eca637 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 26 Oct 2011 13:02:17 +0800 Subject: [PATCH 0114/2585] - removed hyphen which produced invalid email link on github. - referred to ece-guide instead of listing the script's features in the README - added referrals to ece-install and system-info too --- README | 68 +++++++++++++++++----------------------------------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/README b/README index d5ca69e6..ba9c8455 100644 --- a/README +++ b/README @@ -13,60 +13,32 @@ The file structure ressembles where you'd typically put the various scripts and configuration files. That's all it is, only for documentational purposes :-) -========================== -Features of the ece script -========================== -$ ece help -Usage: /usr/bin/ece [-t ] [-i ] - -DESCRIPTION - -t --type - The following types are available: - engine - The Escenic Content Engine, this is the default - and is the assumed type if none is specified. - search - A standalone search indexer and solr instance - rmi-hub - The RMI hub responsible for the internal - communication between the ECE instances. - - -i --instance - The type instance, such as editor1, web1 or search1 - - -p --publication - Needed only for updating publication resources - - -r --resource - Needed only for updating publication resources. - Must be one of: content-type, feature, layout, layout-group - image-version, menu - -v --verbose - Prints out debug statements, useful for debugging. - -The following commands are available: - applog the type's app server log - assemble runs the Assembly Tool *) - clean removes temporary files created by /home/torstein/bin/ece *) - deploy deploys the assembled EAR *) - help prints this help screen - kill uses force to stop the type - log the type's Log4J log - outlog the [ece#engine] script log (system out log) - restart restarts the type - start starts the type - status checks if the type is running - stop stops the type - threaddump write a thread dump to standard out (system out log) - update update publication resources - versions lists the ECE component versions - -*) only applicable if type is 'engine' +=================================== +Features of the /usr/bin/ece script +=================================== +https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/ece-guide.org + +============================================ +Features of the /usr/sbin/ece-install script +============================================ +https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/ece-install-guide.org + +=========================================== +Features of the /usr/bin/system-info script +=========================================== +Gives a overview of your system, especially useful when contacting +Vizrt Online/Escenic Support. + +The script default wise produces a plain text report, but if you pass +-c or --confluence to it, it will generate Confluence Wiki markup +instead. ========================== Feedback ========================== - Hope you enjoy the ece /usr/bin and intit.d scripts. All feedback, come hither! --torstein@escenic.com +torstein@escenic.com From 9abaea1728251f054a5b8f6a71d96163baf355ed Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 26 Oct 2011 13:18:49 +0800 Subject: [PATCH 0115/2585] - added a guide for the system-info, it's so dead easy! --- usr/share/doc/escenic/system-info-guide.org | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 usr/share/doc/escenic/system-info-guide.org diff --git a/usr/share/doc/escenic/system-info-guide.org b/usr/share/doc/escenic/system-info-guide.org new file mode 100644 index 00000000..9119a40d --- /dev/null +++ b/usr/share/doc/escenic/system-info-guide.org @@ -0,0 +1,78 @@ +Welcome to the /usr/bin/system-info Guide + +This wee script gives a overview of your system, especially useful +when contacting Vizrt Online/Escenic Support. + +The script will default wise produce a plain text report, but if you +pass -c or --confluence to it, it will generate Confluence Wiki markup +instead. + +* Example +Here you can see an example of the output generated from system-info: + +#+BEGIN_SRC text +######################################################################### +System details for quanah +######################################################################### + +######################################################################### +Kernel version +######################################################################### +Linux quanah 3.0.0-1-amd64 #1 SMP Sat Aug 27 16:21:11 UTC 2011 x86_64 GNU/Linux + +######################################################################### +Distribution information +######################################################################### +Debian or derivate, version wheezy/sid + +######################################################################### +Database details +######################################################################### +/usr/sbin/mysqld Ver 5.1.58-1 for debian-linux-gnu on x86_64 ((Debian)) + +######################################################################### +Java version +######################################################################### +java version "1.6.0_26" +Java(TM) SE Runtime Environment (build 1.6.0_26-b03) +Java HotSpot(TM) 64-Bit Server VM (build 20.1-b02, mixed mode) + +######################################################################### +Important packages +######################################################################### +ant 1.8.2-4 Java based build tool like make +libapr1 1.4.5-1 Apache Portable Runtime Library +libmysql-java 5.1.16-2 Java database (JDBC) driver for MySQL +libtcnative-1 1.1.22-1 Tomcat native library using the apache portable runtime +maven2 2.2.1-10 Java software project management and comprehension tool +mysql-server 5.1.58-1 MySQL database server (metapackage depending on the latest version) +nginx 1.1.4-2 small, but very powerful and efficient web server and mail proxy +sun-java6-jdk 6.26-3 Sun Java(TM) Development Kit (JDK) 6 +sun-java6-jre 6.26-3 Sun Java(TM) Runtime Environment (JRE) 6 (architecture independent files) +tomcat6-user 6.0.32-6 Servlet and JSP engine -- tools to create user instances +varnish 3.0.1-1~squeeze1 state of the art, high-performance web accelerator + +######################################################################### +Processors +######################################################################### +Processor type: Intel(R) Core(TM)2 Duo CPU P8600 @ 2.40GHz +Number of processors/cores: 1 + +######################################################################### +Memory +######################################################################### +Total memory: 3792504 kB +Free memory: 339096 kB + +######################################################################### +Disk Storage +######################################################################### +Filesystem Type Size Used Avail Use% Mounted on +/dev/sda3 ext3 141G 127G 7.2G 95% / +tmpfs tmpfs 5.0M 8.0K 5.0M 1% /lib/init/rw +tmpfs tmpfs 371M 812K 370M 1% /run +udev tmpfs 1.9G 0 1.9G 0% /dev +tmpfs tmpfs 741M 420K 741M 1% /run/shm +shmfs tmpfs 1.9G 12K 1.9G 1% /lib/init/rw/splashy +/dev/sda2 hfsplus 96G 89G 6.3G 94% /mnt/mac +#+END_SRC From 0aaff3646ab8dd548fa837634fb89535a2f56f21 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 26 Oct 2011 13:19:01 +0800 Subject: [PATCH 0116/2585] - added .HTML version as well --- usr/share/doc/escenic/system-info-guide.html | 327 +++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 usr/share/doc/escenic/system-info-guide.html diff --git a/usr/share/doc/escenic/system-info-guide.html b/usr/share/doc/escenic/system-info-guide.html new file mode 100644 index 00000000..579bb72f --- /dev/null +++ b/usr/share/doc/escenic/system-info-guide.html @@ -0,0 +1,327 @@ + + + + +Welcome to the /usr/bin/system-info Guide + + + + + + + + + + + + + +
    + + +
    + +
    +

    Welcome to the /usr/bin/system-info Guide

    + +

    This wee script gives a overview of your system, especially useful +when contacting Vizrt Online/Escenic Support. +

    +

    +The script will default wise produce a plain text report, but if you +pass -c or –confluence to it, it will generate Confluence Wiki markup +instead. +

    + +
    +

    Table of Contents

    +
    + +
    +
    + +
    +

    1 Example

    +
    + +

    Here you can see an example of the output generated from system-info: +

    + + + +
    #########################################################################
    +System details for quanah
    +#########################################################################
    +
    +#########################################################################
    +Kernel version
    +#########################################################################
    +Linux quanah 3.0.0-1-amd64 #1 SMP Sat Aug 27 16:21:11 UTC 2011 x86_64 GNU/Linux
    +
    +#########################################################################
    +Distribution information
    +#########################################################################
    +Debian or derivate, version wheezy/sid
    +
    +#########################################################################
    +Database details
    +#########################################################################
    +/usr/sbin/mysqld  Ver 5.1.58-1 for debian-linux-gnu on x86_64 ((Debian))
    +
    +#########################################################################
    +Java version
    +#########################################################################
    +java version "1.6.0_26"
    +Java(TM) SE Runtime Environment (build 1.6.0_26-b03)
    +Java HotSpot(TM) 64-Bit Server VM (build 20.1-b02, mixed mode)
    +
    +#########################################################################
    +Important packages
    +#########################################################################
    +ant                                      1.8.2-4                              Java based build tool like make
    +libapr1                                  1.4.5-1                              Apache Portable Runtime Library
    +libmysql-java                            5.1.16-2                             Java database (JDBC) driver for MySQL
    +libtcnative-1                            1.1.22-1                             Tomcat native library using the apache portable runtime
    +maven2                                   2.2.1-10                             Java software project management and comprehension tool
    +mysql-server                             5.1.58-1                             MySQL database server (metapackage depending on the latest version)
    +nginx                                    1.1.4-2                              small, but very powerful and efficient web server and mail proxy
    +sun-java6-jdk                            6.26-3                               Sun Java(TM) Development Kit (JDK) 6
    +sun-java6-jre                            6.26-3                               Sun Java(TM) Runtime Environment (JRE) 6 (architecture independent files)
    +tomcat6-user                             6.0.32-6                             Servlet and JSP engine -- tools to create user instances
    +varnish                                  3.0.1-1~squeeze1                     state of the art, high-performance web accelerator
    +
    +#########################################################################
    +Processors
    +#########################################################################
    +Processor type:  Intel(R) Core(TM)2 Duo CPU P8600 @ 2.40GHz
    +Number of processors/cores: 1
    +
    +#########################################################################
    +Memory
    +#########################################################################
    +Total memory:  3792504 kB
    +Free memory:  339096 kB
    +
    +#########################################################################
    +Disk Storage
    +#########################################################################
    +Filesystem     Type     Size  Used Avail Use% Mounted on
    +/dev/sda3      ext3     141G  127G  7.2G  95% /
    +tmpfs          tmpfs    5.0M  8.0K  5.0M   1% /lib/init/rw
    +tmpfs          tmpfs    371M  812K  370M   1% /run
    +udev           tmpfs    1.9G     0  1.9G   0% /dev
    +tmpfs          tmpfs    741M  420K  741M   1% /run/shm
    +shmfs          tmpfs    1.9G   12K  1.9G   1% /lib/init/rw/splashy
    +/dev/sda2      hfsplus   96G   89G  6.3G  94% /mnt/mac
    +
    + +
    +
    +
    + +
    +
    +

    + Author: Torstein Krause Johansen (tkj@vizrt.com) + Date: 2011-10-26 13:18:12 CST +

    +
    + + From c3365a968da296f831f6393125d55950d01d4aee Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 26 Oct 2011 13:19:52 +0800 Subject: [PATCH 0117/2585] - system-info now has its own guie --- README | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/README b/README index ba9c8455..7a85bc50 100644 --- a/README +++ b/README @@ -26,16 +26,11 @@ https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/ece-ins =========================================== Features of the /usr/bin/system-info script =========================================== -Gives a overview of your system, especially useful when contacting -Vizrt Online/Escenic Support. +https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/system-info-guide.org -The script default wise produces a plain text report, but if you pass --c or --confluence to it, it will generate Confluence Wiki markup -instead. - -========================== +======== Feedback -========================== +======== Hope you enjoy the ece /usr/bin and intit.d scripts. All feedback, come hither! From e7215aa82c20d956916177a8de3595460c373f07 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 26 Oct 2011 14:45:13 +0800 Subject: [PATCH 0118/2585] - the help message of threaddump always listed instance as "default": [ece#engine-dev1] Thread dump (PID 19776) written to system out log. [ece#engine-dev1] Type 'ece -t engine -i default outlog' to see it or view [ece#engine-dev1] /var/log/escenic/engine-dev1.out directly. This has been fixed now: [ece#engine-dev1] Thread dump (PID 19776) written to system out log. [ece#engine-dev1] Type 'ece -t engine -i dev1 outlog' to see it or view [ece#engine-dev1] /var/log/escenic/engine-dev1.out directly. --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index d1c51004..d8171633 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1013,7 +1013,7 @@ function make_thread_dump() if [ -n "$type_pid" ]; then print "Thread dump (PID" $type_pid") written to system out log." - print "Type 'ece -t $type -i default outlog' to see it or view" + print "Type 'ece -t $type -i $instance outlog' to see it or view" print $log_file "directly." if [ -x $java_home/bin/jstack ]; then From 854249102a5c874f95851b19c57cb1de378f24eb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 27 Oct 2011 10:50:21 +0800 Subject: [PATCH 0119/2585] - set_type_port is now only called when the port number is needed, right now this means update_publication_resources and list_versions. Before, ece would also grep through the system out log file for getting the port number, regardless if the command needed it or not. This will no longer happen. If you have a huge system out file, this fix should speed up the general operation of using the ece script. This closes https://github.com/skybert/ece-scripts/issues/25 --- usr/bin/ece | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index d8171633..079af1f9 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -300,9 +300,7 @@ function set_instance_settings() fi set_type_pid - - set_type_port - + # if type is rmi-hub, we don't' need more configuration. if [ $type = "rmi-hub" ]; then return @@ -1187,6 +1185,8 @@ function list_versions() exit 1 fi + set_type_port + version_manager=escenic-admin/browser/Global/neo/io/managers/VersionManager url=http://$host:$port/$version_manager @@ -1208,8 +1208,6 @@ function list_versions() function update_publication_resources() { - url=http://${host}:${port}/escenic-admin/publication-resources - if [ ! -r $resource ]; then print $resource "doesn't exist. I will exit :-(" exit 1 @@ -1220,6 +1218,9 @@ function update_publication_resources() exit 1 fi + set_type_port + url=http://${host}:${port}/escenic-admin/publication-resources + case "$(basename $resource)" in content-type) url=${url}/${publication}/escenic/content-type From 28121315e6a8e962069e99c44ed2014ca10df054 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 28 Oct 2011 16:25:25 +0800 Subject: [PATCH 0120/2585] - As Mogsie says: Solr complains if it doesn't get to create the index directory itself... Hence, simplifying the solr data dir in the engine_dir_list https://github.com/skybert/ece-scripts/issues/10 --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 13d17b89..750080d5 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -134,7 +134,7 @@ http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip http://technet.escenic.com/downloads/release/53/community-engine-3.6.1.0.zip http://technet.escenic.com/downloads/release/53/dashboard-1.0.0.0.zip -http://technet.escenic.com/downloads/release/53/engine-5.3.2.2.zip +http://technet.escenic.com/downloads/release/53/engine-5.3.3.2.zip http://technet.escenic.com/downloads/release/53/forum-3.0.0.0.zip http://technet.escenic.com/downloads/release/53/inpage-1.3.0.0.zip http://technet.escenic.com/downloads/release/53/lucy-dist-4.1.6.0.zip @@ -169,7 +169,7 @@ $common_nursery_dir /var/cache/escenic /var/crash/escenic /var/lib/escenic -/var/lib/escenic/solr/data/index +/var/lib/escenic/solr/data /var/log/escenic /var/run/escenic /var/spool/escenic/migration From 24968abe0684aad7a96bc649b510444c4c78103f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 2 Nov 2011 11:22:57 +0800 Subject: [PATCH 0121/2585] - the test for publication in update_publication_resources was -x (!) --- usr/bin/ece | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 079af1f9..54cd7457 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -906,7 +906,7 @@ function assemble() message="Assembling your EAR file" print $message ".." log $message "..." >> $log_file - + cd $assemblytool_home && \ ant -q ear -DskipRedundancyCheck=true \ 1>>$log_file \ @@ -1213,7 +1213,7 @@ function update_publication_resources() exit 1 fi - if [ -x $publication ]; then + if [ -z $publication ]; then print "You must specify which publication to update (-p )" exit 1 fi From 1de951840bbd1fd46d5dcc52a01312d41e3a634b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 2 Nov 2011 12:03:16 +0800 Subject: [PATCH 0122/2585] - added an edit command to ece in the context of a publication to easy update a publication resource: $ ece -i dev1 -p ip -r layout-group edit Here, ece will use the "dev1" instance and "ip" publication to download the layout-group resource to a temporary directory. ece will then open this file for you with the editor defined in the EDITOR environment variable. After editing & exiting from your editor, ece will update the resource, if the checksum of the file has changed after first downloading it. --- usr/bin/ece | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 54cd7457..4c41fde1 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1208,7 +1208,7 @@ function list_versions() function update_publication_resources() { - if [ ! -r $resource ]; then + if [[ ! -r $resource && -z "$1" ]]; then print $resource "doesn't exist. I will exit :-(" exit 1 fi @@ -1248,15 +1248,41 @@ function update_publication_resources() exit 1 esac + local tmp_dir=$(mktemp -d) + + if [ $1 ]; then + if [[ -z "$EDITOR" || ! -x $(which $EDITOR) ]]; then + print "You must have a valid editor defined in your EDITOR variable" + exit 1 + fi + + run cd $tmp_dir; + run wget $url + md5sum ${resource} > ${resource}.md5sum + exit_on_error "md5sum ${resource}" + + $EDITOR ${resource} + md5sum -c ${resource}.md5sum \ + 1>>$log_file \ + 2>>$log_file + if [ $? -eq 0 ]; then + print "You didn't make any changes to ${resource}, I will exit." + exit 1 + fi + fi + print "Updating the $(basename $resource) resource for the $publication" \ - "publication" + "publication ..." debug POSTing $resource to $url wget -O - \ --post-file ${resource} \ $url \ 1>>$log_file 2>>$log_file - + + if [ -d $tmp_dir ]; then + run rm -rf $tmp_dir + fi } function show_all_log_paths() @@ -1379,6 +1405,9 @@ for el in $command; do update) update_publication_resources ;; + edit) + update_publication_resources "edit_first" + ;; backup) clean_up backup_type From 0c4e05453096aae7e90deb77ca696d0850da206d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 2 Nov 2011 13:29:47 +0800 Subject: [PATCH 0123/2585] - added edit to the help screen --- usr/bin/ece | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/bin/ece b/usr/bin/ece index 4c41fde1..159bc511 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1341,6 +1341,7 @@ function print_help() echo " backup backs up an entire instance (takes time)" echo " clean removes temporary files created by $0 *)" echo " deploy deploys the assembled EAR *)" + echo " edit lets you edit a publication resource" echo " help prints this help screen" echo " kill uses force to stop the type" echo " log the type's log4j log **)" From ee55e316089e1804993ec280d26959ec8b7a2501 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 2 Nov 2011 13:29:58 +0800 Subject: [PATCH 0124/2585] - added autocompletion for the edit command --- etc/bash_completion.d/ece | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index 54d5ae2c..a5810d1e 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -1,10 +1,11 @@ # auto completion for the /usr/bin/ece command. + _ece_commands() { local cur=${COMP_WORDS[COMP_CWORD]} local prev=${COMP_WORDS[COMP_CWORD-1]} - commands="applog assemble backup clean deploy help kill log loglist outlog restart start status stop threaddump update versions" + commands="applog assemble backup clean deploy edit help kill log loglist outlog restart start status stop threaddump update versions" resources="content-type feature layout layout-group image-version menu" types="engine search analysis rmi-hub" @@ -14,17 +15,15 @@ _ece_commands() instances="" dir=/etc/escenic/engine/instance if [ -r $dir ] ; then - cd $dir - for el in *; do - instances=${el}" "$instances + for el in ${dir}/*; do + instances=$(basename ${el})" "$instances done fi publications="" dir=/opt/escenic/assemblytool/publications if [ -r $dir ] ; then - cd $dir - for el in *.properties; do + for el in ${dir}/*.properties; do publications=$(basename $el .properties)" "$publications done fi From 459f77ff04fcc6b89a1b73d9925ac13b9cc67ceb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 2 Nov 2011 16:56:52 +0800 Subject: [PATCH 0125/2585] - added auto completion of option switches - added auto completion of option values, also for the long versions of the option names. --- etc/bash_completion.d/ece | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index a5810d1e..6c5cc56e 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -5,12 +5,15 @@ _ece_commands() local cur=${COMP_WORDS[COMP_CWORD]} local prev=${COMP_WORDS[COMP_CWORD-1]} - commands="applog assemble backup clean deploy edit help kill log loglist outlog restart start status stop threaddump update versions" + commands="applog assemble backup clean deploy edit help kill log loglist \ + outlog restart start status stop threaddump update versions" + options="-i --instance -p --publication -r --publication-resource \ + -t --type" resources="content-type feature layout layout-group image-version menu" types="engine search analysis rmi-hub" # default completions is the list of commands - completions=$commands + completions=$commands" "$options instances="" dir=/etc/escenic/engine/instance @@ -32,16 +35,16 @@ _ece_commands() backup) completions="--exclude-binaries" ;; - -t) + -t|--type) completions=$types ;; - -i) + -i|--instance) completions=$instances ;; - -p) + -p|--publication) completions=$publications ;; - -r) + -r|--publication-resource) completions=$resources ;; esac From cbf67297c8851a6704766adace852943483852f2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 3 Nov 2011 18:57:29 +0800 Subject: [PATCH 0126/2585] - added auto completion of ECOME's security resource --- etc/bash_completion.d/ece | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index 6c5cc56e..c7ce8ee9 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -8,8 +8,9 @@ _ece_commands() commands="applog assemble backup clean deploy edit help kill log loglist \ outlog restart start status stop threaddump update versions" options="-i --instance -p --publication -r --publication-resource \ - -t --type" - resources="content-type feature layout layout-group image-version menu" + -t --type -u --user -w --password" + resources="content-type feature layout layout-group image-version menu \ + security" types="engine search analysis rmi-hub" # default completions is the list of commands From f3e451b39b3471a7ff44c8a6685fdbcc78225463 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 9 Nov 2011 08:02:53 +0800 Subject: [PATCH 0127/2585] - added the chance to change the section parameters for the root section: ece -i dev1 -p mypub -r root-section-parameters edit - added -u/--user and -w/--password options to the script. When set, the commands which use curl and wget behind the scences will use these values for doing the HTTP operations. --- usr/bin/ece | 66 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 159bc511..6d454d62 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -22,6 +22,10 @@ type=engine instance=default command="" backup_exclude_binaries=0 +http_user="" +http_password="" +wget_auth="" +curl_auth="" type_list=" engine @@ -210,6 +214,11 @@ function set_common_settings() $log_dir/messages $log_dir/Escenic-error.log " + + if [[ -n "$http_user" && -n "$http_password" ]]; then + wget_auth="--http-user $http_user --http-password $http_password" + curl_auth="-u ${http_user}:${http_password}" + fi } function set_type_settings() @@ -904,7 +913,7 @@ function assemble() fi message="Assembling your EAR file" - print $message ".." + print $message "..." log $message "..." >> $log_file cd $assemblytool_home && \ @@ -1030,16 +1039,20 @@ function set_type_command_and_instance() next_is_instance=0 next_is_publication=0 next_is_resource=0 + next_is_http_user=0 + next_is_http_password=0 for el in $@; do if [ $el = "-v" -o $el = "--verbose" ]; then verbose=1 continue fi + if [ $el = "-q" -o $el = "--quiet" ]; then quiet=1 continue fi + if [ $el = "--help" ]; then command=help continue @@ -1051,6 +1064,18 @@ function set_type_command_and_instance() continue fi + if [ $next_is_http_user -eq 1 ]; then + http_user=$el + next_is_http_user=0 + continue + fi + + if [ $next_is_http_password -eq 1 ]; then + http_password=$el + next_is_http_password=0 + continue + fi + if [ $next_is_instance -eq 1 ]; then instance=$el next_is_instance=0 @@ -1096,6 +1121,20 @@ function set_type_command_and_instance() else next_is_resource=0 fi + + if [ $el = "-u" -o $el = "--user" ]; then + next_is_http_user=1 + continue + else + next_is_http_user=0 + fi + + if [ $el = "-w" -o $el = "--password" ]; then + next_is_http_password=1 + continue + else + next_is_http_password=0 + fi if [ $el = "--exclude-binaries" ]; then backup_exclude_binaries=1 @@ -1191,7 +1230,7 @@ function list_versions() url=http://$host:$port/$version_manager print "Installed on the ${type} running on ${host}:${port} is:" - wget -O - $url 2>/dev/null | \ + wget $wget_auth -O - $url 2>/dev/null | \ grep "\[\[" | \ sed 's/\[//g' | \ sed 's/\]//g' | \ @@ -1243,6 +1282,11 @@ function update_publication_resources() security) url=${url}/${publication}/escenic/plugin/community/security ;; + root-section-parameters) + do_put=true + url=http://${host}:${port}/escenic-admin + url=${url}/section-parameters-declared/${publication} + ;; *) print "Invalid resource: $(basename $resource) :-(" exit 1 @@ -1257,7 +1301,7 @@ function update_publication_resources() fi run cd $tmp_dir; - run wget $url + run wget $wget_auth $url -O ${resource} md5sum ${resource} > ${resource}.md5sum exit_on_error "md5sum ${resource}" @@ -1275,10 +1319,18 @@ function update_publication_resources() "publication ..." debug POSTing $resource to $url - wget -O - \ - --post-file ${resource} \ - $url \ - 1>>$log_file 2>>$log_file + if [[ -n "$do_put" && $do_put = "true" ]]; then + curl -T ${resource} \ + ${curl_auth} \ + ${url} \ + 1>>$log_file 2>>$log_file + else + wget $wget_auth \ + -O - \ + --post-file ${resource} \ + $url \ + 1>>$log_file 2>>$log_file + fi if [ -d $tmp_dir ]; then run rm -rf $tmp_dir From 2e715cc0014b3f5212d8e7e1acfdb7ce634f42a4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 9 Nov 2011 08:03:26 +0800 Subject: [PATCH 0128/2585] - added root-section-parameters to the auto completion of -r/--publication-resource --- etc/bash_completion.d/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index c7ce8ee9..d34877d1 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -10,7 +10,7 @@ _ece_commands() options="-i --instance -p --publication -r --publication-resource \ -t --type -u --user -w --password" resources="content-type feature layout layout-group image-version menu \ - security" + security root-section-parameters" types="engine search analysis rmi-hub" # default completions is the list of commands From 4e83ca491c5b14ad646aeebebbaec8f11590b9b8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 9 Nov 2011 08:11:53 +0800 Subject: [PATCH 0129/2585] - updated HTML --- usr/share/doc/escenic/ece-guide.html | 115 ++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 20 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.html b/usr/share/doc/escenic/ece-guide.html index 57839ec2..92e080c3 100644 --- a/usr/share/doc/escenic/ece-guide.html +++ b/usr/share/doc/escenic/ece-guide.html @@ -7,7 +7,7 @@ Guide for the /usr/bin/ece Script - + @@ -251,11 +251,12 @@

    Table of Contents

  • 6.2 Specifying instance
  • +
  • 6.3 TAB completion
  • +
  • 7 Backup
  • @@ -666,44 +667,118 @@

    6.2.1 Specifying the in +

    +The script's TAB completion will also help you with completing the +available instances on your system, so you can just type: +

    + + +
    $ ece -i TAB TAB
    +mypub myotherpub
    +
    + + +

    +For this to work, the script assumes that you have installed the +instance specific configuration in /etc/escenic/engine/instance, as is +described in the Escenic Content Engine Installation Guide. +

    + + -
    -

    6.2.2 Using the default instance

    -
    +
    +

    6.3 TAB completion

    +
    -

    However, on development systems and perhaps even test systems, you -probably want to just use one instance on your host, though, and this -is where the "default" instance comes in. +

    The ece script offers TAB completion of all commands, options and +option values. For this to work, you need the completion file loaded +from your .bashrc. +

    +

    +This can either be done by enabling all BASH completions on your +system (some systems have this set up per default), refer to your OS +documentation for this.

    -To use the default instance, you must only have one instance installed -on your system. More specifically, there must no instance specific -configuration files in +Alternatively, to just enable the ece completion, simply add the +following line to your .bashrc:

    -
    /etc/escenic/engine/instance
    +
    . /etc/bash_completion.d/ece
     

    -and that ece.conf, typically located in: +Naturally, the bashcomletion.d needs to be in the above location :-) +

    +
    + +
    + +
    +

    7 Backup

    +
    + +

    One of the many features of the ece script, is that you can take a +snapshot of your current running system, which can then be restored +using the /usr/sbin/ece-install script. +

    +

    +The backup will include the database, the Nursery configuration, the +DB configuration, the Varnish configuration, the nginx congiguration, +all the Escenic software binaries, your publication and your +application servers. +

    +

    +To create the backup, you'll simply do:

    -
    /etc/escenic/ece.conf
    +
    $ ece -i editor1 backup
     
    +

    -must contain all the settings required to run the instance. +It will then create tarball with everything and put it in

    + + +
    /var/backups/escenic
    +
    + +

    -If you've ticked all these boxes, you can issue all ece commands -without the -i parameter. -

    -
    +Sometimes, you only want to take a backup of the data files and are +not so concerned with the Escenic and application server binaries as +these are easily installed from elsewhere. If so, you may do the +following: +

    + + +
    $ ece -i editor1 backup --exclude-binaries
    +
    + + +

    +When running in a production environment, it's advised to first stop +the search instance which runs the Solr indexer, so that the Solr +index isn't corrupted: +

    + + +
    $ ece -i search1 stop
    +$ ece -i editor1 backup
    +
    + + + + + + +
    @@ -712,7 +787,7 @@

    6.2.2 Using the default

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-10-10 14:56:44 CST + Date: 2011-11-09 08:11:44 CST

    From 5a1a200e3e80791261e8aae300f94fdd10936c4e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 9 Nov 2011 08:14:34 +0800 Subject: [PATCH 0130/2585] - don't want to generate the .txt all the time --- usr/share/doc/escenic/ece-guide.txt | 262 +--------------------------- 1 file changed, 4 insertions(+), 258 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.txt b/usr/share/doc/escenic/ece-guide.txt index 8d7f9e5a..25725614 100644 --- a/usr/share/doc/escenic/ece-guide.txt +++ b/usr/share/doc/escenic/ece-guide.txt @@ -1,259 +1,5 @@ - Guide for the /usr/bin/ece Script - ================================= - -Author: Torstein Krause Johansen -Date: 2011-10-07 16:47:42 CST - - -Table of Contents -================= -1 TAB completion - 1.1 Completing ece commands - 1.2 Completing types of servers the ece scripts can operate on - 1.3 Completing the publication resources available -2 Getting an overview of all available options -3 Tailing log files - 3.1 Getting an overview of the log files - 3.2 Commands for tailing the log files - 3.3 Piping the log files to another command -4 Assembling a new EAR -5 Installation - 5.1 Overview of File Paths Used by the ece script -6 Running the ece script - - -1 TAB completion ------------------ -The ece script offers TAB completion, given that your version of BASH -supports this and that you have enabled it (some distributions hasn't -turned on BASH completion per default). - -1.1 Completing ece commands -============================ -You will find yourself using this regularly, both for speed, but also -to remember all the different options and their correct wording: - - - $ ece TAB TAB - applog deploy log start threaddump - assemble help outlog status update - clean kill restart stop versions - -The commands are all described in detail in "ece help" - -1.2 Completing types of servers the ece scripts can operate on -=============================================================== - - - $ ece -t TAB TAB - analysis engine rmi-hub search - - -1.3 Completing the publication resources available -=================================================== -The ece script can update the publication resources for a given -publication. - -To help selecting the correct publication resource, you can make ece -list and complete the available resource names: - - - $ ece -p mypub -r TAB TAB - content-type image-version layout-group - feature layout menu - -2 Getting an overview of all available options ------------------------------------------------ - - - $ ece help - Usage: /home/torstein/bin/ece [-t ] [-i ] - - DESCRIPTION - -t --type - The following types are available: - engine - The Escenic Content Engine, this is the default - and is the assumed type if none is specified. - search - A standalone search indexer and solr instance - rmi-hub - The RMI hub responsible for the internal - communication between the ECE instances. - analysis - The Escenic Analysis Engine also knows as 'Stats' - - -i --instance - The type instance, such as editor1, web1 or search1 - - -p --publication - Needed only for updating publication resources - - -r --resource - Used for updating publication resources. - Must be one of: content-type, feature, layout, layout-group - image-version, menu - - -v --verbose - Prints out debug statements, useful for debugging. - - The following commands are available: - applog the type's app server log - assemble runs the Assembly Tool *) - clean removes temporary files created by /home/torstein/bin/ece *) - deploy deploys the assembled EAR *) - help prints this help screen - kill uses force to stop the type - log the type's Log4J log - outlog the [ece#engine] script log (system out log) - restart restarts the type - start starts the type - status checks if the type is running - stop stops the type - threaddump write a thread dump to standard out (system out log) - update update publication resources - versions lists the ECE component versions - - *) only applicable if type is 'engine' - - -3 Tailing log files --------------------- -The script has a number of options to let you keep track of what's -going on on your system. - -3.1 Getting an overview of the log files -========================================= -The ece script will be more than happy to tell you about the log files -on your system that are of interest to your instance: - - - $ ece -i dev1 loglist - [ece#engine-dev1] System out log: /var/log/escenic/engine-dev1.out - [ece#engine-dev1] App server log: /opt/tomcat-dev1/logs/localhost.2011-09-20.log - [ece#engine-dev1] log4j log: /var/log/escenic/messages - - -3.2 Commands for tailing the log files -======================================= -There are three different logs, which ece will let you tail. When -tailing the log file, ece will also first print out where the file is -located on the file system so you can open it later for further -inspection. - - - - $ ece -i dev1 outlog - [ece#engine-dev1] Tailing the system out log /var/log/escenic/engine-dev1.out - - -The available log file commands are: -outlog: the system out log -applog: the app server log -log: the log4j log - -3.3 Piping the log files to another command -============================================ -If you wish to pass the log files onto another pipe, for instance to -grep all the log files for a certain exception or error message, you -will find the -q parameter useful: - - - $ ece -i dev1 -q loglist | xargs grep IllegalArgumentException - - -This approach is also useful if you wish to tail all the log files at -once: - - - $ ece -i dev1 -q loglist | xargs tail -f - - -4 Assembling a new EAR ------------------------ -The script will quite happily assemble your EAR file. Please note that -you need to run - - - $ ece clean assemble - -whenever you've upgraded either ECE or one of its plugins. The script -will try to take care of you though and warn you of such duplication: - - - [ece#engine] Multiple versions of ECE and/or 3rd party libraries found. - [ece#engine] Remember, you need to run 'ece clean assemble' when - [ece#engine] upgrading either ECE or one of the plugins. - [ece#engine] Press ENTER to run 'ece clean assemble' - -The script will then re-run the assembly by itself. - -If you're just re-running the assembly after just adding a new plugin -or want to re-build your publications after template changes, you can -omit the clean command and just run: - - - $ ece assembly - - -The EAR produced is now ready to be deployed with - - - $ ece [-i ] deploy - -5 Installation ---------------- -The ece script and ece.conf may be used on any Unix like system that -has a fairly recent version of BASH installed. - -5.1 Overview of File Paths Used by the ece script -================================================== -These are recommended files and locations for using the ece script: - - Path Explanation - ----------------------------------+---------------------------------- - /usr/bin/ece The script itself - /etc/escenic/ece.conf The main configuration file - /etc/escenic/ece-.conf Instance specific settings - /var/cache/escenic Directory of assembled EAR files - /tmp Directory for temporary files - -As you can see in ece.conf, there are a number of default locations -dealing with log files, pid files, crash files as well as application -server files. The defaults all follow the File Hierarchy Standard, but -you may of course change these to your liking. - -If you wish to put the .conf files in other places, you may like to know -that the ece script has preset list of locations where it looks for -the .conf files mentioned above, namely: - -- current working directory -- /etc/escenic//instance/ -- /etc/escenic//host/ -- /etc/escenic//common -- /etc/escenic/ -- /etc/escenic -- /../etc - -You may override this list of locations by setting the -this environment variable in your .bashrc or similar: - - - ECE_CONF_LOCATIONS - - -The reason for having so many options is because various Escenic -consultants, partners and customers have requested these locations to -fit their systems. As you can see, fitting everyone's fancy adds up -over time :-) - -6 Running the ece script -------------------------- -You must be normal user to run the ece script, otherwise it will -complain: - - - [ece#engine] Sorry, you cannot be root when running ece - [ece#engine] The root user can only use /etc/init.d/ece - - -As it mentions, the root user may use the init.d script and the -accompanying /etc/default/ece to command the different ECE, EAE and -RMI hub instances on your system. +See: +https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/ece-guide.org +Or for the HTML version: +http://download.escenic.com/ece-scripts/ece-install-guide.html From a1063c14d31df1142bc05c6ec5375774414dc163 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 9 Nov 2011 08:31:02 +0800 Subject: [PATCH 0131/2585] - added section on creating backups --- usr/share/doc/escenic/ece-guide.org | 77 +++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index aa1e8a37..76739185 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -222,22 +222,71 @@ do: $ ece -i editor1 assemble deploy restart #+END_SRC -*** Using the default instance -However, on development systems and perhaps even test systems, you -probably want to just use one instance on your host, though, and this -is where the "default" instance comes in. +The script's TAB completion will also help you with completing the +available instances on your system, so you can just type: +#+BEGIN_SRC sh +$ ece -i TAB TAB +mypub myotherpub +#+END_SRC + +For this to work, the script assumes that you have installed the +instance specific configuration in /etc/escenic/engine/instance, as is +described in the Escenic Content Engine Installation Guide. + + +** TAB completion +The ece script offers TAB completion of all commands, options and +option values. For this to work, you need the completion file loaded +from your .bashrc. + +This can either be done by enabling all BASH completions on your +system (some systems have this set up per default), refer to your OS +documentation for this. -To use the default instance, you must only have one instance installed -on your system. More specifically, there must no instance specific -configuration files in +Alternatively, to just enable the ece completion, simply add the +following line to your .bashrc: #+BEGIN_SRC sh -/etc/escenic/engine/instance -#+END_SRC -and that ece.conf, typically located in: +. /etc/bash_completion.d/ece +#+END_SRC +Naturally, the bash_comletion.d needs to be in the above location :-) +* Backup +One of the many features of the ece script, is that you can take a +snapshot of your current running system, which can then be restored +using the /usr/sbin/ece-install script. + +The backup will include the database, the Nursery configuration, the +DB configuration, the Varnish configuration, the nginx congiguration, +all the Escenic software binaries, your publication and your +application servers. + +To create the backup, you'll simply do: #+BEGIN_SRC sh -/etc/escenic/ece.conf +$ ece -i editor1 backup #+END_SRC -must contain all the settings required to run the instance. -If you've ticked all these boxes, you can issue all ece commands -without the -i parameter. +It will then create tarball with everything and put it in +#+BEGIN_SRC sh +/var/backups/escenic +#+END_SRC + +Sometimes, you only want to take a backup of the data files and are +not so concerned with the Escenic and application server binaries as +these are easily installed from elsewhere. If so, you may do the +following: +#+BEGIN_SRC sh +$ ece -i editor1 backup --exclude-binaries +#+END_SRC + +When running in a production environment, it's advised to first stop +the search instance which runs the Solr indexer, so that the Solr +index isn't corrupted: +#+BEGIN_SRC sh +$ ece -i search1 stop +$ ece -i editor1 backup +#+END_SRC + + + + + + From 81c07b364acc76fc5f3595db27a3dcdd9fbcc24a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 9 Nov 2011 08:31:16 +0800 Subject: [PATCH 0132/2585] - updated HTML --- usr/share/doc/escenic/ece-guide.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.html b/usr/share/doc/escenic/ece-guide.html index 92e080c3..4546c90e 100644 --- a/usr/share/doc/escenic/ece-guide.html +++ b/usr/share/doc/escenic/ece-guide.html @@ -7,7 +7,7 @@ Guide for the /usr/bin/ece Script - + @@ -787,7 +787,7 @@

    7 Backup


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-11-09 08:11:44 CST + Date: 2011-11-09 08:31:08 CST

    From ac3b9dd64fe98f97dfc3ba8e483d5c7563040c76 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 9 Nov 2011 16:45:25 +0800 Subject: [PATCH 0133/2585] - added support for .ecerc. If the user has this file in his/her home directory, he/she can define default values for the settings (like): http_user=admin http_password=nottellingyou These settings can still be overridden on the command line using the -u/--user and -w/--password options. --- usr/bin/ece | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 6d454d62..35d5971d 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1350,6 +1350,15 @@ function show_all_log_paths() fi } +function read_rc_file_if_present() +{ + local rc_file=$HOME/.ecerc + if [ -r $rc_file ]; then + source $rc_file + fi +} + +read_rc_file_if_present set_type_command_and_instance $@ set_id set_common_settings From a8328e922b16a2b843d82759d9984f734e46bd42 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 10 Nov 2011 18:07:50 +0800 Subject: [PATCH 0134/2585] - fixed error in adding a next step in the install_web_server function when profile=all - fixed typo in message printed to non-Debian/Ubuntu users about adding ECE to the desired runlevels. --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 750080d5..3810d47b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2091,7 +2091,7 @@ EOF add_next_step "http://${HOSTNAME}:${port}/binary gives the Adactus endpoint" elif [ $1 -eq 1 ]; then add_next_step "http://${HOSTNAME}:${port}/ gives the Munin interface" - elif [ $2 -eq 1 ]; then + elif [ $1 -eq 2 ]; then add_next_step "http://${HOSTNAME}:${port}/ gives the Munin interface" add_next_step "http://${HOSTNAME}:${port}/binary gives the Adactus endpoint" fi @@ -2291,7 +2291,7 @@ function add_server_to_runlevels() print "Adding the ece init.d script to the default run levels ..." run update-rc.d ece defaults else - add_next_step "Remember to add /etc/intit.d/ece to the desired "\ + add_next_step "Remember to add /etc/init.d/ece to the desired "\ "run levels." # TODO add init.d to the default runlevels, for other # distributions too: From 24429b12c52cf0e2cbd31028262b1d065064fbc5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 10 Nov 2011 18:48:08 +0800 Subject: [PATCH 0135/2585] - added maven_opts variable, for now filled with --batch-mode to avoid download progress status updates. --- usr/sbin/ece-install | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3810d47b..28699ad9 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -36,6 +36,7 @@ log=/var/log/$(basename $0).log conf_file=$HOME/.ece-install.conf common_nursery_dir=/etc/escenic/engine/common ece_scripts_git_source=git://github.com/skybert/ece-scripts.git +maven_opts="--batch-mode" # globals will be set to correct values in run-time. appserver_port=8080 @@ -1072,7 +1073,7 @@ EOF if [ -d /opt/escenic/widget-framework-core-* ]; then print "Basing ${publication_name}.war on the Widget Framework Demo ..." run cd /opt/escenic/widget-framework-core-*/publications/demo-core - run mvn package + run mvn $maven_opts package run cp target/demo-core-*.war ${publication_war} else print "Basing your ${publication_name}.war on ECE/demo-clean ..." @@ -1948,7 +1949,7 @@ EOF cd $wf_maven_dir print "Installing Widget Framework into your Maven repository ..." export JAVA_HOME=$java_home - mvn install \ + mvn $maven_opts install \ 1>>$log 2>>$log # installing the widget-framework-common as a ECE plugin From 98b90ed0d5194002b70a66de328bc07db84c5872 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 10 Nov 2011 19:23:27 +0800 Subject: [PATCH 0136/2585] - the MySQL JDBC driver was installed in the install_database_server, which was wrong. It should be installed (only) in install_ece_third_party_packages --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 28699ad9..8f530344 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1617,7 +1617,7 @@ function install_database_server() print "Debian (or derivative) version with code name $code_name. " print "I will use vanilla MySQL instead." - packages="mysql-server mysql-client libmysql-java" + packages="mysql-server mysql-client" install_packages_if_missing $packages fi fi From 4da9bdaef47ca466c4784532969e5940be413a2d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 12:07:37 +0800 Subject: [PATCH 0137/2585] - for clarity, added sample execution of the script. --- usr/share/doc/escenic/system-info-guide.org | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usr/share/doc/escenic/system-info-guide.org b/usr/share/doc/escenic/system-info-guide.org index 9119a40d..4eff23f7 100644 --- a/usr/share/doc/escenic/system-info-guide.org +++ b/usr/share/doc/escenic/system-info-guide.org @@ -3,6 +3,10 @@ Welcome to the /usr/bin/system-info Guide This wee script gives a overview of your system, especially useful when contacting Vizrt Online/Escenic Support. +#+BEGIN_SRC sh +$ system-info +#+END_SRC + The script will default wise produce a plain text report, but if you pass -c or --confluence to it, it will generate Confluence Wiki markup instead. From b00c62316768360ca47cc1cb54685b6f3de18ce4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 12:07:43 +0800 Subject: [PATCH 0138/2585] - updated HTML --- usr/share/doc/escenic/system-info-guide.html | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/usr/share/doc/escenic/system-info-guide.html b/usr/share/doc/escenic/system-info-guide.html index 579bb72f..d458d9e0 100644 --- a/usr/share/doc/escenic/system-info-guide.html +++ b/usr/share/doc/escenic/system-info-guide.html @@ -1,13 +1,13 @@ - + Welcome to the /usr/bin/system-info Guide - + - + @@ -222,6 +222,13 @@

    Welcome to the /usr/bin/system-info Guide

    This wee script gives a overview of your system, especially useful when contacting Vizrt Online/Escenic Support.

    + + + +
    $ system-info
    +
    + +

    The script will default wise produce a plain text report, but if you pass -c or –confluence to it, it will generate Confluence Wiki markup @@ -320,7 +327,7 @@

    1 Example


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-10-26 13:18:12 CST + Date: 2011-11-11 12:07:19 CST

    From fc5afe7dd984b95d70b7ad5c817a51a64338b705 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 12:10:45 +0800 Subject: [PATCH 0139/2585] - removed .txt versions of the guides --- usr/share/doc/escenic/ece-guide.txt | 5 - usr/share/doc/escenic/ece-install-guide.txt | 688 -------------------- 2 files changed, 693 deletions(-) delete mode 100644 usr/share/doc/escenic/ece-guide.txt delete mode 100644 usr/share/doc/escenic/ece-install-guide.txt diff --git a/usr/share/doc/escenic/ece-guide.txt b/usr/share/doc/escenic/ece-guide.txt deleted file mode 100644 index 25725614..00000000 --- a/usr/share/doc/escenic/ece-guide.txt +++ /dev/null @@ -1,5 +0,0 @@ -See: -https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/ece-guide.org - -Or for the HTML version: -http://download.escenic.com/ece-scripts/ece-install-guide.html diff --git a/usr/share/doc/escenic/ece-install-guide.txt b/usr/share/doc/escenic/ece-install-guide.txt deleted file mode 100644 index e1078ee3..00000000 --- a/usr/share/doc/escenic/ece-install-guide.txt +++ /dev/null @@ -1,688 +0,0 @@ - Welcome to the /usr/sbin/ece-install Guide - ========================================== - -Author: Torstein Krause Johansen -Date: 2011-10-20 18:37:17 CST - - -Table of Contents -================= -1 Supported Operating Systems - 1.1 Debian based operating systems - 1.2 Other GNU/Linux and Unix systems -2 A Note on Running ece-install On Non-GNU/Linux Systems -3 Downloading the ece-install Script -4 Running the ece-install Script -5 Available Server Profiles - 5.1 Common for All Server Profiles - 5.2 Common for Editorial, Presentation and Search Instances - 5.2.1 Sun Java 6 - 5.2.2 Apache Tomcat application server - 5.2.3 Basic Escenic Nursery configuration - 5.2.4 APR, native library for optimal Tomcat I/O performance - 5.2.5 Escenic Assembly environment - 5.2.6 Database driver - 5.2.7 Compression of content - 5.3 Profile - Full Stack on One Host - 5.4 Profile - Editorial Server (Publication Server) - 5.5 Profile - Presentation Server - 5.6 Profile - Database Server - 5.7 Profile - Cache Server - 5.8 Profile - Install Widget Framework - 5.9 Profile - Create Publication - 5.10 Profile - Monitoring Server - 5.11 Profile - Restoring from backup - 5.12 Running interactively - 5.12.1 Start ece-install and choose the Option, "Restore from backup" - 5.12.2 Select Which Backup to Restore - 5.12.3 Choose What to Restore - 5.12.4 Sit Back and Watch ece-install Restore the Data for You - 5.13 Running in FAI mode - 5.14 Data security -6 Full Automatic Install (FAI) - 6.1 Overview of All FAI Parameters - 6.2 Examples - 6.2.1 Installing an Editorial Server & Create a Publication - 6.2.2 Installing Two Presentation Servers On a Single Host - 6.3 Verifying That the Script Is Running In FAI Mode -7 Running More Than One Installation Process -8 Re-running ece-install (and How To Speed It Up) -9 Overview of File Paths Used by the ece-install script -10 Assumptions - 10.1 /etc/esecenic is shared -11 Uninstalling Everything That the ece-install Set Up -12 Example Output FAI - - -1 Supported Operating Systems ------------------------------- -For the best experience, run this script on Debian Squeeze (or newer), -the latest Ubuntu LTS or most other Debian based operating systems, -but it should work without any problems on any Unix like system that -has a recent version of BASH. - -1.1 Debian based operating systems -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If running on a Debian based operating system, ece-install will take -care of installing all required third party software, such as database -& caching server. - -1.2 Other GNU/Linux and Unix systems -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you're installing on another Linux or Unix system, you must first -make sure that these 3rd party components are installed. ece-install -will tell you which ones it cannot find and tell you to install these, -so you can simply run it to find out which ones you need. - -You may also look inside the ece-install script for all calls to the -'assert\_pre\_prequesite' method and you'll get a list of the binaries -ece-install assumes are present. - -2 A Note on Running ece-install On Non-GNU/Linux Systems ---------------------------------------------------------- -Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp -& tar. If you're running a system which has a different set of core -tools, such as any of the BSDs (including Mac OS X) or Solaris, make -sure that GNU's tools take precedence in the command PATH offered to -ece-install. - -3 Downloading the ece-install Script -------------------------------------- -The ece-install script can be downloaded from: -[https://raw.github.com/skybert/ece-scripts/master/usr/sbin/ece-install] -or be downloaded together with the other ece-scripts using git: - - - $ git clone git@github.com:skybert/ece-scripts.git - - -4 Running the ece-install Script ---------------------------------- -You must run the script as the root user. If you start the script as -a regular user, it will complain: - - - [ece-install] You must be root when running ece-install - -To become root on Ubuntu based systems and Mac OS X, do: - - - $ sudo su - -On all other Unix like system, do: - - - $ su - - -You will need access to [http://technet.escenic.com] and put these -credentials in the root user's $HOME/.ece-install.conf, normally this -means /root/.ece-install.conf, the script will also tell you this if -you forget to provide such a configuration file: - - - [ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-( - - -As for the technet credentials, the script will also tell you if it -cannot find what it's looking for: - - - [ece-install] Be sure to set technet_user and technet_password - [ece-install] in /root/.ece-install.conf - - -The minimum .ece-install.conf file is thus: - - - technet_user= - technet_password= - - -If you're installing WF as well, you'll need additional credentials -for this, see the Widget Framwork section below, and if you want to -runn a fully automatic install without any user interaction, you'll -need to specify some FAI specific parameters, see the FAI section -below. - -With the configuration file in place, you're now ready to run it: - - - # bash ece-install [-v|--verbose] - - -ece-install will try to "fail fast", exiting as soon as it detects an -error during its execution: - - - [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-( - [ece-install-1] See /var/log/ece-install.log for further details. - - -As a last resort, if something goes astray with the script, and the -log file cannot give you enough clues, the ultimate way of debugging -this is to run the BASH interpreter with the -x flag: - - - # bash -x ece-install - - -Here you can see everything that BASH does when running the script, -how the wildcard variables expand and so on. Normally, having a tail -on /var/log/ece-install.log should be sufficient, though. - -5 Available Server Profiles ----------------------------- -When starting the script, it will ask you which server profile you -want to install: - - - Hi, which server profile do you wish to install? - - 1 - All in one, full stack on one host, suitable for dev & test environments - 2 - Editorial (publication) server - 3 - Presentation server (ECE + memcached). - 4 - Database server - 5 - Cache server (cache and web server) - 6 - RMI hub - 7 - Search server (Solr + indexer-webapp) - 8 - Install Widget Framework. - 9 - Create a new publication based on WF if available, ECE/clean-demo if not - 10 - A monitoring server (web server + Munin gatherer) - 11 - Restore from backup (DB, data files, binaries, conf & publications) - - Select 1-11 and press ENTER - Your choice [1]> - -5.1 Common for All Server Profiles -=================================== -The script will install a Munin node on the host. It will also -install the Escenic specific Munin plugins on hosts where there are -any Escenic server components. - -5.2 Common for Editorial, Presentation and Search Instances -============================================================ - -5.2.1 Sun Java 6 -~~~~~~~~~~~~~~~~~ - -5.2.2 Apache Tomcat application server -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- Compression of textual content (previously, this was typically set - up with Apache and its mod\_deflate module). -- pooling set up tweaked for high read/write performance. -- proper logging configuration directing solr messages to its own log. -- routing information in the cookies -- application server access log - -5.2.3 Basic Escenic Nursery configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The basic Nursery configuration is taken care of for you, including RMI, - database, search and application server/URIs. - -5.2.4 APR, native library for optimal Tomcat I/O performance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -5.2.5 Escenic Assembly environment -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The reason why ece-install sets this up on each host, is to make the -installation process as smooth as possible. The assembly environment -may be removed after the installation if you want to. - -5.2.6 Database driver -~~~~~~~~~~~~~~~~~~~~~~ - -5.2.7 Compression of content -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This was was previously accomplished by having a web server in fron t -of the application server (or cache server if you used ESI). A typical -system architecture would contain Apache with mod\_deflate. However, -this is no longer necessary as Varnish can handles ESI parsing of -compressed content (and many other things that we before needed Apache -for). Thus, we'll let the application server do the compression for us -now. - -5.3 Profile - Full Stack on One Host -===================================== -This profile is suitable for developers and admins wanting to set up a -test environment. It will install the full stack including caching -server, application server, ECE, assembly host, database, Widget -Framework, as well as creating a publication for you. - -For further details on each of the different server components, see -the different profile descriptions bellow. - - -5.4 Profile - Editorial Server (Publication Server) -==================================================== -Will set up a typical editorial server (often referred to as the -publication server in Escenic literature). - -5.5 Profile - Presentation Server -================================== -This will set up a typical presentation server: -- Memcached, distributed memory cache -- Deployment setup to only deploy escenic-admin and the - publication(s). - -5.6 Profile - Database Server -============================== -If ece-install is run on a support version of Debian or Ubuntu, this -will install the excellent Percona distribution of MySQL with their -set of high performance patches. - -If not, MySQL or Percona must be installed in advance. - -Given that the mysqld is install, this profile will download all the -Escenic components and install the ECE database schema based from the -SQL files contained inside the distribution bundles. To accomplish -this, the script will make a call to drop-and-create-ecedb in the same -directory as the ece-intall script itself. - -The script will fail by itself if the DB already exists: - - - [ece-install] Setting up the ECE database schema ... - ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists - ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists - [ece-install] running tables FAILED, exiting :-( - - -5.7 Profile - Cache Server -=========================== -If ece-install is run on a support version of Debian or Ubuntu, it -will install the latest Varnish 3.0 caching server from the Varnish -APT repository. - -If ece-install is run on a different platform, the admin must install -Varnish 3.x prior to running ece-install. - -The script will configure Varnish for a typical Escenic site: -- it will set up the cache server on port 80 -- will set up an access control lists of IPs which may access the - privileged web applications such as /escenic-admin, /escenic and - /webservice. - - ece-install will also add the host from which you connect, making - sure that if you've SSH-ed into the server to conduct the install, - you'll automatically be included in the "staff" ACL and can access - all the web applications without editing these ACLs (or disabling - security as many does). - -- will set up sticky sessions/session binding -- will set up a backend cluster and allow the user to enter the - different backend servers that will serve the web site. -- will set up configuration to strip away cookies from static - resources, such as CSS, JS and pictures. -- will install the nginx web server for serving static content and - will configure Varnish accordingly. This will be very useful for - Adactus servers wanting to pull content from your ECEs. - -TBD: -- If run on a Linux platform, the script will tweak the kernel - parameters for optimal TCP handling for a web facing server. -- let the /munin run through on port 80, requiring the connecting IPs - to be in the staff network ACL, defined in the Varnish - configuration. - -5.8 Profile - Install Widget Framework -======================================= -You'll need a user name and password for accessing the -repo.escenic.com Maven repository. You should get these credentials -when you bought Widget Framework. If you for some reason do not have -these, please contact support@escenic.com. - -If you don't have these ready in your .ece-install.conf, ece-install will -complain: - - - [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf - [ece-install] If you don't have these, please contact support@escenic.com - -5.9 Profile - Create Publication -================================= -This profile will create a publication for you, only asking you the -name of the publication and which ECE instance to use for its -creation. - -This installation profile will base the publication on the Widget -Framework if its present on the system, if not, ECE's clean demo WAR -is used as a basis. - -5.10 Profile - Monitoring Server -================================= -This will install a Munin gatherer and web server. The latter for -accessing the reports generated by the former. - -TBD: This profile will also install the Nagios interface for -monitoring the different nodes. - -5.11 Profile - Restoring from backup -===================================== -ece-install can restore from a backup made by the ece script: - - - $ ece -i backup - - -As stated in the ece guide, this backup may contain (depending on -what's available on the host where it's run): -- database dump of the instance -- data files (pictures, video files and so on). This is often referred - to as the "multimedia archive" in Escenic literature. -- ECE, cache and web server configuration -- Escenic software - -5.12 Running interactively -=========================== - -5.12.1 Start ece-install and choose the Option, "Restore from backup" -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - # ece-install - - -5.12.2 Select Which Backup to Restore -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - - [ece-install-5] From which dataset do you wish to backup? - 1 - engine-dev1-backup-2011-10-10.tar.gz - 2 - engine-dev1-backup-2011-10-11.tar.gz - [ece-install-5] Enter the number next to the tarball, from 1 to 2 - Your choice [1]> - - -The ece script mentioned above will create backups in - - - /var/backups/escenic - -and the ece-install script will hence expect to find them here. - -5.12.3 Choose What to Restore -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - [ece-install-12] Which part of the system do you wish to restore? - 1 - The database - 2 - The Solr and ECE data files (multimedia archive) - 3 - The ECE configuration files - 4 - The Escenic and Tomcat software binaries + publication templates - Your choice [1]> 2 - - -5.12.4 Sit Back and Watch ece-install Restore the Data for You -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - [ece-install-13] Restoring the Solr & ECE data files on ubiquitous ... - [ece-install-21] - The installation is now complete! It took 0d 0h 0m 20s - [ece-install] Successfully restored Solr & ECE data files - [ece-install] from backup: engine-dev1-backup-2011-10-12.tar.gz - [ece-install] Check /var/lib/escenic to verify they're all there - - -5.13 Running in FAI mode -========================= -If you're running in FAI mode, you can choose between these settings -to decide what to restore and where to find the backup file to -restore from: - - Parameter Default Description - ----------------------------------+---------+---------------------------------------------------- - fai\_restore\_all 0 Restore everything found in the backup file - fai\_restore\_db 0 Install the DB server & restore its contents - fai\_restore\_data\_files 0 Restore the Solr & ECE data files - fai\_restore\_configuration 0 Restore the Solr & ECE configuration files - fai\_restore\_software\_binaries 0 Restore the Escenic and Apache Tomcat software - fai\_restore\_from\_file "" The .tar.gz produced by "ece -i backup" - -For example, to restore everything possible from a given tarball, you -need this in your .ece-install.conf: - - - fai_enabled=1 - - fai_restore_from_backup=1 - fai_restore_all=1 - fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz - - -5.14 Data security -=================== -You should take heed when running restore, so that you're not -restoring a system which you didn't want to change (yes, this mishap -does happen). - -The ece-install script will help you a bit on the way, but the final -responsibility always lies with you as the user. - -If you're trying to restore the DB and the ECE schema already exists, -the restore will fail: - - - [ece-install-8] Restoring the database contents on ubiquitous ... - [ece-install-24] Selecting the most recent database dump ece5db-2011-10-10.sql.gz - ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists - ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exists - [ece-install-24] The command [restoring from var/backups/escenic/ece5db-2011-10-10.sql.gz] FAILED, exiting :-( - [ece-install-24] See /var/log/ece-install.log for further details. - -6 Full Automatic Install (FAI) -------------------------------- -The ece-install script has support for doing a full automatic install -(FAI). You can only install one profile at a time. The profiles are -the parameters with "install" in their name, such as: - - - fai_editor_install - - -6.1 Overview of All FAI Parameters -=================================== -The ece-install script understands for the following settings in the -$HOME/.ece-install.conf file of the root user: - - Parameter Default Description - ----------------------------------+------------------+----------------------------------------------- - fai\_all\_install 0 Install all components on your server. - fai\_cache\_install 0 Install cache server profile - fai\_cache\_backends ${HOSTNAME}:8080 Space separated, e.g. "app1:8080 app2:8080" - fai\_db\_install 0 Install db profile - fai\_db\_host $HOSTNAME Useful for editor & presentation profiles - fai\_db\_port 3306 Useful for editor & presentation profiles - fai\_editor\_install 0 Install the editorial profile - fai\_editor\_name editor1 Name of the editor instance - fai\_editor\_port 8080 HTTP port of the editor instance - fai\_editor\_shutdown 8005 Shutdown port of the editor instance - fai\_enabled 0 Whether or not to run ece-install in FAI mode - fai\_public\_host\_name ${HOSTNAME}:8080 The public address for your website - fai\_presentation\_install 0 Install the presentation server profile - fai\_monitoring\_server\_install 0 Install the monitoring server profile. - fai\_monitoring\_server\_ip 127.0.0.1 The IP of the monitoring server. - fai\_presentation\_name web1 Name of the presentation server instance - fai\_presentation\_port 8080 HTTP port of the presentation server instance - fai\_presentation\_shutdown 8005 Shutdown port of the presentation instance - fai\_publication\_create 0 Create a new publication - fai\_publication\_name mypub Name of the publication - fai\_publication\_use\_instance dev1 Name of local instance to use for creation - fai\_rmi\_install 0 Install RMI hub profile - fai\_search\_name search1 Name of the search instance - fai\_search\_port 8080 HTTP port of the search instance - fai\_search\_shutdown 8005 Shutdown port of the search instance - fai\_wf\_install 0 Install Widget Framework profile - -As you've probably have guessed, 0 means "false" and 1 means "true" :-) - -6.2 Examples -============= - -6.2.1 Installing an Editorial Server & Create a Publication -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To automatically install an editorial server and create a publication -called "jollygood", the minimal configuration in .ece-install.conf -would be: - - - - fai_enabled=1 - fai_editor_install=1 - fai_publication_create=1 - fai_publication_name=jollygood - - -6.2.2 Installing Two Presentation Servers On a Single Host -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you wish to only install two presentation servers called "web1" and -"web2" on your host, you will first run ece-install with: - - - fai_enabled=1 - fai_presentation_install=1 - fai_presentation_name=web1 - - -And then re-issue ece-install with the following configuration: - - - fai_enabled=1 - fai_presentation_install=1 - fai_presentation_name=web2 - fai_presentation_port=8081 - fai_presentation_shutdown=8105 - -Notice that this configuration has some extra options since the -previous run of ece-install could run with the default settings, -whereas the second one needs to override these. - - -6.3 Verifying That the Script Is Running In FAI Mode -===================================================== -When FAI is enabled, ece-install will report: - - - [ece-install] Full Automatic Install (FAI) enabled. - [ece-install] All user input will be read from /root/.ece-install.conf - - -7 Running More Than One Installation Process ---------------------------------------------- -If the script believes there's already an ece-intall process running, -it will abort: - - - [ece-install] There's already one ece-install process running. If you believe - [ece-install] this is wrong, e.g. if a previous run of ece-install was aborted - [ece-install] before it completed, you may delete /var/run/ece-install.pid and - [ece-install] run ece-install again. - - -8 Re-running ece-install (and How To Speed It Up) --------------------------------------------------- -Although the initial thought behind ece-install, is to run it on a -clean system to get up and running as soon as possible. However, you -may want to re-run ece-install on the same host, for instance to add -another instance of ECE, set up Widget Framework or create another -publication. - -ece-install has a number of features which will try to minimise the -time it takes to run it on consecutive runs. If running on Debian -based systems, it will check if you already have installed -pre-requisite 3rd party libraries and only if any are missing will it -ask the package manager to fetch it. - -Likewise, ece-install will see if the Escenic artifacts or application -server that you need are already present in the /tmp/ece-downloads -folder, and only download the missing ones (if any). - -To get a list of the artifacts it'll pull from -[http://technet.escenic.com] and http://tomcat.apache.org search for the -following variables: -- technet\_download\_list -- wf\_download\_list -- tomcat\_download - -9 Overview of File Paths Used by the ece-install script --------------------------------------------------------- -There are of course other paths involved when setting up your system, -but these should be the most interesting ones. - - Path Explanation - ---------------------------------------------+------------------------------------------------------------------ - /etc/apt/sources.list.d/escenic.list 3rd party APT repositories added by ece-install *) - /etc/default/ece The configuration file for the ece init.d script - /etc/escenic/ece-.conf Instance specific settings for /usr/bin/ece - /etc/escenic/ece.conf Common ece.conf file for /usr/bin/ece - /etc/escenic/engine/common Common Nursery configuration layer - /etc/escenic/engine/common/security Common security configuration for all ECE instances. - /etc/escenic/engine/common/trace.properties Log4j configuration, produces instance specific log files. - /etc/escenic/engine/instance/ Instance specific Nursery configuration - /etc/escenic/solr ECE specific Solr configuration - /etc/init.d/mysql[d] For starting and stopping MySQL/Percona - /etc/init.d/varnish For starting and stopping Varnish - /etc/intit.d/ece The init.d script managing _all_ the ECE instances on your host. - /etc/varnish/default.vcl The Varnish configuration - /opt/escenic All ECE components can be found here - /opt/escenic/assemblytool The assembly tool - /opt/escenic/assemblytool/plugins Contains symlinks to all plugins in /opt/escenic - /opt/escenic/engine Symlink pointing to the current ECE - /opt/tomcat Symlink pointing to the install Apache Tomcat (catalina\_home) - /opt/tomcat- Instance specific Tomcat files (catalina\_base) - /usr/bin/ece Script for operating all ECE instances + RMI hub and EAE - /usr/sbin/drop-and-create-ecedb DB script used by the ece-install script - /usr/sbin/ece-install The installation script described in this guide - /var/log/escenic/-.log The instance's log4j log - /var/log/escenic/-.out The instance system out log - /var/log/escenic/solr..log The Solr log (not in standard out!) - /var/run/escenic/-.pid The instance's PID file - -*) Applies only to Debian based systems. - -10 Assumptions ---------------- - -10.1 /etc/esecenic is shared -============================= -It's assumed that the /etc/escenic directory is either on a shared -file system or rsync-ed among the hosts that are running the ECE -instances (the exception being database and cache servers). - - -11 Uninstalling Everything That the ece-install Set Up -------------------------------------------------------- -WARNING: this is potentially dangerous as some of these components may -be used by other pieces of software you have running on your -host. However, this may be useful if you're installing a clean -environment and want to e.g. undo your previous install to install a -different profile. - -Open the ece-install script and look for the "un\_install\_ece" -function, it has copy and pastable commands for undoing most/all -things set up by the script. - - -12 Example Output FAI ----------------------- -The ece-install will often be run in FAI mode where all the settings -are read from ~/.ece-install.conf. Given the following entries in this -file: - - - technet_user=apples - technet_password=andoranges - wf_user=foo - wf_password=bar - - fai_enabled=1 - fai_all_install=1 - - -All the different components described in this guide are installed on -the host that runs the install script. - -The following output is produced from ece-install: -CANNOT INCLUDE FILE ~/tmp/ece-install.out - From 7a804a5e9e0fcf5055734d870ab92df147176134 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 16:27:28 +0800 Subject: [PATCH 0140/2585] - added more logic to get_base_dir_from_bundle to do actually that with .zip files containing a different root directory than their file name. .tar.gz. and tar.bz2 files are still assumed to have the same root directory as their base file name. - this made it possible to add support for using trunk snapshots of the engine distribution. This is very handy for Escenic R&D and QA. Now, you can replace the URL to the official engine distribution on technet with the one from hudson, e.g.: http://hudson.dev.escenic.com/job/Engine-dist/ws/target/engine-trunk-SNAPSHOT-bin.zip instead of http://technet.escenic.com/downloads/release/53/engine-5.3.3.2.zip --- usr/sbin/ece-install | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8f530344..d4dc91ef 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -135,7 +135,7 @@ http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip http://technet.escenic.com/downloads/release/53/community-engine-3.6.1.0.zip http://technet.escenic.com/downloads/release/53/dashboard-1.0.0.0.zip -http://technet.escenic.com/downloads/release/53/engine-5.3.3.2.zip +http://hudson.dev.escenic.com/job/Engine-dist/ws/target/engine-trunk-SNAPSHOT-bin.zip http://technet.escenic.com/downloads/release/53/forum-3.0.0.0.zip http://technet.escenic.com/downloads/release/53/inpage-1.3.0.0.zip http://technet.escenic.com/downloads/release/53/lucy-dist-4.1.6.0.zip @@ -356,13 +356,15 @@ function set_up_engine_and_plugins() cd /opt/escenic/ for el in $technet_download_list; do - if [ $(basename $el | grep ^engine-[0-9] | wc -l) -gt 0 ]; then + if [ $(basename $el | \ + grep -E "^engine-[0-9|^engine-trunk-SNAPSHOT"] | wc -l) -gt 0 ]; then engine_dir=$(get_base_dir_from_bundle $el) + engine_file=$(basename $el) fi done if [ ! -d ${engine_dir} ]; then - run unzip -u $download_dir/${engine_dir}.zip + run unzip -u $download_dir/${engine_file} if [ -h engine ]; then run rm engine fi @@ -657,11 +659,26 @@ function install_ece_third_party_packages function get_base_dir_from_bundle() { - file_name=$(basename $1) - for el in .tar.gz .tar.bz2 .zip; do - file_name=${file_name%$el} - done - + local file_name=$(basename $1) + suffix=${file_name##*.} + + if [ ${suffix} = "zip" ]; then + # we'll look inside the archive to determine the base_dir + file_name=$( + unzip -t ${download_dir}/$file_name | \ + awk '{print $2}' | \ + cut -d'/' -f1 | \ + sort | \ + uniq | \ + grep -v errors | \ + grep [a-z] + ) + else + for el in .tar.gz .tar.bz2 .zip; do + file_name=${file_name%$el} + done + fi + echo $file_name } From f65690f89bf20f14e4a834fd7988898ce6263208 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 16:34:25 +0800 Subject: [PATCH 0141/2585] - added -f/--conf-file option to ece-install. Using this parameter, the callee can specify a configuration file for ece-install, overriding the default /root/.ece-install.conf --- usr/sbin/ece-install | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index d4dc91ef..29456508 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2523,6 +2523,11 @@ function common_post_install() for el in $@; do if [ $el = "-v" -o $el = "--verbose" ]; then debug=1 + elif [ $el = "-f" -o $el = "--conf-file" ]; then + next_is_conf_file=1 + elif [[ -n $next_is_conf_file && $next_is_conf_file -eq 1 ]]; then + conf_file=$el + next_is_conf_file=0 fi done From a3ff72433f0c6ae56ee0317f8d6ba9650d5dbf94 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 16:37:27 +0800 Subject: [PATCH 0142/2585] - updated the ECE bundle to version 5.3.4.3 --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 29456508..ac8a1efc 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -135,7 +135,7 @@ http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip http://technet.escenic.com/downloads/release/53/community-engine-3.6.1.0.zip http://technet.escenic.com/downloads/release/53/dashboard-1.0.0.0.zip -http://hudson.dev.escenic.com/job/Engine-dist/ws/target/engine-trunk-SNAPSHOT-bin.zip +http://technet.escenic.com/downloads/release/53/engine-5.3.4.3.zip http://technet.escenic.com/downloads/release/53/forum-3.0.0.0.zip http://technet.escenic.com/downloads/release/53/inpage-1.3.0.0.zip http://technet.escenic.com/downloads/release/53/lucy-dist-4.1.6.0.zip From e3ab87bf87c729cd68dea0daed830da43f03dabb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 16:43:39 +0800 Subject: [PATCH 0143/2585] - added FAI support for search instances: fai_search_install The full list of search FAI variables is then: fai_search_install fai_search_port fai_search_shutdown fai_search_name --- usr/sbin/ece-install | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index ac8a1efc..0008d57c 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1982,7 +1982,7 @@ EOF " Maven repository" } -function install_search_instance() +function install_search_server() { type=search install_ece_instance "search1" 0 @@ -2547,6 +2547,9 @@ if [ $fai_enabled -eq 1 ]; then elif [ $(get_boolean_conf_value fai_editor_install) -eq 1 ]; then install_profile_number=$PROFILE_EDITORIAL_SERVER install_editorial_server + elif [ $(get_boolean_conf_value fai_search_install) -eq 1 ]; then + install_profile_number=$PROFILE_SEARCH_SERVER + install_search_server elif [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then install_profile_number=$PROFILE_DB_SERVER install_database_server @@ -2602,7 +2605,7 @@ else install_presentation_server ;; $PROFILE_SEARCH_SERVER) - install_search_instance + install_search_server ;; $PROFILE_RMI_HUB) install_rmi_hub From f889d32f0f6c9a37789e99345d04b20790c2cb47 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 17:05:40 +0800 Subject: [PATCH 0144/2585] - hardened reading of conf file from the command line when running the script from a location not being in PATH (i.e. running /my/long/path/ece-install instead of just "ece-install"). - if there's no ece-install PID for some reason, there would be one error message per `print` output as get_seconds_since_start would query the PID file. This has now been fixed. - fixed remove_pid_and_exit_in_error when there would be no PID, for some abnormal reason. - hardened stop_conflicting_processes in case something prior to these (in get_conf_value particularly) have created phony values. --- usr/sbin/ece-install | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0008d57c..b7b98ace 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -58,9 +58,14 @@ next_steps=() function get_seconds_since_start() { - now=`date +%s` - started=`stat -c %Y $pid_file` - seconds=$(( now - started )) + local seconds="n/a" + + if [ -r $pid_file ]; then + now=`date +%s` + started=`stat -c %Y $pid_file` + seconds=$(( now - started )) + fi + echo "$seconds" } @@ -95,7 +100,10 @@ function print_and_log() function remove_pid_and_exit_in_error() { - rm $pid_file + if [ -e $pid_file ]; then + rm $pid_file + fi + exit 1 } @@ -1230,7 +1238,8 @@ function stop_conflicting_processes() { print "Stopping conflicting processes ..." # TODO this is dirty - if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then + if [[ -n "$install_profile_number" && \ + $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]]; then killall java 1>>$log 2>>$log fi } @@ -2527,6 +2536,10 @@ for el in $@; do next_is_conf_file=1 elif [[ -n $next_is_conf_file && $next_is_conf_file -eq 1 ]]; then conf_file=$el + if [ ! -e $conf_file ]; then + conf_file=$(pwd)/${conf_file} + fi + next_is_conf_file=0 fi done From ac5866ebcb1d3040e9c07fee3c0bd3b76547bbda Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 17:11:21 +0800 Subject: [PATCH 0145/2585] - added support for assembling search instances --- usr/bin/ece | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 35d5971d..6db7966b 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -907,8 +907,8 @@ function assemble() fi assemble_attempts=$(( $assemble_attempts + 1 )) - if [ ! $type = "engine" ]; then - print "You cannot assemble $type" + if [[ $type != "engine" && $type != "search" ]]; then + print "You cannot assemble instances of type $type" exit 1 fi From 142351b190e58b2c69b3d2e5e1a12c7d45e8aaf7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 17:22:48 +0800 Subject: [PATCH 0146/2585] - fixed (at least) one error in install_search_server where it wouldn't create the necessary directory for the Lucy Nursery configuration. - added information message about installing a search server, consistent with the other installation profiles. --- usr/sbin/ece-install | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index b7b98ace..37eb63e2 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1993,6 +1993,7 @@ EOF function install_search_server() { + print "Installing a search server on $HOSTNAME ..." type=search install_ece_instance "search1" 0 @@ -2004,7 +2005,7 @@ EOF # TODO update instead of append dir=$common_nursery_dir/com/escenic/framework/search/solr - make_dir dir + make_dir $dir echo "solrServerURI=http://${search_host}:${search_port}/solr" \ >> $dir/SolrSearchEngine.properties @@ -2014,16 +2015,17 @@ EOF >> $dir/DelegatingSearchEngine.properties dir=$common_nursery_dir/com/escenic/lucy + make_dir $dir echo "solrURI=http://${search_host}:${search_port}/solr" \ >> $dir/LucySearchEngine.properties dir=$common_nursery_dir/com/escenic/forum/search/lucy + make_dir $dir echo "solrURI=http://${search_host}:${search_port}/solr" \ >> $dir/SearchEngine.properties set_up_solr assemble_deploy_and_restart_type - } # useful for development and test environments. From 5e38e605494d3bcbbc947ac07b122aa18d0b9315 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 17:28:23 +0800 Subject: [PATCH 0147/2585] - for Escenic R&D and QA: support for engine distributions directly from the Maven repository. --- usr/sbin/ece-install | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 37eb63e2..3f9c1bb8 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -365,7 +365,8 @@ function set_up_engine_and_plugins() for el in $technet_download_list; do if [ $(basename $el | \ - grep -E "^engine-[0-9|^engine-trunk-SNAPSHOT"] | wc -l) -gt 0 ]; then + grep -E "^engine-[0-9|^engine-trunk-SNAPSHOT|^engine-dist"] | \ + wc -l) -gt 0 ]; then engine_dir=$(get_base_dir_from_bundle $el) engine_file=$(basename $el) fi From 5b2986c6e0894f8e35aa32b69e744f3d600702e3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 18:02:06 +0800 Subject: [PATCH 0148/2585] - set_up_engine_and_plugins: fixed error in grep range - experimental support for installing multiple profiles in one go (FAI mode). --- usr/sbin/ece-install | 59 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3f9c1bb8..ebd20cf9 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -365,7 +365,7 @@ function set_up_engine_and_plugins() for el in $technet_download_list; do if [ $(basename $el | \ - grep -E "^engine-[0-9|^engine-trunk-SNAPSHOT|^engine-dist"] | \ + grep -E "^engine-[0-9]|^engine-trunk-SNAPSHOT|^engine-dist" | \ wc -l) -gt 0 ]; then engine_dir=$(get_base_dir_from_bundle $el) engine_file=$(basename $el) @@ -2556,43 +2556,78 @@ if [ $fai_enabled -eq 1 ]; then print "All user input will be read from $conf_file" common_pre_install + + no_fai_profile=1 if [ $(get_boolean_conf_value fai_all_install) -eq 1 ]; then install_profile_number=$PROFILE_ALL_IN_ONE install_all_in_one_environment - elif [ $(get_boolean_conf_value fai_editor_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_editor_install) -eq 1 ]; then install_profile_number=$PROFILE_EDITORIAL_SERVER install_editorial_server - elif [ $(get_boolean_conf_value fai_search_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_search_install) -eq 1 ]; then install_profile_number=$PROFILE_SEARCH_SERVER install_search_server - elif [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then install_profile_number=$PROFILE_DB_SERVER install_database_server - elif [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then install_profile_number=$PROFILE_CACHE_SERVER install_cache_server install_web_server 0 - elif [ $(get_boolean_conf_value fai_rmi_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_rmi_install) -eq 1 ]; then install_profile_number=$PROFILE_RMI_HUB install_rmi_hub - elif [ $(get_boolean_conf_value fai_wf_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_wf_install) -eq 1 ]; then install_profile_number=$PROFILE_WIDGET_FRAMEWORK install_widget_framework - elif [ $(get_boolean_conf_value fai_presentation_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_presentation_install) -eq 1 ]; then install_profile_number=$PROFILE_PRESENTATION_SERVER install_presentation_server - elif [ $(get_boolean_conf_value fai_publication_create) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_publication_create) -eq 1 ]; then install_profile_number=$PROFILE_CREATE_PUBLICATION create_publication - elif [ $(get_boolean_conf_value fai_monitoring_install) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_monitoring_install) -eq 1 ]; then install_profile_number=$PROFILE_MONITORING_SERVER install_munin_gatherer install_web_server 1 - elif [ $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then install_profile_number=$PROFILE_RESTORE_FROM_BACKUP restore_from_backup - else + no_fai_profile=0 + fi + + if [ $no_fai_profile -eq 1 ]; then print "No install profile selected, be sure to have one of the " print "fai__install=1 in your $conf_file" remove_pid_and_exit_in_error From 8ca5b6b8d60cc7f522a5e79a0a54d6339c9377a0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 11 Nov 2011 18:29:51 +0800 Subject: [PATCH 0149/2585] - introduced wget_opts to carry the common wget options - added --quiet to the wget options. --- usr/sbin/ece-install | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index ebd20cf9..259abc0b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -37,6 +37,7 @@ conf_file=$HOME/.ece-install.conf common_nursery_dir=/etc/escenic/engine/common ece_scripts_git_source=git://github.com/skybert/ece-scripts.git maven_opts="--batch-mode" +wget_opts="--continue --inet4-only --quiet" # globals will be set to correct values in run-time. appserver_port=8080 @@ -231,8 +232,7 @@ function download_escenic_components() continue fi - run wget --continue \ - --inet4-only \ + run wget $wget_opts \ --http-user $technet_user \ --http-password $technet_password \ $el @@ -784,7 +784,7 @@ function set_up_app_server fi run cd $download_dir - run wget --continue --inet4-only $tomcat_download + run wget $wget_opts $tomcat_download tomcat_dir=$(get_base_dir_from_bundle $tomcat_download) run cd /opt/ @@ -1808,7 +1808,7 @@ function create_publication() remove_pid_and_exit_in_error fi - print "Getting ready to create a new publiation ..." + print "Getting ready to create a new publication ..." create_publication_definition_and_war # TODO make educated guesses about the available instances on @@ -1959,8 +1959,7 @@ EOF print "Downloading Widget Framework from technet.escenic.com ..." for el in $wf_download_list; do cd $download_dir - wget --continue \ - --inet4-only \ + wget $wget_opts \ --http-user $technet_user \ --http-password $technet_password \ $el \ @@ -2247,9 +2246,7 @@ EOF # install the escenic_jstat munin plugin file=/usr/share/munin/plugins/escenic_jstat_ - run wget \ - --continue \ - --inet4-only \ + run wget $wget_opts \ https://github.com/mogsie/escenic-munin/raw/master/escenic_jstat_ \ -O $file run chmod 755 $file From cdda6e2be935da9ae8e02d79810dd46bc5f5dd2b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 16 Nov 2011 11:57:45 +0800 Subject: [PATCH 0150/2585] - If the user didn't specify which instance to use and if there's only one available instance, use that instead of asking the user to provide it in sanity_check. This means that if there's only one instance available in /etc/escenic/engine/instance, the ece script will set it itself. E.g. when /etc/escenic/engine/instance/ has only editor1, ece status and ece -i editor1 status will be equivalent. --- usr/bin/ece | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index 6db7966b..6582c501 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -437,7 +437,8 @@ function sanity_check() $type = "engine" -a \ $(echo $command | grep help | wc -l) -eq 0 ]; then instance_list=$(get_instance_list) - if [ -n "$instance_list" ]; then + if [[ -n "$instance_list" && \ + $(echo $instance_list | grep ' ' | wc -l) -gt 0 ]]; then print "You must specify the instance with -i " print "e.g. ece -i $(echo $instance_list | cut -d' ' -f1) $command" print "Instances available on $HOSTNAME: $instance_list" @@ -1144,6 +1145,18 @@ function set_type_command_and_instance() # the only thing left at this point, is the command command="$command $el" done + + # If the user didn't specify which instance to use and if there's + # only one available instance, use that instead of asking the user + # to provide it in sanity_check. + if [ $instance = "default" ]; then + local instance_list=$(get_instance_list) + + if [ $(echo $instance_list | grep ' ' | wc -l) -eq 0 ]; then + instance=$instance_list + debug "setting instance=$instance as there's only one" + fi + fi } function clean_up() From edd61d6b89f9a2865840133e8d0e5d3a1383ae1d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 18 Nov 2011 11:36:06 +0800 Subject: [PATCH 0151/2585] - removed TODO about clear strategy when to run "clean" when doing a new assembly as ece now takes care of this for us. --- usr/sbin/ece-install | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 259abc0b..3dbb9ca8 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1732,10 +1732,11 @@ function install_ece_instance() ask_for_instance_name $1 download_escenic_components check_for_required_downloads - + set_up_engine_directories set_up_engine_and_plugins set_up_assembly_tool + set_up_basic_nursery_configuration set_up_instance_specific_nursery_configuration @@ -1778,11 +1779,6 @@ function assemble_deploy_and_restart_type() { print "Assembling, deploying & starting $instance_name ..." - # TODO clear strategy on when to add 'clean' to the command list - # For now, we'll omit it to make the install script run as fast as - # possible. It should in most cases work just fine, the exception - # being if the script is to cater for ECE/plugin upgrades and - # removal of existing publications. ece_command="ece -i $instance_name -t $type assemble deploy restart" su - $ece_user -c "$ece_command" 1>>$log 2>>$log exit_on_error "su - $ece_user -c \"$ece_command\"" From 9362ce27b6acc2d1e494da076be84d579527816d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 21 Nov 2011 18:58:11 +0800 Subject: [PATCH 0152/2585] - added support for bundle based installation (EAR + configuration bundle), drastically speeding up installation of ECE instances. The two new options are: fai__ear fai__conf_archive e.g.: fai_presentation_ear=/tmp/engine.ear fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz You can see in the ece-install output that it's using the supplied archives: [ece-install-4] Setting up the basic Nursery configuration ... [ece-install-4] Using the supplied Nursery & JAAS configuration from bundle: [ece-install-4] /tmp/nursery-skeleton-and-security.tar.gz [..] [ece-install-6] Assembling, deploying & starting pres1 ... [ece-install-7] Using the supplied EAR instead of running an assembly. - new method: verify_that_files_exist_and_are_readable which accepts passing multiple files/directories --- usr/sbin/ece-install | 96 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3dbb9ca8..9a23afa5 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -33,7 +33,7 @@ java_home=/usr/lib/jvm/java-6-sun pid_file=/var/run/$(basename $0).pid download_dir=/tmp/ece-downloads log=/var/log/$(basename $0).log -conf_file=$HOME/.ece-install.conf +conf_file=$HOME/ece-install.conf common_nursery_dir=/etc/escenic/engine/common ece_scripts_git_source=git://github.com/skybert/ece-scripts.git maven_opts="--batch-mode" @@ -469,15 +469,26 @@ function get_asterixes() function set_up_basic_nursery_configuration() { print "Setting up the basic Nursery configuration ..." + + local escenic_home=/opt/escenic - run cp -r /opt/escenic/engine/siteconfig/config-skeleton/* \ + if [ $(is_installing_from_archives) -eq 1 ]; then + print "Using the supplied Nursery & JAAS configuration from bundle:" + print "$ece_instance_conf_archive" + local dir=$(mktemp -d) + escenic_home=$dir + run cd $dir + run tar xzf $ece_instance_conf_archive + fi + + run cp -r $escenic_home/engine/siteconfig/config-skeleton/* \ $common_nursery_dir/ - run cp -r /opt/escenic/engine/security/ \ + run cp -r $escenic_home/engine/security/ \ $common_nursery_dir/ make_dir /etc/escenic/engine/instance - for el in /opt/escenic/assemblytool/plugins/*; do + for el in $escenic_home/assemblytool/plugins/*; do if [ ! -d $el/misc/siteconfig/ ]; then continue fi @@ -1723,6 +1734,57 @@ function update_type_instances_to_start_up() fi } +ece_instance_ear_file="" +ece_instance_conf_archive="" + +function set_archive_files_depending_on_profile() +{ + if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + ece_instance_ear_file=$fai_presentation_ear + ece_instance_conf_archive=$fai_presentation_conf_archive + elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + ece_instance_ear_file=$fai_editor_ear + ece_instance_conf_archive=$fai_editor_conf_archive + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + ece_instance_ear_file=$fai_search_ear + ece_instance_conf_archive=$fai_search_conf_archive + fi +} + + +# Returns 1 if we're installing the ECE instances from a provided EAR +# file and tarball with Nursery & JAAS configuration. +# +function is_installing_from_archives() +{ + log install_profile_number=$install_profile_number + log ece_instance_ear_file=$ece_instance_ear_file + log ece_instance_conf_archive=$ece_instance_conf_archive + + if [[ -z "$ece_instance_ear_file" || -z "$ece_instance_conf_archive" ]]; then + echo 0 + fi + + echo 1 +} + +# verifies that the passed file(s) exist and are readable, depends on +# set_archive_files_depending_on_profile +function verify_that_files_exist_and_are_readable() +{ + debug "Verifying that the file(s) exist(s) and are readable: $@" + + for el in $@; do + if [ ! -e $el ]; then + print "The file" $el "doesn't exist. I will exit now." + remove_pid_and_exit_in_error + elif [ ! -r $el ]; then + print "The file" $el "isn't readable. I will exit now." + remove_pid_and_exit_in_error + fi + done +} + # $1= # $2= function install_ece_instance() @@ -1730,12 +1792,22 @@ function install_ece_instance() install_ece_third_party_packages ask_for_instance_name $1 - download_escenic_components - check_for_required_downloads - set_up_engine_directories - set_up_engine_and_plugins - set_up_assembly_tool + + set_archive_files_depending_on_profile + + # most likely, the user is _not_ installing from archives (EAR + + # configuration bundle), hence the false test goes first. + if [ $(is_installing_from_archives) -eq 0 ]; then + download_escenic_components + check_for_required_downloads + set_up_engine_and_plugins + set_up_assembly_tool + else + verify_that_files_exist_and_are_readable \ + $ece_instance_ear_file \ + $ece_instance_conf_archive + fi set_up_basic_nursery_configuration set_up_instance_specific_nursery_configuration @@ -1780,6 +1852,12 @@ function assemble_deploy_and_restart_type() print "Assembling, deploying & starting $instance_name ..." ece_command="ece -i $instance_name -t $type assemble deploy restart" + if [ $(is_installing_from_archives) -eq 1 ]; then + run cp $ece_instance_ear_file /var/cache/escenic/engine.ear + ece_command="ece -i $instance_name -t $type deploy restart" + print "Using the supplied EAR instead of running an assembly." + fi + su - $ece_user -c "$ece_command" 1>>$log 2>>$log exit_on_error "su - $ece_user -c \"$ece_command\"" } From 3189e2f04e8612da3a9cea71a0cca2ac1e9b856a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 21 Nov 2011 19:24:25 +0800 Subject: [PATCH 0153/2585] - added documentation for the "installation from EAR file" feature. --- usr/share/doc/escenic/ece-install-guide.org | 112 ++++++++++++++------ 1 file changed, 82 insertions(+), 30 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index b4d5b071..59f1266d 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -367,40 +367,92 @@ the parameters with "install" in their name, such as: fai_editor_install #+END_SRC +** Installing from EARs instead of Binaries +It is possible to get ece-install to use a supplied EAR and +configuration archive instead of using the files provided with the +Escenic Content Engine & plugins. + +The EAR to provide is the one you generate with: +#+BEGIN_SRC sh +$ ece -i assemble +#+END_SRC +Normally, the EAR will then be available in: +#+BEGIN_SRC sh +/var/cache/escenic/engine.ear +#+END_SRC + +The configuration bundle must contain: +#+BEGIN_SRC sh +engine/security +engine/siteconfig/bootstrap-skeleton +engine/siteconfig/config-skeleton +assemblytool/plugins//siteconfig +#+END_SRC + +A simple way to create this bundle, is to use a server which has the +assembly environment set up and then do: + +#+BEGIN_SRC sh +$ cd /opt/escenic/engine +$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \ + engine/security \ + engine/siteconfig/config-skeleton/ \ + engine/siteconfig/bootstrap-skeleton/ \ + `find -L assemblytool/plugins/ -name siteconfig` +#+END_SRC + +/tmp/nursery-skeleton-and-security.tar.gz should now have everything +you need. You can now configure your FAI installation to use these by, +e.g.: + +#+BEGIN_SRC sh +fai_presentation_ear=/tmp/engine.ear +fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz +#+END_SRC + +Corresponding configuration options are available for the other server +profiles, see the table below. + ** Overview of All FAI Parameters The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user: -|----------------------------------+------------------+-----------------------------------------------| -| Parameter | Default | Description | -|----------------------------------+------------------+-----------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|----------------------------------+------------------+-----------------------------------------------| +|----------------------------------+------------------+-----------------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+------------------+-----------------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|----------------------------------+------------------+-----------------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 24f3f8e06643421c112d0503bed32c0fdd7970bd Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 21 Nov 2011 19:24:48 +0800 Subject: [PATCH 0154/2585] - updated HTML --- usr/share/doc/escenic/ece-install-guide.html | 171 ++++++++++++++++--- 1 file changed, 144 insertions(+), 27 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index feb244ab..b24855ba 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -273,14 +273,15 @@

    Table of Contents

  • 6 Full Automatic Install (FAI)
  • 7 Running More Than One Installation Process
  • @@ -492,9 +493,9 @@

    5 Available Server Profiles
    Hi, which server profile do you wish to install?
     
    -   1 - All in one, full stack on one host, suitable for dev & test environments
    +   1 - All in one, full stack on one host, suitable for dev & test environments
        2 - Editorial (publication) server
    -   3 - Presentation server (ECE + memcached).
    +   3 - Presentation server (ECE + memcached).
        4 - Database server
        5 - Cache server (cache and web server)
        6 - RMI hub
    @@ -1030,9 +1031,83 @@ 

    6 Full Automatic Install (F
    -

    6.1 Overview of All FAI Parameters

    +

    6.1 Installing from EARs instead of Binaries

    +

    It is possible to get ece-install to use a supplied EAR and +configuration archive instead of using the files provided with the +Escenic Content Engine & plugins. +

    +

    +The EAR to provide is the one you generate with: +

    + + +
    $ ece -i <instance> assemble 
    +
    + +

    +Normally, the EAR will then be available in: +

    + + +
    /var/cache/escenic/engine.ear
    +
    + + +

    +The configuration bundle must contain: +

    + + +
    engine/security
    +engine/siteconfig/bootstrap-skeleton
    +engine/siteconfig/config-skeleton
    +assemblytool/plugins/<plugin>/siteconfig
    +
    + + +

    +A simple way to create this bundle, is to use a server which has the +assembly environment set up and then do: +

    + + + +
    $ cd /opt/escenic/engine
    +$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \
    +  engine/security \
    +  engine/siteconfig/config-skeleton/ \
    +  engine/siteconfig/bootstrap-skeleton/ \
    +  `find -L  assemblytool/plugins/ -name siteconfig`
    +
    + + +

    +/tmp/nursery-skeleton-and-security.tar.gz should now have everything +you need. You can now configure your FAI installation to use these by, +e.g.: +

    + + + +
    fai_presentation_ear=/tmp/engine.ear
    +fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz
    +
    + + +

    +Corresponding configuration options are available for the other server +profiles, see the table below. +

    +
    + +
    + +
    +

    6.2 Overview of All FAI Parameters

    +
    +

    The ece-install script understands for the following settings in the $HOME/.ece-install.conf file of the root user:

    @@ -1045,27 +1120,33 @@

    6.1 Overview of All FAI P fai_all_install0Install all components on your server. -fai_cache_install0Install cache server profile fai_cache_backends${HOSTNAME}:8080Space separated, e.g. "app1:8080 app2:8080" -fai_db_install0Install db profile +fai_cache_install0Install cache server profile fai_db_host$HOSTNAMEUseful for editor & presentation profiles +fai_db_install0Install db profile fai_db_port3306Useful for editor & presentation profiles +fai_editor_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration +fai_editor_ear""EAR to use instead of the Escenic binaries fai_editor_install0Install the editorial profile fai_editor_nameeditor1Name of the editor instance fai_editor_port8080HTTP port of the editor instance fai_editor_shutdown8005Shutdown port of the editor instance fai_enabled0Whether or not to run ece-install in FAI mode -fai_public_host_name${HOSTNAME}:8080The public address for your website -fai_presentation_install0Install the presentation server profile fai_monitoring_server_install0Install the monitoring server profile. fai_monitoring_server_ip127.0.0.1The IP of the monitoring server. +fai_presentation_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration +fai_presentation_ear""EAR to use instead of the Escenic binaries +fai_presentation_install0Install the presentation server profile fai_presentation_nameweb1Name of the presentation server instance fai_presentation_port8080HTTP port of the presentation server instance fai_presentation_shutdown8005Shutdown port of the presentation instance +fai_public_host_name${HOSTNAME}:8080The public address for your website fai_publication_create0Create a new publication fai_publication_namemypubName of the publication fai_publication_use_instancedev1Name of local instance to use for creation fai_rmi_install0Install RMI hub profile +fai_search_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration +fai_search_ear""EAR to use instead of the Escenic binaries fai_search_namesearch1Name of the search instance fai_search_port8080HTTP port of the search instance fai_search_shutdown8005Shutdown port of the search instance @@ -1081,16 +1162,16 @@

    6.1 Overview of All FAI P

    -
    -

    6.2 Examples

    -
    +
    +

    6.3 Examples

    +
    -
    -

    6.2.1 Installing an Editorial Server & Create a Publication

    -
    +
    +

    6.3.1 Installing an Editorial Server & Create a Publication

    +

    To automatically install an editorial server and create a publication called "jollygood", the minimal configuration in .ece-install.conf @@ -1110,9 +1191,9 @@

    6.2.1 Installing an Edi

    -
    -

    6.2.2 Installing Two Presentation Servers On a Single Host

    -
    +
    +

    6.3.2 Installing Two Presentation Servers On a Single Host

    +

    If you wish to only install two presentation servers called "web1" and "web2" on your host, you will first run ece-install with: @@ -1148,9 +1229,9 @@

    6.2.2 Installing Two Pr

    -
    -

    6.3 Verifying That the Script Is Running In FAI Mode

    -
    +
    +

    6.4 Verifying That the Script Is Running In FAI Mode

    +

    When FAI is enabled, ece-install will report:

    @@ -1344,8 +1425,44 @@

    12 Example Output FAI

    The following output is produced from ece-install: -CANNOT INCLUDE FILE ~/tmp/ece-install.out

    + + +
    [ece-install-0] Full Automatic Install (FAI) enabled.
    +[ece-install-0] All user input will be read from /root/ece-install.conf
    +[ece-install-0] I'm logging to /var/log/ece-install.log
    +[ece-install-1] Stopping conflicting processes ...
    +[ece-install-1] Installing 3rd party packages needed by ece-install ...
    +[ece-install-1] Setting up the ece UNIX scripts ...
    +[ece-install-2] Setting up the torstein user's UNIX environment ...
    +[ece-install-2] Installing a presentation server on quanah ...
    +[ece-install-2] Installing 3rd party packages needed by ECE ...
    +[ece-install-4] Setting up the basic Nursery configuration ...
    +[ece-install-4] Using the supplied Nursery & JAAS configuration
    +[ece-install-4] from the bundle: /tmp/nursery-skeleton-and-security.tar.gz
    +[ece-install-4] Setting up the application server ...
    +[ece-install-5] Setting up proper log4j & Java logging configuration ...
    +[ece-install-5] Setting correct permissions on ECE related directories ...
    +[ece-install-6] Assembling, deploying & starting pres1 ...
    +[ece-install-7] I'm using the supplied EAR instead of running an assembly.
    +[ece-install-24] Adding the ece init.d script to the default run levels ...
    +[ece-install-24] Installing a Munin node on quanah ...
    +[ece-install-30] The installation is now complete! It took 0d 0h 0m 30s
    +[ece-install-30] - New ECE instance pres1 installed.
    +[ece-install-30] - Admin interface: http://quanah:9980/escenic-admin/
    +[ece-install-30] - View installed versions with: ece -i pres1 versions
    +[ece-install-30] - Type 'ece help' to see all the options of this script
    +[ece-install-30] - Read its guide: /usr/share/doc/escenic/ece-guide.txt
    +[ece-install-30] - /etc/default/ece lists all instances started at boot time
    +[ece-install-30] - A Munin node has been installed on quanah
    +[ece-install-30] - Install info: /usr/share/doc/escenic/ece-install-guide.txt
    +[ece-install-30] - Guide books: http://documentation.vizrt.com/ece-5.3.html
    +[ece-install-30] Enjoy your time with Escenic Content Engine!
    +[ece-install-30] -Vizrt Online
    +
    +
    + +
    @@ -1354,7 +1471,7 @@

    12 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-10-20 18:36:20 CST + Date: 2011-11-21 19:24:36 CST

    From 6cb28b0dfb6469ace73e2e26675487557c6e9a7c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 21 Nov 2011 19:34:56 +0800 Subject: [PATCH 0155/2585] - .ece-install.conf -> ece-install.conf --- usr/share/doc/escenic/ece-install-guide.org | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 59f1266d..b25e0af0 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -52,21 +52,21 @@ On all other Unix like system, do: #+END_SRC You will need access to http://technet.escenic.com and put these -credentials in the root user's $HOME/.ece-install.conf, normally this -means /root/.ece-install.conf, the script will also tell you this if +credentials in the root user's $HOME/ece-install.conf, normally this +means /root/ece-install.conf, the script will also tell you this if you forget to provide such a configuration file: #+BEGIN_SRC sh -[ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-( +[ece-install] /root/ece-install.conf doesn't exist, I cannot live without it :-( #+END_SRC As for the technet credentials, the script will also tell you if it cannot find what it's looking for: #+BEGIN_SRC sh [ece-install] Be sure to set technet_user and technet_password -[ece-install] in /root/.ece-install.conf +[ece-install] in /root/ece-install.conf #+END_SRC -The minimum .ece-install.conf file is thus: +The minimum ece-install.conf file is thus: #+BEGIN_SRC sh technet_user= technet_password= @@ -239,10 +239,10 @@ repo.escenic.com Maven repository. You should get these credentials when you bought Widget Framework. If you for some reason do not have these, please contact support@escenic.com. -If you don't have these ready in your .ece-install.conf, ece-install will +If you don't have these ready in your ece-install.conf, ece-install will complain: #+BEGIN_SRC sh -[ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf +[ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf [ece-install] If you don't have these, please contact support@escenic.com #+END_SRC ** Profile - Create Publication @@ -332,7 +332,7 @@ restore from: |----------------------------------+---------+------------------------------------------------| For example, to restore everything possible from a given tarball, you -need this in your .ece-install.conf: +need this in your ece-install.conf: #+BEGIN_SRC conf fai_enabled=1 @@ -415,7 +415,7 @@ profiles, see the table below. ** Overview of All FAI Parameters The ece-install script understands for the following settings in the -$HOME/.ece-install.conf file of the root user: +$HOME/ece-install.conf file of the root user: |----------------------------------+------------------+-----------------------------------------------------| | Parameter | Default | Description | @@ -459,7 +459,7 @@ As you've probably have guessed, 0 means "false" and 1 means "true" :-) ** Examples *** Installing an Editorial Server & Create a Publication To automatically install an editorial server and create a publication -called "jollygood", the minimal configuration in .ece-install.conf +called "jollygood", the minimal configuration in ece-install.conf would be: #+BEGIN_SRC sh @@ -495,7 +495,7 @@ whereas the second one needs to override these. When FAI is enabled, ece-install will report: #+BEGIN_SRC sh [ece-install] Full Automatic Install (FAI) enabled. -[ece-install] All user input will be read from /root/.ece-install.conf +[ece-install] All user input will be read from /root/ece-install.conf #+END_SRC * Running More Than One Installation Process @@ -589,7 +589,7 @@ things set up by the script. * Example Output FAI The ece-install will often be run in FAI mode where all the settings -are read from ~/.ece-install.conf. Given the following entries in this +are read from ~/ece-install.conf. Given the following entries in this file: #+BEGIN_SRC sh technet_user=apples From 6288be28f3f872af805ba598a90c8073e18be3ee Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 21 Nov 2011 19:38:30 +0800 Subject: [PATCH 0156/2585] - added section on using a customer configuration file --- usr/share/doc/escenic/ece-install-guide.org | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index b25e0af0..74c773d9 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -532,6 +532,16 @@ following variables: - wf\_download\_list - tomcat\_download +Two other ways of speeding up the installation is (of course) to use +the backup/restore feature or install from a EAR and configuration +bundle, see the FAI section. + +* Using a Custom Configuration File for ece-install +You can specify a different configuration by using the -f parameter: +#+BEGIN_SRC sh +$ ece-install -f ece-install-presentation-server.conf +#+END_SRC + * Overview of File Paths Used by the ece-install script There are of course other paths involved when setting up your system, but these should be the most interesting ones. From 51c8e9632a7b5d5815e992fc0af732e388d163f3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 21 Nov 2011 19:39:05 +0800 Subject: [PATCH 0157/2585] - updated HTML --- usr/share/doc/escenic/ece-install-guide.html | 86 ++++++++++++-------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index b24855ba..8c58f4e7 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -286,14 +286,15 @@

    Table of Contents

  • 7 Running More Than One Installation Process
  • 8 Re-running ece-install (and How To Speed It Up)
  • -
  • 9 Overview of File Paths Used by the ece-install script
  • -
  • 10 Assumptions +
  • 9 Using a Custom Configuration File for ece-install
  • +
  • 10 Overview of File Paths Used by the ece-install script
  • +
  • 11 Assumptions
  • -
  • 11 Uninstalling Everything That the ece-install Set Up
  • -
  • 12 Example Output FAI
  • +
  • 12 Uninstalling Everything That the ece-install Set Up
  • +
  • 13 Example Output FAI
  • @@ -404,13 +405,13 @@

    4 Running the ece-install S

    You will need access to http://technet.escenic.com and put these -credentials in the root user's $HOME/.ece-install.conf, normally this -means root.ece-install.conf, the script will also tell you this if +credentials in the root user's $HOME/ece-install.conf, normally this +means /root/ece-install.conf, the script will also tell you this if you forget to provide such a configuration file:

    -
    [ece-install] /root/.ece-install.conf doesn't exist, I cannot live without it :-(
    +
    [ece-install] /root/ece-install.conf doesn't exist, I cannot live without it :-(
     
    @@ -421,12 +422,12 @@

    4 Running the ece-install S
    [ece-install] Be sure to set technet_user and technet_password
    -[ece-install] in /root/.ece-install.conf
    +[ece-install] in /root/ece-install.conf
     

    -The minimum .ece-install.conf file is thus: +The minimum ece-install.conf file is thus:

    @@ -769,12 +770,12 @@

    5.8 Profile - Install Wid these, please contact support@escenic.com.

    -If you don't have these ready in your .ece-install.conf, ece-install will +If you don't have these ready in your ece-install.conf, ece-install will complain:

    -
    [ece-install] Be sure to set wf_user and wf_password in /root/.ece-install.conf
    +
    [ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf
     [ece-install] If you don't have these, please contact support@escenic.com
     
    @@ -966,7 +967,7 @@

    5.13 Running in FAI mode

    For example, to restore everything possible from a given tarball, you -need this in your .ece-install.conf: +need this in your ece-install.conf:

    @@ -1109,7 +1110,7 @@

    6.2 Overview of All FAI P

    The ece-install script understands for the following settings in the -$HOME/.ece-install.conf file of the root user: +$HOME/ece-install.conf file of the root user:

    @@ -1174,7 +1175,7 @@

    6.3.1 Installing an Edi

    To automatically install an editorial server and create a publication -called "jollygood", the minimal configuration in .ece-install.conf +called "jollygood", the minimal configuration in ece-install.conf would be:

    @@ -1238,7 +1239,7 @@

    6.4 Verifying That the Sc
    [ece-install] Full Automatic Install (FAI) enabled.
    -[ece-install] All user input will be read from /root/.ece-install.conf
    +[ece-install] All user input will be read from /root/ece-install.conf
     
    @@ -1303,14 +1304,35 @@

    8 Re-running ece-install (a +

    +Two other ways of speeding up the installation is (of course) to use +the backup/restore feature or install from a EAR and configuration +bundle, see the FAI section. +

    -

    9 Overview of File Paths Used by the ece-install script

    +

    9 Using a Custom Configuration File for ece-install

    +

    You can specify a different configuration by using the -f parameter: +

    + + +
    $ ece-install -f ece-install-presentation-server.conf
    +
    + + +
    + +
    + +
    +

    10 Overview of File Paths Used by the ece-install script

    +
    +

    There are of course other paths involved when setting up your system, but these should be the most interesting ones.

    @@ -1359,16 +1381,16 @@

    9 Overview of File Paths Us

    -
    -

    10 Assumptions

    -
    +
    +

    11 Assumptions

    +
    -
    -

    10.1 /etc/esecenic is shared

    -
    +
    +

    11.1 /etc/esecenic is shared

    +

    It's assumed that the /etc/escenic directory is either on a shared file system or rsync-ed among the hosts that are running the ECE @@ -1379,9 +1401,9 @@

    10.1 /etc/esecenic is sh

    -
    -

    11 Uninstalling Everything That the ece-install Set Up

    -
    +
    +

    12 Uninstalling Everything That the ece-install Set Up

    +

    WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your @@ -1399,12 +1421,12 @@

    11 Uninstalling Everything

    -
    -

    12 Example Output FAI

    -
    +
    +

    13 Example Output FAI

    +

    The ece-install will often be run in FAI mode where all the settings -are read from ~/.ece-install.conf. Given the following entries in this +are read from ~/ece-install.conf. Given the following entries in this file:

    @@ -1471,7 +1493,7 @@

    12 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-11-21 19:24:36 CST + Date: 2011-11-21 19:38:40 CST

    From 93ea0e615232d5970b89a046f6b280aab468f2fa Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 24 Nov 2011 17:51:07 +0800 Subject: [PATCH 0158/2585] - fixed bug in is_installing_from_archives, for false tests, it would always return two values :-( - fixed exit behaviour (sanity check) to be in accordance with the message given. --- usr/sbin/ece-install | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9a23afa5..5a521c0c 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -38,6 +38,7 @@ common_nursery_dir=/etc/escenic/engine/common ece_scripts_git_source=git://github.com/skybert/ece-scripts.git maven_opts="--batch-mode" wget_opts="--continue --inet4-only --quiet" +apt_opts="--no-install-recommends" # globals will be set to correct values in run-time. appserver_port=8080 @@ -263,9 +264,9 @@ install_packages_if_missing() fi if [ $force_packages -eq 1 ]; then - run apt-get install --assume-yes --force-yes $@ + run apt-get install $apt_opts --assume-yes --force-yes $@ else - run apt-get install --assume-yes $@ + run apt-get install $apt_opts --assume-yes $@ fi fi } @@ -1452,7 +1453,7 @@ function assert_correct_runtime_environment() print "If you blelieve this is wrong, e.g. if a previous run of" print "$(basename $0) was aborted before it completed, you" print "may remove ${pid_file} and run $(basename $0) again." - remove_pid_and_exit_in_error + exit 1 else echo $BASHPID > $pid_file started=`stat -c %Y $pid_file` @@ -1763,6 +1764,7 @@ function is_installing_from_archives() if [[ -z "$ece_instance_ear_file" || -z "$ece_instance_conf_archive" ]]; then echo 0 + return fi echo 1 From 2a3c1a33a4a5b628f828ee14c155ef7222825e6c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 24 Nov 2011 20:07:13 +0800 Subject: [PATCH 0159/2585] - added notes on using sudo (in general on systems using sudo) & nohup (when running in FAI mode). --- usr/share/doc/escenic/ece-install-guide.org | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 74c773d9..f4f089ab 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -83,6 +83,13 @@ With the configuration file in place, you're now ready to run it: # bash ece-install [-v|--verbose] #+END_SRC +If your systmem uses sudo to provide access to the root user, we +recommend becoming root before running ece-install. To become root on +a system using sudo, do: +#+BEGIN_SRC sh +$ sudo su - +#+END_SRC + ece-install will try to "fail fast", exiting as soon as it detects an error during its execution: #+BEGIN_SRC sh @@ -367,6 +374,20 @@ the parameters with "install" in their name, such as: fai_editor_install #+END_SRC +When running in FAI mode, you probably want to redirect standard +output to a log file for easy reading later on: +#+BEGIN_SRC sh +# bash ece-install > ece-install.out & +#+END_SRC + +Alternatively, you may do: +#+BEGIN_SRC sh +# nohup bash ece-install > ece-install.out & +#+END_SRC +The "nohup" at the beginning and the ampersand at the end lets you log +out of the SSH/X session to your Linux/Unix box while the script keeps +running in the background. + ** Installing from EARs instead of Binaries It is possible to get ece-install to use a supplied EAR and configuration archive instead of using the files provided with the From edc2be293a8ba700205812cd13a04b6364d4d501 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 24 Nov 2011 20:13:04 +0800 Subject: [PATCH 0160/2585] - removed duplicate mention of sudo precautions. - added note to initial sudo usage comment on just using sudo --- usr/share/doc/escenic/ece-install-guide.org | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index f4f089ab..640362dd 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -46,9 +46,11 @@ To become root on Ubuntu based systems and Mac OS X, do: #+BEGIN_SRC sh $ sudo su #+END_SRC +Please note that just doing "sudo ece-install" may not work. + On all other Unix like system, do: #+BEGIN_SRC sh - $ su + $ su - #+END_SRC You will need access to http://technet.escenic.com and put these @@ -83,13 +85,6 @@ With the configuration file in place, you're now ready to run it: # bash ece-install [-v|--verbose] #+END_SRC -If your systmem uses sudo to provide access to the root user, we -recommend becoming root before running ece-install. To become root on -a system using sudo, do: -#+BEGIN_SRC sh -$ sudo su - -#+END_SRC - ece-install will try to "fail fast", exiting as soon as it detects an error during its execution: #+BEGIN_SRC sh From 0e28fc098208ba2f6ad094d22c40490666a09b2c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 24 Nov 2011 20:29:40 +0800 Subject: [PATCH 0161/2585] - fixed bug in create_publication_in_db where the Perl command HEAD was used to get the cookie from ECE. However, this Perl command isn't always installed, e.g. on a clean Debian Net install. Now, using "curl -I" instead, which is guaranteed to be installed at this point in the install script - added logging of most of the messages formerly only outputted to the user/system out. This makes it easier to get a complete account of everything that's happened in the install process. --- usr/sbin/ece-install | 191 ++++++++++++++++++++++--------------------- 1 file changed, 99 insertions(+), 92 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 5a521c0c..5969f7f5 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -39,6 +39,7 @@ ece_scripts_git_source=git://github.com/skybert/ece-scripts.git maven_opts="--batch-mode" wget_opts="--continue --inet4-only --quiet" apt_opts="--no-install-recommends" +curl_opts="--silent" # globals will be set to correct values in run-time. appserver_port=8080 @@ -199,14 +200,14 @@ function make_ln() if [ -e $1 -a ! -h $2 ]; then run ln -s $1 $2 elif [ ! -e $1 ]; then - print "Tried to make a symlink to $1, but it doesn't exist" + print_and_log "Tried to make a symlink to $1, but it doesn't exist" remove_pid_and_exit_in_error fi else if [ -e $1 -a ! -h $(basename $1) ]; then run ln -s $1 elif [ ! -e $1 ]; then - print "Tried to make a symlink to $1, but it doesn't exist" + print_and_log "Tried to make a symlink to $1, but it doesn't exist" remove_pid_and_exit_in_error fi fi @@ -225,7 +226,7 @@ function download_escenic_components() return fi - print "Downloading Escenic software from technet.escenic.com ..." + print_and_log "Downloading Escenic software from technet.escenic.com ..." cd $download_dir for el in $technet_download_list; do @@ -274,7 +275,7 @@ install_packages_if_missing() function install_common_os_packages() { - print "Installing 3rd party packages needed by $(basename $0) ..." + print_and_log "Installing 3rd party packages needed by $(basename $0) ..." if [ $on_debian_or_derivative -eq 1 ]; then # Ubuntu doesn't have git (!) but only git-core. @@ -297,7 +298,7 @@ make_dir $ece_directories function set_up_assembly_tool() { - print "Setting up the Assembly Tool ..." + print_and_log "Setting up the Assembly Tool ..." make_dir /opt/escenic/assemblytool/ cd /opt/escenic/assemblytool/ @@ -359,7 +360,7 @@ function set_up_engine_and_plugins() return fi - print "Setting up the Escenic Content Engine & its plugins ..." + print_and_log "Setting up the Escenic Content Engine & its plugins ..." make_dir /opt/escenic cd /opt/escenic/ @@ -402,7 +403,7 @@ function set_up_engine_and_plugins() function set_up_ece_scripts() { - print "Setting up the ece UNIX scripts ..." + print_and_log "Setting up the ece UNIX scripts ..." run cd $download_dir if [ -d ece-scripts ]; then @@ -418,7 +419,7 @@ function set_up_ece_scripts() function set_up_ecedb() { - print "Setting up the ECE database schema ..." + print_and_log "Setting up the ECE database schema ..." make_dir /opt/escenic/engine/plugins run cd /opt/escenic/engine/plugins @@ -469,13 +470,13 @@ function get_asterixes() function set_up_basic_nursery_configuration() { - print "Setting up the basic Nursery configuration ..." + print_and_log "Setting up the basic Nursery configuration ..." local escenic_home=/opt/escenic if [ $(is_installing_from_archives) -eq 1 ]; then - print "Using the supplied Nursery & JAAS configuration from bundle:" - print "$ece_instance_conf_archive" + print_and_log "Using the supplied Nursery & JAAS configuration from" + print_and_log "bundle: $ece_instance_conf_archive" local dir=$(mktemp -d) escenic_home=$dir run cd $dir @@ -560,7 +561,7 @@ function set_up_instance_specific_nursery_configuration() function set_up_proper_logging_configuration() { - print "Setting up proper log4j & Java logging configuration ..." + print_and_log "Setting up proper log4j & Java logging configuration ..." cat > $common_nursery_dir/trace.properties < /opt/escenic/assemblytool/publications/${publication_name}.properties </dev/null | wc -l) \ -lt 1 ]; then @@ -1193,7 +1194,7 @@ function set_up_user_enviornment() function set_up_solr() { - print "Setting up solr ..." + print_and_log "Setting up solr ..." if [ ! -d /etc/escenic/solr ]; then run cp -r /opt/escenic/engine/solr/conf /etc/escenic/solr fi @@ -1210,7 +1211,7 @@ function set_up_solr() # command later, though. function un_install_ece() { - print "Uninstalling ECE ..." + print_and_log "Uninstalling ECE ..." # TODO safety, warnings++ rm -rf /etc/escenic/ \ /opt/*tomcat* \ @@ -1237,6 +1238,7 @@ function un_install_ece() maven2 \ memcached \ munin* \ + nginx \ percona* \ varnish \ sun-java6-jdk \ @@ -1249,7 +1251,7 @@ function un_install_ece() function stop_conflicting_processes() { - print "Stopping conflicting processes ..." + print_and_log "Stopping conflicting processes ..." # TODO this is dirty if [[ -n "$install_profile_number" && \ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]]; then @@ -1259,7 +1261,7 @@ function stop_conflicting_processes() function set_up_varnish() { - print "Setting up Varnish to match your environment ..." + print_and_log "Setting up Varnish to match your environment ..." /etc/init.d/varnish stop 1>>$log 2>>$log # we need to swap standard err and standard out here as varnishd @@ -1436,7 +1438,7 @@ function read_user_input() fi if [ $(is_number $install_profile_number) -eq 0 ]; then - print "The profile number, $install_profile_number, is not a number" + print_and_log "Profile number, $install_profile_number, is not a number" remove_pid_and_exit_in_error fi } @@ -1444,7 +1446,7 @@ function read_user_input() function assert_correct_runtime_environment() { if [ $(whoami) != "root" ]; then - print "You must be root when running $(basename $0)" + print_and_log "You must be root when running $(basename $0)" remove_pid_and_exit_in_error fi @@ -1473,8 +1475,8 @@ function common_pre_install() run source $conf_file if [ -z "$technet_user" -o -z "$technet_password" ]; then - print "Be sure to set technet_user and technet_password " - print "in $conf_file" + print_and_log "Be sure to set technet_user and technet_password " + print_and_log "in $conf_file" remove_pid_and_exit_in_error fi @@ -1507,7 +1509,7 @@ function common_pre_install() function assert_pre_prequesite() { if [ $(which $1 | wc -l) -lt 1 ]; then - print "Please install $1 and then run $(basename $0) again." + print_and_log "Please install $1 and then run $(basename $0) again." remove_pid_and_exit_in_error fi } @@ -1526,10 +1528,10 @@ function add_apt_source() function install_cache_server() { - print "Installing a caching server on $HOSTNAME ..." + print_and_log "Installing a caching server on $HOSTNAME ..." if [ $on_debian_or_derivative -eq 1 ]; then - curl --silent \ + curl ${curl_opts} \ http://repo.varnish-cache.org/debian/GPG-key.txt \ 2>> $log | \ apt-key add - \ @@ -1597,7 +1599,7 @@ function add_next_step() # the ECE DB schema is not set up. function install_database_server() { - print "Installing the database server on $HOSTNAME ..." + print_and_log "Installing the database server on $HOSTNAME ..." source /usr/sbin/drop-and-create-ecedb @@ -1619,7 +1621,7 @@ function install_database_server() fi if [ $supported_code_name -eq 1 ]; then - print "Installing the Percona database ..." + print_and_log "Installing the Percona database ..." if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then gpg --keyserver hkp://keys.gnupg.net \ @@ -1652,9 +1654,10 @@ function install_database_server() install_packages_if_missing $packages force_packages=0 else - print "The Percona APT repsository doesn't have packages for your" - print "Debian (or derivative) version with code name $code_name. " - print "I will use vanilla MySQL instead." + print_and_log "The Percona APT repsository doesn't have packages" + print_and_log "for your Debian (or derivative) version with code" + print_and_log "name $code_name. " + print_and_log "I will use vanilla MySQL instead." packages="mysql-server mysql-client" install_packages_if_missing $packages @@ -1778,10 +1781,10 @@ function verify_that_files_exist_and_are_readable() for el in $@; do if [ ! -e $el ]; then - print "The file" $el "doesn't exist. I will exit now." + print_and_log "The file" $el "doesn't exist. I will exit now." remove_pid_and_exit_in_error elif [ ! -r $el ]; then - print "The file" $el "isn't readable. I will exit now." + print_and_log "The file" $el "isn't readable. I will exit now." remove_pid_and_exit_in_error fi done @@ -1820,7 +1823,7 @@ function install_ece_instance() # special treatment for presentation servers if [ $2 -eq $PROFILE_PRESENTATION_SERVER ]; then file=/etc/escenic/ece-${instance_name}.conf - print "Creating instance specific conf: $file ..." + print_and_log "Creating instance specific conf: $file ..." cat >> $file <>$log 2>>$log @@ -1879,12 +1882,12 @@ function create_publication() { if [ ! -e /opt/escenic/engine -o \ ! -e /opt/escenic/assemblytool ]; then - print "Please install ECE and an assembly environment before" - print "running this installation profile again." + print_and_log "Please install ECE and an assembly environment before" + print_and_log "running this installation profile again." remove_pid_and_exit_in_error fi - print "Getting ready to create a new publication ..." + print_and_log "Getting ready to create a new publication ..." create_publication_definition_and_war # TODO make educated guesses about the available instances on @@ -1916,7 +1919,7 @@ function create_publication() function install_editorial_server() { - print "Installing an editorial server on $HOSTNAME ..." + print_and_log "Installing an editorial server on $HOSTNAME ..." type=engine install_ece_instance "editor1" 0 } @@ -1944,7 +1947,7 @@ EOF add_next_step "Restart all your instances to make the hub see them." - print "Starting the RMI-hub on $HOSTNAME ..." + print_and_log "Starting the RMI-hub on $HOSTNAME ..." ece_command="ece -t rmi-hub restart" su - $ece_user -c "$ece_command" 1>>$log 2>>$log } @@ -1956,7 +1959,7 @@ EOF function get_conf_value() { if [ ! -e "$conf_file" ]; then - print $conf_file "doesn't exist." + print_and_log $conf_file "doesn't exist." remove_pid_and_exit_in_error fi @@ -1984,6 +1987,7 @@ function get_boolean_conf_value() function install_widget_framework() { + print_and_log "Installing Widget Framework on $HOSTNAME ..." # TODO java.lang.NoClassDefFoundError: # Lcom/escenic/framework/captcha/ReCaptchaConfig; @@ -1991,12 +1995,12 @@ function install_widget_framework() wf_password=$(get_conf_value wf_password) if [ -z "$wf_user" -o -z "$wf_password" ]; then - print "Missing wf_user and wf_password in $conf_file" - print "If you don't have these, please contact support@escenic.com" + print_and_log "Missing wf_user and wf_password in ${conf_file}. If you" + print_and_log "don't have these, please contact support@escenic.com" remove_pid_and_exit_in_error fi - print "Creating a Maven settings file: $HOME/.m2/settings.xml ..." + print_and_log "Creating a Maven settings file: $HOME/.m2/settings.xml ..." make_dir $HOME/.m2 cat > $HOME/.m2/settings.xml < @@ -2032,7 +2036,7 @@ function install_widget_framework() EOF - print "Downloading Widget Framework from technet.escenic.com ..." + print_and_log "Downloading Widget Framework from technet.escenic.com ..." for el in $wf_download_list; do cd $download_dir wget $wget_opts \ @@ -2047,10 +2051,13 @@ EOF done assert_pre_prequesite mvn - wf_maven_dir=$(echo /opt/escenic/widget-framework-core-*/maven) - cd $wf_maven_dir - print "Installing Widget Framework into your Maven repository ..." export JAVA_HOME=$java_home + wf_maven_dir=$(echo /opt/escenic/widget-framework-core-*/maven) + run cd $wf_maven_dir + + print_and_log "Installing Widget Framework into your Maven repository ..." + log "JAVA_HOME=$JAVA_HOME" + mvn $maven_opts install \ 1>>$log 2>>$log @@ -2069,12 +2076,12 @@ EOF function install_search_server() { - print "Installing a search server on $HOSTNAME ..." + print_and_log "Installing a search server on $HOSTNAME ..." type=search install_ece_instance "search1" 0 file=/etc/escenic/ece-${instance_name}.conf - print "Creating instance specific conf: $file ..." + print_and_log "Creating instance specific conf: $file ..." cat >> $file <> $file <_install=1 in your $conf_file" + print_and_log "No install profile selected, be sure to have one of the " + print_and_log "fai__install=1 in your $conf_file" remove_pid_and_exit_in_error fi From 3c1f510613d21329beb8125f8d5712286b4762c4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 24 Nov 2011 21:15:17 +0800 Subject: [PATCH 0162/2585] - fitting output onto standard terminal width --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 5969f7f5..94171339 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2070,8 +2070,8 @@ EOF cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ - add_next_step "Widget Framework has been installed into your local"\ -" Maven repository" + add_next_step "Widget Framework has been installed into your "\ +" Maven repo" } function install_search_server() From c93983b4c86850a0fbed05a3be9d2809f0df25ae Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 25 Nov 2011 12:06:30 +0800 Subject: [PATCH 0163/2585] - added get_info_for_type which provides the 'info' command giving the user information about the selected instance like this: $ ece info --instance editor2 [ece#engine-editor2] Current instance: editor2 [ece#engine-editor2] Instances available on quanah: dev1 editor2 pres1 [ece#engine-editor2] Conf files parsed: /etc/escenic/ece-editor2.conf /etc/escenic/ece.conf [ece#engine-editor2] ECE location: /opt/escenic/engine [ece#engine-editor2] Assembly Tool location: /opt/escenic/assemblytool [ece#engine-editor2] Java location: /usr/lib/jvm/java-6-sun [ece#engine-editor2] Application server: tomcat [ece#engine-editor2] Tomcat home: /opt/tomcat [ece#engine-editor2] Tomcat base: /opt/tomcat-editor2 --- etc/bash_completion.d/ece | 2 +- usr/bin/ece | 44 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index d34877d1..69431562 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -5,7 +5,7 @@ _ece_commands() local cur=${COMP_WORDS[COMP_CWORD]} local prev=${COMP_WORDS[COMP_CWORD-1]} - commands="applog assemble backup clean deploy edit help kill log loglist \ + commands="applog assemble backup clean deploy edit help info kill log loglist \ outlog restart start status stop threaddump update versions" options="-i --instance -p --publication -r --publication-resource \ -t --type -u --user -w --password" diff --git a/usr/bin/ece b/usr/bin/ece index 6582c501..91d88d69 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -154,12 +154,15 @@ function ensure_that_required_fields_are_set() fi } +ece_conf_files_read=() + function read_conf_file() { for el in $conf_file_location_list; do if [ -r $el/$1 ]; then debug "found $1 in $el, reading it" source $el/$1 + ece_conf_files_read=($el/$1 ${ece_conf_files_read}) break fi done @@ -435,7 +438,7 @@ function sanity_check() # system. if [ $instance = "default" -a \ $type = "engine" -a \ - $(echo $command | grep help | wc -l) -eq 0 ]; then + $(echo $command | egrep 'help' | wc -l) -eq 0 ]; then instance_list=$(get_instance_list) if [[ -n "$instance_list" && \ $(echo $instance_list | grep ' ' | wc -l) -gt 0 ]]; then @@ -872,6 +875,39 @@ function kill_type() exit_on_error "kill_type" } +function get_info_for_type() +{ + print "Current instance: ${instance}" + print "Instances available on $HOSTNAME: $(get_instance_list)" + print "Conf files parsed: ${ece_conf_files_read[@]}" + + if [ -n "${ece_home}" ]; then + print "ECE location: $ece_home" + fi + if [[ -n "${assemblytool_home}" && -e "${assemblytool_home}" ]]; then + print "Assembly Tool location: $assemblytool_home" + fi + if [ -n "${java_home}" ]; then + print "Java location: $java_home" + fi + if [ -n "${appserver}" ]; then + print "Application server: $appserver" + case "$appserver" in + tomcat) + if [ -n "${tomcat_home}" ]; then + print "Tomcat home:" $tomcat_home + print "Tomcat base:" $tomcat_base + fi + ;; + resin) + if [ -n "${resin_home}" ]; then + print "Resin home:" $resin_home + fi + ;; + esac + fi +} + function restart_type() { stop_type @@ -881,7 +917,7 @@ function restart_type() # properly before starting the new one. sleep 8 - # sometimes the JVM refuses to shot down when doing a graceful + # sometimes the JVM refuses to shut down when doing a graceful # kill, therefore, if it's still running after the sleep above, we # use brute force to kill it. set_type_pid @@ -1417,6 +1453,7 @@ function print_help() echo " deploy deploys the assembled EAR *)" echo " edit lets you edit a publication resource" echo " help prints this help screen" + echo " info prints various info about the selected ECE instance" echo " kill uses force to stop the type" echo " log the type's log4j log **)" echo " loglist list all the log paths" @@ -1447,6 +1484,9 @@ for el in $command; do restart) restart_type ;; + info) + get_info_for_type + ;; log) tail_messages_log ;; From bf4a2026b4d4704838b97028a20388569d09f8dc Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 25 Nov 2011 12:51:20 +0800 Subject: [PATCH 0164/2585] - removed any default instances, both engine and search instances. This to prevent confusion when installing a fresh system, getting errors at boot time (or when invoking the init.d) because the local instances don't correspond to the ones given in /etc/default/ece. --- etc/default/ece | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/default/ece b/etc/default/ece index a6983116..d021aaa4 100644 --- a/etc/default/ece +++ b/etc/default/ece @@ -4,8 +4,8 @@ # instances. If you only have one # instance, use "default" as its # name. If you don't have any instances of that type on this host, # then just set the variable to an empty string. -engine_instance_list="editor1 web1" -search_instance_list="search1" +engine_instance_list="" +search_instance_list="" analysis_instance_list="" # You only need one hub in an ECE cluster, so only set this one to From 4be921766831ad28854ab12610c3de4afd64eaf7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 28 Nov 2011 12:29:16 +0800 Subject: [PATCH 0165/2585] - removed get_asterixes, it hasn't been used since its inception and having it lingering around till it does is a waste of pixels. The git log has it. --- usr/sbin/ece-install | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 94171339..069fbe60 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -454,20 +454,6 @@ function set_up_ecedb() add_next_step "DB is now set up on $HOSTNAME:3306" } -# returns a string of asterixes with the same lenght as the inputted -# string. -function get_asterixes() -{ - s="" - j=$(echo $1 | wc -c) - - # wc -c always returns the length + 1, hence we start on index=1 - for (( i=1; i<$j; i++ )); do - s=${s}"-" - done - echo "$s" -} - function set_up_basic_nursery_configuration() { print_and_log "Setting up the basic Nursery configuration ..." From 06b0c7e4f25fe3ea62f38a1f973055cf4ef9f64e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 28 Nov 2011 16:19:24 +0800 Subject: [PATCH 0166/2585] - added "ece flush" to flush all the ECE caches. This is the same as going to /escenic-admin -> "Clear all caches" -> clicking on "Confirm". - added get_escenic_admin_url and made every method that needs the /escenic-admin call it. --- etc/bash_completion.d/ece | 2 +- usr/bin/ece | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index 69431562..8a8b6a4d 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -6,7 +6,7 @@ _ece_commands() local prev=${COMP_WORDS[COMP_CWORD-1]} commands="applog assemble backup clean deploy edit help info kill log loglist \ - outlog restart start status stop threaddump update versions" + flush outlog restart start status stop threaddump update versions" options="-i --instance -p --publication -r --publication-resource \ -t --type -u --user -w --password" resources="content-type feature layout layout-group image-version menu \ diff --git a/usr/bin/ece b/usr/bin/ece index 91d88d69..aa6b06a4 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1275,8 +1275,8 @@ function list_versions() set_type_port - version_manager=escenic-admin/browser/Global/neo/io/managers/VersionManager - url=http://$host:$port/$version_manager + version_manager=browser/Global/neo/io/managers/VersionManager + url=$(get_escenic_admin_url)/$version_manager print "Installed on the ${type} running on ${host}:${port} is:" wget $wget_auth -O - $url 2>/dev/null | \ @@ -1306,8 +1306,7 @@ function update_publication_resources() exit 1 fi - set_type_port - url=http://${host}:${port}/escenic-admin/publication-resources + local url=$(get_escenic_admin_url)/publication-resources case "$(basename $resource)" in content-type) @@ -1333,7 +1332,7 @@ function update_publication_resources() ;; root-section-parameters) do_put=true - url=http://${host}:${port}/escenic-admin + url=$(get_escenic_admin_url) url=${url}/section-parameters-declared/${publication} ;; *) @@ -1407,6 +1406,29 @@ function read_rc_file_if_present() fi } +function get_escenic_admin_url() +{ + set_type_port + local url=http://${host}:${port}/escenic-admin + echo ${url} +} + +function flush_caches() +{ + if [ ${type} != "engine" ]; then + print "You cannot flush the caches of a ${type} instance" + return + fi + + print "Flushing all of ${instance}'s caches on $HOSTNAME" + local url=$(get_escenic_admin_url)/do/publication/clearallcaches + wget $wget_auth \ + -O - \ + --post-data='confirm=Confirm' \ + $url \ + 1>>$log_file 2>>$log_file +} + read_rc_file_if_present set_type_command_and_instance $@ set_id @@ -1452,6 +1474,7 @@ function print_help() echo " clean removes temporary files created by $0 *)" echo " deploy deploys the assembled EAR *)" echo " edit lets you edit a publication resource" + echo " flush flushes all ECE caches ('Clear all caches') *)" echo " help prints this help screen" echo " info prints various info about the selected ECE instance" echo " kill uses force to stop the type" @@ -1527,6 +1550,9 @@ for el in $command; do clean_up backup_type ;; + flush) + flush_caches + ;; help) print_help ;; From bfc4d4cad8c3923b94e1ccb3be2a94acb9be4ab5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Dec 2011 13:38:47 +0800 Subject: [PATCH 0167/2585] - added mention on when the --instance/-i can be omitted and the importance of specifying it when using /usr/bin/ece in your own scripting. --- usr/share/doc/escenic/ece-guide.org | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index 76739185..514220d0 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -211,9 +211,7 @@ complain: As it mentions, the root user may use the init.d script and the accompanying /etc/default/ece to command the different ECE, EAE and RMI hub instances on your system. -** Specifying instance - -*** Specifying the instance +** Specifying the instance The script is made for being easy to use with multiple instances on the same host. You specify the instance you want to operate on using the -i parameter. E.g. to assemble and deploy the editor1 instance, you'd @@ -233,6 +231,12 @@ For this to work, the script assumes that you have installed the instance specific configuration in /etc/escenic/engine/instance, as is described in the Escenic Content Engine Installation Guide. +If you only have one instance installed, you may omit the +--instance/-i parameter. However, if you're running ece from another +script (e.g. an init.d or a deployment script), we recommend you to +always specify the --instance/-i parameter to make sure that future +additions of instances don't break your old scripts which assumed that +only one instance was installed. ** TAB completion The ece script offers TAB completion of all commands, options and From 7bfb081684f736bd2108f1784e435ae6412c45b7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Dec 2011 13:40:59 +0800 Subject: [PATCH 0168/2585] - avoid wiki stupidifcation --- usr/share/doc/escenic/ece-guide.org | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index 514220d0..3bd8cca3 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -214,7 +214,7 @@ RMI hub instances on your system. ** Specifying the instance The script is made for being easy to use with multiple instances on the same host. You specify the instance you want to operate on using the --i parameter. E.g. to assemble and deploy the editor1 instance, you'd + -i parameter. E.g. to assemble and deploy the editor1 instance, you'd do: #+BEGIN_SRC sh $ ece -i editor1 assemble deploy restart @@ -232,7 +232,7 @@ instance specific configuration in /etc/escenic/engine/instance, as is described in the Escenic Content Engine Installation Guide. If you only have one instance installed, you may omit the ---instance/-i parameter. However, if you're running ece from another + --instance/-i parameter. However, if you're running ece from another script (e.g. an init.d or a deployment script), we recommend you to always specify the --instance/-i parameter to make sure that future additions of instances don't break your old scripts which assumed that From d1237ac1f7828eaa9d53e3345cb878520400de73 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Dec 2011 13:41:52 +0800 Subject: [PATCH 0169/2585] - argh!, org/github/wiki --- usr/share/doc/escenic/ece-guide.org | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usr/share/doc/escenic/ece-guide.org b/usr/share/doc/escenic/ece-guide.org index 3bd8cca3..ebfb7676 100644 --- a/usr/share/doc/escenic/ece-guide.org +++ b/usr/share/doc/escenic/ece-guide.org @@ -213,8 +213,8 @@ accompanying /etc/default/ece to command the different ECE, EAE and RMI hub instances on your system. ** Specifying the instance The script is made for being easy to use with multiple instances on -the same host. You specify the instance you want to operate on using the - -i parameter. E.g. to assemble and deploy the editor1 instance, you'd +the same host. You specify the instance you want to operate on using +the -i parameter. E.g. to assemble and deploy the editor1 instance, you'd do: #+BEGIN_SRC sh $ ece -i editor1 assemble deploy restart @@ -231,8 +231,8 @@ For this to work, the script assumes that you have installed the instance specific configuration in /etc/escenic/engine/instance, as is described in the Escenic Content Engine Installation Guide. -If you only have one instance installed, you may omit the - --instance/-i parameter. However, if you're running ece from another +If you only have one instance installed, you may omit +the --instance/-i parameter. However, if you're running ece from another script (e.g. an init.d or a deployment script), we recommend you to always specify the --instance/-i parameter to make sure that future additions of instances don't break your old scripts which assumed that From 4eb18b4c81cd040b31d212932c25653b8ed9637e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Dec 2011 10:55:48 +0800 Subject: [PATCH 0170/2585] - added configuration option fai_publication_war so that the user can at install time provide the WAR to base the new publication on. Default is still the WF demo (if WF installed) or the demo shipped with the ECE distribution. - fixed bug in set_up_user_enviornment where the BASH auto completion of ece wasn't actually written to the user's .bashrc --- usr/sbin/ece-install | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 069fbe60..32b833d3 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1095,7 +1095,10 @@ context-root: ${publication_name} EOF publication_war=/opt/escenic/assemblytool/publications/${publication_name}.war - if [ -d /opt/escenic/widget-framework-core-* ]; then + if [[ $fai_enabled -eq 1 && -n "${fai_publication_war}" ]]; then + print_and_log "Basing ${publication_name}.war on the one specified in $conf_file" + run cp ${fai_publication_war} ${publication_war} + elif [ -d /opt/escenic/widget-framework-core-* ]; then print_and_log "Basing ${publication_name}.war on Widget Framework Demo ..." run cd /opt/escenic/widget-framework-core-*/publications/demo-core run mvn $maven_opts package @@ -1172,9 +1175,10 @@ function set_up_user_enviornment() export JAVA_HOME=$java_home - if [ $(grep bash_completion.d/ece /home/$ece_user/.bashrc 2>/dev/null | \ + bashrc=/home/${ece_user}/.bashrc + if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ wc -l) -lt 1 ]; then - run echo ". /etc/bash_completion.d/ece" + echo ". /etc/bash_completion.d/ece" > ${bashrc} fi } From 84e22c430a8912ffc429204168277f193f910c30 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Dec 2011 11:06:52 +0800 Subject: [PATCH 0171/2585] - added fai_publication_war to the list of available FAI options. --- usr/share/doc/escenic/ece-install-guide.org | 73 +++++++++++---------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 640362dd..e248c140 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -433,42 +433,43 @@ profiles, see the table below. The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: -|----------------------------------+------------------+-----------------------------------------------------| -| Parameter | Default | Description | -|----------------------------------+------------------+-----------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|----------------------------------+------------------+-----------------------------------------------------| +|----------------------------------+----------------------+-----------------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+----------------------+-----------------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to to base the new publication on | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|----------------------------------+----------------------+-----------------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 9fe5ed916c0644de6949691e601a1fa58ec31c66 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Dec 2011 11:07:14 +0800 Subject: [PATCH 0172/2585] - updated HTML --- usr/share/doc/escenic/ece-install-guide.html | 33 ++++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 8c58f4e7..4d455d22 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -394,12 +394,15 @@

    4 Running the ece-install S
    $ sudo su
     
    +

    +Please note that just doing "sudo ece-install" may not work. +

    On all other Unix like system, do:

    -
    $ su
    +
    $ su -
     
    @@ -1028,6 +1031,29 @@

    6 Full Automatic Install (F

    +

    +When running in FAI mode, you probably want to redirect standard +output to a log file for easy reading later on: +

    + + +
    # bash ece-install > ece-install.out &
    +
    + + +

    +Alternatively, you may do: +

    + + +
    # nohup bash ece-install > ece-install.out &
    +
    + +

    +The "nohup" at the beginning and the ampersand at the end lets you log +out of the SSH/X session to your Linux/Unix box while the script keeps +running in the background. +

    @@ -1145,6 +1171,7 @@

    6.2 Overview of All FAI P

    + @@ -1493,7 +1520,7 @@

    13 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-11-21 19:38:40 CST + Date: 2011-12-02 11:07:01 CST

    From 6f459b257f257412f19e7e7c635e65d736a76faf Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 5 Dec 2011 15:42:17 +0800 Subject: [PATCH 0173/2585] - now tests to see if the CommunityEngine.properties file is present in /etc/escenic, if not, an informative error message is given: [ece-install#23] Could not find/write to /etc/escenic/engine/common/com/escenic/community/CommunityEngine.properties" [ece-install#23] Have you installed the Community Engine plugin?" --- usr/sbin/ece-install | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 32b833d3..f4fcff1d 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -511,8 +511,14 @@ updateConnector=/connector/UpdateConnector EOF file=$common_nursery_dir/com/escenic/community/CommunityEngine.properties - sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file - exit_on_error "sed on $file" + if [ -w ${file} ]; then + sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file + exit_on_error "sed on $file" + else + print_and_log "Could not find/write to ${file}," + print_and_log "have you installed the Community Engine plugin?" + remove_pid_and_exit_in_error + fi file=$common_nursery_dir/com/escenic/webstart/StudioConfig.properties cat >> $file < Date: Mon, 5 Dec 2011 16:20:48 +0800 Subject: [PATCH 0174/2585] - since ece-install now is supporting installing multiple profiles at the same time (from the same invocation of ece-install), the install of the DB profile will be executed first and the RMI hub last. This way, it's possible to define both installing a DB and an ECE instance like the editorial server profile in the same ece-install.conf: fai_enabled=1 fai_db_install=1 fai_editor_install=1 --- usr/sbin/ece-install | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index f4fcff1d..fee4dcac 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2641,6 +2641,12 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi + if [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then + install_profile_number=$PROFILE_DB_SERVER + install_database_server + no_fai_profile=0 + fi + if [ $(get_boolean_conf_value fai_editor_install) -eq 1 ]; then install_profile_number=$PROFILE_EDITORIAL_SERVER install_editorial_server @@ -2653,12 +2659,6 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi - if [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then - install_profile_number=$PROFILE_DB_SERVER - install_database_server - no_fai_profile=0 - fi - if [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then install_profile_number=$PROFILE_CACHE_SERVER install_cache_server @@ -2666,12 +2666,6 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi - if [ $(get_boolean_conf_value fai_rmi_install) -eq 1 ]; then - install_profile_number=$PROFILE_RMI_HUB - install_rmi_hub - no_fai_profile=0 - fi - if [ $(get_boolean_conf_value fai_wf_install) -eq 1 ]; then install_profile_number=$PROFILE_WIDGET_FRAMEWORK install_widget_framework @@ -2703,6 +2697,12 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi + if [ $(get_boolean_conf_value fai_rmi_install) -eq 1 ]; then + install_profile_number=$PROFILE_RMI_HUB + install_rmi_hub + no_fai_profile=0 + fi + if [ $no_fai_profile -eq 1 ]; then print_and_log "No install profile selected, be sure to have one of the " print_and_log "fai__install=1 in your $conf_file" From b95f401e5a602a0d75e0cf411028b7475eba6832 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Dec 2011 12:07:49 +0800 Subject: [PATCH 0175/2585] - added script for creating Debian and RedHat packages - added DEBIAN control file --- usr/local/src/create-packages | 23 +++++++++++++++++++++++ usr/local/src/debian/DEBIAN/control | 10 ++++++++++ 2 files changed, 33 insertions(+) create mode 100644 usr/local/src/create-packages create mode 100644 usr/local/src/debian/DEBIAN/control diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages new file mode 100644 index 00000000..f0276e3a --- /dev/null +++ b/usr/local/src/create-packages @@ -0,0 +1,23 @@ +#! /usr/bin/env bash + +rm -rf debian/{etc,usr} + +target_dir=debian +package_name=escenic-content-engine-installer +version=5.3.3.3 + +cp -r ../../../etc $target_dir + +mkdir -p $target_dir/usr +cp -r ../../../usr/bin $target_dir/usr +cp -r ../../../usr/sbin $target_dir/usr +cp -r ../../../usr/share $target_dir/usr + +sed -i "s#VERSION#${version}#g" $target_dir/DEBIAN/control + +dpkg-deb --build debian +mv debian.deb ${package_name}-${version}.deb + +if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then + fakeroot alien --to-rpm --scripts ${package_name}-${version}.deb +fi diff --git a/usr/local/src/debian/DEBIAN/control b/usr/local/src/debian/DEBIAN/control new file mode 100644 index 00000000..8e74d6ec --- /dev/null +++ b/usr/local/src/debian/DEBIAN/control @@ -0,0 +1,10 @@ +Package: escenic-content-engine-installer +Version: 5.3.3.3 +Section: base +Priority: optional +Architecture: all +Maintainer: Torstein Krause Johansen +Description: The Escenic Content Engine Installer. + This script lets you install the Escenic Content Engine, its plugins + and Widget Framwork. It will also allow you to install the database, + cache and web servers. From 9560ee98ed6f3b7115c86a5e29635f8cb0ae0782 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Dec 2011 12:56:00 +0800 Subject: [PATCH 0176/2585] - implemented a "package" command which generates DEB and RPM packages for the EAR installed. 'ece package' will use the tomcat_base & instance information available on its own host. Example run: $ ece -i editor2 package [ece#engine-editor2] RPM package of your editor2 engine instance with with build [ece#engine-editor2] version 1323752034 is now available in /var/cache/escenic [ece#engine-editor2] DEB package of your editor2 engine instance with with build [ece#engine-editor2] version 1323752034 is now available in /var/cache/escenic $ ls /var/cache/escenic/*.{rpm,deb} -rw-r--r-- 1 torstein torstein 106M Dec 13 12:54 /var/cache/escenic/escenic-engine-editor2-1323752034-2.noarch.rpm -rw-r--r-- 1 torstein torstein 106M Dec 13 12:54 /var/cache/escenic/escenic-engine-editor2-1323752034.deb --- usr/bin/ece | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index aa6b06a4..fccdaa3b 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -994,6 +994,90 @@ function assemble() debug $assemblytool_home/dist/engine.ear "is now ready" } +# creates DEB and RPM packages suitable for deploying any kind of ECE +# instance that was previously assembled. +function create_packages() +{ + local ear=$cache_dir/engine.ear + if [ ! -e $ear ]; then + print "$ear does not exist. Did you run '"`basename $0`" assemble'?" + exit 1 + fi + + local dir=$(mktemp -d) + local package_dir=$dir/debian + local package_name=escenic-${type}-${instance} + # TODO the version might be a bit on the extreme side. Could come + # from the SCM (if available) + local version=$(date +%s) + + run mkdir -p $package_dir/DEBIAN + cat > $package_dir/DEBIAN/control < +Description: The Escenic Content Engine of type ${type} + for the ${instance} instance. Built on $HOSTNAME +EOF + + case $appserver in + tomcat) + local tomcat_dir=${package_dir}${tomcat_base} + local tomcat_escenic_dir=${tomcat_dir}/escenic + + # lib + run mkdir -p ${tomcat_escenic_dir} + run cd ${tomcat_escenic_dir} + run jar xf ${ear} lib + + # war + run mkdir -p ${tomcat_dir}/webapps + run cd ${tomcat_dir}/webapps + for el in $(jar tf ${ear} | grep .war$); do + if [ -n "${deploy_webapp_white_list}" ]; then + for ele in $deploy_webapp_white_list; do + if [ ${el} = ${ele} ]; then + run jar xf ${ear} ${el} + fi + done + else + run jar xf ${ear} ${el} + fi + done + + # copy these from the current configuration: bin, conf + run cp -r ${tomcat_base}/{bin,conf} ${tomcat_dir} + + # these just need to be there: logs, temp + run mkdir ${tomcat_dir}/{logs,temp} + + # build the packages + cd ${dir} + run dpkg-deb --build debian + run mv debian.deb ${package_name}-${version}.deb + + if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then + run fakeroot alien --to-rpm --scripts ${package_name}-${version}.deb + mv *.rpm ${cache_dir} + print "RPM package of your $instance $type instance with with build" + print "version $version is now available in ${cache_dir}" + fi + + mv *.deb ${cache_dir} + print "DEB package of your $instance $type instance with with build" + print "version $version is now available in ${cache_dir}" + ;; + *) + print "Only supported on Tomcat so far" + ;; + esac + + run -rf ${dir} +} + function get_log4j_log() { for el in $log_file_list; do @@ -1534,6 +1618,9 @@ for el in $command; do assemble) assemble ;; + package) + create_packages + ;; clean) clean_up ;; From dc516b5052e485d852dbdf4f8dde68e6e7b9671d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Dec 2011 12:56:30 +0800 Subject: [PATCH 0177/2585] - added package to possible completions --- etc/bash_completion.d/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index 8a8b6a4d..78a4a254 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -6,7 +6,7 @@ _ece_commands() local prev=${COMP_WORDS[COMP_CWORD-1]} commands="applog assemble backup clean deploy edit help info kill log loglist \ - flush outlog restart start status stop threaddump update versions" + flush outlog package restart start status stop threaddump update versions" options="-i --instance -p --publication -r --publication-resource \ -t --type -u --user -w --password" resources="content-type feature layout layout-group image-version menu \ From fe5d58ca14d32d5db22d8b6424931f708c073f3e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Dec 2011 13:32:25 +0800 Subject: [PATCH 0178/2585] - added simple postinst script --- usr/local/src/debian/DEBIAN/postinst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 usr/local/src/debian/DEBIAN/postinst diff --git a/usr/local/src/debian/DEBIAN/postinst b/usr/local/src/debian/DEBIAN/postinst new file mode 100644 index 00000000..d22cabda --- /dev/null +++ b/usr/local/src/debian/DEBIAN/postinst @@ -0,0 +1,4 @@ +#! /usr/bin/env bash +set -e + +/usr/sbin/ece-install From 28c1b63ebc2d539e98534062c0681721e9c3461c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Dec 2011 13:36:41 +0800 Subject: [PATCH 0179/2585] - making sure installed scripts are executable --- usr/local/src/debian/DEBIAN/postinst | 4 ++++ 1 file changed, 4 insertions(+) mode change 100644 => 100755 usr/local/src/debian/DEBIAN/postinst diff --git a/usr/local/src/debian/DEBIAN/postinst b/usr/local/src/debian/DEBIAN/postinst old mode 100644 new mode 100755 index d22cabda..2b37fbca --- a/usr/local/src/debian/DEBIAN/postinst +++ b/usr/local/src/debian/DEBIAN/postinst @@ -1,4 +1,8 @@ #! /usr/bin/env bash set -e +chmod +x /usr/sbin/ece-install /usr/bin/{ece,system-info} /usr/sbin/ece-install + +exit 0 + From 6d91a315477cd3a3b078176a94ff0d82b498c74a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Dec 2011 21:02:35 +0800 Subject: [PATCH 0180/2585] - simplified the postinst --- usr/local/src/debian/DEBIAN/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/local/src/debian/DEBIAN/postinst b/usr/local/src/debian/DEBIAN/postinst index 2b37fbca..fe550beb 100755 --- a/usr/local/src/debian/DEBIAN/postinst +++ b/usr/local/src/debian/DEBIAN/postinst @@ -2,7 +2,7 @@ set -e chmod +x /usr/sbin/ece-install /usr/bin/{ece,system-info} -/usr/sbin/ece-install +echo "You can now run ece-install to install Escenic Content Engine & friends" exit 0 From 8cabf4091a8f60ee48f5bd3384658c15ae4254d4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Dec 2011 12:34:39 +0800 Subject: [PATCH 0181/2585] - improved package names in "ece package", the command will now create names like: escenic-content-engine-- escenic-search-engine-- escenic-analysis-engine-- based on the instance type - fixed bug in the deploy_webapp_white_list filtering inside create_packages. - fixed cleanup(), now all EARs, DEBs and RPMs in /var/cache/escenic are cleaned when doing "ece clean". --- usr/bin/ece | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index fccdaa3b..cbc99cc4 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1006,7 +1006,18 @@ function create_packages() local dir=$(mktemp -d) local package_dir=$dir/debian - local package_name=escenic-${type}-${instance} + local package_name="" + + if [ $type = "engine" ]; then + package_name="escenic-content-engine-${instance}" + elif [ $type = "search" ]; then + package_name="escenic-search-engine-${instance}" + elif [ $type = "rmi-hub" ]; then + package_name="escenic-rmi-hub" + elif [ $type = "analysis" ]; then + package_name="escenic-analysis-engine-${instance}" + fi + # TODO the version might be a bit on the extreme side. Could come # from the SCM (if available) local version=$(date +%s) @@ -1039,7 +1050,7 @@ EOF for el in $(jar tf ${ear} | grep .war$); do if [ -n "${deploy_webapp_white_list}" ]; then for ele in $deploy_webapp_white_list; do - if [ ${el} = ${ele} ]; then + if [ ${el} = ${ele}.war ]; then run jar xf ${ear} ${el} fi done @@ -1057,25 +1068,28 @@ EOF # build the packages cd ${dir} run dpkg-deb --build debian - run mv debian.deb ${package_name}-${version}.deb + local deb_package=${package_name}-${version}.deb + run mv debian.deb ${deb_package} if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then - run fakeroot alien --to-rpm --scripts ${package_name}-${version}.deb + run fakeroot alien --to-rpm --scripts ${deb_package} mv *.rpm ${cache_dir} print "RPM package of your $instance $type instance with with build" - print "version $version is now available in ${cache_dir}" + print "version $version is now available:" + print $(echo /var/cache/escenic/${package_name}-${version}*.rpm) fi mv *.deb ${cache_dir} print "DEB package of your $instance $type instance with with build" - print "version $version is now available in ${cache_dir}" + print "version $version is now available:" + print /var/cache/escenic/${deb_package} ;; *) - print "Only supported on Tomcat so far" + print "Package creation is only supported on Tomcat so far." ;; esac - - run -rf ${dir} + + run rm -rf ${dir} } function get_log4j_log() @@ -1295,14 +1309,12 @@ function clean_up() 2>>$log_file fi - if [ -e /var/cache/escenic/ -a \ - `ls /var/cache/escenic | grep ece- | wc -l` -gt 0 ]; then - print "Cleaning up "`basename $0`" files in /var/cache/escenic/ ..." - rm -rf /var/cache/escenic/* \ + if [ -d /var/cache/escenic ]; then + print "Cleaning up ear, deb and rpm files in /var/cache/escenic ..." + rm -rf /var/cache/escenic/*.{rpm,deb,ear} \ 1>>$log_file \ 2>>$log_file fi - } function set_id() From 694c73f2c02c121db88d35724d05f2ddf405f7d3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 18:36:37 +0800 Subject: [PATCH 0182/2585] - ece will now honour appserver_port when set in any of the .conf files. - default wise, the /etc/escenic/ece.conf will ship with appserver_port=8080 - the appserver_port is unset, ece will (continue to) use the system out log on Tomcat to determine the port. - for other appservers, if the appserver_port is unset, ece will default to port 8080. - this should hopefully close the rather old https://github.com/skybert/ece-scripts/issues/25 :-) --- etc/escenic/ece.conf | 9 +++++++- usr/bin/ece | 49 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/etc/escenic/ece.conf b/etc/escenic/ece.conf index be5ded7d..3f5ddc0c 100644 --- a/etc/escenic/ece.conf +++ b/etc/escenic/ece.conf @@ -1,4 +1,4 @@ -# ece script configruation -*- sh -*- +# ece script configruation -*- conf -*- # version: $Revision: #1 $ $Date: 2011/02/18 $ # author: torstein@escenic.com @@ -33,6 +33,13 @@ java_home=/usr/lib/jvm/java-6-sun ######################################################################## appserver=tomcat +######################################################################## +# App server port. If this variable is unset, ece will try to figure +# out the port number by itself. ece-.conf should override +# this when there's more than one ECE instance on the same host. +######################################################################## +appserver_port=8080 + ######################################################################## # Section II: Variables that are sensible defaults but may be changed # to accommodate the conventions or tastes of your OS or company. diff --git a/usr/bin/ece b/usr/bin/ece index cbc99cc4..a871108e 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -13,6 +13,7 @@ verbose=0 force_ipv4=0 quiet=0 +everything_but_the_kitchen_sink=0 ###################################################################### # Dear user, don't touch anyting in this script, especially below this @@ -259,7 +260,11 @@ function set_type_pid() function set_type_port() { host=localhost - if [ $appserver != "tomcat" ]; then + # port set in ece[-instance].conf takes precedence + if [ -n "${appserver_port}" ]; then + port=${appserver_port} + debug "appserver_port set in .conf files=${port}" + elif [ $appserver != "tomcat" ]; then debug "Only tomcat is supported for reading host/port right now, " debug "trying to make an educated guess" port=8080 @@ -1036,8 +1041,8 @@ EOF case $appserver in tomcat) - local tomcat_dir=${package_dir}${tomcat_base} - local tomcat_escenic_dir=${tomcat_dir}/escenic + local tomcat_base_dir=${package_dir}${tomcat_base} + local tomcat_escenic_dir=${tomcat_base_dir}/escenic # lib run mkdir -p ${tomcat_escenic_dir} @@ -1045,8 +1050,8 @@ EOF run jar xf ${ear} lib # war - run mkdir -p ${tomcat_dir}/webapps - run cd ${tomcat_dir}/webapps + run mkdir -p ${tomcat_base_dir}/webapps + run cd ${tomcat_base_dir}/webapps for el in $(jar tf ${ear} | grep .war$); do if [ -n "${deploy_webapp_white_list}" ]; then for ele in $deploy_webapp_white_list; do @@ -1059,11 +1064,36 @@ EOF fi done + # putting this block here so that anything overridden in + # tomcat_base takes precedence over tomcat_home + if [ ${everything_but_the_kitchen_sink} -eq 1 ]; then + ( + local tomcat_home_dir=${package_dir}${tomcat_home} + run mkdir -p ${tomcat_home_dir} + run cd ${tomcat_home_dir} + run cp -r ${tomcat_home}/{bin,lib} ${tomcat_home_dir} + + local etc_escenic_dir=${package_dir}/etc/escenic + run mkdir -p ${etc_escenic_dir} + cd ${etc_escenic_dir} + run cp -r /etc/escenic/{ece,ece-${instance}}.conf \ + ${etc_escenic_dir} + + run mkdir -p ${etc_escenic_dir}/engine + run cp -r /etc/escenic/engine/common \ + ${etc_escenic_dir} + + run mkdir -p ${etc_escenic_dir}/engine/instance + run cp -r /etc/escenic/engine/instance/${instance} \ + ${etc_escenic_dir} + ) + fi + # copy these from the current configuration: bin, conf - run cp -r ${tomcat_base}/{bin,conf} ${tomcat_dir} + run cp -r ${tomcat_base}/{bin,conf} ${tomcat_base_dir} # these just need to be there: logs, temp - run mkdir ${tomcat_dir}/{logs,temp} + run mkdir ${tomcat_base_dir}/{logs,temp} # build the packages cd ${dir} @@ -1183,6 +1213,11 @@ function set_type_command_and_instance() continue fi + if [ $el = "--full" ]; then + everything_but_the_kitchen_sink=1 + continue + fi + if [ $el = "-q" -o $el = "--quiet" ]; then quiet=1 continue From c094801e6e352f55f089fdf0a9e096dbed37dc21 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 18:36:58 +0800 Subject: [PATCH 0183/2585] - added Emacs mode line --- etc/bash_completion.d/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index 78a4a254..ad53165e 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -1,4 +1,4 @@ -# auto completion for the /usr/bin/ece command. +# auto completion for the /usr/bin/ece command. Emacs: -*- sh -*- mode _ece_commands() { From 51a3a8ba883397200f6265987a3768a197b44e41 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 18:37:28 +0800 Subject: [PATCH 0184/2585] - fixed permissions on DEBIAN/postinst --- usr/local/src/create-packages | 3 +++ 1 file changed, 3 insertions(+) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index f0276e3a..f362f618 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -15,6 +15,9 @@ cp -r ../../../usr/share $target_dir/usr sed -i "s#VERSION#${version}#g" $target_dir/DEBIAN/control +# fix permissions +chmod 755 $target_dir/DEBIAN/postinst + dpkg-deb --build debian mv debian.deb ${package_name}-${version}.deb From c390397302af3611836c1f0762889bc8acac35f4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 19:22:13 +0800 Subject: [PATCH 0185/2585] - to support the change in ece (694c73f2c02c121db88d35724d05f2ddf405f7d3), ece-install will also populate the appserver_port parameter in ece-.conf at the time of installation. Change done in set_up_app_server - fixed bug in is_installing_from_archives where it would return 1 (true) whenever installing (presentation) server interactively. - fixed bug in install_ece_instance where the presentation server server specific appendices to ece-.conf wouldn't be written because it was checking against a hard value '2', instead of INSTALL_PRESENTATION_SERVER constant. --- usr/sbin/ece-install | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index fee4dcac..a37505c5 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -146,7 +146,7 @@ http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip http://technet.escenic.com/downloads/release/53/community-engine-3.6.1.0.zip http://technet.escenic.com/downloads/release/53/dashboard-1.0.0.0.zip -http://technet.escenic.com/downloads/release/53/engine-5.3.4.3.zip +http://technet.escenic.com/downloads/release/53/engine-5.3.4.7.zip http://technet.escenic.com/downloads/release/53/forum-3.0.0.0.zip http://technet.escenic.com/downloads/release/53/inpage-1.3.0.0.zip http://technet.escenic.com/downloads/release/53/lucy-dist-4.1.6.0.zip @@ -810,6 +810,7 @@ function set_up_app_server set_ece_instance_conf tomcat_base $tomcat_base set_ece_instance_conf tomcat_home /opt/tomcat + set_ece_instance_conf appserver_port $appserver_port run cd $tomcat_base/lib make_ln $jdbc_driver @@ -1761,7 +1762,9 @@ function is_installing_from_archives() log ece_instance_ear_file=$ece_instance_ear_file log ece_instance_conf_archive=$ece_instance_conf_archive - if [[ -z "$ece_instance_ear_file" || -z "$ece_instance_conf_archive" ]]; then + if [[ -z "$ece_instance_ear_file" || \ + -z "$ece_instance_conf_archive" || \ + $fai_enabled -eq 0 ]]; then echo 0 return fi @@ -1845,7 +1848,7 @@ function install_presentation_server() { print_and_log "Installing a presentation server on $HOSTNAME ..." type=engine - install_ece_instance "web1" 1 + install_ece_instance "web1" $PROFILE_PRESENTATION_SERVER } function assemble_deploy_and_restart_type() From e2b39bbe513d1d2969825ad7a48d8cc13f366ac4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 19:44:34 +0800 Subject: [PATCH 0186/2585] - now sets name & email dynamically in the DEB/RPM packages depending on the user running the create-packages script. --- usr/local/src/create-packages | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index f362f618..34330868 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -4,7 +4,26 @@ rm -rf debian/{etc,usr} target_dir=debian package_name=escenic-content-engine-installer -version=5.3.3.3 + +function get_version() +{ + engine_file=$(grep http ../../sbin/ece-install | grep "/engine") + version=$(basename $engine_file .zip | cut -d'-' -f2) + echo $version +} + +function get_name() +{ + name=$(grep $USER /etc/passwd | cut -d':' -f5 | cut -d',' -f1) + echo "${name}" +} + +function get_email() +{ + echo ${USER}@${HOSTNAME} +} + +version=$(get_version) cp -r ../../../etc $target_dir @@ -13,7 +32,11 @@ cp -r ../../../usr/bin $target_dir/usr cp -r ../../../usr/sbin $target_dir/usr cp -r ../../../usr/share $target_dir/usr +cp $target_dir/DEBIAN/control $target_dir/DEBIAN/control.orig + sed -i "s#VERSION#${version}#g" $target_dir/DEBIAN/control +sed -i "s#MAINTAINER_NAME#$(get_name)#g" $target_dir/DEBIAN/control +sed -i "s#MAINTAINER_EMAIL#$(get_email)#g" $target_dir/DEBIAN/control # fix permissions chmod 755 $target_dir/DEBIAN/postinst @@ -24,3 +47,6 @@ mv debian.deb ${package_name}-${version}.deb if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then fakeroot alien --to-rpm --scripts ${package_name}-${version}.deb fi + +# cleanup +cp $target_dir/DEBIAN/control.orig $target_dir/DEBIAN/control From 759cc08c5b70910eb732ba5db1e57b826520664c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 19:44:54 +0800 Subject: [PATCH 0187/2585] - now prompts the user if he/she wants to run the installation script after installation. --- usr/local/src/debian/DEBIAN/postinst | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/usr/local/src/debian/DEBIAN/postinst b/usr/local/src/debian/DEBIAN/postinst index fe550beb..d8b24474 100755 --- a/usr/local/src/debian/DEBIAN/postinst +++ b/usr/local/src/debian/DEBIAN/postinst @@ -1,8 +1,19 @@ #! /usr/bin/env bash -set -e +# set -e chmod +x /usr/sbin/ece-install /usr/bin/{ece,system-info} -echo "You can now run ece-install to install Escenic Content Engine & friends" + +echo "Do you wish to run the Escenic Content Engine installer now?" + +answer=no +echo -n "Your choice: yes/[no] " +read answer + +if [ "$answer" = "yes" ]; then + ece-install +else + echo "Run the Escenic Content Engine installer by typing 'ece-install'" +fi exit 0 From 03d984be0f2d3bf5f0fbc13e26374182ec9ca11b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 19:49:14 +0800 Subject: [PATCH 0188/2585] - added guard on dpkg-deb --- usr/local/src/create-packages | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 34330868..7a1a7236 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -41,6 +41,11 @@ sed -i "s#MAINTAINER_EMAIL#$(get_email)#g" $target_dir/DEBIAN/control # fix permissions chmod 755 $target_dir/DEBIAN/postinst +if [ ! -x /usr/bin/dpkg-deb ]; then + echo "You must have dpkg-deb installed" + exit 1 +fi + dpkg-deb --build debian mv debian.deb ${package_name}-${version}.deb From 590086f8b3e055e4cf266769cd1c08b41b4fa499 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 19:51:09 +0800 Subject: [PATCH 0189/2585] - added user friendly messages if the host system has insufficient packages for creating any of the packages. --- usr/local/src/create-packages | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 7a1a7236..c94d7daa 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -42,7 +42,7 @@ sed -i "s#MAINTAINER_EMAIL#$(get_email)#g" $target_dir/DEBIAN/control chmod 755 $target_dir/DEBIAN/postinst if [ ! -x /usr/bin/dpkg-deb ]; then - echo "You must have dpkg-deb installed" + echo "You must have dpkg-deb installed :-(" exit 1 fi @@ -51,6 +51,8 @@ mv debian.deb ${package_name}-${version}.deb if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then fakeroot alien --to-rpm --scripts ${package_name}-${version}.deb +else + echo "You must have 'alien' and 'fakeroot' installed to create RPMs" fi # cleanup From 4f7edae396e5757c435fe35831cc28c3468a65a0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Dec 2011 20:00:54 +0800 Subject: [PATCH 0190/2585] - added more description --- usr/local/src/debian/DEBIAN/control | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/usr/local/src/debian/DEBIAN/control b/usr/local/src/debian/DEBIAN/control index 8e74d6ec..78ba42f0 100644 --- a/usr/local/src/debian/DEBIAN/control +++ b/usr/local/src/debian/DEBIAN/control @@ -1,10 +1,14 @@ Package: escenic-content-engine-installer -Version: 5.3.3.3 +Version: VERSION Section: base Priority: optional Architecture: all -Maintainer: Torstein Krause Johansen +Maintainer: MAINTAINER_NAME Description: The Escenic Content Engine Installer. - This script lets you install the Escenic Content Engine, its plugins - and Widget Framwork. It will also allow you to install the database, - cache and web servers. + The ece-install script lets you install the Escenic Content Engine, + its plugins and Widget Framwork. It will also allow you to install + the database, cache and web servers. + . + The script can run in both interactive or full automatic mode (FAI). See the + documentation in /usr/share/doc/escenic/ for further details. + \ No newline at end of file From 31c3e377fdc81fdbd28f643a117114fce26d9b67 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 11 Jan 2012 18:04:38 +0800 Subject: [PATCH 0191/2585] - added the of two new variables: escenic_root_dir and escenic_conf_dir which can be overridden in ece-install.conf. The defaults will work fine in most cases and the default paths are the ones recommended by Escenic/Vizrt Online and are the ones used in the documentation: escenic_root_dir=/opt/escenic escenic_conf_dir=/etc/escenic --- usr/sbin/ece-install | 130 ++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 64 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a37505c5..940f64d6 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -15,13 +15,15 @@ ##################################################################### # User definable variables (the defaults are fine in most # cases). These are the most likely variables you want to change and -# they can all be set in the .ece-isntall.conf file +# they can all be set in the ece-install.conf file ##################################################################### ece_user=escenic ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 tomcat_download=http://apache.stu.edu.tw/tomcat/tomcat-6/v6.0.33/bin/apache-tomcat-6.0.33.tar.gz +escenic_root_dir=/opt/escenic +escenic_conf_dir=/etc/escenic # The script will install the sun-java6-jdk package on Debian based # systems and this is the path of the JAVA home with this package. If @@ -175,9 +177,9 @@ PROFILE_RESTORE_FROM_BACKUP=11 # directories. engine_dir_list=" $common_nursery_dir -/etc/escenic/engine/family/default -/etc/escenic/engine/host/localhost -/opt/escenic +$escenic_conf_dir/engine/family/default +$escenic_conf_dir/engine/host/localhost +$escenic_root_dir /var/cache/escenic /var/crash/escenic /var/lib/escenic @@ -300,27 +302,27 @@ function set_up_assembly_tool() { print_and_log "Setting up the Assembly Tool ..." - make_dir /opt/escenic/assemblytool/ - cd /opt/escenic/assemblytool/ + make_dir $escenic_root_dir/assemblytool/ + cd $escenic_root_dir/assemblytool/ if [ -e $download_dir/assemblytool*zip ]; then run unzip -u $download_dir/assemblytool*zip fi # adding an instance layer to the Nursery configuration - cp -r /opt/escenic/engine/siteconfig/bootstrap-skeleton \ - /opt/escenic/assemblytool/conf - cd /opt/escenic/assemblytool/conf/ + cp -r $escenic_root_dir/engine/siteconfig/bootstrap-skeleton \ + $escenic_root_dir/assemblytool/conf + cd $escenic_root_dir/assemblytool/conf/ cp -r layers/host layers/instance cat > layers/instance/Files.properties <> Nursery.properties echo "layer.06 = /layers/instance/Layer" >> Nursery.properties # set up which plugins to use - cd /opt/escenic/assemblytool/ + cd $escenic_root_dir/assemblytool/ make_dir plugins cd plugins find ../../ -maxdepth 1 -type d | \ @@ -341,7 +343,7 @@ EOF make_ln $directory done - run cd /opt/escenic/assemblytool/ + run cd $escenic_root_dir/assemblytool/ run ant -q initialize sed -i 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ assemble.properties \ @@ -362,8 +364,8 @@ function set_up_engine_and_plugins() print_and_log "Setting up the Escenic Content Engine & its plugins ..." - make_dir /opt/escenic - cd /opt/escenic/ + make_dir $escenic_root_dir + cd $escenic_root_dir/ for el in $technet_download_list; do if [ $(basename $el | \ @@ -385,9 +387,9 @@ function set_up_engine_and_plugins() debug "${engine_dir} is already there, skipping to next step." fi - # we now extract all the plugins. We extract them in /opt/escenic + # we now extract all the plugins. We extract them in $escenic_root_dir # as we want to re-use them between minor updates of ECE. - cd /opt/escenic/ + cd $escenic_root_dir/ for el in $download_dir/*.zip; do if [ $(basename $el | grep ^engine-.*.zip | wc -l) -gt 0 ]; then continue @@ -421,8 +423,8 @@ function set_up_ecedb() { print_and_log "Setting up the ECE database schema ..." - make_dir /opt/escenic/engine/plugins - run cd /opt/escenic/engine/plugins + make_dir $escenic_root_dir/engine/plugins + run cd $escenic_root_dir/engine/plugins find ../../ -maxdepth 1 -type d | \ grep -v assemblytool | \ @@ -449,7 +451,7 @@ function set_up_ecedb() create_ecedb cd ~/ - run rm -rf /opt/escenic/engine/plugins + run rm -rf $escenic_root_dir/engine/plugins add_next_step "DB is now set up on $HOSTNAME:3306" } @@ -458,25 +460,25 @@ function set_up_basic_nursery_configuration() { print_and_log "Setting up the basic Nursery configuration ..." - local escenic_home=/opt/escenic + local escenic_root_dir=$escenic_root_dir if [ $(is_installing_from_archives) -eq 1 ]; then print_and_log "Using the supplied Nursery & JAAS configuration from" print_and_log "bundle: $ece_instance_conf_archive" local dir=$(mktemp -d) - escenic_home=$dir + escenic_root_dir=$dir run cd $dir run tar xzf $ece_instance_conf_archive fi - run cp -r $escenic_home/engine/siteconfig/config-skeleton/* \ + run cp -r $escenic_root_dir/engine/siteconfig/config-skeleton/* \ $common_nursery_dir/ - run cp -r $escenic_home/engine/security/ \ + run cp -r $escenic_root_dir/engine/security/ \ $common_nursery_dir/ - make_dir /etc/escenic/engine/instance + make_dir $escenic_conf_dir/engine/instance - for el in $escenic_home/assemblytool/plugins/*; do + for el in $escenic_root_dir/assemblytool/plugins/*; do if [ ! -d $el/misc/siteconfig/ ]; then continue fi @@ -532,7 +534,7 @@ EOF function set_up_instance_specific_nursery_configuration() { - for el in /etc/escenic/engine/instance/*; do + for el in $escenic_conf_dir/engine/instance/*; do i=$(( i + 1 )) if [ $(basename $el) = $instance_name ]; then rmi_port="8${i}23" @@ -541,7 +543,7 @@ function set_up_instance_specific_nursery_configuration() done nursery_context=neo/io/managers/HubConnectionManager.properties - file=/etc/escenic/engine/instance/$instance_name/$nursery_context + file=$escenic_conf_dir/engine/instance/$instance_name/$nursery_context make_dir $(dirname $file) # we don't touch it if the file already exists. @@ -620,7 +622,7 @@ function set_conf_file_value() # the value already is set, it will replace it with this one. function set_ece_instance_conf() { - instance_conf_file=/etc/escenic/ece-$instance_name.conf + instance_conf_file=$escenic_conf_dir/ece-$instance_name.conf set_conf_file_value $1 $2 $instance_conf_file } @@ -1093,26 +1095,26 @@ function create_publication_definition_and_war() fi print_and_log "Setting up the ${publication_name} publication ..." - make_dir /opt/escenic/assemblytool/publications/ - run cd /opt/escenic/assemblytool/publications/ - cat > /opt/escenic/assemblytool/publications/${publication_name}.properties < $escenic_root_dir/assemblytool/publications/${publication_name}.properties <> /etc/escenic/engine/common/Initial.properties <> $escenic_conf_dir/engine/common/Initial.properties <> $file <>$log 2>>$log - cd /opt/escenic/ + cd $escenic_root_dir/ unzip -u $download_dir/$(basename $el) \ 1>>$log 2>>$log done assert_pre_prequesite mvn export JAVA_HOME=$java_home - wf_maven_dir=$(echo /opt/escenic/widget-framework-core-*/maven) + wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) run cd $wf_maven_dir print_and_log "Installing Widget Framework into your Maven repository ..." @@ -2062,7 +2064,7 @@ EOF # installing the widget-framework-common as a ECE plugin wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) - cd /opt/escenic/assemblytool/plugins + cd $escenic_root_dir/assemblytool/plugins if [ ! -h $(basename $wf_dist_dir) ]; then ln -s $wf_dist_dir fi @@ -2079,7 +2081,7 @@ function install_search_server() type=search install_ece_instance "search1" 0 - file=/etc/escenic/ece-${instance_name}.conf + file=$escenic_conf_dir/ece-${instance_name}.conf print_and_log "Creating instance specific conf: $file ..." cat >> $file < Date: Wed, 11 Jan 2012 18:23:46 +0800 Subject: [PATCH 0192/2585] - updated documentation, added description of escenic_root_dir and escenic_conf_dir to the file path chapter. --- usr/share/doc/escenic/ece-install-guide.org | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index e248c140..d7873521 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -596,6 +596,20 @@ but these should be the most interesting ones. *) Applies only to Debian based systems. +The root directory of all Escenic software binaries, /opt/escenic, +together with the root directory for the configuration files may be +overwritten in ece-install.conf by setting these variables: + +#+BEGIN_SRC sh +escenic_root_dir=/opt/escenic-parallel-env +escenic_conf_dir=/etc/escenic-parallel-env +#+END_SRC + +Note, this is only needed if you are running two completely separate +environments on the same host. A use case is if you're setting up a +test environment and want to separate stacks of Escenic Content Engine +and plugins. + * Assumptions ** /etc/esecenic is shared It's assumed that the /etc/escenic directory is either on a shared From 235c17b0ad42c003692fdebcf37f2bc71b85a3ac Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 11 Jan 2012 18:25:30 +0800 Subject: [PATCH 0193/2585] - updated HTML --- usr/share/doc/escenic/ece-install-guide.html | 121 +++++++++++-------- 1 file changed, 70 insertions(+), 51 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 4d455d22..0c873aed 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -414,7 +414,7 @@

    4 Running the ece-install S

    -
    [ece-install] /root/ece-install.conf doesn't exist, I cannot live without it :-(
    +
    [ece-install] /root/ece-install.conf doesn't exist, I cannot live without it :-(
     
    @@ -425,7 +425,7 @@

    4 Running the ece-install S
    [ece-install] Be sure to set technet_user and technet_password
    -[ece-install] in /root/ece-install.conf
    +[ece-install] in /root/ece-install.conf
     
    @@ -434,8 +434,8 @@

    4 Running the ece-install S

    -
    technet_user=<user>
    -technet_password=<password>
    +
    technet_user=<user>
    +technet_password=<password>
     
    @@ -451,7 +451,7 @@

    4 Running the ece-install S

    -
    # bash ece-install [-v|--verbose]
    +
    # bash ece-install [-v|--verbose]
     
    @@ -473,7 +473,7 @@

    4 Running the ece-install S

    -
    # bash -x ece-install
    +
    # bash -x ece-install
     
    @@ -497,9 +497,9 @@

    5 Available Server Profiles
    Hi, which server profile do you wish to install?
     
    -   1 - All in one, full stack on one host, suitable for dev & test environments
    +   1 - All in one, full stack on one host, suitable for dev & test environments
        2 - Editorial (publication) server
    -   3 - Presentation server (ECE + memcached).
    +   3 - Presentation server (ECE + memcached).
        4 - Database server
        5 - Cache server (cache and web server)
        6 - RMI hub
    @@ -691,10 +691,10 @@ 

    5.6 Profile - Database Se

    -
    [ece-install] Setting up the ECE database schema ...
    -ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    -ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists
    -[ece-install] running tables FAILED, exiting :-(
    +
    [ece-install] Setting up the ECE database schema ...
    +ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists
    +ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists
    +[ece-install] running tables FAILED, exiting :-(
     
    @@ -778,8 +778,8 @@

    5.8 Profile - Install Wid

    -
    [ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf
    -[ece-install] If you don't have these, please contact support@escenic.com
    +
    [ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf
    +[ece-install] If you don't have these, please contact support@escenic.com
     
    @@ -974,11 +974,11 @@

    5.13 Running in FAI mode

    -
    fai_enabled=1
    +
    fai_enabled=1
     
    -fai_restore_from_backup=1
    -fai_restore_all=1
    -fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz
    +fai_restore_from_backup=1
    +fai_restore_all=1
    +fai_restore_from_file=/var/backups/escenic/engine-dev1-backup-2011-10-10.tar.gz
     
    @@ -1037,7 +1037,7 @@

    6 Full Automatic Install (F

    -
    # bash ece-install > ece-install.out &
    +
    # bash ece-install > ece-install.out &
     
    @@ -1046,7 +1046,7 @@

    6 Full Automatic Install (F

    -
    # nohup bash ece-install > ece-install.out &
    +
    # nohup bash ece-install > ece-install.out &
     

    @@ -1102,11 +1102,11 @@

    6.1 Installing from EARs
    $ cd /opt/escenic/engine
    -$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \
    -  engine/security \
    -  engine/siteconfig/config-skeleton/ \
    -  engine/siteconfig/bootstrap-skeleton/ \
    -  `find -L  assemblytool/plugins/ -name siteconfig`
    +$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \
    +  engine/security \
    +  engine/siteconfig/config-skeleton/ \
    +  engine/siteconfig/bootstrap-skeleton/ \
    +  `find -L  assemblytool/plugins/ -name siteconfig`
     
    @@ -1118,8 +1118,8 @@

    6.1 Installing from EARs -
    fai_presentation_ear=/tmp/engine.ear
    -fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz
    +
    fai_presentation_ear=/tmp/engine.ear
    +fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz
     
    @@ -1208,10 +1208,10 @@

    6.3.1 Installing an Edi -
    fai_enabled=1
    -fai_editor_install=1
    -fai_publication_create=1
    -fai_publication_name=jollygood
    +
    fai_enabled=1
    +fai_editor_install=1
    +fai_publication_create=1
    +fai_publication_name=jollygood
     
    @@ -1228,9 +1228,9 @@

    6.3.2 Installing Two Pr

    -
    fai_enabled=1
    -fai_presentation_install=1
    -fai_presentation_name=web1
    +
    fai_enabled=1
    +fai_presentation_install=1
    +fai_presentation_name=web1
     
    @@ -1239,11 +1239,11 @@

    6.3.2 Installing Two Pr

    -
    fai_enabled=1
    -fai_presentation_install=1
    -fai_presentation_name=web2
    -fai_presentation_port=8081
    -fai_presentation_shutdown=8105
    +
    fai_enabled=1
    +fai_presentation_install=1
    +fai_presentation_name=web2
    +fai_presentation_port=8081
    +fai_presentation_shutdown=8105
     

    @@ -1284,10 +1284,10 @@

    7 Running More Than One Ins

    -
    [ece-install] There's already one ece-install process running. If you believe
    -[ece-install] this is wrong, e.g. if a previous run of ece-install was aborted
    -[ece-install] before it completed, you may delete /var/run/ece-install.pid and
    -[ece-install] run ece-install again.
    +
    [ece-install] There's already one ece-install process running. If you believe
    +[ece-install] this is wrong, e.g. if a previous run of ece-install was aborted
    +[ece-install] before it completed, you may delete /var/run/ece-install.pid and
    +[ece-install] run ece-install again.
     
    @@ -1404,6 +1404,25 @@

    10 Overview of File Paths

    *) Applies only to Debian based systems.

    +

    +The root directory of all Escenic software binaries, /opt/escenic, +together with the root directory for the configuration files may be +overwritten in ece-install.conf by setting these variables: +

    + + + +
    escenic_root_dir=/opt/escenic-parallel-env
    +escenic_conf_dir=/etc/escenic-parallel-env
    +
    + + +

    +Note, this is only needed if you are running two completely separate +environments on the same host. A use case is if you're setting up a +test environment and want to separate stacks of Escenic Content Engine +and plugins. +

    @@ -1458,13 +1477,13 @@

    13 Example Output FAI

    -
    technet_user=apples
    -technet_password=andoranges
    -wf_user=foo
    -wf_password=bar
    +
    technet_user=apples
    +technet_password=andoranges
    +wf_user=foo
    +wf_password=bar
     
    -fai_enabled=1
    -fai_all_install=1
    +fai_enabled=1
    +fai_all_install=1
     
    @@ -1520,7 +1539,7 @@

    13 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2011-12-02 11:07:01 CST + Date: 2012-01-11 18:24:53 CST

    From d9d517e0d75439b1e47d5fbd8d4f81fde85da3d7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 11 Jan 2012 18:56:52 +0800 Subject: [PATCH 0194/2585] - added the following variables to override installation, log, run & data directories from ece-install.conf: escenic_log_dir=/var/log/escenic escenic_data_dir=/var/lib/escenic escenic_run_dir=/var/run/escenic escenic_backups_dir=/var/backups/escenic appserver_root_dir=/opt in addition, there's already escenic_root_dir and escenic_conf_dir - changed the entire script to use these variables - changed the sed expressions in set_up_assembly_tool TODO: test that there's no false friends/mis-replaced paths in the script. --- usr/sbin/ece-install | 64 +++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 940f64d6..987f3599 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -22,8 +22,16 @@ ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 tomcat_download=http://apache.stu.edu.tw/tomcat/tomcat-6/v6.0.33/bin/apache-tomcat-6.0.33.tar.gz + +# These variables govern where software is installed and data and run +# time files are written. escenic_root_dir=/opt/escenic escenic_conf_dir=/etc/escenic +escenic_log_dir=/var/log/escenic +escenic_data_dir=/var/lib/escenic +escenic_run_dir=/var/run/escenic +escenic_backups_dir=/var/backups/escenic +appserver_root_dir=/opt # The script will install the sun-java6-jdk package on Debian based # systems and this is the path of the JAVA home with this package. If @@ -182,10 +190,10 @@ $escenic_conf_dir/engine/host/localhost $escenic_root_dir /var/cache/escenic /var/crash/escenic -/var/lib/escenic -/var/lib/escenic/solr/data -/var/log/escenic -/var/run/escenic +$escenic_data_dir +$escenic_data_dir/solr/data +$escenic_log_dir +$escenic_run_dir /var/spool/escenic/migration " @@ -345,12 +353,12 @@ EOF run cd $escenic_root_dir/assemblytool/ run ant -q initialize - sed -i 's/#\ engine.root\ =\ \./engine.root=\/opt\/escenic\/engine/g' \ + sed -i "s~#\ engine.root\ =\ \.~engine.root=${escenic_root_dir}/engine~g" \ assemble.properties \ 1>>$log 2>>$log exit_on_error "sed on assemble.properties" - sed -i 's/\#\# plugins\ =\ \/path\/to\/plugins/plugins=\/opt\/escenic\/assemblytool\/plugins/g' \ + sed -i "s~\#\# plugins\ =\ /path/to/plugins~plugins=${escenic_root_dir}/assemblytool/plugins~g" \ assemble.properties \ 1>>$log 2>>$log exit_on_error "sed on assemble.properties" @@ -504,7 +512,7 @@ function set_up_basic_nursery_configuration() cat > $common_nursery_dir/ServerConfig.properties < $common_nursery_dir/neo/io/managers/ContentManager.properties < $common_nursery_dir/trace.properties < @@ -1198,8 +1206,8 @@ function set_up_solr() run cp -r $escenic_root_dir/engine/solr/conf $escenic_conf_dir/solr fi - make_dir /var/lib/escenic/solr/ - run cd /var/lib/escenic/solr/ + make_dir $escenic_data_dir/solr/ + run cd $escenic_data_dir/solr/ if [ ! -h conf ]; then run ln -s $escenic_conf_dir/solr conf fi @@ -1213,13 +1221,13 @@ function un_install_ece() print_and_log "Uninstalling ECE ..." # TODO safety, warnings++ rm -rf $escenic_conf_dir/ \ - /opt/*tomcat* \ + ${appserver_root_dir}/*tomcat* \ $escenic_root_dir \ - /var/lib/escenic \ - /var/run/escenic/ \ + $escenic_data_dir \ + $escenic_run_dir/ \ $escenic_conf_dir/ \ /usr/bin/ece \ - /var/log/escenic/ \ + $escenic_log_dir/ \ /etc/apt/sources.list.d/escenic.list \ /var/run/ece-install.pid # $HOME/.m2 \ @@ -2158,7 +2166,7 @@ server { # Typical endpoint for Adactus/Mobilize to get the videos to # transcode. location /binary { - root /var/lib/escenic; + root $escenic_data_dir; index index.html; } } @@ -2191,7 +2199,7 @@ server { # Typical endpoint for Adactus/Mobilize to get the videos to # transcode. location /binary { - root /var/lib/escenic; + root $escenic_data_dir; index index.html; } } @@ -2354,9 +2362,9 @@ EOF done # we need to hack a bit since escenic_jstat_ looks for - # instance PIDs in /var/run/escenic ece-.pid. It's + # instance PIDs in $escenic_run_dir ece-.pid. It's # now -.pid - file=/var/run/escenic/$type-${instance_name}.pid + file=$escenic_run_dir/$type-${instance_name}.pid if [ ! -e $file ]; then run touch $file fi @@ -2420,7 +2428,7 @@ function restore_from_backup() restore_data_files=0 restore_conf=0 backup_file="" - backup_dir=/var/backups/escenic + backup_dir=$escenic_backups_dir if [ $(get_boolean_conf_value fai_enabled) -eq 1 -a \ $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then @@ -2539,7 +2547,7 @@ function restore_from_backup() run tar -C / -xf $backup_file var/lib/escenic add_next_step "Successfully restored Solr & ECE data files" add_next_step "Backup file used: $(basename $backup_file)" - add_next_step "Check /var/lib/escenic to verify they're all there." + add_next_step "Check $escenic_data_dir to verify they're all there." fi if [ $restore_conf -eq 1 -o $restore_all -eq 1 ]; then @@ -2557,7 +2565,7 @@ function restore_from_backup() run tar -C / -xf $backup_file opt add_next_step "Successfully restored Escenic & Tomcat binaries" add_next_step "Backup file used: $(basename $backup_file)" - add_next_step "Check /opt to verify that they're all there". + add_next_step "Check ${appserver_root_dir} to verify that they're all there". install_ece_third_party_packages set_up_engine_directories From d14793dd73bfe4009db0421ab250b514d59fc915 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 12 Jan 2012 12:55:47 +0800 Subject: [PATCH 0195/2585] - fixed syntax error in set_up_app_server, regression after yesterday's parameterisation. --- usr/sbin/ece-install | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 987f3599..3790ef7b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -304,8 +304,6 @@ function install_common_os_packages() done } -make_dir $ece_directories - function set_up_assembly_tool() { print_and_log "Setting up the Assembly Tool ..." @@ -819,7 +817,7 @@ function set_up_app_server done set_ece_instance_conf tomcat_base $tomcat_base - set_ece_instance_conf tomcat_home ${appserver_root_dir/tomcat + set_ece_instance_conf tomcat_home ${appserver_root_dir}/tomcat set_ece_instance_conf appserver_port $appserver_port run cd $tomcat_base/lib From 750839ce1436b958afcb7d1bd5ec61115296b6aa Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 12 Jan 2012 14:17:47 +0800 Subject: [PATCH 0196/2585] - added these options to ece-install.conf: escenic_spool_dir=/var/spool/escenic escenic_cache_dir=/var/cache/escenic escenic_crash_dir=/var/crash/escenic - moved the setting of engine_dir_list to common_pre_install as this list is comprised of variables settable in ece-install.conf - fixed corner case if you install nginx, and the sites available file is not available. This is really obscure, but it happened for me, so it can happen for others too. --- usr/sbin/ece-install | 48 ++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3790ef7b..ab21abd8 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -31,6 +31,9 @@ escenic_log_dir=/var/log/escenic escenic_data_dir=/var/lib/escenic escenic_run_dir=/var/run/escenic escenic_backups_dir=/var/backups/escenic +escenic_spool_dir=/var/spool/escenic +escenic_cache_dir=/var/cache/escenic +escenic_crash_dir=/var/crash/escenic appserver_root_dir=/opt # The script will install the sun-java6-jdk package on Debian based @@ -44,7 +47,6 @@ pid_file=/var/run/$(basename $0).pid download_dir=/tmp/ece-downloads log=/var/log/$(basename $0).log conf_file=$HOME/ece-install.conf -common_nursery_dir=/etc/escenic/engine/common ece_scripts_git_source=git://github.com/skybert/ece-scripts.git maven_opts="--batch-mode" wget_opts="--continue --inet4-only --quiet" @@ -181,22 +183,6 @@ PROFILE_CREATE_PUBLICATION=9 PROFILE_MONITORING_SERVER=10 PROFILE_RESTORE_FROM_BACKUP=11 -# Because of issue VF-3559, we also create the default family and host -# directories. -engine_dir_list=" -$common_nursery_dir -$escenic_conf_dir/engine/family/default -$escenic_conf_dir/engine/host/localhost -$escenic_root_dir -/var/cache/escenic -/var/crash/escenic -$escenic_data_dir -$escenic_data_dir/solr/data -$escenic_log_dir -$escenic_run_dir -/var/spool/escenic/migration -" - function make_dir() { if [ ! -d $1 ]; then @@ -1479,6 +1465,26 @@ function common_pre_install() run source $conf_file + # These variables are placed here as all the directories can be + # overridden in ece-install.conf + common_nursery_dir=$escenic_conf_dir/engine/common + + # Because of issue VF-3559, we also + # create the default family and host directories. + engine_dir_list=" + $common_nursery_dir + $escenic_conf_dir/engine/family/default + $escenic_conf_dir/engine/host/localhost + $escenic_root_dir + $escenic_cache_dir + $escenic_crash_dir + $escenic_data_dir + $escenic_data_dir/solr/data + $escenic_log_dir + $escenic_run_dir + $escenic_spool_dir/migration + " + if [ -z "$technet_user" -o -z "$technet_password" ]; then print_and_log "Be sure to set technet_user and technet_password " print_and_log "in $conf_file" @@ -1865,7 +1871,7 @@ function assemble_deploy_and_restart_type() ece_command="ece -i $instance_name -t $type assemble deploy restart" if [ $(is_installing_from_archives) -eq 1 ]; then - run cp $ece_instance_ear_file /var/cache/escenic/engine.ear + run cp $ece_instance_ear_file $escenic_cache_dir/engine.ear ece_command="ece -i $instance_name -t $type deploy restart" print_and_log "Using the supplied EAR instead of running an assembly." fi @@ -2152,7 +2158,11 @@ function install_web_server() fi file=/etc/nginx/sites-available/default - run mv $file $file.orig + # in some very unusual cases, this file will not exist (can occur + # when re-running ece-install several times). + if [ -e $file ]; then + run mv $file $file.orig + fi if [ $1 -eq 0 ]; then port=81 From 6245ab208a7f31d6011cd83d967ebb517ca49160 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 12 Jan 2012 14:20:10 +0800 Subject: [PATCH 0197/2585] - added all directory paths --- usr/share/doc/escenic/ece-install-guide.org | 80 ++++++++++++--------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index d7873521..7f726e50 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -32,24 +32,24 @@ ece-install. The ece-install script can be downloaded from: https://raw.github.com/skybert/ece-scripts/master/usr/sbin/ece-install or be downloaded together with the other ece-scripts using git: -#+BEGIN_SRC sh +#+BEGIN_SRC conf $ git clone git@github.com:skybert/ece-scripts.git #+END_SRC * Running the ece-install Script You must run the script as the root user. If you start the script as a regular user, it will complain: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install] You must be root when running ece-install #+END_SRC To become root on Ubuntu based systems and Mac OS X, do: -#+BEGIN_SRC sh +#+BEGIN_SRC conf $ sudo su #+END_SRC Please note that just doing "sudo ece-install" may not work. On all other Unix like system, do: -#+BEGIN_SRC sh +#+BEGIN_SRC conf $ su - #+END_SRC @@ -57,19 +57,19 @@ You will need access to http://technet.escenic.com and put these credentials in the root user's $HOME/ece-install.conf, normally this means /root/ece-install.conf, the script will also tell you this if you forget to provide such a configuration file: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install] /root/ece-install.conf doesn't exist, I cannot live without it :-( #+END_SRC As for the technet credentials, the script will also tell you if it cannot find what it's looking for: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install] Be sure to set technet_user and technet_password [ece-install] in /root/ece-install.conf #+END_SRC The minimum ece-install.conf file is thus: -#+BEGIN_SRC sh +#+BEGIN_SRC conf technet_user= technet_password= #+END_SRC @@ -81,13 +81,13 @@ need to specify some FAI specific parameters, see the FAI section below. With the configuration file in place, you're now ready to run it: -#+BEGIN_SRC sh +#+BEGIN_SRC conf # bash ece-install [-v|--verbose] #+END_SRC ece-install will try to "fail fast", exiting as soon as it detects an error during its execution: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-( [ece-install-1] See /var/log/ece-install.log for further details. #+END_SRC @@ -95,7 +95,7 @@ error during its execution: As a last resort, if something goes astray with the script, and the log file cannot give you enough clues, the ultimate way of debugging this is to run the BASH interpreter with the -x flag: -#+BEGIN_SRC sh +#+BEGIN_SRC conf # bash -x ece-install #+END_SRC @@ -106,7 +106,7 @@ on /var/log/ece-install.log should be sufficient, though. * Available Server Profiles When starting the script, it will ask you which server profile you want to install: -#+BEGIN_SRC sh +#+BEGIN_SRC conf Hi, which server profile do you wish to install? 1 - All in one, full stack on one host, suitable for dev & test environments @@ -243,7 +243,7 @@ these, please contact support@escenic.com. If you don't have these ready in your ece-install.conf, ece-install will complain: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf [ece-install] If you don't have these, please contact support@escenic.com #+END_SRC @@ -264,7 +264,7 @@ monitoring the different nodes. ** Profile - Restoring from backup ece-install can restore from a backup made by the ece script: -#+BEGIN_SRC sh +#+BEGIN_SRC conf $ ece -i backup #+END_SRC @@ -293,7 +293,7 @@ Your choice [1]> #+END_SRC The ece script mentioned above will create backups in -#+BEGIN_SRC sh +#+BEGIN_SRC conf /var/backups/escenic #+END_SRC and the ece-install script will hence expect to find them here. @@ -365,18 +365,18 @@ ERROR 1050 (42S01) at line 25: Table '`ece5db`.`AccessControlList`' already exis The ece-install script has support for doing a full automatic install (FAI). You can only install one profile at a time. The profiles are the parameters with "install" in their name, such as: -#+BEGIN_SRC sh +#+BEGIN_SRC conf fai_editor_install #+END_SRC When running in FAI mode, you probably want to redirect standard output to a log file for easy reading later on: -#+BEGIN_SRC sh +#+BEGIN_SRC conf # bash ece-install > ece-install.out & #+END_SRC Alternatively, you may do: -#+BEGIN_SRC sh +#+BEGIN_SRC conf # nohup bash ece-install > ece-install.out & #+END_SRC The "nohup" at the beginning and the ampersand at the end lets you log @@ -389,16 +389,16 @@ configuration archive instead of using the files provided with the Escenic Content Engine & plugins. The EAR to provide is the one you generate with: -#+BEGIN_SRC sh +#+BEGIN_SRC conf $ ece -i assemble #+END_SRC Normally, the EAR will then be available in: -#+BEGIN_SRC sh +#+BEGIN_SRC conf /var/cache/escenic/engine.ear #+END_SRC The configuration bundle must contain: -#+BEGIN_SRC sh +#+BEGIN_SRC conf engine/security engine/siteconfig/bootstrap-skeleton engine/siteconfig/config-skeleton @@ -408,7 +408,7 @@ assemblytool/plugins//siteconfig A simple way to create this bundle, is to use a server which has the assembly environment set up and then do: -#+BEGIN_SRC sh +#+BEGIN_SRC conf $ cd /opt/escenic/engine $ tar czf /tmp/nursery-skeleton-and-security.tar.gz \ engine/security \ @@ -421,7 +421,7 @@ $ tar czf /tmp/nursery-skeleton-and-security.tar.gz \ you need. You can now configure your FAI installation to use these by, e.g.: -#+BEGIN_SRC sh +#+BEGIN_SRC conf fai_presentation_ear=/tmp/engine.ear fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz #+END_SRC @@ -479,7 +479,7 @@ To automatically install an editorial server and create a publication called "jollygood", the minimal configuration in ece-install.conf would be: -#+BEGIN_SRC sh +#+BEGIN_SRC conf fai_enabled=1 fai_editor_install=1 fai_publication_create=1 @@ -489,14 +489,14 @@ fai_publication_name=jollygood *** Installing Two Presentation Servers On a Single Host If you wish to only install two presentation servers called "web1" and "web2" on your host, you will first run ece-install with: -#+BEGIN_SRC sh +#+BEGIN_SRC conf fai_enabled=1 fai_presentation_install=1 fai_presentation_name=web1 #+END_SRC And then re-issue ece-install with the following configuration: -#+BEGIN_SRC sh +#+BEGIN_SRC conf fai_enabled=1 fai_presentation_install=1 fai_presentation_name=web2 @@ -510,7 +510,7 @@ whereas the second one needs to override these. ** Verifying That the Script Is Running In FAI Mode When FAI is enabled, ece-install will report: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install] Full Automatic Install (FAI) enabled. [ece-install] All user input will be read from /root/ece-install.conf #+END_SRC @@ -518,7 +518,7 @@ When FAI is enabled, ece-install will report: * Running More Than One Installation Process If the script believes there's already an ece-intall process running, it will abort: -#+BEGIN_SRC sh +#+BEGIN_SRC conf [ece-install] There's already one ece-install process running. If you believe [ece-install] this is wrong, e.g. if a previous run of ece-install was aborted [ece-install] before it completed, you may delete /var/run/ece-install.pid and @@ -555,7 +555,7 @@ bundle, see the FAI section. * Using a Custom Configuration File for ece-install You can specify a different configuration by using the -f parameter: -#+BEGIN_SRC sh +#+BEGIN_SRC conf $ ece-install -f ece-install-presentation-server.conf #+END_SRC @@ -596,13 +596,23 @@ but these should be the most interesting ones. *) Applies only to Debian based systems. -The root directory of all Escenic software binaries, /opt/escenic, -together with the root directory for the configuration files may be -overwritten in ece-install.conf by setting these variables: +* Overriding the Escenic directories +All of the Escenic specific directories may be overwritten in +ece-install.conf. Here's an example of changing all the paths possible +with the same suffix. -#+BEGIN_SRC sh -escenic_root_dir=/opt/escenic-parallel-env -escenic_conf_dir=/etc/escenic-parallel-env +#+BEGIN_SRC conf +suffix=escenic-parallel + +escenic_root_dir=/opt/$suffix +escenic_conf_dir=/etc/$suffix +escenic_log_dir=/var/log/$suffix +escenic_data_dir=/var/lib/$suffix +escenic_run_dir=/var/run/$suffix +escenic_backups_dir=/var/backups/$suffix +escenic_spool_dir=/var/spool/$suffix +escenic_cache_dir=/var/cache/$suffix +escenic_crash_dir=/var/crash/$suffix #+END_SRC Note, this is only needed if you are running two completely separate @@ -632,7 +642,7 @@ things set up by the script. The ece-install will often be run in FAI mode where all the settings are read from ~/ece-install.conf. Given the following entries in this file: -#+BEGIN_SRC sh +#+BEGIN_SRC conf technet_user=apples technet_password=andoranges wf_user=foo From fdc32d7a2a91f528f234a6b8de194c4f8233dee4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 12 Jan 2012 14:20:35 +0800 Subject: [PATCH 0198/2585] - updated HTML --- usr/share/doc/escenic/ece-install-guide.html | 173 ++++++++++--------- 1 file changed, 95 insertions(+), 78 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 0c873aed..2aee0d45 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -288,13 +288,14 @@

    Table of Contents

  • 8 Re-running ece-install (and How To Speed It Up)
  • 9 Using a Custom Configuration File for ece-install
  • 10 Overview of File Paths Used by the ece-install script
  • -
  • 11 Assumptions +
  • 11 Overriding the Escenic directories
  • +
  • 12 Assumptions
  • -
  • 12 Uninstalling Everything That the ece-install Set Up
  • -
  • 13 Example Output FAI
  • +
  • 13 Uninstalling Everything That the ece-install Set Up
  • +
  • 14 Example Output FAI
  • @@ -366,7 +367,7 @@

    3 Downloading the ece-insta

    -
    $ git clone git@github.com:skybert/ece-scripts.git
    +
    $ git clone git@github.com:skybert/ece-scripts.git
     
    @@ -383,7 +384,7 @@

    4 Running the ece-install S

    -
    [ece-install] You must be root when running ece-install
    +
    [ece-install] You must be root when running ece-install
     

    @@ -391,7 +392,7 @@

    4 Running the ece-install S

    -
    $ sudo su
    +
    $ sudo su
     

    @@ -402,7 +403,7 @@

    4 Running the ece-install S

    -
    $ su -
    +
    $ su -
     
    @@ -414,7 +415,7 @@

    4 Running the ece-install S

    -
    [ece-install] /root/ece-install.conf doesn't exist, I cannot live without it :-(
    +
    [ece-install] /root/ece-install.conf doesn't exist, I cannot live without it :-(
     
    @@ -424,8 +425,8 @@

    4 Running the ece-install S

    -
    [ece-install] Be sure to set technet_user and technet_password
    -[ece-install] in /root/ece-install.conf
    +
    [ece-install] Be sure to set technet_user and technet_password
    +[ece-install] in /root/ece-install.conf
     
    @@ -434,7 +435,7 @@

    4 Running the ece-install S

    -
    technet_user=<user>
    +
    technet_user=<user>
     technet_password=<password>
     
    @@ -451,7 +452,7 @@

    4 Running the ece-install S

    -
    # bash ece-install [-v|--verbose]
    +
    # bash ece-install [-v|--verbose]
     
    @@ -461,8 +462,8 @@

    4 Running the ece-install S

    -
    [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-(
    -[ece-install-1] See /var/log/ece-install.log for further details.
    +
    [ece-install-1] The command [cd /does/not/exist] FAILED, exiting :-(
    +[ece-install-1] See /var/log/ece-install.log for further details.
     
    @@ -473,7 +474,7 @@

    4 Running the ece-install S

    -
    # bash -x ece-install
    +
    # bash -x ece-install
     
    @@ -495,22 +496,22 @@

    5 Available Server Profiles

    -
    Hi, which server profile do you wish to install?
    +
    Hi, which server profile do you wish to install?
     
    -   1 - All in one, full stack on one host, suitable for dev & test environments
    -   2 - Editorial (publication) server
    -   3 - Presentation server (ECE + memcached).
    -   4 - Database server
    -   5 - Cache server (cache and web server)
    -   6 - RMI hub
    -   7 - Search server (Solr + indexer-webapp)
    -   8 - Install Widget Framework.
    -   9 - Create a new publication based on WF if available, ECE/clean-demo if not
    -   10 - A monitoring server (web server + Munin gatherer)
    -   11 - Restore from backup (DB, data files, binaries, conf & publications)
    +   1 - All in one, full stack on one host, suitable for dev & test environments
    +   2 - Editorial (publication) server
    +   3 - Presentation server (ECE + memcached).
    +   4 - Database server
    +   5 - Cache server (cache and web server)
    +   6 - RMI hub
    +   7 - Search server (Solr + indexer-webapp)
    +   8 - Install Widget Framework.
    +   9 - Create a new publication based on WF if available, ECE/clean-demo if not
    +   10 - A monitoring server (web server + Munin gatherer)
    +   11 - Restore from backup (DB, data files, binaries, conf & publications)
     
    -Select 1-11 and press ENTER
    -Your choice [1]> 
    +Select 1-11 and press ENTER
    +Your choice [1]> 
     
    @@ -778,8 +779,8 @@

    5.8 Profile - Install Wid

    -
    [ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf
    -[ece-install] If you don't have these, please contact support@escenic.com
    +
    [ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf
    +[ece-install] If you don't have these, please contact support@escenic.com
     
    @@ -825,7 +826,7 @@

    5.11 Profile - Restoring

    -
    $ ece -i <instance> backup
    +
    $ ece -i <instance> backup
     
    @@ -892,7 +893,7 @@

    5.12.2 Select Which Ba

    -
    /var/backups/escenic
    +
    /var/backups/escenic
     

    @@ -1027,7 +1028,7 @@

    6 Full Automatic Install (F

    -
    fai_editor_install
    +
    fai_editor_install
     
    @@ -1037,7 +1038,7 @@

    6 Full Automatic Install (F

    -
    # bash ece-install > ece-install.out &
    +
    # bash ece-install > ece-install.out &
     
    @@ -1046,7 +1047,7 @@

    6 Full Automatic Install (F

    -
    # nohup bash ece-install > ece-install.out &
    +
    # nohup bash ece-install > ece-install.out &
     

    @@ -1070,7 +1071,7 @@

    6.1 Installing from EARs

    -
    $ ece -i <instance> assemble 
    +
    $ ece -i <instance> assemble 
     

    @@ -1078,7 +1079,7 @@

    6.1 Installing from EARs

    -
    /var/cache/escenic/engine.ear
    +
    /var/cache/escenic/engine.ear
     
    @@ -1087,10 +1088,10 @@

    6.1 Installing from EARs

    -
    engine/security
    -engine/siteconfig/bootstrap-skeleton
    -engine/siteconfig/config-skeleton
    -assemblytool/plugins/<plugin>/siteconfig
    +
    engine/security
    +engine/siteconfig/bootstrap-skeleton
    +engine/siteconfig/config-skeleton
    +assemblytool/plugins/<plugin>/siteconfig
     
    @@ -1101,12 +1102,12 @@

    6.1 Installing from EARs -
    $ cd /opt/escenic/engine
    -$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \
    -  engine/security \
    -  engine/siteconfig/config-skeleton/ \
    -  engine/siteconfig/bootstrap-skeleton/ \
    -  `find -L  assemblytool/plugins/ -name siteconfig`
    +
    $ cd /opt/escenic/engine
    +$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \
    +  engine/security \
    +  engine/siteconfig/config-skeleton/ \
    +  engine/siteconfig/bootstrap-skeleton/ \
    +  `find -L  assemblytool/plugins/ -name siteconfig`
     
    @@ -1118,7 +1119,7 @@

    6.1 Installing from EARs -
    fai_presentation_ear=/tmp/engine.ear
    +
    fai_presentation_ear=/tmp/engine.ear
     fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz
     
    @@ -1208,7 +1209,7 @@

    6.3.1 Installing an Edi -
    fai_enabled=1
    +
    fai_enabled=1
     fai_editor_install=1
     fai_publication_create=1
     fai_publication_name=jollygood
    @@ -1228,7 +1229,7 @@ 

    6.3.2 Installing Two Pr

    -
    fai_enabled=1
    +
    fai_enabled=1
     fai_presentation_install=1
     fai_presentation_name=web1
     
    @@ -1239,7 +1240,7 @@

    6.3.2 Installing Two Pr

    -
    fai_enabled=1
    +
    fai_enabled=1
     fai_presentation_install=1
     fai_presentation_name=web2
     fai_presentation_port=8081
    @@ -1265,8 +1266,8 @@ 

    6.4 Verifying That the Sc

    -
    [ece-install] Full Automatic Install (FAI) enabled.
    -[ece-install] All user input will be read from /root/ece-install.conf
    +
    [ece-install] Full Automatic Install (FAI) enabled.
    +[ece-install] All user input will be read from /root/ece-install.conf
     
    @@ -1284,7 +1285,7 @@

    7 Running More Than One Ins

    -
    [ece-install] There's already one ece-install process running. If you believe
    +
    [ece-install] There's already one ece-install process running. If you believe
     [ece-install] this is wrong, e.g. if a previous run of ece-install was aborted
     [ece-install] before it completed, you may delete /var/run/ece-install.pid and
     [ece-install] run ece-install again.
    @@ -1348,7 +1349,7 @@ 

    9 Using a Custom Configurat

    -
    $ ece-install -f ece-install-presentation-server.conf
    +
    $ ece-install -f ece-install-presentation-server.conf
     
    @@ -1404,16 +1405,32 @@

    10 Overview of File Paths

    *) Applies only to Debian based systems.

    -

    -The root directory of all Escenic software binaries, /opt/escenic, -together with the root directory for the configuration files may be -overwritten in ece-install.conf by setting these variables: + + + + +

    +

    11 Overriding the Escenic directories

    +
    + +

    All of the Escenic specific directories may be overwritten in +ece-install.conf. Here's an example of changing all the paths possible +with the same suffix.

    -
    escenic_root_dir=/opt/escenic-parallel-env
    -escenic_conf_dir=/etc/escenic-parallel-env
    +
    suffix=escenic-parallel
    +
    +escenic_root_dir=/opt/$suffix
    +escenic_conf_dir=/etc/$suffix
    +escenic_log_dir=/var/log/$suffix
    +escenic_data_dir=/var/lib/$suffix
    +escenic_run_dir=/var/run/$suffix
    +escenic_backups_dir=/var/backups/$suffix
    +escenic_spool_dir=/var/spool/$suffix
    +escenic_cache_dir=/var/cache/$suffix
    +escenic_crash_dir=/var/crash/$suffix
     
    @@ -1427,16 +1444,16 @@

    10 Overview of File Paths

    -
    -

    11 Assumptions

    -
    +
    +

    12 Assumptions

    +
    -
    -

    11.1 /etc/esecenic is shared

    -
    +
    +

    12.1 /etc/esecenic is shared

    +

    It's assumed that the /etc/escenic directory is either on a shared file system or rsync-ed among the hosts that are running the ECE @@ -1447,9 +1464,9 @@

    11.1 /etc/esecenic is sh

    -
    -

    12 Uninstalling Everything That the ece-install Set Up

    -
    +
    +

    13 Uninstalling Everything That the ece-install Set Up

    +

    WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your @@ -1467,9 +1484,9 @@

    12 Uninstalling Everything

    -
    -

    13 Example Output FAI

    -
    +
    +

    14 Example Output FAI

    +

    The ece-install will often be run in FAI mode where all the settings are read from ~/ece-install.conf. Given the following entries in this @@ -1477,7 +1494,7 @@

    13 Example Output FAI

    -
    technet_user=apples
    +
    technet_user=apples
     technet_password=andoranges
     wf_user=foo
     wf_password=bar
    @@ -1539,7 +1556,7 @@ 

    13 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2012-01-11 18:24:53 CST + Date: 2012-01-12 14:20:21 CST

    From ef5b30b07396f311cb7e1c3208ed3d471dbfd91f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 13 Jan 2012 11:17:27 +0800 Subject: [PATCH 0199/2585] - now runs unzip with --quiet - replaced short with long --update option switch to unzip -> self documenting. - added country code switch for (Debian) mirror, will/can be used for the other mirrors too. mirror_country_suffix=jp --- usr/sbin/ece-install | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index ab21abd8..e9ad2520 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -36,6 +36,8 @@ escenic_cache_dir=/var/cache/escenic escenic_crash_dir=/var/crash/escenic appserver_root_dir=/opt +mirror_country_suffix=jp + # The script will install the sun-java6-jdk package on Debian based # systems and this is the path of the JAVA home with this package. If # you're using a different system or have other preferences, change @@ -298,7 +300,7 @@ function set_up_assembly_tool() cd $escenic_root_dir/assemblytool/ if [ -e $download_dir/assemblytool*zip ]; then - run unzip -u $download_dir/assemblytool*zip + run unzip --quiet --update $download_dir/assemblytool*zip fi # adding an instance layer to the Nursery configuration @@ -369,7 +371,7 @@ function set_up_engine_and_plugins() done if [ ! -d ${engine_dir} ]; then - run unzip -u $download_dir/${engine_file} + run unzip --quiet --update $download_dir/${engine_file} if [ -h engine ]; then run rm engine fi @@ -389,7 +391,7 @@ function set_up_engine_and_plugins() continue fi - run unzip -u $el + run unzip --quiet --update $el done ece_software_setup_completed=1 @@ -638,7 +640,7 @@ function install_ece_third_party_packages grep non-free | \ wc -l) if [ $has_non_free -eq 0 ]; then - add_apt_source "deb http://ftp.tw.debian.org/debian/ $(lsb_release -s -c) contrib non-free" + add_apt_source "deb http://ftp.${mirror_country_suffix}.debian.org/debian/ $(lsb_release -s -c) contrib non-free" fi fi @@ -1484,7 +1486,7 @@ function common_pre_install() $escenic_run_dir $escenic_spool_dir/migration " - + if [ -z "$technet_user" -o -z "$technet_password" ]; then print_and_log "Be sure to set technet_user and technet_password " print_and_log "in $conf_file" @@ -2059,7 +2061,7 @@ EOF 1>>$log 2>>$log cd $escenic_root_dir/ - unzip -u $download_dir/$(basename $el) \ + unzip --quiet --update $download_dir/$(basename $el) \ 1>>$log 2>>$log done From 4bc5f848962cd22f03943b5ddb880070a1414e56 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 13 Jan 2012 11:25:09 +0800 Subject: [PATCH 0200/2585] - fixed regression from d9d517e0d75439b1e47d5fbd8d4f81fde85da3d7, tomcat_base became ${appserver_root_dir}-${instance}, instead of ${appserver_root_dir}/tomcat-${instance}. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index e9ad2520..dfd3d2ad 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -796,7 +796,7 @@ function set_up_app_server fi run ln -sf ${tomcat_dir} tomcat - tomcat_base=${appserver_root_dir}-${instance_name} + tomcat_base=${appserver_root_dir}/tomcat-${instance_name} make_dir $tomcat_base run cp -r ${appserver_root_dir}/${tomcat_dir}/conf $tomcat_base From e73a2f1c938ba18e8e388157e1c757351c7111bf Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 13 Jan 2012 11:25:49 +0800 Subject: [PATCH 0201/2585] - added header signature --- usr/local/src/create-packages | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index c94d7daa..89f2056f 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -1,5 +1,10 @@ #! /usr/bin/env bash +# script fro creating DEB and RPM packages of the ece-install and ece +# scripts. + +# by tkj@vizrt.com + rm -rf debian/{etc,usr} target_dir=debian From 47d96443a3959ba4cde612b41f0036140e78d257 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 13 Jan 2012 17:10:23 +0800 Subject: [PATCH 0202/2585] - --quiet --udpate -> -q -u --- usr/sbin/ece-install | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index dfd3d2ad..6c8084e4 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -300,7 +300,7 @@ function set_up_assembly_tool() cd $escenic_root_dir/assemblytool/ if [ -e $download_dir/assemblytool*zip ]; then - run unzip --quiet --update $download_dir/assemblytool*zip + run unzip -q -u $download_dir/assemblytool*zip fi # adding an instance layer to the Nursery configuration @@ -327,7 +327,7 @@ EOF continue fi - # nuisance to get the community engine, but not the engine + # nuisance to get the community engine, but not the engine if [ $(echo $directory | grep engine | wc -l) -gt 0 ]; then if [ $(echo $directory | grep community | wc -l) -lt 1 ]; then continue @@ -371,7 +371,7 @@ function set_up_engine_and_plugins() done if [ ! -d ${engine_dir} ]; then - run unzip --quiet --update $download_dir/${engine_file} + run unzip -q -u $download_dir/${engine_file} if [ -h engine ]; then run rm engine fi @@ -391,7 +391,7 @@ function set_up_engine_and_plugins() continue fi - run unzip --quiet --update $el + run unzip -q -u $el done ece_software_setup_completed=1 @@ -510,9 +510,12 @@ EOF if [ -w ${file} ]; then sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file exit_on_error "sed on $file" + elif [ ! -e ${file} ]; then + print_and_log "Could not find ${file}," + print_and_log "I assume you have not installed Community Engine." else - print_and_log "Could not find/write to ${file}," - print_and_log "have you installed the Community Engine plugin?" + print_and_log "Could not write to ${file}," + print_and_log "Community Engine might not work because of this. " remove_pid_and_exit_in_error fi @@ -1167,9 +1170,14 @@ function set_up_user_enviornment() { print_and_log "Setting up the ${ece_user} user's UNIX environment ..." - if [ $(grep JAVA_HOME /home/$ece_user/.bashrc 2>/dev/null | wc -l) \ + local bashrc=/home/${ece_user}/.bashrc + if [ $(uname -s) = "Darwin" ]; then + bashrc=/Users/${ece_user}/.bashrc + fi + + if [ $(grep JAVA_HOME ${bashrc} 2>/dev/null | wc -l) \ -lt 1 ]; then - run echo JAVA_HOME=$java_home >> /home/$ece_user/.bashrc + run echo JAVA_HOME=$java_home >> ${bashrc} fi if [ $(grep JAVA_HOME /root/.bashrc 2>/dev/null | wc -l) -lt 1 ]; then @@ -1178,11 +1186,14 @@ function set_up_user_enviornment() export JAVA_HOME=$java_home - bashrc=/home/${ece_user}/.bashrc if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ wc -l) -lt 1 ]; then echo ". /etc/bash_completion.d/ece" > ${bashrc} fi + + if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then + echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" > ${bashrc} + fi } function set_up_solr() @@ -2061,7 +2072,7 @@ EOF 1>>$log 2>>$log cd $escenic_root_dir/ - unzip --quiet --update $download_dir/$(basename $el) \ + unzip -q -u $download_dir/$(basename $el) \ 1>>$log 2>>$log done From 12f380218e29f35b5a2fb44382181dc54358684d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 13 Jan 2012 17:45:12 +0800 Subject: [PATCH 0203/2585] - removed adding JAVA_HOME to .bashrc, this shouldn't be needed as ece sorts this out by itself. - moved from common_pre_install set_up_ece_scripts to install_ece_instance as this makes more sense and non-standard installs would otherwise get ece.conf copied to the wrong place. --- usr/sbin/ece-install | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 6c8084e4..5733c826 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -410,7 +410,7 @@ function set_up_ece_scripts() fi run cp -r ece-scripts/usr/* /usr/ - run cp -r ece-scripts/etc/* /etc/ + run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ } function set_up_ecedb() @@ -511,7 +511,7 @@ EOF sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file exit_on_error "sed on $file" elif [ ! -e ${file} ]; then - print_and_log "Could not find ${file}," + print_and_log "I Could not find ECOME configuration file," print_and_log "I assume you have not installed Community Engine." else print_and_log "Could not write to ${file}," @@ -1175,24 +1175,13 @@ function set_up_user_enviornment() bashrc=/Users/${ece_user}/.bashrc fi - if [ $(grep JAVA_HOME ${bashrc} 2>/dev/null | wc -l) \ - -lt 1 ]; then - run echo JAVA_HOME=$java_home >> ${bashrc} - fi - - if [ $(grep JAVA_HOME /root/.bashrc 2>/dev/null | wc -l) -lt 1 ]; then - run echo JAVA_HOME=$java_home >> /root/.bashrc - fi - - export JAVA_HOME=$java_home - if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ wc -l) -lt 1 ]; then - echo ". /etc/bash_completion.d/ece" > ${bashrc} + echo ". /etc/bash_completion.d/ece" >> ${bashrc} fi if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then - echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" > ${bashrc} + echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" >> ${bashrc} fi } @@ -1525,7 +1514,6 @@ function common_pre_install() make_dir $download_dir stop_conflicting_processes install_common_os_packages - set_up_ece_scripts create_user_and_group_if_not_present set_up_user_enviornment } @@ -1824,6 +1812,7 @@ function install_ece_instance() ask_for_instance_name $1 set_up_engine_directories + set_up_ece_scripts set_archive_files_depending_on_profile From 6e17d34e0faa3ee170993fdefd4fb0fcbdde231a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 13 Jan 2012 18:42:56 +0800 Subject: [PATCH 0204/2585] - now, set_up_ece_scripts also updates the ece.conf according to the directory path variables. --- usr/sbin/ece-install | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 5733c826..9a879d6a 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -411,6 +411,20 @@ function set_up_ece_scripts() run cp -r ece-scripts/usr/* /usr/ run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ + + local file=${escenic_conf_dir}/ece.conf + set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file + set_conf_file_value backup_dir ${escenic_backups_dir} $file + set_conf_file_value cache_dir ${escenic_cache_dir} ${file} + set_conf_file_value ece_home ${escenic_root_dir}/engine ${file} + set_conf_file_value heap_dump_dir ${escenic_crash_dir} ${file} + set_conf_file_value log_dir ${escenic_log_dir} ${file} + set_conf_file_value pid_dir ${escenic_run_dir} ${file} + set_conf_file_value rmi_hub_conf ${escenic_conf_dir}/rmi-hub ${file} + set_conf_file_value solr_home ${escenic_data_dir}/solr ${file} + set_conf_file_value ece_security_configuration_dir \ + ${escenic_conf_dir}/engine/common/security \ + ${file} } function set_up_ecedb() @@ -511,7 +525,7 @@ EOF sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file exit_on_error "sed on $file" elif [ ! -e ${file} ]; then - print_and_log "I Could not find ECOME configuration file," + print_and_log "I Could not find an ECOME configuration file," print_and_log "I assume you have not installed Community Engine." else print_and_log "Could not write to ${file}," From 1a40822ed80eeba09ee1bec479a48e2d5d8d2a86 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 16 Jan 2012 11:19:03 +0800 Subject: [PATCH 0205/2585] - regression: now copies bash completion - now, the directories used in the bash completion is populated from --- usr/sbin/ece-install | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9a879d6a..b830a754 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -411,7 +411,8 @@ function set_up_ece_scripts() run cp -r ece-scripts/usr/* /usr/ run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ - + run cp -r ece-scripts/bash_completion.d/ece /etc/bash_completion.d/ + local file=${escenic_conf_dir}/ece.conf set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file set_conf_file_value backup_dir ${escenic_backups_dir} $file @@ -425,6 +426,9 @@ function set_up_ece_scripts() set_conf_file_value ece_security_configuration_dir \ ${escenic_conf_dir}/engine/common/security \ ${file} + + run sed -i "s#/etc/escenic#${escenic_conf_dir}#g" /etc/bash_completion.d/ece + run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece } function set_up_ecedb() From 4508a8a8dacc023c460b8735179bff140b7fbc59 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 16 Jan 2012 11:26:39 +0800 Subject: [PATCH 0206/2585] - set_up_assembly_tool to fixes the path to the Nursery configuration according to escenic_conf_dir which may have been overridden by the user. --- usr/sbin/ece-install | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index b830a754..742a896c 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -314,6 +314,13 @@ fileSystemRoot = $escenic_conf_dir/engine/instance/\${com.escenic.instance}/ EOF echo "" >> Nursery.properties echo "layer.06 = /layers/instance/Layer" >> Nursery.properties + + # fixing the path to the Nursery configuration according to + # escenic_conf_dir which may have been overridden by the user. + for el in $(find $escenic_root_dir/assemblytool/conf -name Files.properties) + do + sed -i "s#/etc/escenic#${escenic_conf_dir}#g" $el + done # set up which plugins to use cd $escenic_root_dir/assemblytool/ From 949515b27937b9687ed6c820461954e07a321235 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 16 Jan 2012 11:58:53 +0800 Subject: [PATCH 0207/2585] - fixed path to bash_completion.d --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 742a896c..891df663 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -418,7 +418,7 @@ function set_up_ece_scripts() run cp -r ece-scripts/usr/* /usr/ run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ - run cp -r ece-scripts/bash_completion.d/ece /etc/bash_completion.d/ + run cp -r ece-scripts/etc/bash_completion.d/ece /etc/bash_completion.d/ local file=${escenic_conf_dir}/ece.conf set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file From 2b91b89d753fb122a4022a2c7a6019210e846f1a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 17 Jan 2012 12:23:11 +0800 Subject: [PATCH 0208/2585] - removed common_settings_list it isn't used anymore (and haven't for a long time). --- usr/bin/ece | 7 ------- 1 file changed, 7 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index a871108e..8f6ac818 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -34,13 +34,6 @@ search rmi-hub " -common_settings_list=" -ece_home -java_home -log_dir -pid_dir -" - hub_required_fields=" ece_server_hostname rmi_hub_conf From 8539feaaa661c960ea0cf0232c086973a9f19d82 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 17 Jan 2012 12:36:43 +0800 Subject: [PATCH 0209/2585] - introduced escenic_conf_dir (yes, escenic, not ece as this holds both RMI hub, Analysis Engine and ECE configuration files). - the default is /etc/escenic, but this can be overridden in any of the .conf files. Since this is a bit chicken and the egg problem, in order to load a .conf file which overrides it, you must set the ECE_CONF_LOCATIONS environment variable, e.g.: export ECE_CONF_LOCATIONS=/etc/escenic-parallel In which you define escenic_conf_dir=/etc/escenic-parallel to be used by all other calls to the script. --- etc/escenic/ece.conf | 6 ++++-- usr/bin/ece | 30 ++++++++++++++++-------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/etc/escenic/ece.conf b/etc/escenic/ece.conf index 3f5ddc0c..09cb959b 100644 --- a/etc/escenic/ece.conf +++ b/etc/escenic/ece.conf @@ -50,10 +50,12 @@ appserver_port=8080 ######################################################################## assemblytool_home=/opt/escenic/assemblytool cache_dir=/var/cache/escenic -ece_security_configuration_dir=/etc/escenic/engine/common/security +# The root of all Escenic configuration files (not only ECE) +escenic_conf_dir=/etc/escenic +ece_security_configuration_dir=${escenic_conf_dir}/engine/common/security log_dir=/var/log/escenic pid_dir=/var/run/escenic -rmi_hub_conf=/etc/escenic/rmi-hub +rmi_hub_conf=${escenic_conf_dir}/rmi-hub rmi_hub_home=$ece_home/contrib/rmi-hub solr_home=/var/lib/escenic/solr tmp_dir=/tmp diff --git a/usr/bin/ece b/usr/bin/ece index 8f6ac818..ed3dc60b 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -14,6 +14,7 @@ verbose=0 force_ipv4=0 quiet=0 everything_but_the_kitchen_sink=0 +escenic_conf_dir=/etc/escenic ###################################################################### # Dear user, don't touch anyting in this script, especially below this @@ -44,6 +45,7 @@ engine_required_fields=" appserver assemblytool_home cache_dir +escenic_conf_dir ece_home ece_security_configuration_dir enable_heap_dump @@ -164,13 +166,13 @@ function read_conf_file() # If the system is installed using the recommended paths, the method # will return a list of the instances configured in -# /etc/escenic/engine/instance +# ${escenic_conf_dir}/engine/instance function get_instance_list() { instance_list="" - if [ -r /etc/escenic/engine/instance ]; then - instance_list=$(ls /etc/escenic/engine/instance) + if [ -r ${escenic_conf_dir}/engine/instance ]; then + instance_list=$(ls ${escenic_conf_dir}/engine/instance) fi echo $instance_list @@ -185,11 +187,11 @@ function set_common_settings() else conf_file_location_list=" `dirname $0` - /etc/escenic/$type/instance/$instance - /etc/escenic/$type/host/`echo $HOSTNAME | tr '[A-Z]' '[a-z]'` - /etc/escenic/$type/common - /etc/escenic/$type - /etc/escenic + ${escenic_conf_dir}/$type/instance/$instance + ${escenic_conf_dir}/$type/host/`echo $HOSTNAME | tr '[A-Z]' '[a-z]'` + ${escenic_conf_dir}/$type/common + ${escenic_conf_dir}/$type + ${escenic_conf_dir} `dirname $0`/../etc " fi @@ -666,7 +668,7 @@ function backup_type() archive_file=$backup_dir/${type}-${instance}-backup-$(date --iso).tar possible_backup=" -/etc/escenic +${escenic_conf_dir} /opt/escenic /etc/init.d/ece /etc/default/ece @@ -703,7 +705,7 @@ function backup_type() print "- All Escenic binaries & publication templates in /opt/escenic" fi - print "- All configuration in /etc/escenic and /etc/default/ece" + print "- All configuration in ${escenic_conf_dir} and /etc/default/ece" print "- All bootstrap scripts from /etc/init.d" print "Enjoy!" } @@ -1066,18 +1068,18 @@ EOF run cd ${tomcat_home_dir} run cp -r ${tomcat_home}/{bin,lib} ${tomcat_home_dir} - local etc_escenic_dir=${package_dir}/etc/escenic + local etc_escenic_dir=${package_dir}${escenic_conf_dir} run mkdir -p ${etc_escenic_dir} cd ${etc_escenic_dir} - run cp -r /etc/escenic/{ece,ece-${instance}}.conf \ + run cp -r ${escenic_conf_dir}/{ece,ece-${instance}}.conf \ ${etc_escenic_dir} run mkdir -p ${etc_escenic_dir}/engine - run cp -r /etc/escenic/engine/common \ + run cp -r ${escenic_conf_dir}/engine/common \ ${etc_escenic_dir} run mkdir -p ${etc_escenic_dir}/engine/instance - run cp -r /etc/escenic/engine/instance/${instance} \ + run cp -r ${escenic_conf_dir}/engine/instance/${instance} \ ${etc_escenic_dir} ) fi From 9f20b03bc5cc0c1a7424cba2a82c8d63298277af Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 17 Jan 2012 12:38:24 +0800 Subject: [PATCH 0210/2585] - ece now supports escenic_conf_dir, so the ece-install now sets this in the install ece.conf file. --- usr/sbin/ece-install | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 891df663..6c17d8fc 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -425,6 +425,7 @@ function set_up_ece_scripts() set_conf_file_value backup_dir ${escenic_backups_dir} $file set_conf_file_value cache_dir ${escenic_cache_dir} ${file} set_conf_file_value ece_home ${escenic_root_dir}/engine ${file} + set_conf_file_value escenic_conf_dir ${escenic_conf_dir} ${file} set_conf_file_value heap_dump_dir ${escenic_crash_dir} ${file} set_conf_file_value log_dir ${escenic_log_dir} ${file} set_conf_file_value pid_dir ${escenic_run_dir} ${file} From a31c5674ea0631aa88c42f335634120e015c5135 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 17 Jan 2012 19:45:18 +0800 Subject: [PATCH 0211/2585] - added configuration of DB creation parameters: fai_db_install=1 fai_db_user=myuser fai_db_password=mypassword fai_db_schema=mydb fai_db_host=localhost fai_db_drop_old_db_first=1 --- usr/share/doc/escenic/ece-install-guide.org | 78 +++++++++++---------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 7f726e50..7f7c502a 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -433,43 +433,47 @@ profiles, see the table below. The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: -|----------------------------------+----------------------+-----------------------------------------------------| -| Parameter | Default | Description | -|----------------------------------+----------------------+-----------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to to base the new publication on | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|----------------------------------+----------------------+-----------------------------------------------------| +|----------------------------------+----------------------+----------------------------------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+----------------------+----------------------------------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_drop_old_db_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_db\_schema | ece5db | Useful for DB installation profile | +| fai\_db\_user | ece5user | Useful for DB installation profile | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to to base the new publication on | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|----------------------------------+----------------------+----------------------------------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 2e55412c5074723a417390e4f679159a14c5757f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 17 Jan 2012 19:46:01 +0800 Subject: [PATCH 0212/2585] - added configuration of DB parameters: fai_db_install=1 fai_db_user=myuser fai_db_password=mypassword fai_db_schema=mydb fai_db_host=localhost fai_db_drop_old_db_first=1 --- usr/sbin/ece-install | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 6c17d8fc..4de97e73 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -465,15 +465,42 @@ function set_up_ecedb() fi done - # TODO in future, it'll be possible to override db_user, - # db_password and db_schema here by reading these via - # get_conf_value + # the user may override standard DB settings in ece-install.conf + if [ -n "${fai_db_host}" ]; then + db_host=${fai_db_host} + fi + if [ -n "${fai_db_user}" ]; then + db_user=${fai_db_user} + fi + if [ -n "${fai_db_password}" ]; then + db_password=${fai_db_password} + fi + if [ -n "${fai_db_schema}" ]; then + db_schema=${fai_db_schema} + fi + if [ -n "${fai_db_drop_old_db_first}" ]; then + drop_db_first=${fai_db_drop_old_db_first} + print_and_log "WARNING: I hope you know what you're doing!" + print_and_log "WARNING: fai_db_drop_old_db_first is 1 (true)" + fi + ece_home=${escenic_root_dir}/engine + + # the port isn't fully supported. The user must himself update the + # mysql configuration to run it on a non standard port. + + if [ -n "${fai_db_port}" ]; then + db_port=${fai_db_port} + else + db_port=3306 + fi + + pre_install_new_ecedb create_ecedb cd ~/ run rm -rf $escenic_root_dir/engine/plugins - add_next_step "DB is now set up on $HOSTNAME:3306" + add_next_step "DB is now set up on ${db_host}:${db_port}" } function set_up_basic_nursery_configuration() @@ -877,11 +904,11 @@ function set_up_app_server maxIdle="8" maxWait="2000" initialSize="20" - username="ece5user" - password="ece5password" + username="${db_user}" + password="${db_password}" driverClassName="com.mysql.jdbc.Driver" -url="jdbc:mysql://localhost:3306/ece5db?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8" +url="jdbc:mysql://${db_host}:${db_port}/${db_schema}?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8" removeAbandoned="true" removeAbandonedTimeout="120" logAbandoned="true" From 3f1feb6952654723b1f37742b08403a1a33d56b8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 17 Jan 2012 19:49:40 +0800 Subject: [PATCH 0213/2585] - pernickety --- usr/sbin/ece-install | 1 - 1 file changed, 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 4de97e73..2bbd0767 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -487,7 +487,6 @@ function set_up_ecedb() # the port isn't fully supported. The user must himself update the # mysql configuration to run it on a non standard port. - if [ -n "${fai_db_port}" ]; then db_port=${fai_db_port} else From 45f07261e5fe1a6983e719642159c1eb53a921f4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 18 Jan 2012 10:45:42 +0800 Subject: [PATCH 0214/2585] - forgot to set DB host/port in set_up_app_server/ECE_UPDATE_DS from local variables. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 2bbd0767..65113946 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -930,7 +930,7 @@ url="jdbc:mysql://${db_host}:${db_port}/${db_schema}?autoReconnect=true&useU username="ece5user" password="ece5password" driverClassName="com.mysql.jdbc.Driver" - url="jdbc:mysql://localhost:3306/ece5db?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8" + url="jdbc:mysql://${db_host}:${db_port}/${db_schema}?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8" removeAbandoned="true" removeAbandonedTimeout="120" logAbandoned="true" From de4aa1f723e06e77a6aa362be538e884c7c287e4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 18 Jan 2012 12:11:03 +0800 Subject: [PATCH 0215/2585] - added set_db_settings_from_fai_conf, called from set_up_ecedb - added set_db_defaults_if_not_set, called from both set_up_app_server and set_up_ecedb. This method sets defaults for all DB variables if not set: * host - localhost * port - 3306 * schema - ece5db * user - ece5usr * password - read-the-source-luke --- usr/sbin/ece-install | 110 +++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 65113946..5e129476 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -439,6 +439,63 @@ function set_up_ece_scripts() run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece } + +# Method used both from interactive mode to set any missing values +# (defaults) +function set_db_defaults_if_not_set() +{ + if [ -z "$db_host" ]; then + db_host=localhost + fi + + if [ -z "$db_port" ]; then + db_port=3306 + fi + + if [ -z "$db_user" ]; then + db_user=ece5user + fi + + if [ -z "$db_password" ]; then + db_user=ece5password + fi + + if [ -z "$db_schema" ]; then + db_schema=ece5db + fi +} + +function set_db_settings_from_fai_conf() +{ + # the port isn't fully supported. The user must himself update the + # mysql configuration to run it on a non standard port. + if [ -n "${fai_db_port}" ]; then + db_port=${fai_db_port} + fi + + if [ -n "${fai_db_host}" ]; then + db_host=${fai_db_host} + fi + + if [ -n "${fai_db_user}" ]; then + db_user=${fai_db_user} + fi + + if [ -n "${fai_db_password}" ]; then + db_password=${fai_db_password} + fi + + if [ -n "${fai_db_schema}" ]; then + db_schema=${fai_db_schema} + fi + + if [ -n "${fai_db_drop_old_db_first}" ]; then + drop_db_first=${fai_db_drop_old_db_first} + print_and_log "WARNING: I hope you know what you're doing!" + print_and_log "WARNING: fai_db_drop_old_db_first is 1 (true)" + fi +} + function set_up_ecedb() { print_and_log "Setting up the ECE database schema ..." @@ -466,33 +523,11 @@ function set_up_ecedb() done # the user may override standard DB settings in ece-install.conf - if [ -n "${fai_db_host}" ]; then - db_host=${fai_db_host} - fi - if [ -n "${fai_db_user}" ]; then - db_user=${fai_db_user} - fi - if [ -n "${fai_db_password}" ]; then - db_password=${fai_db_password} - fi - if [ -n "${fai_db_schema}" ]; then - db_schema=${fai_db_schema} - fi - if [ -n "${fai_db_drop_old_db_first}" ]; then - drop_db_first=${fai_db_drop_old_db_first} - print_and_log "WARNING: I hope you know what you're doing!" - print_and_log "WARNING: fai_db_drop_old_db_first is 1 (true)" - fi - ece_home=${escenic_root_dir}/engine + set_db_settings_from_fai_conf + set_db_defaults_if_not_set - # the port isn't fully supported. The user must himself update the - # mysql configuration to run it on a non standard port. - if [ -n "${fai_db_port}" ]; then - db_port=${fai_db_port} - else - db_port=3306 - fi - + # the methods in drop-and-create-ecedb needs ece_home to be set + ece_home=${escenic_root_dir}/engine pre_install_new_ecedb create_ecedb @@ -798,25 +833,20 @@ function set_up_app_server if [ $fai_enabled -eq 0 ]; then print "Another question: Where does the database run?" - print "Press ENTER to accept the default ($HOSTNAME:3306)" - print "Or enter: :, e.g.: 'db1:3306'" - echo -n "Your choice [$HOSTNAME:3306]> " + print "Press ENTER to accept the default ($HOSTNAME:3306:ece5db)" + print "Or enter: ::, e.g.: 'db1:3306:mydb'" + echo -n "Your choice [$HOSTNAME:3306:schema]> " read user_database - else - user_database=$(get_conf_value fai_db_host) - if [ -n "${user_database}" ]; then - user_database=$user_database":"$(get_conf_value fai_db_port) - fi - fi - - if [ -z "$user_database" ]; then - db_host=$HOSTNAME - db_port=3306 - else + db_host=$(echo $user_database | cut -d':' -f1) db_port=$(echo $user_database | cut -d':' -f2) + db_schema=$(echo $user_database | cut -d':' -f3) + else + set_db_settings_from_fai_conf fi + set_db_defaults_if_not_set + if [ $fai_enabled -eq 0 ]; then print "Last question: Where does the search instance run?" print "Press ENTER to accept the default ($HOSTNAME:8080)" From 905c6a9e041f14bc118d4012eadc8d7e9461db54 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 18 Jan 2012 13:15:58 +0800 Subject: [PATCH 0216/2585] - updated HTML --- usr/share/doc/escenic/ece-install-guide.html | 78 ++++++++++---------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 2aee0d45..ef9e1c5d 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ Welcome to the /usr/sbin/ece-install Guide - + @@ -1141,45 +1141,49 @@

    6.2 Overview of All FAI P

    fai_publication_create0Create a new publication
    fai_publication_namemypubName of the publication
    fai_publication_use_instancedev1Name of local instance to use for creation
    fai_publication_war"WF or ECE demo WAR"WAR to to base the new publication on
    fai_rmi_install0Install RMI hub profile
    fai_search_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration
    fai_search_ear""EAR to use instead of the Escenic binaries
    -+ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ParameterDefaultDescription
    ParameterDefaultDescription
    fai_all_install0Install all components on your server.
    fai_cache_backends${HOSTNAME}:8080Space separated, e.g. "app1:8080 app2:8080"
    fai_cache_install0Install cache server profile
    fai_db_host$HOSTNAMEUseful for editor & presentation profiles
    fai_db_install0Install db profile
    fai_db_port3306Useful for editor & presentation profiles
    fai_editor_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration
    fai_editor_ear""EAR to use instead of the Escenic binaries
    fai_editor_install0Install the editorial profile
    fai_editor_nameeditor1Name of the editor instance
    fai_editor_port8080HTTP port of the editor instance
    fai_editor_shutdown8005Shutdown port of the editor instance
    fai_enabled0Whether or not to run ece-install in FAI mode
    fai_monitoring_server_install0Install the monitoring server profile.
    fai_monitoring_server_ip127.0.0.1The IP of the monitoring server.
    fai_presentation_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration
    fai_presentation_ear""EAR to use instead of the Escenic binaries
    fai_presentation_install0Install the presentation server profile
    fai_presentation_nameweb1Name of the presentation server instance
    fai_presentation_port8080HTTP port of the presentation server instance
    fai_presentation_shutdown8005Shutdown port of the presentation instance
    fai_public_host_name${HOSTNAME}:8080The public address for your website
    fai_publication_create0Create a new publication
    fai_publication_namemypubName of the publication
    fai_publication_use_instancedev1Name of local instance to use for creation
    fai_publication_war"WF or ECE demo WAR"WAR to to base the new publication on
    fai_rmi_install0Install RMI hub profile
    fai_search_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration
    fai_search_ear""EAR to use instead of the Escenic binaries
    fai_search_namesearch1Name of the search instance
    fai_search_port8080HTTP port of the search instance
    fai_search_shutdown8005Shutdown port of the search instance
    fai_wf_install0Install Widget Framework profile
    fai_all_install0Install all components on your server.
    fai_cache_backends${HOSTNAME}:8080Space separated, e.g. "app1:8080 app2:8080"
    fai_cache_install0Install cache server profile
    fai_db_host$HOSTNAMEUseful for editor & presentation profiles
    fai_db_dropolddbfirst0Warning: this will drop the old database before installing a new one
    fai_db_install0Install db profile
    fai_db_passwordread-the-source-lukeUseful for DB installation profile
    fai_db_port3306Useful for editor & presentation profiles
    fai_db_schemaece5dbUseful for DB installation profile
    fai_db_userece5userUseful for DB installation profile
    fai_editor_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration
    fai_editor_ear""EAR to use instead of the Escenic binaries
    fai_editor_install0Install the editorial profile
    fai_editor_nameeditor1Name of the editor instance
    fai_editor_port8080HTTP port of the editor instance
    fai_editor_shutdown8005Shutdown port of the editor instance
    fai_enabled0Whether or not to run ece-install in FAI mode
    fai_monitoring_server_install0Install the monitoring server profile.
    fai_monitoring_server_ip127.0.0.1The IP of the monitoring server.
    fai_presentation_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration
    fai_presentation_ear""EAR to use instead of the Escenic binaries
    fai_presentation_install0Install the presentation server profile
    fai_presentation_nameweb1Name of the presentation server instance
    fai_presentation_port8080HTTP port of the presentation server instance
    fai_presentation_shutdown8005Shutdown port of the presentation instance
    fai_public_host_name${HOSTNAME}:8080The public address for your website
    fai_publication_create0Create a new publication
    fai_publication_namemypubName of the publication
    fai_publication_use_instancedev1Name of local instance to use for creation
    fai_publication_war"WF or ECE demo WAR"WAR to to base the new publication on
    fai_rmi_install0Install RMI hub profile
    fai_search_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration
    fai_search_ear""EAR to use instead of the Escenic binaries
    fai_search_namesearch1Name of the search instance
    fai_search_port8080HTTP port of the search instance
    fai_search_shutdown8005Shutdown port of the search instance
    fai_wf_install0Install Widget Framework profile
    @@ -1556,7 +1560,7 @@

    14 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2012-01-12 14:20:21 CST + Date: 2012-01-18 13:15:34 CST

    From f8507592eb6c65bfc91021b0d668e9b57a4bce75 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 18 Jan 2012 15:40:56 +0800 Subject: [PATCH 0217/2585] - fai_db_drop_old_db_first=0 would give warning (only =1 should do). Now fixed. --- usr/sbin/ece-install | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 5e129476..9f19161f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -491,8 +491,10 @@ function set_db_settings_from_fai_conf() if [ -n "${fai_db_drop_old_db_first}" ]; then drop_db_first=${fai_db_drop_old_db_first} - print_and_log "WARNING: I hope you know what you're doing!" - print_and_log "WARNING: fai_db_drop_old_db_first is 1 (true)" + if [ $fai_db_drop_old_db_first -eq 1 ]; then + print_and_log "WARNING: I hope you know what you're doing!" + print_and_log "WARNING: fai_db_drop_old_db_first is 1 (true)" + fi fi } From 493d5f215fa419befd9b9fb3b82cb3b8070b7ab7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 18 Jan 2012 18:35:27 +0800 Subject: [PATCH 0218/2585] - added fai_publication_war_uri_list configuration option: Publication WARs used for setting up Assembly tool. This is a list of URIs, the following formats are supported: * http://my.build.server.com/build/234234/mypub.war * https://my.build.server.com/build/234234/mypub.war * file:///var/cache/build/234234/mypub.war * /var/cache/build/234234/mypub.war When set, AT will set up these publication definitions and download/copy the WAR file to /publications. --- usr/sbin/ece-install | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9f19161f..b4db5e91 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -355,6 +355,39 @@ EOF assemble.properties \ 1>>$log 2>>$log exit_on_error "sed on assemble.properties" + + make_dir $escenic_root_dir/assemblytool/publications + + # set up user publication definitions + if [ -n "${fai_publication_war_uri_list}" ]; then + run cd $escenic_root_dir/assemblytool/publications + + for el in ${fai_publication_war_uri_list}; do + if [[ $el == http* ]]; then + run wget $wget_opts $el + elif [[ $el == file://* ]]; then + local file_with_path=$(echo $el | sed "s#file://##g") + run cp $file_with_path . + else + run cp ${el} . + fi + + if [ ! -e $(basename $el) ]; then + print_and_log "Failed to get user publication $el." + print_and_log "I will skip it and continue to the next one." + continue + fi + + local short_name=$(basename $el .war) + cat > ${short_name}.properties < Date: Wed, 18 Jan 2012 18:35:53 +0800 Subject: [PATCH 0219/2585] * usr/share/doc/escenic/ece-install-guide.org: add documentation for fai_publication_war_uri_list --- usr/share/doc/escenic/ece-install-guide.org | 83 +++++++++++---------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 7f7c502a..a5fd6387 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -433,47 +433,48 @@ profiles, see the table below. The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: -|----------------------------------+----------------------+----------------------------------------------------------------------| -| Parameter | Default | Description | -|----------------------------------+----------------------+----------------------------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_drop_old_db_first | 0 | Warning: this will drop the old database before installing a new one | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_db\_schema | ece5db | Useful for DB installation profile | -| fai\_db\_user | ece5user | Useful for DB installation profile | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to to base the new publication on | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|----------------------------------+----------------------+----------------------------------------------------------------------| +|----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_db\_schema | ece5db | Useful for DB installation profile | +| fai\_db\_user | ece5user | Useful for DB installation profile | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 6c66f9d94705724edd91046bbc4a13e0afff6e05 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 19 Jan 2012 11:35:16 +0800 Subject: [PATCH 0220/2585] - forgot setting user name and password through variables in ECE_UPDATE_DS --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index b4db5e91..c7c6849b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -992,8 +992,8 @@ url="jdbc:mysql://${db_host}:${db_port}/${db_schema}?autoReconnect=true&useU maxIdle="8" maxWait="2000" initialSize="20" - username="ece5user" - password="ece5password" + username="${db_user}" + password="${db_password}" driverClassName="com.mysql.jdbc.Driver" url="jdbc:mysql://${db_host}:${db_port}/${db_schema}?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8" removeAbandoned="true" From 6431a21b2d4d8116e9fde632f7bd1969830e3540 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 19 Jan 2012 13:12:55 +0800 Subject: [PATCH 0221/2585] - changed set_conf_file_value to accept any number of value arguments: e.g.: set_conf_file_value \ mykey \ myvalue1 myvalue2 myvalue3 \ /etc/myfile.conf - added get_deploy_white_list() which returns correct white lists for presentation and search servers. For presentation servers, this means that all publications defined in /publications are included in the white list. - install_search_server and install_ece_instance now use get_deploy_white_list() --- usr/sbin/ece-install | 75 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 16 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c7c6849b..14697756 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -718,18 +718,34 @@ EOF # Parameters: # $1 is the property -# $2 is the value -# $3 is the file +# $2 ... to the (last - 1) is the value +# The last argument is the file +# +# e.g.: +# set_conf_file_value \ +# mykey \ +# myvalue1 myvalue2 myvalue3 \ +# /etc/myfile.conf + function set_conf_file_value() { - if [ -r $3 ]; then - if [ $(grep ^$1 $3 | wc -l) -gt 0 ]; then - sed -i "s~$1=.*~$1=\"$2\"~g" $3 + if [ $# -lt 3 ]; then + return + fi + + local file=${@:${#@}} + local key=${@:1:1} + local value_end_index=$(( $# - 2 )) + local value=${@:2:${value_end_index}} + + if [ -r $file ]; then + if [ $(grep ^$1 $file | wc -l) -gt 0 ]; then + sed -i "s~$key=.*~$key=\"$value\"~g" $file else - echo "$1=\"$2\"" >> $3 + echo "$key=\"$value\"" >> $file fi else - echo "$1=\"$2\"" >> $3 + echo "$key=\"$value\"" >> $file fi } @@ -1952,13 +1968,15 @@ function install_ece_instance() set_up_app_server set_up_proper_logging_configuration - # special treatment for presentation servers - if [ $2 -eq $PROFILE_PRESENTATION_SERVER ]; then + # We set a WAR white list for all profiles except all in one + if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE -a \ + $install_profile_number -ne $PROFILE_EDITORIAL_SERVER ]; then file=$escenic_conf_dir/ece-${instance_name}.conf - print_and_log "Creating instance specific conf: $file ..." - cat >> $file <> $file < Date: Thu, 19 Jan 2012 13:22:58 +0800 Subject: [PATCH 0222/2585] - now, EAE is included in the plugin (symlinks) Assembly Tool knows about. --- usr/sbin/ece-install | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 14697756..b8d98329 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -334,13 +334,16 @@ EOF continue fi - # nuisance to get the community engine, but not the engine - if [ $(echo $directory | grep engine | wc -l) -gt 0 ]; then - if [ $(echo $directory | grep community | wc -l) -lt 1 ]; then - continue - fi + # nuisance to get the community engine and analysis engine, + # but not the engine + if [ $(echo $directory | \ + grep engine | \ + grep -v community | \ + grep -v analysis | \ + wc -l) -gt 0 ]; then + continue fi - + make_ln $directory done From 4f10b3137ca09ccc1da6d7b57f24310ee9347313 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 19 Jan 2012 13:26:31 +0800 Subject: [PATCH 0223/2585] - added missing fai_search_install from FAI parameter overview. --- usr/share/doc/escenic/ece-install-guide.org | 77 +++++++++++---------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index a5fd6387..a48dc6b3 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -434,46 +434,47 @@ The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: |----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| Parameter | Default | Description | +| Parameter | Default | Description | |----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | -| fai\_db\_install | 0 | Install db profile | +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_install | 0 | Install db profile | | fai\_db\_password | read-the-source-luke | Useful for DB installation profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_db\_schema | ece5db | Useful for DB installation profile | -| fai\_db\_user | ece5user | Useful for DB installation profile | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | -| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_db\_schema | ece5db | Useful for DB installation profile | +| fai\_db\_user | ece5user | Useful for DB installation profile | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | |----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 11bcfd229ec4e0ca25ec0ec96ebe4949bcc11c64 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 19 Jan 2012 20:31:14 +0800 Subject: [PATCH 0224/2585] - implemented run_hook, adding the possibility to plugin in hooks at different places in ece-install. The default location for ece-install scripts is $HOME/ece-scripts.d of the root user. The hook names are inspired by the Debian package scripts: * .preinst (pre install) * .postinst (post install) - added two hooks: * install_analysis_server.preinst * install_analysis_server.postinst - first implementation of EAE profile. The ece-install.conf option fai_analysis_install=1 is now available, together with _host and _port. This profile will configure the EAE Reporter with the values from fai_analysis_host and fai_analysis_port, with localhost and 8080 as faill backs. --- usr/sbin/ece-install | 79 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index b8d98329..121b0613 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -21,8 +21,8 @@ ece_user=escenic ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 -tomcat_download=http://apache.stu.edu.tw/tomcat/tomcat-6/v6.0.33/bin/apache-tomcat-6.0.33.tar.gz - +tomcat_download=http://ftp.mirror.tw/pub/apache/tomcat/tomcat-6/v6.0.35/bin/apache-tomcat-6.0.35.tar.gz + # These variables govern where software is installed and data and run # time files are written. escenic_root_dir=/opt/escenic @@ -36,6 +36,7 @@ escenic_cache_dir=/var/cache/escenic escenic_crash_dir=/var/crash/escenic appserver_root_dir=/opt +# country code for selecting the correct (APT) mirror. mirror_country_suffix=jp # The script will install the sun-java6-jdk package on Debian based @@ -55,6 +56,10 @@ wget_opts="--continue --inet4-only --quiet" apt_opts="--no-install-recommends" curl_opts="--silent" +# hook scripts +ece_install_scripts_dir=$HOME/ece-install.d + + # globals will be set to correct values in run-time. appserver_port=8080 on_debian_or_derivative=0 @@ -184,6 +189,7 @@ PROFILE_WIDGET_FRAMEWORK=8 PROFILE_CREATE_PUBLICATION=9 PROFILE_MONITORING_SERVER=10 PROFILE_RESTORE_FROM_BACKUP=11 +PROFILE_ANALYSIS_SERVER=12 function make_dir() { @@ -849,7 +855,7 @@ function get_escaped_bash_string() echo $result } -function set_up_app_server +function set_up_app_server() { print_and_log "Setting up the application server ..." @@ -874,6 +880,9 @@ function set_up_app_server elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then appserver_port=$(get_conf_value fai_search_port) shutdown_port=$(get_conf_value fai_search_shutdown) + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then + appserver_port=$(get_conf_value fai_analysis_port) + shutdown_port=$(get_conf_value fai_analysis_shutdown) fi fi @@ -1983,7 +1992,11 @@ function install_ece_instance() fi set_correct_permissions - assemble_deploy_and_restart_type + + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + assemble_deploy_and_restart_type + fi + update_type_instances_to_start_up set_conf_file_value ece_unix_user $ece_user /etc/default/ece set_conf_file_value ece_unix_group $ece_group /etc/default/ece @@ -2101,6 +2114,58 @@ function install_editorial_server() install_ece_instance "editor1" 0 } +function run_hook() +{ + if [ -e $ece_install_scripts_dir -a \ + ${ece_install_scripts_dir}/{$1} ]; then + print_and_log "Started hook $1 ..." + bash ${ece_install_scripts_dir}/${1} + print_and_log "Finished hook $1 ..." + fi +} + +function install_analysis_server() +{ + run_hook install_analysis_server.preinst + + print_and_log "Installing an analysis server on $HOSTNAME ..." + type=analysis + + install_ece_instance "analysis1" + + run cp ${escenic_root_dir}/analysis-engine-*/wars/*.war \ + ${tomcat_base}/webapps + + local analysis_host=localhost + if [ -n "${fai_analysis_host}" ]; then + analysis_host=${fai_analysis_host} + fi + + + # deploy the EAE WARs + run cp ${escenic_root_dir}/analysis-engine-*/wars/*.war \ + ${tomcat_base}/webapps + + local ece_command="ece -i ${instance_name} -t ${type} start" + su - $ece_user -c "$ece_command" \ + 1>>$log 2>>$log + exit_on_error "su - $ece_user -c \"$ece_command\"" + + local seconds=3 + print_and_log "Waiting ${seconds} seconds for EAE to come up ..." + sleep 10 + + # set QS host:port in EAE Reports + print_and_log "Configuring EAE Reports ..." + local file=${tomcat_base}/webapps/analysis-reports/WEB-INF/config/reports.cfg + sed -i "s#host:port#${analysis_host}:${appserver_port}#g" $file \ + 1>>$log 2>>$log + exit_on_error "Configuring EAE Reports in $file" + + run touch ${tomcat_base}/webapps/analysis-reports/WEB-INF/web.xml + run_hook install_analysis_server.postinst +} + function install_rmi_hub() { make_dir $escenic_conf_dir/rmi-hub @@ -2888,6 +2953,12 @@ if [ $fai_enabled -eq 1 ]; then install_rmi_hub no_fai_profile=0 fi + + if [ $(get_boolean_conf_value fai_analysis_install) -eq 1 ]; then + install_profile_number=$PROFILE_ANALYSIS_SERVER + install_analysis_server + no_fai_profile=0 + fi if [ $no_fai_profile -eq 1 ]; then print_and_log "No install profile selected, be sure to have one of the " From 77500d7cba2c45165d131540cffbcf441ddb76f3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 20 Jan 2012 11:03:33 +0800 Subject: [PATCH 0225/2585] - with with -> with --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index ed3dc60b..ebb3e0b1 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1099,7 +1099,7 @@ EOF if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then run fakeroot alien --to-rpm --scripts ${deb_package} mv *.rpm ${cache_dir} - print "RPM package of your $instance $type instance with with build" + print "RPM package of your $instance $type instance with build" print "version $version is now available:" print $(echo /var/cache/escenic/${package_name}-${version}*.rpm) fi From 9e00bee71768311277a6fda59cf4fb4394b5cdfb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 20 Jan 2012 11:35:13 +0800 Subject: [PATCH 0226/2585] - argh! webapp white list contained escenic_admin instead of escenic-admin. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 121b0613..0041d0c6 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2013,7 +2013,7 @@ function install_ece_instance() function get_deploy_white_list() { - local white_list="escenic_admin" + local white_list="escenic-admin" if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER \ -a -n "${publication_name}" ]; then white_list="${white_list} ${publication_name} " From c8f057b9812dc09a6a636cc9ea4874d6e3e0d960 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 20 Jan 2012 12:11:05 +0800 Subject: [PATCH 0227/2585] - optimisation in set_correct_permissions, when setting the permissions of the escenic_data_dir, it will now check if the root directory has correct permissions and if yes, skip all its child directories. This is because existing multimedia archives normally will be several GBs and the script will hence linger unnecessarily long on this step - even though it's strictly speaking "more correct" to do this thoroughly. --- usr/sbin/ece-install | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0041d0c6..3c56ce39 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1138,10 +1138,24 @@ function set_correct_permissions() print_and_log "Setting correct permissions on ECE related directories ..." for el in $engine_dir_list; do - if [ -d $el ]; then - run chown -R ${ece_user}:${ece_group} $el \ - 1>>$log 2>>$log + if [ ! -d $el ]; then + continue fi + + if [ ${el} = ${escenic_data_dir} ]; then + local correct_permission=$(find ${el} \ + -maxdepth 0 \ + -user ${ece_user} | \ + wc -l) + if [ $correct_permission -gt 0 ]; then + print_and_log "Data directory root, $el," + print_and_log "has correct permissions, skiping sub directories." + continue + fi + fi + + run chown -R ${ece_user}:${ece_group} $el \ + 1>>$log 2>>$log done if [ -d "$tomcat_base" ]; then From 7807c6a6e0670fe9aac47cc59e30f250df18146e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 20 Jan 2012 19:10:32 +0800 Subject: [PATCH 0228/2585] - moved set_correct_permissions to two places & removed the old places: it's now called inside assemble_deploy_and_restart_type which is where it's really needed during the course of the ece-install life span. The second place is in common_post_install - applied patch by Stefan Norman: * new method is_using_conf_archive * renamed is_installing_from_archives to is_installing_from_ear * this is to make use of the passed configuration bundle without also having to use an EAR * added LDAP support: Note: if this is the only thing which will allow ece-install to work on ECE 5.2, I'll keep it. If there will be others, I'll remove it as I don't want the script to more than necessary and it's really scoped for ECE 5.3 and up. --- usr/sbin/ece-install | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3c56ce39..df77b83c 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -162,7 +162,7 @@ function is_number() technet_download_list=" http://technet.escenic.com/downloads/assemblytool-2.0.2.zip -http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.6.0.zip +http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.7.0.zip http://technet.escenic.com/downloads/release/53/community-engine-3.6.1.0.zip http://technet.escenic.com/downloads/release/53/dashboard-1.0.0.0.zip http://technet.escenic.com/downloads/release/53/engine-5.3.4.7.zip @@ -587,7 +587,7 @@ function set_up_basic_nursery_configuration() local escenic_root_dir=$escenic_root_dir - if [ $(is_installing_from_archives) -eq 1 ]; then + if [ $(is_using_conf_archive) -eq 1 ]; then print_and_log "Using the supplied Nursery & JAAS configuration from" print_and_log "bundle: $ece_instance_conf_archive" local dir=$(mktemp -d) @@ -631,6 +631,10 @@ function set_up_basic_nursery_configuration() databaseProductName=MySQL filePublicationRoot=$escenic_data_dir/engine/ webPublicationRoot=http://${public_host_name}/ + +# These two LDAP settings can be ignored on systems with ECE >= 5.3 +ldapProductName=OpenLdap +ldapProductVersion=2.2.26 EOF cat > $common_nursery_dir/neo/io/managers/ContentManager.properties < Date: Fri, 20 Jan 2012 19:29:51 +0800 Subject: [PATCH 0229/2585] - added print_and_log - made use of run when moving RPM files, this makes it easier to spot bugs in "ece package" --- usr/bin/ece | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index ebb3e0b1..de6da17d 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -105,6 +105,12 @@ function log() echo "$id" `date` $@ 2>/dev/null >> $log_file } +function print_and_log() +{ + print "$@" + log "$@" +} + function verify_that_directory_and_file_are_writeable() { dir=`dirname $1` @@ -454,7 +460,8 @@ function deploy() { ear=$cache_dir/engine.ear if [ ! -e $ear ]; then - print "$ear does not exist. Did you run '"`basename $0`" assemble'?" + print_and_log "$ear does not exist. " + print_and_log "Did you run '"`basename $0`" -i" $instance "assemble'?" exit 1 fi @@ -1000,7 +1007,8 @@ function create_packages() { local ear=$cache_dir/engine.ear if [ ! -e $ear ]; then - print "$ear does not exist. Did you run '"`basename $0`" assemble'?" + print_and_log "$ear does not exist. " + print_and_log "Did you run '"`basename $0`" -i" $instance "assemble'?" exit 1 fi @@ -1098,7 +1106,7 @@ EOF if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then run fakeroot alien --to-rpm --scripts ${deb_package} - mv *.rpm ${cache_dir} + run mv *.rpm ${cache_dir} print "RPM package of your $instance $type instance with build" print "version $version is now available:" print $(echo /var/cache/escenic/${package_name}-${version}*.rpm) From 1a228ac177491b1b1d427322ced631ccc4038a2b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 23 Jan 2012 16:57:36 +0800 Subject: [PATCH 0230/2585] - now running the "clean" ece target in assemble_deploy_and_restart_type, this should fix problems with profiles getting WARs and libraries from the previous run of installing a profile. This is only an issue when running multiple installation profiles in the same ece-install process. --- usr/sbin/ece-install | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index df77b83c..0c869369 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2075,7 +2075,9 @@ function assemble_deploy_and_restart_type() set_correct_permissions - ece_command="ece -i $instance_name -t $type assemble deploy restart" + # need to run clean here since we might be running multiple + # profiles in the same ece-install process. + ece_command="ece -i $instance_name -t $type clean assemble deploy restart" if [ $(is_installing_from_ear) -eq 1 ]; then run cp $ece_instance_ear_file $escenic_cache_dir/engine.ear ece_command="ece -i $instance_name -t $type deploy restart" From 99c619612527772b597e88dcb5613c3e8b928b79 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 23 Jan 2012 17:12:41 +0800 Subject: [PATCH 0231/2585] - hooks can now pick up ece-install member variables by looking in $escenic_cache_dir/ece-install.env - fixed bug in run_hook where the desired hook wasn't evaluated properly. --- usr/sbin/ece-install | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0c869369..e4ed8252 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -58,7 +58,7 @@ curl_opts="--silent" # hook scripts ece_install_scripts_dir=$HOME/ece-install.d - +ece_install_env=${escenic_cache_dir}/$(basename $0).env # globals will be set to correct values in run-time. appserver_port=8080 @@ -2148,7 +2148,15 @@ function install_editorial_server() function run_hook() { if [ -e $ece_install_scripts_dir -a \ - ${ece_install_scripts_dir}/{$1} ]; then + -e ${ece_install_scripts_dir}/${1} ]; then + + # Dumping all set variables (no functions) to a file from + # which the hooks can pick them up. We do this to avoid + # running "export" in front of all local variables which may + # or may not be useful. Furthermore, we filter out upper case + # variables as these are environment variables. + set | grep ^[a-z] | grep \= > ${ece_install_env} + print_and_log "Started hook $1 ..." bash ${ece_install_scripts_dir}/${1} print_and_log "Finished hook $1 ..." From 4ffe3c3e2259631d50533854aac53abf19ec1d59 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 23 Jan 2012 19:50:30 +0800 Subject: [PATCH 0232/2585] - added JNDI configuration based on installation profile, currently switching on EAE and the rest (either or) - added set_correct_permissions to install_analysis_server. - sets a number of best practice defaults in the EAE reports.cfg and logger.cfg configuration files. - added global appserver_host, just like we already have _port - added hack to set_conf_file_value, dont_quote_conf_values=1, since EAE isn't able to handle quoted values in its .cfg files (!) --- usr/sbin/ece-install | 187 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 153 insertions(+), 34 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index e4ed8252..2a0e601c 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -61,6 +61,7 @@ ece_install_scripts_dir=$HOME/ece-install.d ece_install_env=${escenic_cache_dir}/$(basename $0).env # globals will be set to correct values in run-time. +appserver_host=localhost appserver_port=8080 on_debian_or_derivative=0 on_debian=0 @@ -729,6 +730,11 @@ org.apache.catalina.core.ContainerBase.[Catalina].[localhost].handlers = 2localh EOF } +# don't quote values when setting conf file values with +# set_conf_file_value. This is a hack-ish variable due to EAE's +# handling of .cfg files. +dont_quote_conf_values=0 + # Parameters: # $1 is the property # $2 ... to the (last - 1) is the value @@ -739,7 +745,6 @@ EOF # mykey \ # myvalue1 myvalue2 myvalue3 \ # /etc/myfile.conf - function set_conf_file_value() { if [ $# -lt 3 ]; then @@ -750,15 +755,34 @@ function set_conf_file_value() local key=${@:1:1} local value_end_index=$(( $# - 2 )) local value=${@:2:${value_end_index}} + + local parent_dir=$(dirname $file) + if [ ! -d ${parent_dir} ]; then + print ${parent_dir} "doesn't exist, something is wrong :-(" + remove_pid_and_exit_in_error + fi if [ -r $file ]; then if [ $(grep ^$1 $file | wc -l) -gt 0 ]; then - sed -i "s~$key=.*~$key=\"$value\"~g" $file + if [ $dont_quote_conf_values -eq 0 ]; then + sed -i "s~$key=.*~$key=\"$value\"~g" $file + else + sed -i "s~$key=.*~$key=$value~g" $file + fi else - echo "$key=\"$value\"" >> $file + if [ $dont_quote_conf_values -eq 0 ]; then + echo "$key=\"$value\"" >> $file + else + echo "$key=$value" >> $file + fi + fi else - echo "$key=\"$value\"" >> $file + if [ $dont_quote_conf_values -eq 0 ]; then + echo "$key=\"$value\"" >> $file + else + echo "$key=$value" >> $file + fi fi } @@ -991,7 +1015,11 @@ function set_up_app_server() type="org.apache.catalina.UserDatabase" description="User database that can be updated and saved" factory="org.apache.catalina.users.MemoryUserDatabaseFactory" - pathname="conf/tomcat-users.xml" /> + pathname="conf/tomcat-users.xml" + /> +EOF + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + cat >> $tomcat_base/conf/server.xml < + /> + /> +EOF + else + cat >> $tomcat_base/conf/server.xml < + +EOF + fi + + cat >> $tomcat_base/conf/server.xml < @@ -1047,7 +1127,8 @@ url="jdbc:mysql://${db_host}:${db_port}/${db_schema}?autoReconnect=true&useU connectionTimeout="20000" URIEncoding="UTF-8" compression="on" - redirectPort="8543" /> + redirectPort="8543" + /> WEB-INF/web.xml +EOF + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + cat >> $tomcat_base/conf/context.xml < - + type="javax.sql.DataSource" + /> - + type="javax.sql.DataSource" + /> - + override="false" + /> - + override="false" + /> - + override="false" + /> - + override="false" + /> - + override="false" + /> EOF - + else + cat >> $tomcat_base/conf/context.xml < + + +EOF + fi } function create_user_and_group_if_not_present() @@ -1411,7 +1509,7 @@ function stop_conflicting_processes() # TODO this is dirty if [[ -n "$install_profile_number" && \ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]]; then - killall java 1>>$log 2>>$log + killall -9 java 1>>$log 2>>$log fi } @@ -2175,16 +2273,19 @@ function install_analysis_server() run cp ${escenic_root_dir}/analysis-engine-*/wars/*.war \ ${tomcat_base}/webapps - local analysis_host=localhost if [ -n "${fai_analysis_host}" ]; then - analysis_host=${fai_analysis_host} + appserver_host=${fai_analysis_host} + fi + if [ -n "${fai_analysis_port}" ]; then + appserver_port=${fai_analysis_port} fi - # deploy the EAE WARs run cp ${escenic_root_dir}/analysis-engine-*/wars/*.war \ ${tomcat_base}/webapps + set_correct_permissions + local ece_command="ece -i ${instance_name} -t ${type} start" su - $ece_user -c "$ece_command" \ 1>>$log 2>>$log @@ -2194,12 +2295,30 @@ function install_analysis_server() print_and_log "Waiting ${seconds} seconds for EAE to come up ..." sleep 10 - # set QS host:port in EAE Reports + # EAE cannot handle .cfg files with quotes values (!) + dont_quote_conf_values=1 + print_and_log "Configuring EAE Reports ..." local file=${tomcat_base}/webapps/analysis-reports/WEB-INF/config/reports.cfg - sed -i "s#host:port#${analysis_host}:${appserver_port}#g" $file \ - 1>>$log 2>>$log - exit_on_error "Configuring EAE Reports in $file" + set_conf_file_value queryServiceUrl \ + http://${appserver_host}:${appserver_port}/analysis-qs/QueryService \ + ${file} + + print_and_log "Configuring EAE Logger ..." + local file=${tomcat_base}/webapps/analysis-logger/WEB-INF/config/logger.cfg + set_conf_file_value databaseQueueManager.reinsertcount \ + 6 \ + ${file} + set_conf_file_value pageview.maintenance.cron.expr '0 0 4 * * ? *' $file + set_conf_file_value pageview.aggr.hour.cron.expr '0 10 * * * ?' $file + set_conf_file_value imageResponse false $file + set_conf_file_value pageview.maintenance.older.than.months 6 $file + set_conf_file_value pageview.aggr.day.older.than.days 7 $file + set_conf_file_value pageview.aggr.day.cron.expr '0 0 5 * * ? *' $file + set_conf_file_value pageview.aggr.hour.older.than.hours 2 $file + set_conf_file_value pageview.maintenance.older.than.days 0 $file + + dont_quote_conf_values=0 run touch ${tomcat_base}/webapps/analysis-reports/WEB-INF/web.xml run_hook install_analysis_server.postinst From 357515d2859fb70e9d70644566b794bc981872c5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 24 Jan 2012 18:48:28 +0800 Subject: [PATCH 0233/2585] - fixed the setting of non default configuration file (-f, --conf-file). It will now correctly add PWD if the path isn't an absolute one. The logic was there before too, but somehow it wasn't working (anymore). This fixes the problem that ece-install all of a sudden prints out n/a in the time elapsed string of the script identifier ([ece-install-n/a]) as well as returning errors whenever calling get_conf_value: ./ece-install: line 2381: [: [ece-install-n/a] ece-install-DEV.conf doesn't exist.: integer expression expected --- usr/sbin/ece-install | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 2a0e601c..bbfbe7af 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -3024,10 +3024,14 @@ for el in $@; do next_is_conf_file=1 elif [[ -n $next_is_conf_file && $next_is_conf_file -eq 1 ]]; then conf_file=$el - if [ ! -e $conf_file ]; then - conf_file=$(pwd)/${conf_file} - fi - + case ${conf_file} in + /*) + ;; + *) + conf_file=$(pwd)/${conf_file} + ;; + esac + next_is_conf_file=0 fi done From f7cb3416542fb7ed656de662c1cc5e7a3d8a901a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 24 Jan 2012 19:25:33 +0800 Subject: [PATCH 0234/2585] - added hook overview, generated by create_hook_overview shell script --- usr/share/doc/escenic/ece-install-hooks.org | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 usr/share/doc/escenic/ece-install-hooks.org diff --git a/usr/share/doc/escenic/ece-install-hooks.org b/usr/share/doc/escenic/ece-install-hooks.org new file mode 100644 index 00000000..1665ae0b --- /dev/null +++ b/usr/share/doc/escenic/ece-install-hooks.org @@ -0,0 +1,3 @@ + +- install_analysis_server.preinst +- install_analysis_server.postinst From c3d695f0fdd511990cbc9b79f7e8c4afb5ad59f3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 24 Jan 2012 19:25:44 +0800 Subject: [PATCH 0235/2585] - added --- .../usr/share/doc/escenic/create_hook_overview | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 usr/local/src/debian/usr/share/doc/escenic/create_hook_overview diff --git a/usr/local/src/debian/usr/share/doc/escenic/create_hook_overview b/usr/local/src/debian/usr/share/doc/escenic/create_hook_overview new file mode 100644 index 00000000..b68ce6f3 --- /dev/null +++ b/usr/local/src/debian/usr/share/doc/escenic/create_hook_overview @@ -0,0 +1,14 @@ +#! /usr/bin/env bash + +dir=~/src/ece-scripts + +echo "" > $dir/usr/share/doc/escenic/ece-install-hooks.org + +grep run_hook $dir/usr/sbin/ece-install | \ + grep -v function | \ + awk '{print $2}' | \ + while read l; do + echo "- $l" >> $dir/usr/share/doc/escenic/ece-install-hooks.org +done + + From 9d0f5ac9e901e452c5b352a689e731c5d09cba07 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 24 Jan 2012 19:26:18 +0800 Subject: [PATCH 0236/2585] - added --- usr/share/doc/escenic/create_hook_overview | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 usr/share/doc/escenic/create_hook_overview diff --git a/usr/share/doc/escenic/create_hook_overview b/usr/share/doc/escenic/create_hook_overview new file mode 100644 index 00000000..b68ce6f3 --- /dev/null +++ b/usr/share/doc/escenic/create_hook_overview @@ -0,0 +1,14 @@ +#! /usr/bin/env bash + +dir=~/src/ece-scripts + +echo "" > $dir/usr/share/doc/escenic/ece-install-hooks.org + +grep run_hook $dir/usr/sbin/ece-install | \ + grep -v function | \ + awk '{print $2}' | \ + while read l; do + echo "- $l" >> $dir/usr/share/doc/escenic/ece-install-hooks.org +done + + From 02a523f16adead16009cd75e18ce1852e653367a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 24 Jan 2012 19:27:43 +0800 Subject: [PATCH 0237/2585] - added section on writing your own hooks --- usr/share/doc/escenic/ece-install-guide.org | 40 +++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index a48dc6b3..25c0ec67 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -626,6 +626,46 @@ environments on the same host. A use case is if you're setting up a test environment and want to separate stacks of Escenic Content Engine and plugins. +* Extending ece-install by Writing Hooks +ece-install has a number of hooks on which you can hook on your own +scripts. The scripts are to reside in $HOME/ece-conf.d/ and have names +inspired by Debian's package scripts: + +#+BEGIN_SRC conf +. +#+END_SRC + +e.g.: + +#+BEGIN_SRC conf +install_analysis_server.preinst +#+END_SRC + +Will be run before the body of the hook, just the corresponding +.postinst hook will be run after. + +** Accessing ece-install variables +Before running the hook, ece-install will make all its local variables +available in /var/run/escenic/ece-install.env, which can then be used +by the hook scripts. + +** Example hook +Here is an example hook which will be run after the EAE is installed. + +#+BEGIN_SRC sh +# Put this in is $HOME/ece-install.d/install_analysis_server.postinst + +# read ece-install's current variables +source /var/run/escenic/ece-install.env + +# do something useful +echo "Hello from $0, EAE is installed in ${tomcat_base}" > /tmp/hello.txt +#+END_SRC + +** Available hooks +Currently, the following hooks are available: +#+INCLUDE: "ece-install-hooks.org" + * Assumptions ** /etc/esecenic is shared It's assumed that the /etc/escenic directory is either on a shared From 40e25b78c27f4b0834612ad6a0d8bf4ccb8e6225 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 24 Jan 2012 19:28:09 +0800 Subject: [PATCH 0238/2585] - changed path of .env variable file to escenic_run_dir --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index bbfbe7af..1873f120 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -58,7 +58,7 @@ curl_opts="--silent" # hook scripts ece_install_scripts_dir=$HOME/ece-install.d -ece_install_env=${escenic_cache_dir}/$(basename $0).env +ece_install_env=${escenic_run_dir}/$(basename $0).env # globals will be set to correct values in run-time. appserver_host=localhost From ca2b1e01e8ae9f0e469f58ab1ec8257b5f57fc11 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 25 Jan 2012 14:02:08 +0800 Subject: [PATCH 0239/2585] - added log_call_stack - remove_pid_and_exit_in_error now calls log_call_stack - fixed error handling when running script as normal user, it will now not produce any errors becaues it cannot write to the run and log files. --- usr/sbin/ece-install | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 1873f120..597d44bc 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -121,12 +121,26 @@ function print_and_log() log "$@" } +function log_call_stack() +{ + log "Call stack:" + + # skipping i=0 as this is log_call_stack itself + for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do + echo -n ${FUNCNAME[$i]}:${BASH_LINENO[$i-1]} " " >> $log + sed -n "${BASH_LINENO[$i-1]}p" $0 >> $log + done +} + function remove_pid_and_exit_in_error() { if [ -e $pid_file ]; then rm $pid_file fi + print "Have a look in $log for the error call stack." + log_call_stack + exit 1 } @@ -1700,8 +1714,8 @@ function read_user_input() function assert_correct_runtime_environment() { if [ $(whoami) != "root" ]; then - print_and_log "You must be root when running $(basename $0)" - remove_pid_and_exit_in_error + print "You must be root when running $(basename $0)" + exit 1 fi if [ -e $pid_file ]; then From 272f2dc22f9c92d9999f1c6f0aa25ae80c477c60 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 25 Jan 2012 14:27:39 +0800 Subject: [PATCH 0240/2585] - fixed problem in log_call_stack if you call ece-install without an absolute path. --- usr/sbin/ece-install | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 597d44bc..affd571d 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -127,8 +127,12 @@ function log_call_stack() # skipping i=0 as this is log_call_stack itself for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do - echo -n ${FUNCNAME[$i]}:${BASH_LINENO[$i-1]} " " >> $log - sed -n "${BASH_LINENO[$i-1]}p" $0 >> $log + echo -n ${0}:${BASH_LINENO[$i-1]}:"inside "${FUNCNAME[$i]}"() " >> $log + if [ -e $0 ]; then + sed -n "${BASH_LINENO[$i-1]}p" $0 >> log + else + echo "" >> $log + fi done } @@ -138,7 +142,7 @@ function remove_pid_and_exit_in_error() rm $pid_file fi - print "Have a look in $log for the error call stack." + print "See $log for the error call stack." log_call_stack exit 1 From 2b0aa70fc8d1f1cb84e4900466e4b4750fab8db3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 25 Jan 2012 15:46:26 +0800 Subject: [PATCH 0241/2585] - used BASH_SOURCE[index] instead of $0, this makes the call stack work when calling external scripts too! --- usr/sbin/ece-install | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index affd571d..14d25c1a 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -127,9 +127,9 @@ function log_call_stack() # skipping i=0 as this is log_call_stack itself for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do - echo -n ${0}:${BASH_LINENO[$i-1]}:"inside "${FUNCNAME[$i]}"() " >> $log - if [ -e $0 ]; then - sed -n "${BASH_LINENO[$i-1]}p" $0 >> log + echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:"inside "${FUNCNAME[$i]}"() " >> $log + if [ -e ${BASH_SOURCE[$i]} ]; then + sed -n "${BASH_LINENO[$i-1]}p" ${BASH_SOURCE[$i]} >> $log else echo "" >> $log fi From ae8faa4fcb2f2882fad23ec8ee491d3170f8eb3e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 26 Jan 2012 11:46:14 +0800 Subject: [PATCH 0242/2585] - fai_analysis_name is now supported - changed the reading of the conf variables and default fallback in ask_for_instance_name --- usr/sbin/ece-install | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 14d25c1a..7b65ca9f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -127,7 +127,7 @@ function log_call_stack() # skipping i=0 as this is log_call_stack itself for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do - echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:"inside "${FUNCNAME[$i]}"() " >> $log + echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:${FUNCNAME[$i]}"() => " >> $log if [ -e ${BASH_SOURCE[$i]} ]; then sed -n "${BASH_LINENO[$i-1]}p" ${BASH_SOURCE[$i]} >> $log else @@ -1975,11 +1975,13 @@ function ask_for_instance_name() read instance_name else if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - instance_name=$(get_conf_value fai_editor_name) + instance_name=${fai_editor_name-$1} elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - instance_name=$(get_conf_value fai_presentation_name) + instance_name=${fai_presentation_name-$1} elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - instance_name=$(get_conf_value fai_search_name) + instance_name=${fai_search_name-$1} + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then + instance_name=${fai_analysis_name-$1} fi fi From 5186698e97f0aad54db0e705ce587531e3eaa65f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 26 Jan 2012 15:15:45 +0800 Subject: [PATCH 0243/2585] - removed stop_conflicting_processes, it was a pretty dodgy method and it's not *really* needed. It's better that the script fails than that it's allowed to kill away. --- usr/sbin/ece-install | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 7b65ca9f..1fb06275 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -123,13 +123,15 @@ function print_and_log() function log_call_stack() { - log "Call stack:" + log "Call stack (top most is the last one, main is the first):" # skipping i=0 as this is log_call_stack itself for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do - echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:${FUNCNAME[$i]}"() => " >> $log + echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:${FUNCNAME[$i]}"()" >> $log if [ -e ${BASH_SOURCE[$i]} ]; then - sed -n "${BASH_LINENO[$i-1]}p" ${BASH_SOURCE[$i]} >> $log + echo -n " => " >> $log + sed -n "${BASH_LINENO[$i-1]}p" ${BASH_SOURCE[$i]} | \ + sed "s#^[ \t]*##g" >> $log else echo "" >> $log fi @@ -142,7 +144,6 @@ function remove_pid_and_exit_in_error() rm $pid_file fi - print "See $log for the error call stack." log_call_stack exit 1 @@ -1521,16 +1522,6 @@ function un_install_ece() apt-get clean } -function stop_conflicting_processes() -{ - print_and_log "Stopping conflicting processes ..." - # TODO this is dirty - if [[ -n "$install_profile_number" && \ - $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]]; then - killall -9 java 1>>$log 2>>$log - fi -} - function set_up_varnish() { print_and_log "Setting up Varnish to match your environment ..." @@ -1791,7 +1782,6 @@ function common_pre_install() fi make_dir $download_dir - stop_conflicting_processes install_common_os_packages create_user_and_group_if_not_present set_up_user_enviornment From e138858a1e0c490fcd8944a3f392407549b85477 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 26 Jan 2012 18:09:30 +0800 Subject: [PATCH 0244/2585] - .env never got set correctly if overriden via escenci_run_dir in .conf. Fixed now. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 1fb06275..29bc961f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -58,7 +58,6 @@ curl_opts="--silent" # hook scripts ece_install_scripts_dir=$HOME/ece-install.d -ece_install_env=${escenic_run_dir}/$(basename $0).env # globals will be set to correct values in run-time. appserver_host=localhost @@ -1756,6 +1755,7 @@ function common_pre_install() $escenic_run_dir $escenic_spool_dir/migration " + ece_install_env=${escenic_run_dir}/$(basename $0).env if [ -z "$technet_user" -o -z "$technet_password" ]; then print_and_log "Be sure to set technet_user and technet_password " From a3606e4ec5383171d0d8c7bffee19941b94a5c20 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 27 Jan 2012 11:09:46 +0800 Subject: [PATCH 0245/2585] - added tmp_dir to the _required_fields list variables. --- usr/bin/ece | 3 +++ 1 file changed, 3 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index de6da17d..7a79f945 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -53,17 +53,20 @@ enable_remote_debugging enable_remote_monitoring java_home solr_home +tmp_dir " search_required_fields=" appserver java_home solr_home +tmp_dir " analysis_required_fields=" appserver java_home +tmp_dir " #################################################################### # Will exit the ece execution if the last operation failed. While From c1a1283007229fff4a4e8493323117423f380fb6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 27 Jan 2012 12:21:26 +0800 Subject: [PATCH 0246/2585] - the editor server profile deployed indexer-webapp, which is wrong as long as we've got the search profile and "install all" profile. This has now been fixed. - added get_publication_short_name_list, used by the get_deploy_white_list when switching on presentation and editor profiles. --- usr/sbin/ece-install | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 29bc961f..77b04593 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2118,8 +2118,7 @@ function install_ece_instance() set_up_proper_logging_configuration # We set a WAR white list for all profiles except all in one - if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE -a \ - $install_profile_number -ne $PROFILE_EDITORIAL_SERVER ]; then + if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE ]; then file=$escenic_conf_dir/ece-${instance_name}.conf print_and_log "Creating deployment white list in $file ..." set_conf_file_value \ @@ -2146,25 +2145,38 @@ function install_ece_instance() add_next_step "/etc/default/ece lists all instances started at boot time" } +function get_publication_short_name_list() +{ + local short_name_list="" + + local publication_def_dir=${escenic_root_dir}/assemblytool/publications + if [ $(ls ${publication_def_dir} | grep .properties$ | wc -l) -eq 0 ]; then + echo ${short_name_list} + return + fi + + for el in $(find ${publication_def_dir} -name "*.properties"); do + local short_name=$(basename $el .properties) + short_name_list="${short_name_list} ${short_name}" + done + + echo ${short_name_list} + } + function get_deploy_white_list() { local white_list="escenic-admin" + if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER \ -a -n "${publication_name}" ]; then white_list="${white_list} ${publication_name} " elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then white_list="${white_list} solr indexer-webapp" elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - local publication_def_dir=${escenic_root_dir}/assemblytool/publications - if [ $(ls ${publication_def_dir} | grep .properties$ | wc -l) -eq 0 ]; then - echo ${white_list} - return - fi - - for el in $(find ${publication_def_dir} -name "*.properties"); do - local short_name=$(basename $el .properties) - white_list="${white_list} ${short_name}" - done + white_list="${white_list} "$(get_publication_short_name_list) + elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + white_list="${white_list} escenic studio indexer-webservice" + white_list="${white_list} "$(get_publication_short_name_list) fi echo ${white_list} From af9d6474d29568fe55ed091428e53d99d3a92215 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 27 Jan 2012 09:58:32 +0100 Subject: [PATCH 0247/2585] Hardened by quoting some variables --- usr/bin/ece | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index de6da17d..382506d7 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -299,18 +299,18 @@ function set_instance_settings() # at this point in the script flow, we have now read all the # possible combinations of conf files (common, type and instance) # and can now ensure that required fields are set and apply them. - if [ $type = "rmi-hub" ]; then + if [ "$type" = "rmi-hub" ]; then ensure_that_required_fields_are_set $hub_required_fields - elif [ $type = "search" ]; then + elif [ "$type" = "search" ]; then ensure_that_required_fields_are_set $search_required_fields - elif [ $type = "engine" ]; then + elif [ "$type" = "engine" ]; then ensure_that_required_fields_are_set $engine_required_fields fi # We respect ece_server if it's set in any of the configuration # files. If it's not set there, it sets some sensible defaults. - if [ -z $ece_server ]; then - if [ $instance = "default" ]; then + if [ -z "$ece_server" ]; then + if [ "$instance" = "default" ]; then ece_server=${HOSTNAME} else ece_server=${HOSTNAME}-${instance} @@ -320,7 +320,7 @@ function set_instance_settings() set_type_pid # if type is rmi-hub, we don't' need more configuration. - if [ $type = "rmi-hub" ]; then + if [ "$type" = "rmi-hub" ]; then return fi From 4d2cda07139423f83dc4af9550aedd7b4635f4ad Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 27 Jan 2012 10:07:33 +0100 Subject: [PATCH 0248/2585] Cleanup --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index 382506d7..877ce04a 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1333,7 +1333,7 @@ function set_type_command_and_instance() function clean_up() { - if [ $type = "engine" ]; then + if [ "$type" == "engine" ]; then print "Cleaning up generated files in $assemblytool_home ..." run cd $assemblytool_home run ant clean From a59effed550c0678b7a48785b8ca0c1f896c4f36 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 27 Jan 2012 10:43:06 +0100 Subject: [PATCH 0249/2585] Hardening: variables in if tests escaped with quotes. Style: single = replaced with double ==. --- usr/bin/ece | 160 ++++++++++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 877ce04a..fdb90f81 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -88,14 +88,14 @@ function run() #################################################################### function debug() { - if [ $verbose -eq 1 ]; then + if [ "$verbose" -eq 1 ]; then print $@ fi } function print() { - if [ $quiet -eq 0 ]; then + if [ "$quiet" -eq 0 ]; then echo "$id" $@ fi } @@ -151,7 +151,7 @@ function ensure_that_required_fields_are_set() requirements_failed=1 done - if [ $requirements_failed -eq 1 ]; then + if [ "$requirements_failed" -eq 1 ]; then exit 1 fi } @@ -206,7 +206,7 @@ function set_common_settings() # instance specific ones. read_conf_file `basename $0`.conf - if [ $instance = "default" ]; then + if [ "$instance" == "default" ]; then log_file=$log_dir/$type.out pid_file=$pid_dir/$type.pid else @@ -235,7 +235,7 @@ function set_type_settings() function set_type_pid() { - if [ $(uname) = "Linux" ]; then + if [ "$(uname)" == "Linux" ]; then ps_options="auxww" else ps_options="-ef" @@ -244,7 +244,7 @@ function set_type_pid() # Get a hold of the PID of the process. Although we've got the PID # file, we stil get the PID from the OS here as we used it for # sanity checking later on, see stop_type(). - if [ $type = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then # we need to be able to differentiate between an ECE instance # and an rmi hub, for this we use java.security.policy which # the hub doesn't have. @@ -265,7 +265,7 @@ function set_type_port() if [ -n "${appserver_port}" ]; then port=${appserver_port} debug "appserver_port set in .conf files=${port}" - elif [ $appserver != "tomcat" ]; then + elif [ "$appserver" != "tomcat" ]; then debug "Only tomcat is supported for reading host/port right now, " debug "trying to make an educated guess" port=8080 @@ -299,18 +299,18 @@ function set_instance_settings() # at this point in the script flow, we have now read all the # possible combinations of conf files (common, type and instance) # and can now ensure that required fields are set and apply them. - if [ "$type" = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then ensure_that_required_fields_are_set $hub_required_fields - elif [ "$type" = "search" ]; then + elif [ "$type" == "search" ]; then ensure_that_required_fields_are_set $search_required_fields - elif [ "$type" = "engine" ]; then + elif [ "$type" == "engine" ]; then ensure_that_required_fields_are_set $engine_required_fields fi # We respect ece_server if it's set in any of the configuration # files. If it's not set there, it sets some sensible defaults. if [ -z "$ece_server" ]; then - if [ "$instance" = "default" ]; then + if [ "$instance" == "default" ]; then ece_server=${HOSTNAME} else ece_server=${HOSTNAME}-${instance} @@ -320,7 +320,7 @@ function set_instance_settings() set_type_pid # if type is rmi-hub, we don't' need more configuration. - if [ "$type" = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then return fi @@ -339,63 +339,63 @@ function set_instance_settings() -Djava.security.auth.login.config=$ece_security_configuration_dir/jaas.config\ -Djava.security.policy=$ece_security_configuration_dir/java.policy" - if [ $instance != "default" ]; then + if [ "$instance" != "default" ]; then ece_args=$ece_args" -Dcom.escenic.instance=$instance" fi - if [ $ece_server_hostname ]; then + if [ "$ece_server_hostname" ]; then ece_args=$ece_args" -Djava.rmi.server.hostname=$ece_server_hostname" fi - if [ $apr_lib_dir ]; then + if [ "$apr_lib_dir" ]; then ece_args=$ece_args" -Djava.library.path=$apr_lib_dir" fi - if [ $min_heap_size ]; then + if [ "$min_heap_size" ]; then ece_args=$ece_args" -Xms$min_heap_size" fi - if [ $max_heap_size ]; then + if [ "$max_heap_size" ]; then ece_args=$ece_args" -Xmx$max_heap_size" fi - if [ $min_permgen_size ]; then + if [ "$min_permgen_size" ]; then ece_args=$ece_args" -XX:PermSize=$min_permgen_size" fi - if [ $max_permgen_size ]; then + if [ "$max_permgen_size" ]; then ece_args=$ece_args" -XX:MaxPermSize=$max_permgen_size" fi - if [ $jvm_encoding ]; then + if [ "$jvm_encoding" ]; then ece_args=$ece_args" -Dsun.jnu.encoding=$jvm_encoding" ece_args=$ece_args" -Dfile.encoding=$jvm_encoding" fi - if [ $enable_remote_debugging -eq 1 ]; then + if [ "$enable_remote_debugging" -eq 1 ]; then ece_args=$ece_args" -Xdebug -Xnoagent -Djava.compiler=NONE \ -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$remote_debugging_port" fi - if [ $enable_remote_monitoring -eq 1 ]; then + if [ "$enable_remote_monitoring" -eq 1 ]; then ece_args=$ece_args" -Dcom.sun.management.jmxremote" ece_args=$ece_args" -Dcom.sun.management.jmxremote.authenticate=false" ece_args=$ece_args" -Dcom.sun.management.jmxremote.ssl=false" ece_args=$ece_args" -Dcom.sun.management.jmxremote.port=$remote_monitoring_port" fi - if [ $enable_heap_dump -eq 1 ]; then + if [ "$enable_heap_dump" -eq 1 ]; then ece_args=$ece_args" -XX:+HeapDumpOnOutOfMemoryError" ece_args=$ece_args" -XX:HeapDumpPath=$heap_dump_dir" fi - if [ $force_ipv4 -eq 1 ]; then + if [ "$force_ipv4" -eq 1 ]; then ece_args=$ece_args" -Djava.net.preferIPv4Stack=true" fi # Resin needs some more arguments as its XML parser is not # compatible with ECE. - if [ $appserver = "resin" ]; then + if [ "$appserver" == "resin" ]; then ece_args="$ece_args\ -Dorg.xml.sax.driver=org.apache.xerces.parsers.SAXParser -Djavax.xml.stream.XMLInputFactory=com.sun.xml.stream.ZephyrParserFactory @@ -404,8 +404,8 @@ function set_instance_settings() -Djavax.xml.transform.TransformerFactory=org.apache.xalan.processor.TransformerFactoryImpl" debug "resin_home=$resin_home" - elif [ $appserver = "tomcat" ]; then - if [ -z $tomcat_base ]; then + elif [ "$appserver" == "tomcat" ]; then + if [ -z "$tomcat_base" ]; then tomcat_base=$tomcat_home else export CATALINA_BASE=$tomcat_base @@ -425,7 +425,7 @@ function set_instance_settings() function sanity_check() { - if [ $(whoami) = "root" ]; then + if [ "$(whoami)" == "root" ]; then print "Sorry, you cannot be root when running $(basename $0)" print "The root user can only use /etc/init.d/ece" exit 1 @@ -442,8 +442,8 @@ function sanity_check() # checks if the user is using the default instance when he/she # really wants to start one of the instances installed on the # system. - if [ $instance = "default" -a \ - $type = "engine" -a \ + if [ "$instance" == "default" -a \ + "$type" == "engine" -a \ $(echo $command | egrep 'help' | wc -l) -eq 0 ]; then instance_list=$(get_instance_list) if [[ -n "$instance_list" && \ @@ -459,7 +459,7 @@ function sanity_check() function deploy() { ear=$cache_dir/engine.ear - if [ ! -e $ear ]; then + if [ ! -e "$ear" ]; then print_and_log "$ear does not exist. " print_and_log "Did you run '"`basename $0`" -i" $instance "assemble'?" exit 1 @@ -539,14 +539,14 @@ function deploy() deploy_this_war=0 for el in $deploy_webapp_white_list; do - if [ $el = $name ]; then + if [ "$el" == $name ]; then debug "found $war in white list, will deploy it" deploy_this_war=1 fi done fi - if [ $deploy_this_war -eq 0 ]; then + if [ "$deploy_this_war" -eq 0 ]; then continue fi @@ -613,15 +613,15 @@ function backup_type() print $message log $message - if [ $type = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then ensure_that_required_fields_are_set $hub_required_fields - elif [ $type = "search" ]; then + elif [ "$type" == "search" ]; then # TODO trim & tun the default parameters for the search # instance. ensure_that_required_fields_are_set $engine_required_fields - elif [ $type = "engine" ]; then + elif [ "$type" == "engine" ]; then ensure_that_required_fields_are_set $engine_required_fields - elif [ $type = "analysis" ]; then + elif [ "$type" == "analysis" ]; then ensure_that_required_fields_are_set $analysis_required_fields fi @@ -634,7 +634,7 @@ function backup_type() exit 1 fi - if [ $appserver = "tomcat" ]; then + if [ "$appserver" == "tomcat" ]; then if [ ! -d $tomcat_base/conf ]; then print $tomcat_base/conf "doesn't exist :-(" print "check your ece.conf the $instance instance of type $type" @@ -684,14 +684,14 @@ ${escenic_conf_dir} " actual_backup="" for el in $possible_backup; do - if [ $el = "/opt/escenic" -a $backup_exclude_binaries -eq 1 ]; then + if [ "$el" == "/opt/escenic" -a $backup_exclude_binaries -eq 1 ]; then continue - elif [ -r $el ]; then + elif [ -r "$el" ]; then actual_backup="$el $actual_backup" fi done - if [ $appserver = "tomcat" -a $backup_exclude_binaries -eq 0 ]; then + if [ "$appserver" == "tomcat" -a "$backup_exclude_binaries" -eq 0 ]; then actual_backup="$(get_actual_file $tomcat_home) $tomcat_base $actual_backup" fi @@ -707,7 +707,7 @@ ${escenic_conf_dir} print "- Database snapshot" print "- All Solr & Escenic data files from /var/lib/escenic" - if [ $backup_exclude_binaries -eq 0 ]; then + if [ "$backup_exclude_binaries" -eq 0 ]; then print "- All app servers in /opt" print "- All Escenic binaries & publication templates in /opt/escenic" fi @@ -725,7 +725,7 @@ function start_type() print $message log $message - if [ $type = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then ensure_that_required_fields_are_set $hub_required_fields if [ -r $rmi_hub_conf ]; then @@ -751,13 +751,13 @@ function start_type() echo $pid > $pid_file exit 0 - elif [ $type = "search" ]; then + elif [ "$type" == "search" ]; then # TODO trim & tun the default parameters for the search # instance. ensure_that_required_fields_are_set $engine_required_fields - elif [ $type = "engine" ]; then + elif [ "$type" == "engine" ]; then ensure_that_required_fields_are_set $engine_required_fields - elif [ $type = "analysis" ]; then + elif [ "$type" == "analysis" ]; then ensure_that_required_fields_are_set $analysis_required_fields fi @@ -943,7 +943,7 @@ assemble_attempts=0 function assemble() { - if [ $assemble_attempts -gt 1 ]; then + if [ "$assemble_attempts" -gt 1 ]; then print "I've tried to assemble twice now and FAILED :-(" print "You probably have multiple versions of one or more plugins" print "Check your plugins directory, sort it out and try again." @@ -951,7 +951,7 @@ function assemble() fi assemble_attempts=$(( $assemble_attempts + 1 )) - if [[ $type != "engine" && $type != "search" ]]; then + if [[ "$type" != "engine" && "$type" != "search" ]]; then print "You cannot assemble instances of type $type" exit 1 fi @@ -983,7 +983,7 @@ function assemble() fi done - if [ $duplicates_found -eq 1 ]; then + if [ "$duplicates_found" -eq 1 ]; then print "Multiple versions of ECE and/or 3rd party libraries found." print "Remember, you need to run '$(basename $0) clean assemble' when " print "upgrading either ECE or one of the plugins." @@ -1016,13 +1016,13 @@ function create_packages() local package_dir=$dir/debian local package_name="" - if [ $type = "engine" ]; then + if [ "$type" == "engine" ]; then package_name="escenic-content-engine-${instance}" - elif [ $type = "search" ]; then + elif [ "$type" == "search" ]; then package_name="escenic-search-engine-${instance}" - elif [ $type = "rmi-hub" ]; then + elif [ "$type" == "rmi-hub" ]; then package_name="escenic-rmi-hub" - elif [ $type = "analysis" ]; then + elif [ "$type" == "analysis" ]; then package_name="escenic-analysis-engine-${instance}" fi @@ -1058,7 +1058,7 @@ EOF for el in $(jar tf ${ear} | grep .war$); do if [ -n "${deploy_webapp_white_list}" ]; then for ele in $deploy_webapp_white_list; do - if [ ${el} = ${ele}.war ]; then + if [ ${el} == ${ele}.war ]; then run jar xf ${ear} ${el} fi done @@ -1069,7 +1069,7 @@ EOF # putting this block here so that anything overridden in # tomcat_base takes precedence over tomcat_home - if [ ${everything_but_the_kitchen_sink} -eq 1 ]; then + if [ "${everything_but_the_kitchen_sink}" -eq 1 ]; then ( local tomcat_home_dir=${package_dir}${tomcat_home} run mkdir -p ${tomcat_home_dir} @@ -1157,9 +1157,9 @@ function tail_out_log() function get_app_log() { - if [ $appserver = "tomcat" ]; then + if [ "$appserver" == "tomcat" ]; then log_file=$tomcat_base/logs/localhost.`date +%F`.log - elif [ $appserver = "resin" -a -e $resin_home/log/jvm-default.log ]; then + elif [ "$appserver" == "resin" -a -e $resin_home/log/jvm-default.log ]; then log_file=$resin_home/log/jvm-default.log else print "I don't know where the logs for $appserver are." @@ -1174,7 +1174,7 @@ function get_app_log() function tail_app_log() { - if [ $type = "rmi-hub" ]; then + if [ "$type" == "rmi-hub" ]; then print "There is no application server log for $type" exit 1 fi @@ -1211,105 +1211,105 @@ function set_type_command_and_instance() next_is_http_password=0 for el in $@; do - if [ $el = "-v" -o $el = "--verbose" ]; then + if [ "$el" == "-v" -o "$el" == "--verbose" ]; then verbose=1 continue fi - if [ $el = "--full" ]; then + if [ "$el" == "--full" ]; then everything_but_the_kitchen_sink=1 continue fi - if [ $el = "-q" -o $el = "--quiet" ]; then + if [ "$el" == "-q" -o $el == "--quiet" ]; then quiet=1 continue fi - if [ $el = "--help" ]; then + if [ "$el" == "--help" ]; then command=help continue fi - if [ $next_is_type -eq 1 ]; then + if [ "$next_is_type" -eq 1 ]; then type=$el next_is_type=0 continue fi - if [ $next_is_http_user -eq 1 ]; then + if [ "$next_is_http_user" -eq 1 ]; then http_user=$el next_is_http_user=0 continue fi - if [ $next_is_http_password -eq 1 ]; then + if [ "$next_is_http_password" -eq 1 ]; then http_password=$el next_is_http_password=0 continue fi - if [ $next_is_instance -eq 1 ]; then + if [ "$next_is_instance" -eq 1 ]; then instance=$el next_is_instance=0 continue fi - if [ $next_is_publication -eq 1 ]; then + if [ "$next_is_publication" -eq 1 ]; then publication=$el next_is_publication=0 continue fi - if [ $next_is_resource -eq 1 ]; then + if [ "$next_is_resource" -eq 1 ]; then resource=$el next_is_resource=0 continue fi - if [ $el = "-t" -o $el = "--type" ]; then + if [ "$el" == "-t" -o "$el" == "--type" ]; then next_is_type=1 continue else next_is_type=0 fi - if [ $el = "-i" -o $el = "--instance" ]; then + if [ "$el" == "-i" -o "$el" == "--instance" ]; then next_is_instance=1 continue else next_is_instance=0 fi - if [ $el = "-p" -o $el = "--publication" ]; then + if [ "$el" == "-p" -o "$el" == "--publication" ]; then next_is_publication=1 continue else next_is_publication=0 fi - if [ $el = "-r" -o $el = "--publication-resource" ]; then + if [ "$el" == "-r" -o "$el" == "--publication-resource" ]; then next_is_resource=1 continue else next_is_resource=0 fi - if [ $el = "-u" -o $el = "--user" ]; then + if [ "$el" == "-u" -o "$el" == "--user" ]; then next_is_http_user=1 continue else next_is_http_user=0 fi - if [ $el = "-w" -o $el = "--password" ]; then + if [ "$el" == "-w" -o "$el" == "--password" ]; then next_is_http_password=1 continue else next_is_http_password=0 fi - if [ $el = "--exclude-binaries" ]; then + if [ "$el" == "--exclude-binaries" ]; then backup_exclude_binaries=1 continue fi @@ -1321,7 +1321,7 @@ function set_type_command_and_instance() # If the user didn't specify which instance to use and if there's # only one available instance, use that instead of asking the user # to provide it in sanity_check. - if [ $instance = "default" ]; then + if [ "$instance" == "default" ]; then local instance_list=$(get_instance_list) if [ $(echo $instance_list | grep ' ' | wc -l) -eq 0 ]; then @@ -1357,7 +1357,7 @@ function clean_up() function set_id() { - if [ $instance = "default" ]; then + if [ "$instance" == "default" ]; then id="["`basename $0`"#${type}]" else id="["`basename $0`"#${type}-${instance}]" @@ -1501,7 +1501,7 @@ function update_publication_resources() "publication ..." debug POSTing $resource to $url - if [[ -n "$do_put" && $do_put = "true" ]]; then + if [[ -n "$do_put" && "$do_put" == "true" ]]; then curl -T ${resource} \ ${curl_auth} \ ${url} \ @@ -1521,7 +1521,7 @@ function update_publication_resources() function show_all_log_paths() { - if [ $quiet -eq 0 ]; then + if [ "$quiet" -eq 0 ]; then print "System out log: "$log_file print "App server log: "$(get_app_log) print "log4j log: "$(get_log4j_log) @@ -1549,7 +1549,7 @@ function get_escenic_admin_url() function flush_caches() { - if [ ${type} != "engine" ]; then + if [ "${type}" != "engine" ]; then print "You cannot flush the caches of a ${type} instance" return fi From 8132a06dc6b1b58f0d9b75e7c7941a0459fe84b6 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 27 Jan 2012 11:02:18 +0100 Subject: [PATCH 0250/2585] If no /etc/escenic/engine/instances exist, assume the default anyway. Maybe check for ece.conf files instead? --- usr/bin/ece | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index fdb90f81..3f3bef96 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1322,10 +1322,14 @@ function set_type_command_and_instance() # only one available instance, use that instead of asking the user # to provide it in sanity_check. if [ "$instance" == "default" ]; then + debug "Trying to determine instance name" local instance_list=$(get_instance_list) if [ $(echo $instance_list | grep ' ' | wc -l) -eq 0 ]; then instance=$instance_list + if [ -z "$instance" ] ; then + instance=default + fi debug "setting instance=$instance as there's only one" fi fi From f04a1024ee8e4c30db41f1cccf323600cf94d585 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 27 Jan 2012 11:14:50 +0100 Subject: [PATCH 0251/2585] Added output of file name that failed being writeable, to help figure out why stuff fails --- usr/bin/ece | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 3f3bef96..317da94b 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -115,11 +115,11 @@ function verify_that_directory_and_file_are_writeable() { dir=`dirname $1` if [ ! -e $dir ]; then - print $dir " doesn't exist" + print $1: $dir " doesn't exist" exit 1 fi if [ ! -w $dir ]; then - print $dir " isn't writable for user $USER" + print $1: $dir " isn't writable for user $USER" exit 1 fi From 3d8eea97386e28e7c1107c86aacdbc3d3dd0905c Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 27 Jan 2012 11:18:24 +0100 Subject: [PATCH 0252/2585] Added log_dir and pid_dir as required inputs from ece.conf --- usr/bin/ece | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 317da94b..2d0f17b9 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -53,6 +53,8 @@ enable_remote_debugging enable_remote_monitoring java_home solr_home +log_dir +pid_dir " search_required_fields=" From 46739914f5b55175723abdef90791af63a926ac5 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 27 Jan 2012 11:34:12 +0100 Subject: [PATCH 0253/2585] Verify that the ece_security_configuration_dir is set to a real directory --- usr/bin/ece | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 2d0f17b9..eb992fd9 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -759,6 +759,12 @@ function start_type() ensure_that_required_fields_are_set $engine_required_fields elif [ "$type" == "engine" ]; then ensure_that_required_fields_are_set $engine_required_fields + if [ ! -d "$ece_security_configuration_dir" ] ; then + print "ece_security_configuration_dir $ece_security_configuration_dir" + print "did not exist." + print "Exiting :-(" + exit 1 + fi elif [ "$type" == "analysis" ]; then ensure_that_required_fields_are_set $analysis_required_fields fi From ca09d47770cf5f9d328edbab3eb4f5976d054664 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 30 Jan 2012 18:30:56 +0800 Subject: [PATCH 0254/2585] - stop_type will now re-attempt the killing of the JVM 6 times (instead of just one) after sleeping one second to allow the JVM to shut down cleanly. After the graceful attempts to shut down the app server, it's instead shutdown forcefully (kill -9). This is similar behaviour to that of Tomcat's shutdown.sh (which again calls cataline.sh with 'stop') with the added benefit that it should work for all application servers (and rmi-hub for that matter). - made use of the run() wrapper a few places in stop_type --- usr/bin/ece | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 7b09a326..e069dce7 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -428,11 +428,11 @@ function set_instance_settings() function sanity_check() { - if [ "$(whoami)" == "root" ]; then - print "Sorry, you cannot be root when running $(basename $0)" - print "The root user can only use /etc/init.d/ece" - exit 1 - fi +# if [ "$(whoami)" == "root" ]; then +# print "Sorry, you cannot be root when running $(basename $0)" +# print "The root user can only use /etc/init.d/ece" +# exit 1 +# fi verify_that_directory_and_file_are_writeable $log_file verify_that_directory_and_file_are_writeable $pid_file @@ -846,20 +846,35 @@ function stop_type() print "removing dangling PID file $pid_file. " print "In future, be sure to use $0 to start " print "and stop your $type" - kill $type_pid 1>>$log_file 2>>$log_file + run kill $type_pid rm $pid_file return fi fi + + for i in {0..5}; do + set_type_pid + + if [ -n "$type_pid" ]; then + run kill $type_pid + else + debug "Previous gracious kill attempt of $instance succeeded." + break + fi + + sleep 1 + done + + set_type_pid + if [ -n "$type_pid" ]; then + print_and_log "I could not stop $type instance $instance gracefully," + print_and_log "I will now use force." + run kill -9 $type_pid + fi - kill $type_pid \ - 1>>$log_file \ - 2>>$log_file if [ -e $pid_file ]; then - rm $pid_file \ - 1>>$log_file \ - 2>>$log_file + run rm $pid_file fi else print "The $instance instance of $type on $HOSTNAME is NOT running" From 4aeaabefa10d080b05b76ec169de9821778bff8e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 30 Jan 2012 19:10:14 +0800 Subject: [PATCH 0255/2585] - removed the sleep 8 in restart_type as stop_type now guarantees that the type instance is stopped - replaced some legacy print & log with print_and_log --- usr/bin/ece | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 7d2eac71..384d792c 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -727,8 +727,7 @@ function start_type() { unset CLASSPATH message="Starting the $instance instance of $type on $HOSTNAME ..." - print $message - log $message + print_and_log $message if [ "$type" == "rmi-hub" ]; then ensure_that_required_fields_are_set $hub_required_fields @@ -944,11 +943,6 @@ function get_info_for_type() function restart_type() { stop_type - - # I've tested this with various values, 8 seconds seems to - # guarantee that the previous JVM process has stopped - # properly before starting the new one. - sleep 8 # sometimes the JVM refuses to shut down when doing a graceful # kill, therefore, if it's still running after the sleep above, we @@ -956,9 +950,7 @@ function restart_type() set_type_pid if [ -n "$type_pid" ]; then message="The $instance instance of $type failed to stop gracefully" - debug $message - log $message >> $log_file - print $message + print_and_log $message kill_type fi From 35c55e6f13d992b2e3d284e3a52ccc4151d1cbc8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 1 Feb 2012 10:20:11 +0800 Subject: [PATCH 0256/2585] - fixed resource link to eae-qs in context.xml for profile=analysis --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 77b04593..281e1e14 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1226,7 +1226,7 @@ EOF /> From a77be0670e23a64ab4e4c24b1fb675c3978ca2c5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 1 Feb 2012 12:29:05 +0800 Subject: [PATCH 0257/2585] - added apr_lib_dir --- etc/escenic/ece.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/etc/escenic/ece.conf b/etc/escenic/ece.conf index 09cb959b..166b5e01 100644 --- a/etc/escenic/ece.conf +++ b/etc/escenic/ece.conf @@ -90,6 +90,13 @@ tomcat_home=/usr/share/tomcat6 # binaries, just with different data. # tomcat_base=/opt/tomcat-editor1 +##################################################################### +# If you've compiled or installed APR support (native Tomcat +# libraries, using the Apache APR library), you specify the install +# directory here, if not comment it out. +##################################################################### +apr_lib_dir=/usr/lib + ######################################################################## # JVM related (the Java Virtual Machine) ######################################################################## From 7eb5c07c9c1d92b7d331e926da5f986d0ec242d7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 1 Feb 2012 17:40:32 +0800 Subject: [PATCH 0258/2585] - for the search profile, it's now configurable where the indexer-webservice shall reside. Introduced new FAI parameter: fai_search_indexer_ws_uri. You can e.g. use it like this if you're installing an editor and a search instance at the same time: fai_editor_name=editor2 fai_editor_port=8080 fai_search_install=1 fai_search_indexer_ws_uri=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ - search profile no longer gets data sources in {server,context}.xml - editor/presentation profiles no longer get search profile specific JNDI resources. The only "escenic/*" JNDI parameter set for editor/presentation instances is escenic/solr-base-uri: | JNDI resource | Used by which components | Configured in which Nursery component | |----------------------------------------+--------------------------+----------------------------------------| | escenic/indexer-webservice | indexer-webapp | /com/escenic/indexer/IndexerConfig | | escenic/index-update-uri | indexer-webapp | /com/escenic/indexer/IndexerConfig | | escenic/solr-base-uri | indexer-webapp | /com/escenic/indexer/IndexerConfig | | | engine-presentation | /neo/xredsys/presentation/TagFacade | | | phoenix-webstudio | /com/escenic/solr/SolrServer | | escenic/head-tail-storage-file | indexer-webapp | /com/escenic/indexer/IndexerConfig | | escenic/failing-documents-storage-file | indexer-webapp | /com/escenic/indexer/StatisticsService | --- usr/sbin/ece-install | 51 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 281e1e14..1b1964f0 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -957,7 +957,24 @@ function set_up_app_server() set_db_defaults_if_not_set if [ $fai_enabled -eq 0 ]; then - print "Last question: Where does the search instance run?" + print "Awfully sorry to bug you with so many questions, but:" + print "What's the URI to the indexer-webservice? (this is typically" + print "something like http://editor1/indexer-webservice/index/)" + echo -n "Your choice [http://${HOSTNAME}:8080/indexer-webservice/index/]> " + read user_indexer_ws_uri + else + user_indexer_ws_uri=${fai_search_indexer_ws_uri} + fi + + if [ -n "$user_indexer_ws_uri" ]; then + indexer_ws_uri=$user_indexer_ws_uri + else + indexer_ws_uri=http://${HOSTNAME}:${appserver_port}/indexer-webservice/index/ + fi + + + if [ $fai_enabled -eq 0 ]; then + print "Last question, I promise!: Where does the search instance run?" print "Press ENTER to accept the default ($HOSTNAME:8080)" print "If you're in doubt, just press ENTER :-)" print "Or enter: :, e.g.: 'search1:8080'" @@ -1036,7 +1053,8 @@ function set_up_app_server() pathname="conf/tomcat-users.xml" /> EOF - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER \ + -a $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then cat >> $tomcat_base/conf/server.xml < EOF - else + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then cat >> $tomcat_base/conf/server.xml <WEB-INF/web.xml EOF - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ + $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then cat >> $tomcat_base/conf/context.xml < +EOF + fi + + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + cat >> $tomcat_base/conf/context.xml < +EOF + fi + + if [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + cat >> $tomcat_base/conf/context.xml < @@ -1217,7 +1246,7 @@ EOF /> EOF - else + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then cat >> $tomcat_base/conf/context.xml < Date: Wed, 1 Feb 2012 18:06:01 +0800 Subject: [PATCH 0259/2585] - embarassing --- usr/bin/ece | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 384d792c..74d8e7f4 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -430,11 +430,11 @@ function set_instance_settings() function sanity_check() { -# if [ "$(whoami)" == "root" ]; then -# print "Sorry, you cannot be root when running $(basename $0)" -# print "The root user can only use /etc/init.d/ece" -# exit 1 -# fi + if [ "$(whoami)" == "root" ]; then + print "Sorry, you cannot be root when running $(basename $0)" + print "The root user can only use /etc/init.d/ece" + exit 1 + fi verify_that_directory_and_file_are_writeable $log_file verify_that_directory_and_file_are_writeable $pid_file From 1b052a7683fc7c5f9e43e7b1da5423180836744f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 2 Feb 2012 15:24:02 +0800 Subject: [PATCH 0260/2585] - ece deploy now cleans up the temporary directory it uses, this closes Vizrt issues VF-2959 and VF-2491 - ece clean no longer cleans up the /tmp/ece-* directories as these no longer are created by "ece deploy" - added run wrapper to "ece clean"'s cleaning of packages --- usr/bin/ece | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 74d8e7f4..a05aef36 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -471,7 +471,7 @@ function deploy() fi # extract EAR to a temporary area - dir=$tmp_dir/`basename $0`-`date +%s` + local dir=$(mktemp -d) (mkdir -p $dir && cd $dir && jar xf $ear) exit_on_error "extracting $ear to $dir" @@ -579,6 +579,8 @@ function deploy() print "Deployment is only implemented for Resin and Tomcat so far." ;; esac + + run rm -rf ${dir} } # Returns the file (can be a directory) passed to the function only @@ -1361,19 +1363,9 @@ function clean_up() run ant clean fi - tmp_dir_prefix="`basename $0`-" - if [ `ls $tmp_dir | grep $tmp_dir_prefix | wc -l` -gt 0 ]; then - print "Cleaning up generated files in $tmp_dir ..." - rm -rf $tmp_dir/$tmp_dir_prefix[0-9]* \ - 1>>$log_file \ - 2>>$log_file - fi - if [ -d /var/cache/escenic ]; then print "Cleaning up ear, deb and rpm files in /var/cache/escenic ..." - rm -rf /var/cache/escenic/*.{rpm,deb,ear} \ - 1>>$log_file \ - 2>>$log_file + run rm -rf /var/cache/escenic/*.{rpm,deb,ear} fi } From 79ab854f3128d8f9096308c9b74baeceb26d1e41 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 2 Feb 2012 17:59:26 +0800 Subject: [PATCH 0261/2585] - new FAI variable: fai_search_for_editor. If set to 1 (default 0), the set_up_solr method will set up Solr for use for editorial servers, whereas the value of 0 (default) will make it suitable for presentation servers. - set_up_solr now has .preinst and .postinst hooks -> to be used by - users who want their own customisation's to the solr configuration files. - now uses dir_suffix for the directories, merely changed for educational purposes (how to override this is already in the ece-install-guide.org - changed URL to Tomcat download, the old one was broken - changed URL to ece-scripts to be the escenic fork. - fix in install_ece_instance to make installing search profiles: it will no longer run assemble_deploy_and_restart_type if the profile is search (as it's done later for search profiles inside install_search_server). --- usr/sbin/ece-install | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 1b1964f0..dccd6b88 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -7,7 +7,7 @@ # with the "ece -i backup" command. # # Always check for the latest version at -# http://github.com/skybert/ece-scripts +# http://github.com/escenic/ece-scripts # echo comments and suggestions > tkj@vizrt.com # echo frustrations > /dev/null @@ -21,19 +21,20 @@ ece_user=escenic ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 -tomcat_download=http://ftp.mirror.tw/pub/apache/tomcat/tomcat-6/v6.0.35/bin/apache-tomcat-6.0.35.tar.gz - +tomcat_download=http://ftp.stut.edu.tw/var/ftp/pub/OpenSource/apache/tomcat/tomcat-6/v6.0.35/bin/apache-tomcat-6.0.35.tar.gz + # These variables govern where software is installed and data and run # time files are written. -escenic_root_dir=/opt/escenic -escenic_conf_dir=/etc/escenic -escenic_log_dir=/var/log/escenic -escenic_data_dir=/var/lib/escenic -escenic_run_dir=/var/run/escenic -escenic_backups_dir=/var/backups/escenic -escenic_spool_dir=/var/spool/escenic -escenic_cache_dir=/var/cache/escenic -escenic_crash_dir=/var/crash/escenic +dir_suffix=escenic +escenic_root_dir=/opt${dir_suffix} +escenic_conf_dir=/etc${dir_suffix} +escenic_log_dir=/var/log${dir_suffix} +escenic_data_dir=/var/lib${dir_suffix} +escenic_run_dir=/var/run${dir_suffix} +escenic_backups_dir=/var/backups${dir_suffix} +escenic_spool_dir=/var/spool${dir_suffix} +escenic_cache_dir=/var/cache${dir_suffix} +escenic_crash_dir=/var/crash${dir_suffix} appserver_root_dir=/opt # country code for selecting the correct (APT) mirror. @@ -1495,6 +1496,8 @@ function set_up_user_enviornment() function set_up_solr() { + run_hook set_up_solr.preinst + print_and_log "Setting up solr ..." if [ ! -d $escenic_conf_dir/solr ]; then run cp -r $escenic_root_dir/engine/solr/conf $escenic_conf_dir/solr @@ -1505,6 +1508,16 @@ function set_up_solr() if [ ! -h conf ]; then run ln -s $escenic_conf_dir/solr conf fi + + local editorial_search_instance=${fai_search_for_editor-0} + local file=$escenic_conf_dir/solr/solrconfig.xml + if [ $editorial_search_instance -eq 1 ]; then + run sed -i "s#[0-9]*#5000#g" $file + else + run sed -i "s#[0-9]*#60000#g" $file + fi + + run_hook set_up_solr.postinst } # So far, I've used this method for copy/past-ing it into the shell @@ -2156,7 +2169,8 @@ function install_ece_instance() $file fi - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ + $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then assemble_deploy_and_restart_type fi From e665597c5a89a6644087ec36d0f1d74e1cd294c8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 2 Feb 2012 18:08:31 +0800 Subject: [PATCH 0262/2585] - add documentation for fai_search_indexer_ws_uri and fai_search_for_editor --- usr/share/doc/escenic/ece-install-guide.org | 97 +++++++++++---------- 1 file changed, 53 insertions(+), 44 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 25c0ec67..74f80bec 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -433,49 +433,52 @@ profiles, see the table below. The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: -|----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| Parameter | Default | Description | -|----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_db\_schema | ece5db | Useful for DB installation profile | -| fai\_db\_user | ece5user | Useful for DB installation profile | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | -| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|----------------------------------+----------------------+---------------------------------------------------------------------------------------------------------------------------------------| +|----------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| Parameter | Default | Description | +|----------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_db\_schema | ece5db | Useful for DB installation profile | +| fai\_db\_user | ece5user | Useful for DB installation profile | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | +| fai\_search\_indexer\_ws\_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | +| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|----------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +#+TBLFM: $2=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ 0 As you've probably have guessed, 0 means "false" and 1 means "true" :-) @@ -664,7 +667,13 @@ echo "Hello from $0, EAE is installed in ${tomcat_base}" > /tmp/hello.txt ** Available hooks Currently, the following hooks are available: -#+INCLUDE: "ece-install-hooks.org" + +#+BEGIN_SRC sh +install_analysis_server.preinst +install_analysis_server.postinst +set_up_solr.preinst +set_up_solr.postinst +#+END_SRC * Assumptions ** /etc/esecenic is shared From 62c3905d0dace7a8ecaa02ba4fbcffa593f8bb44 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 3 Feb 2012 10:59:57 +0800 Subject: [PATCH 0263/2585] - Removed an important slash when in yesterday's introduction of dir_suffix. Man, search and replace is hard. --- usr/sbin/ece-install | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index dccd6b88..500fb7b5 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -26,15 +26,15 @@ tomcat_download=http://ftp.stut.edu.tw/var/ftp/pub/OpenSource/apache/tomcat/tomc # These variables govern where software is installed and data and run # time files are written. dir_suffix=escenic -escenic_root_dir=/opt${dir_suffix} -escenic_conf_dir=/etc${dir_suffix} -escenic_log_dir=/var/log${dir_suffix} -escenic_data_dir=/var/lib${dir_suffix} -escenic_run_dir=/var/run${dir_suffix} -escenic_backups_dir=/var/backups${dir_suffix} -escenic_spool_dir=/var/spool${dir_suffix} -escenic_cache_dir=/var/cache${dir_suffix} -escenic_crash_dir=/var/crash${dir_suffix} +escenic_root_dir=/opt/${dir_suffix} +escenic_conf_dir=/etc/${dir_suffix} +escenic_log_dir=/var/log/${dir_suffix} +escenic_data_dir=/var/lib/${dir_suffix} +escenic_run_dir=/var/run/${dir_suffix} +escenic_backups_dir=/var/backups/${dir_suffix} +escenic_spool_dir=/var/spool/${dir_suffix} +escenic_cache_dir=/var/cache/${dir_suffix} +escenic_crash_dir=/var/crash/${dir_suffix} appserver_root_dir=/opt # country code for selecting the correct (APT) mirror. From 91aa998d2f7a5006815955f136e9139dc9a9b4f0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 3 Feb 2012 16:08:30 +0800 Subject: [PATCH 0264/2585] - new feature: The solr configuration can now be included in the fai__conf_archive and used instead of the one supplied with ECE. As mentioned in the install guide, the archive should be created like this (or similar, the paths are the important thing): $ cd /opt/escenic $ tar czf /tmp/nursery-skeleton-solr-and-security.tar.gz \ engine/security \ engine/siteconfig/config-skeleton/ \ engine/solr/conf \ engine/siteconfig/bootstrap-skeleton/ \ `find -L assemblytool/plugins/ -name siteconfig` And then used like this in the ece-install.conf file: fai_search_conf_archive=/tmp/nursery-skeleton-solr-and-security.tar.gz - included profile=search for adding the ece init.d script to the default run levels. - (I believe this must be a) regression bug: /etc/init.d/ece and /etc/default/ece wasn't copied out. Fixed. - updated download link for Tomcat - now cleans up the temporary directory used for the conf archive. --- usr/sbin/ece-install | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 500fb7b5..3386fa14 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -21,7 +21,7 @@ ece_user=escenic ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 -tomcat_download=http://ftp.stut.edu.tw/var/ftp/pub/OpenSource/apache/tomcat/tomcat-6/v6.0.35/bin/apache-tomcat-6.0.35.tar.gz +tomcat_download=http://ftp.mirror.tw/pub/apache/tomcat/tomcat-6/v6.0.35/bin/apache-tomcat-6.0.35.tar.gz # These variables govern where software is installed and data and run # time files are written. @@ -481,6 +481,8 @@ function set_up_ece_scripts() run cp -r ece-scripts/usr/* /usr/ run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ run cp -r ece-scripts/etc/bash_completion.d/ece /etc/bash_completion.d/ + run cp -r ece-scripts/etc/init.d/* /etc/init.d/ + run cp -r ece-scripts/etc/default/* /etc/default/ local file=${escenic_conf_dir}/ece.conf set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file @@ -610,9 +612,9 @@ function set_up_basic_nursery_configuration() if [ $(is_using_conf_archive) -eq 1 ]; then print_and_log "Using the supplied Nursery & JAAS configuration from" print_and_log "bundle: $ece_instance_conf_archive" - local dir=$(mktemp -d) - escenic_root_dir=$dir - run cd $dir + local a_tmp_dir=$(mktemp -d) + escenic_root_dir=$a_tmp_dir + run cd $a_tmp_dir run tar xzf $ece_instance_conf_archive fi @@ -631,6 +633,8 @@ function set_up_basic_nursery_configuration() run cp -r $el/misc/siteconfig/* $common_nursery_dir/ done + run rm -r ${a_tmp_dir} + public_host_name=$HOSTNAME:${appserver_port} if [ $fai_enabled -eq 1 ]; then if [ -n "$fai_public_host_name" ]; then @@ -1500,9 +1504,22 @@ function set_up_solr() print_and_log "Setting up solr ..." if [ ! -d $escenic_conf_dir/solr ]; then - run cp -r $escenic_root_dir/engine/solr/conf $escenic_conf_dir/solr + if [ $(is_using_conf_archive) -eq 1 ]; then + print_and_log "Using the supplied Solr configuration from" + print_and_log "bundle: $ece_instance_conf_archive" + local a_tmp_dir=$(mktemp -d) + run cd $a_tmp_dir + run tar xzf $ece_instance_conf_archive engine/solr/conf + run cp -r engine/solr/conf $escenic_conf_dir/solr + run rm -r $a_tmp_dir + else + print_and_log "Installing default Solr conf shipped with ECE ..." + run cp -r $escenic_root_dir/engine/solr/conf $escenic_conf_dir/solr + fi + else + print_and_log "$escenic_conf_dir/solr already exists, not touching it." fi - + make_dir $escenic_data_dir/solr/ run cd $escenic_data_dir/solr/ if [ ! -h conf ]; then @@ -2861,7 +2878,6 @@ function add_server_to_runlevels() # no need to add init.d scripts to the runlevel(s) for these # profiles if [ $install_profile_number -eq $PROFILE_WIDGET_FRAMEWORK -o \ - $install_profile_number -eq $PROFILE_SEARCH_SERVER -o \ $install_profile_number -eq $PROFILE_DB_SERVER -o \ $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then return From de9000232537136dfe1f8ca62893cb8fb5fb6b3f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 3 Feb 2012 16:12:14 +0800 Subject: [PATCH 0265/2585] - updated documentation on fai__conf_archive, it now includes solr. --- usr/share/doc/escenic/ece-install-guide.org | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 74f80bec..6543f911 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -409,26 +409,31 @@ A simple way to create this bundle, is to use a server which has the assembly environment set up and then do: #+BEGIN_SRC conf -$ cd /opt/escenic/engine -$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \ +$ cd /opt/escenic +$ tar czf /tmp/nursery-skeleton-solr-and-security.tar.gz \ engine/security \ engine/siteconfig/config-skeleton/ \ + engine/solr/conf \ engine/siteconfig/bootstrap-skeleton/ \ - `find -L assemblytool/plugins/ -name siteconfig` + `find -L assemblytool/plugins/ -name siteconfig` #+END_SRC -/tmp/nursery-skeleton-and-security.tar.gz should now have everything +/tmp/nursery-skeleton-solr-and-security.tar.gz should now have everything you need. You can now configure your FAI installation to use these by, e.g.: #+BEGIN_SRC conf fai_presentation_ear=/tmp/engine.ear -fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz +fai_presentation_conf_archive=/tmp/nursery-skeleton-solr-and-security.tar.gz #+END_SRC Corresponding configuration options are available for the other server profiles, see the table below. +The inclusion of the engine/solr directory makes it easy for users to +provide their own, optimised Solr configuration. In this context, also +note that a post install hook, set_up_solr.postinst, is available. + ** Overview of All FAI Parameters The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: From 07016d990891fe25218c12727e6b9466ce9bc52e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 3 Feb 2012 16:17:45 +0800 Subject: [PATCH 0266/2585] - mentioned that the inclusion of engine/solr/conf in the bundle is optional --- usr/share/doc/escenic/ece-install-guide.org | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 6543f911..a42eb87b 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -405,6 +405,11 @@ engine/siteconfig/config-skeleton assemblytool/plugins//siteconfig #+END_SRC +and optionally also: +#+BEGIN_SRC sh +engine/solr/conf +#+END_SRC + A simple way to create this bundle, is to use a server which has the assembly environment set up and then do: From bb48cdb975a351fb466f313f9a5c2dcbc21cb8f0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 6 Feb 2012 11:45:45 +0800 Subject: [PATCH 0267/2585] - fixed regression introduced in 91aa998d2f7a5006815955f136e9139dc9a9b4f0, this closes https://github.com/skybert/ece-scripts/issues/32 --- usr/sbin/ece-install | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3386fa14..f83d4d56 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -633,8 +633,10 @@ function set_up_basic_nursery_configuration() run cp -r $el/misc/siteconfig/* $common_nursery_dir/ done - run rm -r ${a_tmp_dir} - + if [ -n "${a_tmp_dir}" ]; then + run rm -rf ${a_tmp_dir} + fi + public_host_name=$HOSTNAME:${appserver_port} if [ $fai_enabled -eq 1 ]; then if [ -n "$fai_public_host_name" ]; then From 374c1ccd08015a4cc2138197f8f19b06f7217847 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 6 Feb 2012 12:11:16 +0800 Subject: [PATCH 0268/2585] - replaced the use of get_conf_value when setting the FAI app server HTTP and shutdown ports, now it uses standard BASH evaluation of fai__ with fallback. --- usr/sbin/ece-install | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index f83d4d56..3e8f28bc 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -925,17 +925,17 @@ function set_up_app_server() fi else if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - appserver_port=$(get_conf_value fai_editor_port) - shutdown_port=$(get_conf_value fai_editor_shutdown) + appserver_port=${fai_editor_port-8080} + shutdown_port=${fai_editor_shutdown-8005} elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - appserver_port=$(get_conf_value fai_presentation_port) - shutdown_port=$(get_conf_value fai_presentation_shutdown) + appserver_port=${fai_presentation_port-8080} + shutdown_port=${fai_presentation_shutdown-8005} elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - appserver_port=$(get_conf_value fai_search_port) - shutdown_port=$(get_conf_value fai_search_shutdown) + appserver_port=${fai_search_port-8080} + shutdown_port=${fai_search_shutdown-8005} elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then - appserver_port=$(get_conf_value fai_analysis_port) - shutdown_port=$(get_conf_value fai_analysis_shutdown) + appserver_port=${fai_analysis_port-8080} + shutdown_port=${fai_analysis_shutdown-8005} fi fi From a7f864a09bf1cf3263d5b19081fafd56b04902d3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 6 Feb 2012 18:36:04 +0800 Subject: [PATCH 0269/2585] - new feature: support for specifyng the EAE DB's details with dedicated fai_analysis_db_ fai_analysis_db_host fai_analysis_db_password fai_analysis_db_port fai_analysis_db_schema The need for this arises when installing many server profiles with the same invocation of ece-install. Normally, when only installing one profile at a time, the EAE (analysis) installation profile heeds the standard FAI parameters: fai_db_host fai_db_password fai_db_port fai_db_schema However, if you're installing also an editor profile in the same go, i.e. from the same ece-install.conf, you want an additional set of DB parameters. The EAE profile is very special in that it has its own DB and other settings from the ECE instances and it's therefore the only profile which has named DB FAI parameters, e.g. there's no fai_editor_db_host and so on. - improved set_db_defaults_if_not_set by using standard BASH fallback default values. - defined default DB settings to be used instead of hard coding the values. - added more comments to the install_analysis_server method explaining the hack needed to EAE's cfg files. --- usr/sbin/ece-install | 61 ++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3e8f28bc..a8f5d8db 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -503,54 +503,54 @@ function set_up_ece_scripts() run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece } +default_db_port=3306 +default_db_host=localhost +default_db_user=ece5user +default_db_password=ece5password +default_db_schema=ece5db # Method used both from interactive mode to set any missing values # (defaults) function set_db_defaults_if_not_set() { if [ -z "$db_host" ]; then - db_host=localhost + db_host=${default_db_host} fi if [ -z "$db_port" ]; then - db_port=3306 + db_port=${default_db_port} fi if [ -z "$db_user" ]; then - db_user=ece5user + db_user=${default_db_user} fi if [ -z "$db_password" ]; then - db_user=ece5password + db_password=${default_db_password} fi if [ -z "$db_schema" ]; then - db_schema=ece5db + db_schema=${default_db_schema} fi } function set_db_settings_from_fai_conf() { - # the port isn't fully supported. The user must himself update the - # mysql configuration to run it on a non standard port. - if [ -n "${fai_db_port}" ]; then - db_port=${fai_db_port} - fi - - if [ -n "${fai_db_host}" ]; then - db_host=${fai_db_host} - fi + # Note: the port isn't fully supported. The user must himself + # update the mysql configuration to run it on a non standard port. - if [ -n "${fai_db_user}" ]; then - db_user=${fai_db_user} - fi - - if [ -n "${fai_db_password}" ]; then - db_password=${fai_db_password} - fi - - if [ -n "${fai_db_schema}" ]; then - db_schema=${fai_db_schema} + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + db_port=${fai_db_port-${default_db_port}} + db_host=${fai_db_host-${default_db_host}} + db_user=${fai_db_user-${default_db_user}} + db_password=${fai_db_password-${default_db_password}} + db_schema=${fai_db_schema-${default_db_schema}} + else + db_port=${fai_analysis_db_port-${default_db_port}} + db_host=${fai_analysis_db_host-${default_db_host}} + db_user=${fai_analysis_db_user-${default_db_user}} + db_password=${fai_analysis_db_password-${default_db_password}} + db_schema=${fai_analysis_db_schema-${default_db_schema}} fi if [ -n "${fai_db_drop_old_db_first}" ]; then @@ -949,9 +949,10 @@ function set_up_app_server() if [ $fai_enabled -eq 0 ]; then print "Another question: Where does the database run?" - print "Press ENTER to accept the default ($HOSTNAME:3306:ece5db)" - print "Or enter: ::, e.g.: 'db1:3306:mydb'" - echo -n "Your choice [$HOSTNAME:3306:schema]> " + print "Press ENTER to accept the default " \ + "($HOSTNAME:${default_db_port}:${default_db_schema})" + print "Or enter: ::, e.g.: 'db1:${default_db_port}:mydb'" + echo -n "Your choice [$HOSTNAME:${default_db_port}:schema]> " read user_database db_host=$(echo $user_database | cut -d':' -f1) @@ -2402,8 +2403,12 @@ function install_analysis_server() set_conf_file_value pageview.aggr.hour.older.than.hours 2 $file set_conf_file_value pageview.maintenance.older.than.days 0 $file + # important to turn this off here, it's only for the EAE .cfg + # files, see above. dont_quote_conf_values=0 - + + # touching web.xml to trigger a re-deploy of the EAE Reports + # application. run touch ${tomcat_base}/webapps/analysis-reports/WEB-INF/web.xml run_hook install_analysis_server.postinst } From 856a5da306e30b3369fe439e102fa141f40ea533 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 7 Feb 2012 13:13:31 +0800 Subject: [PATCH 0270/2585] - tomcat logging configuration is now only tweaked for profile=all and profile=search. All other profiles get the logging configuration that comes with Tocmat. This means that only profile=search & profile=all get a dedicated solr log. - fixed regression introduced in 7eb5c07c9c1d92b7d331e926da5f986d0ec242d7, when adding different JNDI configuration based on profile (the bug was that e.g. editor profiles wouldn't get a closing element in their context.xml). - removed additional setting of deployment white list for profile=search as install_ece_instance now has support for this through the use of get_deploy_white_list. --- usr/sbin/ece-install | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a8f5d8db..f9ee3467 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -726,7 +726,12 @@ EOF make_ln $common_nursery_dir/trace.properties run ln -sf trace.properties log4j.properies - cat > $tomcat_base/conf/logging.properties < $tomcat_base/conf/logging.properties < +EOF + else + cat >> $tomcat_base/conf/context.xml < EOF fi } @@ -2569,13 +2582,6 @@ function install_search_server() type=search install_ece_instance "search1" 0 - file=$escenic_conf_dir/ece-${instance_name}.conf - print_and_log "Creating instance specific conf: $file ..." - set_conf_file_value \ - deploy_webapp_white_list \ - $(get_deploy_white_list) \ - $file - # TODO update instead of append dir=$common_nursery_dir/com/escenic/framework/search/solr make_dir $dir From 80936cd57f58d270e860c7fad84c9305c9e61c0a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 7 Feb 2012 16:14:50 +0800 Subject: [PATCH 0271/2585] - new feature: the "ece info" command now also includes the status of the application server. A Typical output now looks like this: $ ece -i dev1 info [ece#engine-dev1] Current instance: dev1 [ece#engine-dev1] Instances available on quanah: dev1 [ece#engine-dev1] Conf files parsed: /etc/escenic/ece-dev1.conf /etc/escenic/ece.conf [ece#engine-dev1] ECE location: /opt/escenic/engine [ece#engine-dev1] Assembly Tool location: /opt/escenic/assemblytool [ece#engine-dev1] Java location: /usr/lib/jvm/java-6-sun [ece#engine-dev1] Application server: tomcat new -> [ece#engine-dev1] Appplication server status: UP 0d 0h 0m 2s [ece#engine-dev1] Tomcat home: /opt/tomcat [ece#engine-dev1] Tomcat base: /opt/tomcat-dev1 The "ece status" works as before. - changed get_status to use echo returns instead of printing out the results. This way it's possible to make use of the get_status from other places were you don't want the program ID string as a part of the result. --- usr/bin/ece | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index a05aef36..7cc4da5f 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -926,6 +926,8 @@ function get_info_for_type() fi if [ -n "${appserver}" ]; then print "Application server: $appserver" + print "Appplication server status:" $(get_status) + case "$appserver" in tomcat) if [ -n "${tomcat_home}" ]; then @@ -1217,7 +1219,7 @@ function make_thread_dump() kill -QUIT $type_pid >> $log_file fi else - get_status + print "$(get_status)" fi } @@ -1385,19 +1387,19 @@ function set_id() function get_status() { if [ -z "$type_pid" ]; then - print "DOWN" + echo "DOWN" exit 0 elif [ -r $pid_file ]; then if [ "$type_pid" != `cat $pid_file` ]; then - print "Is running, but was not started with `basename $0`" - print "system PID $ece_id differs from the PID in $pid_file" + echo "Is running, but was not started with `basename $0`" + echo "system PID $ece_id differs from the PID in $pid_file" exit 1 fi else - print $pid_file "did not exist, " - print "the ${instance} instance of ${type} is running with PID $type_pid" - print "but hasn't been started properly with `basename $0`" + echo $pid_file "did not exist, " + echo "the ${instance} instance of ${type} is running with PID $type_pid" + echo "but hasn't been started properly with `basename $0`" exit 1 fi @@ -1411,7 +1413,7 @@ function get_status() minutes=$(( seconds_left / 60 )) seconds_left=$(( seconds_left - $minutes * 60 )) - print "UP" ${days}d ${hours}h ${minutes}m ${seconds_left}s + echo "UP" ${days}d ${hours}h ${minutes}m ${seconds_left}s } function list_versions() @@ -1647,7 +1649,7 @@ for el in $command; do start_type ;; status) - get_status + print "$(get_status)" ;; stop) stop_type From 01248b5dd929aa02d896da1738ddf07959e0f7e4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 7 Feb 2012 16:24:42 +0800 Subject: [PATCH 0272/2585] - improved "ece info" to also include PID and port. Also introduced a kind of grouping in the appserver info output: $ ece -i dev1 info [ece#engine-dev1] Current instance: dev1 [ece#engine-dev1] Instances available on quanah: dev1 [ece#engine-dev1] Conf files parsed: /etc/escenic/ece-dev1.conf /etc/escenic/ece.conf [ece#engine-dev1] ECE location: /opt/escenic/engine [ece#engine-dev1] Assembly Tool location: /opt/escenic/assemblytool [ece#engine-dev1] Java location: /usr/lib/jvm/java-6-sun [ece#engine-dev1] Application server: [ece#engine-dev1] |-> Status: UP 0d 0h 10m 30s [ece#engine-dev1] |-> Port: 8080 [ece#engine-dev1] |-> PID: 25518 [ece#engine-dev1] |-> Type: tomcat [ece#engine-dev1] |-> Tomcat home: /opt/tomcat [ece#engine-dev1] |-> Tomcat base: /opt/tomcat-dev1 --- usr/bin/ece | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 7cc4da5f..e3b47195 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -925,19 +925,24 @@ function get_info_for_type() print "Java location: $java_home" fi if [ -n "${appserver}" ]; then - print "Application server: $appserver" - print "Appplication server status:" $(get_status) + print "Application server:" + set_type_port + set_type_pid + print "|-> Status:" $(get_status) + print "|-> Port:" $port + print "|-> PID:" $type_pid + print "|-> Type: " $appserver case "$appserver" in tomcat) if [ -n "${tomcat_home}" ]; then - print "Tomcat home:" $tomcat_home - print "Tomcat base:" $tomcat_base + print "|-> Tomcat home:" $tomcat_home + print "|-> Tomcat base:" $tomcat_base fi ;; resin) if [ -n "${resin_home}" ]; then - print "Resin home:" $resin_home + print " |-> Resin home:" $resin_home fi ;; esac From 24f46c7769e88fe7358445647d2b3c02711927a9 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 7 Feb 2012 19:30:50 +0800 Subject: [PATCH 0273/2585] - new feature: ece-install now sets up the virtual hosts configuration for Tomcat application servers if the profile=editorial, profile=all or profile=presentation. The user can define one domain for each publication in the following FAI parameter: fai_publication_domain_mapping_list=" firepub#fire.escenic.com ildpub#ild.escenic.com " This will produce the following stanzas in server.xml: --- usr/sbin/ece-install | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index f9ee3467..b022c5cf 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1196,6 +1196,37 @@ EOF xmlValidation="false" xmlNamespaceAware="false"> +EOF + if [[ ($install_profile_number == $PROFILE_EDITORIAL_SERVER || \ + $install_profile_number == $PROFILE_PRESENTATION_SERVER || \ + $install_profile_number == $PROFILE_ALL_IN_ONE) && \ + -n "$fai_publication_domain_mapping_list" ]]; then + + declare -A publication_domain_map + + for el in ${fai_publication_domain_mapping_list}; do + local old_ifs=$IFS + # the entries in the fai_publication_domain_mapping_list + # are on the form: # + IFS='#' + read publication domain <<< "$el" + publication_domain_map[${publication}]="${domain}" + IFS=$old_ifs + done + + for el in ${!publication_domain_map[@]}; do + cat >> $tomcat_base/conf/server.xml < + + +EOF + done + fi + + cat >> $tomcat_base/conf/server.xml < @@ -2826,7 +2857,6 @@ allow ${escaped_munin_gather_ip} EOF fi - # install the escenic_jstat munin plugin file=/usr/share/munin/plugins/escenic_jstat_ run wget $wget_opts \ From ad23222a4e430aaba83a594c200287c0f5de5b7a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 7 Feb 2012 19:41:21 +0800 Subject: [PATCH 0274/2585] - updated documentation, added section on virtual hosts --- usr/share/doc/escenic/ece-install-guide.org | 128 +++++++++++++------- 1 file changed, 83 insertions(+), 45 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index a42eb87b..c9159163 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -439,55 +439,93 @@ The inclusion of the engine/solr directory makes it easy for users to provide their own, optimised Solr configuration. In this context, also note that a post install hook, set_up_solr.postinst, is available. +** Setting up virtual hosts +Setting up virtual host definitions in the application server makes a +some things easier, such as ECE plugins which set cookies based on +information they get from the app server. + +ece-install can set up the virtual hosts configuration for Tomcat +application servers if the profile is editorial, all presentation. + +To use this feature, you must define one domain for each publication +in the following FAI parameter: +#+BEGIN_SRC sh +fai_publication_domain_mapping_list=" +firepub#fire.escenic.com +ildpub#ild.escenic.com +" +#+END_SRC + +This will produce the following stanzas in server.xml: + +#+BEGIN_SRC nxml + + + + + + +#+END_SRC + +You must of course make sure that these host names are resolvable to +the intended IPs on your system using /etc/hosts or a local DNS. + ** Overview of All FAI Parameters The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: -|----------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| Parameter | Default | Description | -|----------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_db\_schema | ece5db | Useful for DB installation profile | -| fai\_db\_user | ece5user | Useful for DB installation profile | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | -| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | -| fai\_search\_indexer\_ws\_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | -| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | -|----------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +|-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| Parameter | Default | Description | +|-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_db\_schema | ece5db | Useful for DB installation profile | +| fai\_db\_user | ece5user | Useful for DB installation profile | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | +| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | +| fai\_search\_indexer\_ws\_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | +| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_wf\_install | 0 | Install Widget Framework profile | +|-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| #+TBLFM: $2=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ 0 As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 960dcbdf43ba9373690195c53ea66299e2e698be Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 7 Feb 2012 19:41:37 +0800 Subject: [PATCH 0275/2585] - updated HTML export --- usr/share/doc/escenic/ece-install-guide.html | 414 +++++++++++++------ 1 file changed, 294 insertions(+), 120 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index ef9e1c5d..e49f7282 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -1,13 +1,13 @@ - + Welcome to the /usr/sbin/ece-install Guide + - + @@ -67,6 +67,10 @@ margin: 70px 70px 0px 50px; } +pre { + background-color: grey; +} + h1, h2 { color: #e88424; } @@ -224,84 +228,92 @@

    Welcome to the /usr/sbin/ece-install Guide

    Table of Contents

    -

    1 Supported Operating Systems

    +

    1 Supported Operating Systems

    For the best experience, run this script on Debian Squeeze (or newer), @@ -313,7 +325,7 @@

    1 Supported Operating Syste

    -

    1.1 Debian based operating systems

    +

    1.1 Debian based operating systems

    If running on a Debian based operating system, ece-install will take @@ -325,7 +337,7 @@

    1.1 Debian based operatin

    -

    1.2 Other GNU/Linux and Unix systems

    +

    1.2 Other GNU/Linux and Unix systems

    If you're installing on another Linux or Unix system, you must first @@ -344,7 +356,7 @@

    1.2 Other GNU/Linux and U

    -

    2 A Note on Running ece-install On Non-GNU/Linux Systems

    +

    2 A Note on Running ece-install On Non-GNU/Linux Systems

    Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp @@ -358,7 +370,7 @@

    2 A Note on Running ece-ins

    -

    3 Downloading the ece-install Script

    +

    3 Downloading the ece-install Script

    The ece-install script can be downloaded from: @@ -376,7 +388,7 @@

    3 Downloading the ece-insta

    -

    4 Running the ece-install Script

    +

    4 Running the ece-install Script

    You must run the script as the root user. If you start the script as @@ -488,7 +500,7 @@

    4 Running the ece-install S

    -

    5 Available Server Profiles

    +

    5 Available Server Profiles

    When starting the script, it will ask you which server profile you @@ -518,7 +530,7 @@

    5 Available Server Profiles

    -

    5.1 Common for All Server Profiles

    +

    5.1 Common for All Server Profiles

    The script will install a Munin node on the host. It will also @@ -530,14 +542,14 @@

    5.1 Common for All Server

    -

    5.2 Common for Editorial, Presentation and Search Instances

    +

    5.2 Common for Editorial, Presentation and Search Instances

    -

    5.2.1 Sun Java 6

    +

    5.2.1 Sun Java 6

    @@ -545,7 +557,7 @@

    5.2.1 Sun Java 6

    -

    5.2.2 Apache Tomcat application server

    +

    5.2.2 Apache Tomcat application server

      @@ -567,7 +579,7 @@

      5.2.2 Apache Tomcat app

    -

    5.2.3 Basic Escenic Nursery configuration

    +

    5.2.3 Basic Escenic Nursery configuration

    The basic Nursery configuration is taken care of for you, including RMI, @@ -577,7 +589,7 @@

    5.2.3 Basic Escenic Nur

    -

    5.2.4 APR, native library for optimal Tomcat I/O performance

    +

    5.2.4 APR, native library for optimal Tomcat I/O performance

    @@ -585,7 +597,7 @@

    5.2.4 APR, native libra

    -

    5.2.5 Escenic Assembly environment

    +

    5.2.5 Escenic Assembly environment

    The reason why ece-install sets this up on each host, is to make the @@ -597,7 +609,7 @@

    5.2.5 Escenic Assembly

    -

    5.2.6 Database driver

    +

    5.2.6 Database driver

    @@ -606,7 +618,7 @@

    5.2.6 Database driver <

    -

    5.2.7 Compression of content

    +

    5.2.7 Compression of content

    This was was previously accomplished by having a web server in fron t @@ -623,7 +635,7 @@

    5.2.7 Compression of co

    -

    5.3 Profile - Full Stack on One Host

    +

    5.3 Profile - Full Stack on One Host

    This profile is suitable for developers and admins wanting to set up a @@ -641,7 +653,7 @@

    5.3 Profile - Full Stack

    -

    5.4 Profile - Editorial Server (Publication Server)

    +

    5.4 Profile - Editorial Server (Publication Server)

    Will set up a typical editorial server (often referred to as the @@ -652,7 +664,7 @@

    5.4 Profile - Editorial S

    -

    5.5 Profile - Presentation Server

    +

    5.5 Profile - Presentation Server

    This will set up a typical presentation server: @@ -670,7 +682,7 @@

    5.5 Profile - Presentatio

    -

    5.6 Profile - Database Server

    +

    5.6 Profile - Database Server

    If ece-install is run on a support version of Debian or Ubuntu, this @@ -704,7 +716,7 @@

    5.6 Profile - Database Se

    -

    5.7 Profile - Cache Server

    +

    5.7 Profile - Cache Server

    If ece-install is run on a support version of Debian or Ubuntu, it @@ -765,7 +777,7 @@

    5.7 Profile - Cache Serve

    -

    5.8 Profile - Install Widget Framework

    +

    5.8 Profile - Install Widget Framework

    You'll need a user name and password for accessing the @@ -788,7 +800,7 @@

    5.8 Profile - Install Wid

    -

    5.9 Profile - Create Publication

    +

    5.9 Profile - Create Publication

    This profile will create a publication for you, only asking you the @@ -804,7 +816,7 @@

    5.9 Profile - Create Publ

    -

    5.10 Profile - Monitoring Server

    +

    5.10 Profile - Monitoring Server

    This will install a Munin gatherer and web server. The latter for @@ -819,7 +831,7 @@

    5.10 Profile - Monitorin

    -

    5.11 Profile - Restoring from backup

    +

    5.11 Profile - Restoring from backup

    ece-install can restore from a backup made by the ece script: @@ -851,14 +863,14 @@

    5.11 Profile - Restoring

    -

    5.12 Running interactively

    +

    5.12 Running interactively

    -

    5.12.1 Start ece-install and choose the Option, "Restore from backup"

    +

    5.12.1 Start ece-install and choose the Option, "Restore from backup"

    @@ -873,7 +885,7 @@

    5.12.1 Start ece-insta

    -

    5.12.2 Select Which Backup to Restore

    +

    5.12.2 Select Which Backup to Restore

    @@ -904,7 +916,7 @@

    5.12.2 Select Which Ba

    -

    5.12.3 Choose What to Restore

    +

    5.12.3 Choose What to Restore

    @@ -924,7 +936,7 @@

    5.12.3 Choose What to

    -

    5.12.4 Sit Back and Watch ece-install Restore the Data for You

    +

    5.12.4 Sit Back and Watch ece-install Restore the Data for You

    @@ -944,7 +956,7 @@

    5.12.4 Sit Back and Wa

    -

    5.13 Running in FAI mode

    +

    5.13 Running in FAI mode

    If you're running in FAI mode, you can choose between these settings @@ -988,7 +1000,7 @@

    5.13 Running in FAI mode

    -

    5.14 Data security

    +

    5.14 Data security

    You should take heed when running restore, so that you're not @@ -1019,7 +1031,7 @@

    5.14 Data security

    -

    6 Full Automatic Install (FAI)

    +

    6 Full Automatic Install (FAI)

    The ece-install script has support for doing a full automatic install @@ -1059,7 +1071,7 @@

    6 Full Automatic Install (F

    -

    6.1 Installing from EARs instead of Binaries

    +

    6.1 Installing from EARs instead of Binaries

    It is possible to get ece-install to use a supplied EAR and @@ -1095,6 +1107,15 @@

    6.1 Installing from EARs

    +

    +and optionally also: +

    + + +
    engine/solr/conf
    +
    + +

    A simple way to create this bundle, is to use a server which has the assembly environment set up and then do: @@ -1102,17 +1123,18 @@

    6.1 Installing from EARs -
    $ cd /opt/escenic/engine
    -$ tar czf /tmp/nursery-skeleton-and-security.tar.gz \
    +
    $ cd /opt/escenic
    +$ tar czf /tmp/nursery-skeleton-solr-and-security.tar.gz \
       engine/security \
       engine/siteconfig/config-skeleton/ \
    +  engine/solr/conf \
       engine/siteconfig/bootstrap-skeleton/ \
    -  `find -L  assemblytool/plugins/ -name siteconfig`
    +  `find -L assemblytool/plugins/ -name siteconfig`
     

    -/tmp/nursery-skeleton-and-security.tar.gz should now have everything +/tmp/nursery-skeleton-solr-and-security.tar.gz should now have everything you need. You can now configure your FAI installation to use these by, e.g.:

    @@ -1120,7 +1142,7 @@

    6.1 Installing from EARs
    fai_presentation_ear=/tmp/engine.ear
    -fai_presentation_conf_archive=/tmp/nursery-skeleton-and-security.tar.gz
    +fai_presentation_conf_archive=/tmp/nursery-skeleton-solr-and-security.tar.gz
     
    @@ -1128,14 +1150,73 @@

    6.1 Installing from EARs Corresponding configuration options are available for the other server profiles, see the table below.

    +

    +The inclusion of the engine/solr directory makes it easy for users to +provide their own, optimised Solr configuration. In this context, also +note that a post install hook, setupsolr.postinst, is available. +

    -

    6.2 Overview of All FAI Parameters

    +

    6.2 Setting up virtual hosts

    +

    Setting up virtual host definitions in the application server makes a +some things easier, such as ECE plugins which set cookies based on +information they get from the app server. +

    +

    +ece-install can set up the virtual hosts configuration for Tomcat +application servers if the profile is editorial, all presentation. +

    +

    +To use this feature, you must define one domain for each publication +in the following FAI parameter: +

    + + +
    fai_publication_domain_mapping_list="
    +firepub#fire.escenic.com
    +ildpub#ild.escenic.com
    +"
    +
    + + +

    +This will produce the following stanzas in server.xml: +

    + + + +
    <Host name="fire.escenic.com" appBase="webapps" autoDeploy="false">
    +  <Context displayName="fire.escenic.com"
    +           docBase="firepub"
    +           path="/"
    +  />
    +</Host>
    +<Host name="ild.escenic.com" appBase="webapps" autoDeploy="false">
    +  <Context displayName="ild.escenic.com"
    +           docBase="ildpub"
    +           path="/"
    +  />
    +</Host>
    +
    + + +

    +You must of course make sure that these host names are resolvable to +the intended IPs on your system using /etc/hosts or a local DNS. +

    +
    + +
    + +
    +

    6.3 Overview of All FAI Parameters

    +
    +

    The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user:

    @@ -1151,7 +1232,7 @@

    6.2 Overview of All FAI P fai_cache_backends${HOSTNAME}:8080Space separated, e.g. "app1:8080 app2:8080" fai_cache_install0Install cache server profile fai_db_host$HOSTNAMEUseful for editor & presentation profiles -fai_db_dropolddbfirst0Warning: this will drop the old database before installing a new one +fai_db_drop_old_db_first0Warning: this will drop the old database before installing a new one fai_db_install0Install db profile fai_db_passwordread-the-source-lukeUseful for DB installation profile fai_db_port3306Useful for editor & presentation profiles @@ -1176,10 +1257,15 @@

    6.2 Overview of All FAI P fai_publication_create0Create a new publication fai_publication_namemypubName of the publication fai_publication_use_instancedev1Name of local instance to use for creation -fai_publication_war"WF or ECE demo WAR"WAR to to base the new publication on +fai_publication_war"WF or ECE demo WAR"WAR to base the new publication on +fai_publication_war_uri_list""Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, ///tmp/mypub.war and /tmp/mypub.war. +fai_publication_domain_mapping_list""Mapping between publication names and their corresponding domains on the form: one#one.com other#other.com fai_rmi_install0Install RMI hub profile fai_search_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration fai_search_ear""EAR to use instead of the Escenic binaries +fai_search_for_editor0If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. +fai_search_indexer_ws_urihttp://${HOSTNAME}:8080/indexer-webservice/index/URI of the indexer-webservice that the search instance shall use for knowing what to index. +fai_search_install0Install the search server profile (Solr + indexer) fai_search_namesearch1Name of the search instance fai_search_port8080HTTP port of the search instance fai_search_shutdown8005Shutdown port of the search instance @@ -1195,16 +1281,16 @@

    6.2 Overview of All FAI P

    -
    -

    6.3 Examples

    -
    +
    +

    6.4 Examples

    +
    -
    -

    6.3.1 Installing an Editorial Server & Create a Publication

    -
    +
    +

    6.4.1 Installing an Editorial Server & Create a Publication

    +

    To automatically install an editorial server and create a publication called "jollygood", the minimal configuration in ece-install.conf @@ -1224,9 +1310,9 @@

    6.3.1 Installing an Edi

    -
    -

    6.3.2 Installing Two Presentation Servers On a Single Host

    -
    +
    +

    6.4.2 Installing Two Presentation Servers On a Single Host

    +

    If you wish to only install two presentation servers called "web1" and "web2" on your host, you will first run ece-install with: @@ -1262,9 +1348,9 @@

    6.3.2 Installing Two Pr

    -
    -

    6.4 Verifying That the Script Is Running In FAI Mode

    -
    +
    +

    6.5 Verifying That the Script Is Running In FAI Mode

    +

    When FAI is enabled, ece-install will report:

    @@ -1281,7 +1367,7 @@

    6.4 Verifying That the Sc

    -

    7 Running More Than One Installation Process

    +

    7 Running More Than One Installation Process

    If the script believes there's already an ece-intall process running, @@ -1301,7 +1387,7 @@

    7 Running More Than One Ins

    -

    8 Re-running ece-install (and How To Speed It Up)

    +

    8 Re-running ece-install (and How To Speed It Up)

    Although the initial thought behind ece-install, is to run it on a @@ -1324,7 +1410,7 @@

    8 Re-running ece-install (a

    To get a list of the artifacts it'll pull from -http://technet.escenic.com and http://tomcat.apache.org search for the +http://technet.escenic.com and http://tomcat.apache.org search for the following variables:

    • technet_download_list @@ -1346,7 +1432,7 @@

      8 Re-running ece-install (a

    -

    9 Using a Custom Configuration File for ece-install

    +

    9 Using a Custom Configuration File for ece-install

    You can specify a different configuration by using the -f parameter: @@ -1362,7 +1448,7 @@

    9 Using a Custom Configurat

    -

    10 Overview of File Paths Used by the ece-install script

    +

    10 Overview of File Paths Used by the ece-install script

    There are of course other paths involved when setting up your system, @@ -1414,7 +1500,7 @@

    10 Overview of File Paths

    -

    11 Overriding the Escenic directories

    +

    11 Overriding the Escenic directories

    All of the Escenic specific directories may be overwritten in @@ -1449,16 +1535,104 @@

    11 Overriding the Escenic

    -

    12 Assumptions

    +

    12 Extending ece-install by Writing Hooks

    +

    ece-install has a number of hooks on which you can hook on your own +scripts. The scripts are to reside in $HOME/ece-conf.d/ and have names +inspired by Debian's package scripts: +

    + + + +
    <hook name>.<phase>
    +
    + + +

    +e.g.: +

    + + + +
    install_analysis_server.preinst
    +
    + + +

    +Will be run before the body of the hook, just the corresponding +.postinst hook will be run after. +

    -

    12.1 /etc/esecenic is shared

    +

    12.1 Accessing ece-install variables

    +

    Before running the hook, ece-install will make all its local variables +available in /var/run/escenic/ece-install.env, which can then be used +by the hook scripts. +

    +
    + +
    + +
    +

    12.2 Example hook

    +
    + +

    Here is an example hook which will be run after the EAE is installed. +

    + + + +
    # Put this in is $HOME/ece-install.d/install_analysis_server.postinst
    +
    +# read ece-install's current variables
    +source /var/run/escenic/ece-install.env
    +
    +# do something useful
    +echo "Hello from $0, EAE is installed in ${tomcat_base}" > /tmp/hello.txt
    +
    + + +
    + +
    + +
    +

    12.3 Available hooks

    +
    + +

    Currently, the following hooks are available: +

    + + + +
    install_analysis_server.preinst
    +install_analysis_server.postinst
    +set_up_solr.preinst
    +set_up_solr.postinst
    +
    + + +
    +
    + +
    + +
    +

    13 Assumptions

    +
    + + +
    + +
    +

    13.1 /etc/esecenic is shared

    +
    +

    It's assumed that the /etc/escenic directory is either on a shared file system or rsync-ed among the hosts that are running the ECE instances (the exception being database and cache servers). @@ -1468,9 +1642,9 @@

    12.1 /etc/esecenic is sh

    -
    -

    13 Uninstalling Everything That the ece-install Set Up

    -
    +
    +

    14 Uninstalling Everything That the ece-install Set Up

    +

    WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your @@ -1488,9 +1662,9 @@

    13 Uninstalling Everything

    -
    -

    14 Example Output FAI

    -
    +
    +

    15 Example Output FAI

    +

    The ece-install will often be run in FAI mode where all the settings are read from ~/ece-install.conf. Given the following entries in this @@ -1560,7 +1734,7 @@

    14 Example Output FAI

    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2012-01-18 13:15:34 CST + Date: 2012-02-07 19:40:37 CST

    From 74c8d0bc485b704065c613801e2d0c50fc60b5de Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 8 Feb 2012 13:09:42 +0800 Subject: [PATCH 0276/2585] - added exit 0 at the end of the script. --- usr/sbin/ece-install | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index b022c5cf..18667081 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -3301,4 +3301,5 @@ else common_post_install fi +exit 0 From 4411daa4eb0ffa9e31a034c2e9792363950b5104 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 8 Feb 2012 19:26:56 +0800 Subject: [PATCH 0277/2585] - new feature: set_up_app_server will now call new method ensure_domain_is_known_to_local_host. This method will add entries to /etc/hosts if the domain names mentioned in fai_publication_domain_mapping_list are not resolvable to your local host (neither localhost or the IP of your $HOSTNAME). This can be quickly identified in /etc/hosts by search by the comments pre-pended to the host entries: # added by ece-install @ Wed Feb 8 19:21:49 CST 2012 127.0.1.1 fire.escenic.com # added by ece-install @ Wed Feb 8 19:21:51 CST 2012 127.0.1.1 ild.escenic.com --- usr/sbin/ece-install | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 18667081..8bb9769a 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -908,7 +908,7 @@ function get_base_dir_from_bundle() # $1 : the string function get_escaped_bash_string() { - result=$(echo $1 | \ + local result=$(echo $1 | \ sed -e 's/\$/\\$/g' \ -e 's/\*/\\*/g' \ -e 's#/#\\/#g' \ @@ -916,6 +916,40 @@ function get_escaped_bash_string() echo $result } +# $1 the domain +# +# The method will ensure that the passed domain is resolvable by the +# host on which ece-install is run. +function ensure_domain_is_known_to_local_host() +{ + if [ -z "$1" ]; then + return 1 + fi + + local localhost_ip=$(ping -c 1 localhost 2>/dev/null | \ + head -1 | \ + cut -d'(' -f2 | \ + cut -d')' -f1) + local hostname_ip=$(ping -c 1 $HOSTNAME 2>/dev/null | \ + head -1 | \ + cut -d'(' -f2 | \ + cut -d')' -f1) + local domain_ip=$(ping -c 1 $1 2>/dev/null | \ + head -1 | \ + cut -d'(' -f2 | \ + cut -d')' -f1) + + if [[ $domain_ip != $localhost_ip && $domain_ip != hostname_ip ]]; then + print_and_log "The domain name ${1} is not resolvable to this host" + print_and_log "I will remedy this by adding it to /etc/hosts" + cat >> /etc/hosts < Date: Wed, 8 Feb 2012 19:27:11 +0800 Subject: [PATCH 0278/2585] - documented behaviour of ensure_domain_is_known_to_local_host --- usr/share/doc/escenic/ece-install-guide.org | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index c9159163..e5d23e77 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -473,8 +473,16 @@ This will produce the following stanzas in server.xml: #+END_SRC -You must of course make sure that these host names are resolvable to -the intended IPs on your system using /etc/hosts or a local DNS. +Furthermore, if these host names are not resolvable to your local +host (neither localhost or the IP of your $HOSTNAME), ece-install will +add entries for these domains in the machine's /etc/hosts: +#+BEGIN_SRC sh +# added by ece-install @ Wed Feb 8 19:21:49 CST 2012 +127.0.1.1 fire.escenic.com + +# added by ece-install @ Wed Feb 8 19:21:51 CST 2012 +127.0.1.1 ild.escenic.com +#+END_SRC ** Overview of All FAI Parameters The ece-install script understands for the following settings in the From 9fb69dd1de87de0fb746bc6952b5f66afdd68f71 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 9 Feb 2012 12:59:20 +0800 Subject: [PATCH 0279/2585] - added documentation on fai_keep_off_etc_hosts=1 - sorted setting keys in the FAI overview table --- usr/share/doc/escenic/ece-install-guide.org | 88 +++++++++++---------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index e5d23e77..32b169c4 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -484,55 +484,59 @@ add entries for these domains in the machine's /etc/hosts: 127.0.1.1 ild.escenic.com #+END_SRC +If you do not want ece-install to touch your /etc/hosts, you can set +fai_keep_off_etc_hosts=1 in your ece-install.conf + ** Overview of All FAI Parameters The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| Parameter | Default | Description | +| Parameter | Default | Description | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_db\_schema | ece5db | Useful for DB installation profile | -| fai\_db\_user | ece5user | Useful for DB installation profile | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | -| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_db\_schema | ece5db | Useful for DB installation profile | +| fai\_db\_user | ece5user | Useful for DB installation profile | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | +| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | | fai\_search\_indexer\_ws\_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | -| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | +| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_keep\_off\_etc\_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | +| fai\_wf\_install | 0 | Install Widget Framework profile | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| #+TBLFM: $2=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ 0 From 156c6b231aff730037b27294e54ee4430f23ba12 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 9 Feb 2012 13:02:16 +0800 Subject: [PATCH 0280/2585] - added fai_keep_off_etc_hosts: If you do not want ece-install to touch your /etc/hosts, you can set fai_keep_off_etc_hosts=1 in your ece-install.conf Right now, the only time ece-install will do this is when setting the virtual host configuration in Tomcat, via the fai_publication_domain_mapping_list. In future, this keep off setting may be used for other places too, if it's logical that ece-install otherwise would manipulate the host's /etc/hosts. --- usr/sbin/ece-install | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8bb9769a..d424c2b9 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -939,7 +939,10 @@ function ensure_domain_is_known_to_local_host() cut -d'(' -f2 | \ cut -d')' -f1) - if [[ $domain_ip != $localhost_ip && $domain_ip != hostname_ip ]]; then + local keep_off_etc_hosts=${fai_keep_off_etc_hosts-0} + if [[ $domain_ip != $localhost_ip && \ + $domain_ip != hostname_ip && \ + $keep_off_etc_hosts -ne 1 ]]; then print_and_log "The domain name ${1} is not resolvable to this host" print_and_log "I will remedy this by adding it to /etc/hosts" cat >> /etc/hosts < Date: Thu, 9 Feb 2012 13:02:32 +0800 Subject: [PATCH 0281/2585] - more documentation on fai_keep_off_etc_hosts --- usr/share/doc/escenic/ece-install-guide.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 32b169c4..5a9dc8d8 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -485,7 +485,7 @@ add entries for these domains in the machine's /etc/hosts: #+END_SRC If you do not want ece-install to touch your /etc/hosts, you can set -fai_keep_off_etc_hosts=1 in your ece-install.conf +fai\_keep\_off\_etc\_hosts=1 in your ece-install.conf ** Overview of All FAI Parameters The ece-install script understands for the following settings in the From aea57f25056ca83e4269dff49fd5876340bc4d26 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 9 Feb 2012 13:14:58 +0800 Subject: [PATCH 0282/2585] - new feature: garbage collection log is now set up for all types and instances. For example the dev1 instance of type engine will get its log in: /var/log/escenic/engine-dev1-gc.log --- usr/bin/ece | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index e3b47195..0f8dd362 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -342,7 +342,10 @@ function set_instance_settings() -Dsolr.solr.home=$solr_home\ -Djava.awt.headless=true\ -Djava.security.auth.login.config=$ece_security_configuration_dir/jaas.config\ - -Djava.security.policy=$ece_security_configuration_dir/java.policy" + -Djava.security.policy=$ece_security_configuration_dir/java.policy\ + -XX:+PrintGCTimeStamps\ + -XX:+PrintGCDetails\ + -Xloggc:${log_dir}/${type}-${instance}-gc.log" if [ "$instance" != "default" ]; then ece_args=$ece_args" -Dcom.escenic.instance=$instance" From 8ad0fba44cb814177af376e0152a6b7f7bbe9446 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 9 Feb 2012 13:17:40 +0800 Subject: [PATCH 0283/2585] - added documentation on why garbage collection is added to the default JVM parameters. --- usr/bin/ece | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 0f8dd362..a361bcf3 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -338,6 +338,8 @@ function set_instance_settings() # * java.awt.headless is to avoid potential problems with graphics # handling/generation, causing 0x0 bitmaps etc. # * java.security for configuring the Java security framework with ECE. + # * garbage collection log: this is paramount to keep an eye on + # when running in production. ece_args="-Descenic.server=$ece_server\ -Dsolr.solr.home=$solr_home\ -Djava.awt.headless=true\ From afc39cc789e9799f27d7d37857c4173df6e192d7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 10 Feb 2012 16:39:37 +0800 Subject: [PATCH 0284/2585] - fix for Ubuntu 11.10 & upcoming Debian stable; since the Sun Java packages have been removed, we can no longer rely on that installing the java-6-sun-* packages actually does that. To handle migration issues (other packages depending on this package), the Debian (/and Ubuntu?) developers have decided to let this package now provide Open JDK. However, ECE doesn't support this one yet (there are weird issues with serialization objects which we use in the ECE <-> ECS communication), so until then, we must insist on using Sun Java. ece-install will now, if Sun Java isn't already installed, pull down the binary distribution from Oracle, build Debian packages of it and set up a local repository. This seems to work well on both Ubuntu and Debian. Tested on: Ubuntu 11.10 server edition and Debian Squeeze testing/unstable. This feature stands on the shoulders of http://github.com/flexiondotorg/oab-java6 , which again stands on the shoulders of https://github.com/rraptorr/sun-java6 This closes https://github.com/skybert/ece-scripts/issues/30 - fixed problem with ant pulling down OpenJDK on systems where Sun Java isn't already installed. - removed some redundant 1 & 2 redirects (the run wrapper takes care of this). - introduced the run wrapper a few more places: removing lines of code feels good! - new feature in create_publication running in interactive mode: added a list of instances installed on the host when the script prompts the user for which ECE instance to user for creating the publication. --- usr/sbin/ece-install | 128 +++++++++++++++++++++++++++++++------------ 1 file changed, 94 insertions(+), 34 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index d424c2b9..3b5be0ca 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -379,7 +379,7 @@ EOF assemble.properties \ 1>>$log 2>>$log exit_on_error "sed on assemble.properties" - + sed -i "s~\#\# plugins\ =\ /path/to/plugins~plugins=${escenic_root_dir}/assemblytool/plugins~g" \ assemble.properties \ 1>>$log 2>>$log @@ -472,8 +472,10 @@ function set_up_ece_scripts() run cd $download_dir if [ -d ece-scripts ]; then - (cd ece-scripts - git pull 1>>$log 2>>$log) + ( + run cd ece-scripts + run git pull + ) else run git clone $ece_scripts_git_source fi @@ -831,17 +833,54 @@ function set_ece_instance_conf() set_conf_file_value $1 $2 $instance_conf_file } +function create_java_deb_packages_and_repo() +{ + install_packages_if_missing $git_package + + print $(lsb_release -i | cut -d':' -f2) \ + $(lsb_release -r | cut -d':' -f2) \ + "doesn't have official Sun/Oracle Java packages," + print "I'm creating packages & local repo for you ..." + + local tmp_dir=$(mktemp -d) + ( + run cd $tmp_dir + run git clone https://github.com/flexiondotorg/oab-java6.git + run cd oab-java6 + run bash oab-java6.sh + run rm -rf $tmp_dir + ) + + add_next_step "Local APT repository with Sun/Oracle Java packages" + add_next_step "has been installed at /var/local/oab/deb and added" + add_next_step "to your APT system with /etc/apt/sources.list.d/oab.list" +} + +function has_sun_java_installed() +{ + if [[ -x /usr/lib/jvm/java-6-sun/bin/java || \ + $(java -version 2>&1 > /dev/null | grep HotSpot | wc -l) -gt 0 ]]; then + echo 1 + else + echo 0 + fi +} + # Installs third party packages needed by the ECE (i.e. Java related). # Also see install_common_os_packages for packages common to all # servers in the architecture. function install_ece_third_party_packages { - print_and_log "Installing 3rd party packages needed by ECE ..." + run_hook install_ece_third_party_packages.preinst if [ $on_debian_or_derivative -eq 1 ]; then - if [ $on_ubuntu -eq 1 ]; then - add_apt_source "deb http://archive.canonical.com/ $(lsb_release -s -c) partner" + # Sun Java was removed in Ubuntu in 11.10 + local version_needs_local_java_deb=1110 + local version=$(lsb_release -s -r | sed "s#\.##g") + + add_apt_source \ + "deb http://archive.canonical.com/ $(lsb_release -s -c) partner" elif [ $on_debian -eq 1 ]; then code_name=$(lsb_release -s -c) has_non_free=$(grep $(lsb_release -s -c) \ @@ -851,13 +890,28 @@ function install_ece_third_party_packages grep non-free | \ wc -l) if [ $has_non_free -eq 0 ]; then - add_apt_source "deb http://ftp.${mirror_country_suffix}.debian.org/debian/ $(lsb_release -s -c) contrib non-free" + add_apt_source "deb http://ftp.${mirror_country_suffix}.debian.org/debian/" \ + $(lsb_release -s -c) \ + "contrib non-free" fi + + # Sun Java was removed in Debian 6.0.4 (TODO verify this version) + local version_needs_local_java_deb=604 + local version=$(lsb_release -s -r | sed "s#\.##g") + fi + + if [ $version -ge $version_needs_local_java_deb -a \ + $(has_sun_java_installed) -eq 0 ]; then + create_java_deb_packages_and_repo fi echo "sun-java6-jdk shared/accepted-sun-dlj-v1-1 boolean true" | \ debconf-set-selections + # install sun-java6-jdk first so that ant doesn't pull down OpenJDK + local packages="sun-java6-jdk" + install_packages_if_missing $packages + packages=" ant ant-contrib @@ -867,9 +921,9 @@ function install_ece_third_party_packages libmysql-java memcached maven2 - sun-java6-jdk wget" + print_and_log "Installing 3rd party packages needed by ECE ..." install_packages_if_missing $packages fi @@ -1091,9 +1145,7 @@ function set_up_app_server() escenic_loader=",$\{catalina.base\}/escenic/lib/*.jar" old=$(get_escaped_bash_string ${common_loader}) new=${escaped_common_loader}$(get_escaped_bash_string ${escenic_loader}) - sed -i "s#${old}#${new}#g" $file \ - 1>>$log 2>>$log - exit_on_error "sed on $file" + run sed -i "s#${old}#${new}#g" $file cat > $tomcat_base/conf/server.xml < @@ -1400,8 +1452,7 @@ function set_correct_permissions() fi fi - run chown -R ${ece_user}:${ece_group} $el \ - 1>>$log 2>>$log + run chown -R ${ece_user}:${ece_group} $el done if [ -d "$tomcat_base" ]; then @@ -1675,7 +1726,7 @@ function un_install_ece() function set_up_varnish() { print_and_log "Setting up Varnish to match your environment ..." - /etc/init.d/varnish stop 1>>$log 2>>$log + run /etc/init.d/varnish stop # we need to swap standard err and standard out here as varnishd # -V for some reason writes to standard error. @@ -1813,7 +1864,7 @@ EOF set resp.http.X-Cache-Backend = req.backend; } EOF - /etc/init.d/varnish start 1>>$log 2>>$log + run /etc/init.d/varnish start } function read_user_input() @@ -2056,9 +2107,8 @@ function install_database_server() print_and_log "Installing the Percona database ..." if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then - gpg --keyserver hkp://keys.gnupg.net \ - --recv-keys 1C4CBDCDCD2EFD2A \ - 1>>$log 2>>$log + run gpg --keyserver hkp://keys.gnupg.net \ + --recv-keys 1C4CBDCDCD2EFD2A # There has been twice now, during six months, that # the key cannot be retrieved from @@ -2371,6 +2421,20 @@ function ensure_that_instance_is_running() fi } +# If the system is installed using the recommended paths, the method +# will return a list of the instances configured in +# ${escenic_conf_dir}/engine/instance +function get_instance_list() +{ + local instance_list="" + + if [ -r ${escenic_conf_dir}/engine/instance ]; then + instance_list=$(ls ${escenic_conf_dir}/engine/instance) + fi + + echo $instance_list +} + function create_publication() { if [ ! -e $escenic_root_dir/engine -o \ @@ -2385,11 +2449,13 @@ function create_publication() # TODO make educated guesses about the available instances on # $HOSTNAME by looking in $escenic_conf_dir/ece-*.conf - - default_instance=dev1 + + local instance_list=$(get_instance_list) + local default_instance=$(echo ${instance_list} | cut -d' ' -f1) if [ $fai_enabled -eq 0 ]; then print "Which ECE instance do you wish to use to create it?" + print "These instances are available: $instance_list" echo -n "Your choice [$default_instance]> " read instance_name else @@ -2461,8 +2527,7 @@ function install_analysis_server() set_correct_permissions local ece_command="ece -i ${instance_name} -t ${type} start" - su - $ece_user -c "$ece_command" \ - 1>>$log 2>>$log + su - $ece_user -c "$ece_command" 1>>$log 2>>$log exit_on_error "su - $ece_user -c \"$ece_command\"" local seconds=3 @@ -2617,15 +2682,12 @@ EOF print_and_log "Downloading Widget Framework from technet.escenic.com ..." for el in $wf_download_list; do cd $download_dir - wget $wget_opts \ + run wget $wget_opts \ --http-user $technet_user \ --http-password $technet_password \ - $el \ - 1>>$log 2>>$log - - cd $escenic_root_dir/ - unzip -q -u $download_dir/$(basename $el) \ - 1>>$log 2>>$log + $el + run cd $escenic_root_dir/ + run unzip -q -u $download_dir/$(basename $el) done assert_pre_prequesite mvn @@ -2636,8 +2698,7 @@ EOF print_and_log "Installing Widget Framework into your Maven repository ..." log "JAVA_HOME=$JAVA_HOME" - mvn $maven_opts install \ - 1>>$log 2>>$log + run mvn $maven_opts install # installing the widget-framework-common as a ECE plugin wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) @@ -2773,7 +2834,7 @@ server { EOF fi - /etc/init.d/nginx restart 1>>$log 2>>$log + run /etc/init.d/nginx restart if [ $1 -eq 0 ]; then add_next_step "http://${HOSTNAME}:${port}/binary gives the Adactus endpoint" @@ -2954,8 +3015,7 @@ EOF fi if [ $on_debian_or_derivative -eq 1 ]; then - service munin-node restart \ - 1>>$log 2>>$log + run service munin-node restart fi add_next_step "A Munin node has been installed on $HOSTNAME" From 5aa363811fd80485c227af8a8876917e32b2429b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 10 Feb 2012 16:55:32 +0800 Subject: [PATCH 0285/2585] - removed two TODOs, they're HAVEDONEs! --- usr/sbin/ece-install | 5 ----- 1 file changed, 5 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3b5be0ca..16771a6e 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2447,9 +2447,6 @@ function create_publication() print_and_log "Getting ready to create a new publication ..." create_publication_definition_and_war - # TODO make educated guesses about the available instances on - # $HOSTNAME by looking in $escenic_conf_dir/ece-*.conf - local instance_list=$(get_instance_list) local default_instance=$(echo ${instance_list} | cut -d' ' -f1) @@ -2468,8 +2465,6 @@ function create_publication() type=engine ensure_that_instance_is_running $instance_name - # TODO set the appserver_port based on the instance_name chosen - # above. create_publication_in_db $publication_war assemble_deploy_and_restart_type From b909d696fbf92b5dac9981c194938a99a89715e8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 10 Feb 2012 17:01:22 +0800 Subject: [PATCH 0286/2585] - Java packages in sqeeze/6.0 are fine, putting test for self build packages to 6.1 (perhaps changed to 7.0 later when wheezy's version number is announced). --- usr/sbin/ece-install | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 16771a6e..a90a4b11 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -895,8 +895,10 @@ function install_ece_third_party_packages "contrib non-free" fi - # Sun Java was removed in Debian 6.0.4 (TODO verify this version) - local version_needs_local_java_deb=604 + # Sun Java will be removed in the next Debian stable, + # wheezy (either 6.1 or 7.0 not announced yet, current in + # squeeze/6.0, hence setting 6.1 here). + local version_needs_local_java_deb=610 local version=$(lsb_release -s -r | sed "s#\.##g") fi From 16263129c30d165da50c7b767b068b235180f19c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 14 Feb 2012 20:08:41 +0800 Subject: [PATCH 0287/2585] - first working version of automatic setup of icinga (enhanced Nagios fork) server and client. The server is a part of the profile=monitoring. When installing the clients, which means pretty much all the server profiles, the parameter fai_monitoring_host_ip is respected as the one which may enter and collect the Nagios data (icinga uses Nagios's plugins, the client has only vanilla Nagios software installed) and the Munin data. When installing the monitoring server, the following parameters are implemented and have value: fai_monitoring_install=1 fai_monitoring_host_list="fire:192.168.1.109 wind:192.168.1.108" fai_monitoring_ece_host_list="fire wind" This will create logical service groups for "ece-hosts", as well as setting up general overview of the fire and wind hosts. The meaning of ece-hosts can be expanded upon to have more triggers/probles/nagios:service definitions to gather information especially interesting for ECE hosts. Likewise, I envision (not yet implemented) to have host groups fai_monitoring_cache_list, fai_monitoring_search_list, fai_monitoring_db_list --- usr/sbin/ece-install | 178 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 150 insertions(+), 28 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a90a4b11..819fc0a9 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2856,7 +2856,6 @@ function install_munin_gatherer() return fi - # TODO ask user/read fai_munin_node_list if [ $fai_enabled -eq 0 ]; then print "Which nodes shall this Munin monitor gather from?" print "Separate your hosts with a space, e.g.: 'editor01 db01 web01'" @@ -2867,7 +2866,7 @@ function install_munin_gatherer() node_list=$user_munin_nodes fi else - node_list=$(get_conf_value fai_munin_node_list) + node_list=${fai_munin_node_list} fi if [ -z "$node_list" ]; then @@ -2876,7 +2875,7 @@ function install_munin_gatherer() for el in $node_list; do print_and_log "Adding ${el} to the Muning gatherer on ${HOSTNAME} ..." - file=/etc/munin/munin-conf.d/escenic.conf + local file=/etc/munin/munin-conf.d/escenic.conf cat >> $file <:, e.g.: fire:192.168.1.100 +function set_up_icinga_host() +{ + local old_ifs=$IFS + IFS=':' + read host_name ip <<< "$1" + local file=/etc/icinga/objects/${host_name}_icinga.cfg + IFS=$old_ifs + + + if [ $(grep "host_name $host_name" $file | wc -l) -gt 0 ]; then + print "Icinga host" $host_name "already defined, skipping it." + return + fi + + cat >> $file <> $file <> $file + for el in $host_group_member_list; do + echo -n " ${el}," >> $file + done + cat >> $file < Date: Tue, 14 Feb 2012 20:13:50 +0800 Subject: [PATCH 0288/2585] - support for fai_monitoring_admin_password, with fallback. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 819fc0a9..9fde345e 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2921,7 +2921,7 @@ function monitoring_install_icinga_server() # set password for the icingaadmin print "Setting user/pass for icinga admin ..." local file=/etc/icinga/htpasswd.users - run htpasswd -b -c $file icingaadmin admin + run htpasswd -b -c $file icingaadmin ${fai_monitoring_admin_password-admin} # enable remote commands local file=/etc/icinga/icinga.cfg From 80b8e86dbae6cc2e131dda6e2d2415f9897a9309 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 15 Feb 2012 12:05:05 +0800 Subject: [PATCH 0289/2585] - calls to add_apt_source must be on one line, or else the grep string might be wrong. --- usr/sbin/ece-install | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9fde345e..ee862508 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -883,23 +883,21 @@ function install_ece_third_party_packages "deb http://archive.canonical.com/ $(lsb_release -s -c) partner" elif [ $on_debian -eq 1 ]; then code_name=$(lsb_release -s -c) - has_non_free=$(grep $(lsb_release -s -c) \ + has_non_free=$(grep $(lsb_release -s -c) \ /etc/apt/sources.list | \ egrep -v "^#|deb-src" | \ grep -v security | \ grep non-free | \ wc -l) if [ $has_non_free -eq 0 ]; then - add_apt_source "deb http://ftp.${mirror_country_suffix}.debian.org/debian/" \ - $(lsb_release -s -c) \ - "contrib non-free" + add_apt_source "deb http://ftp.${mirror_country_suffix}.debian.org/debian/ $(lsb_release -s -c) contrib non-free" fi # Sun Java will be removed in the next Debian stable, # wheezy (either 6.1 or 7.0 not announced yet, current in # squeeze/6.0, hence setting 6.1 here). local version_needs_local_java_deb=610 - local version=$(lsb_release -s -r | sed "s#\.##g") + local version=$(lsb_release -s -r | sed "s#\.##g") fi if [ $version -ge $version_needs_local_java_deb -a \ @@ -932,6 +930,9 @@ function install_ece_third_party_packages for el in ant mvn java; do assert_pre_prequesite $el done + + exit 0 + } function get_base_dir_from_bundle() @@ -3049,25 +3050,22 @@ EOF fi # install the escenic_jstat munin plugin - file=/usr/share/munin/plugins/escenic_jstat_ + local file=/usr/share/munin/plugins/escenic_jstat_ run wget $wget_opts \ https://github.com/mogsie/escenic-munin/raw/master/escenic_jstat_ \ -O $file run chmod 755 $file - if [ ! -e $escenic_conf_dir -o \ - $(ls $escenic_conf_dir/ | grep ece- | wc -l) -lt 1 ]; then + local instance_list=$(get_instance_list) + if [ -z ${instance_list} ]; then print_and_log "No ECE instances found on $HOSTNAME, so I'm not adding" print_and_log "additional Munin configuration" add_next_step "A Munin node has been installed on $HOSTNAME" return fi - escenic_jstat_modules="_gc _gcoverhead _heap _uptime" - for el in $escenic_conf_dir/ece-*.conf; do - current_instance=$(basename ${el} | \ - sed -e 's/ece-//g' -e 's/engine-//g' | \ - cut -d'.' -f1) + local escenic_jstat_modules="_gc _gcoverhead _heap _uptime" + for current_instance in $instance_list; do for module in $escenic_jstat_modules; do cd /usr/share/munin/plugins/ make_ln escenic_jstat_ escenic_jstat_${current_instance}${module} From a94b41970685145de9954b7ffae7b1673f9b9397 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 15 Feb 2012 18:54:38 +0800 Subject: [PATCH 0290/2585] - fixed the permissions of the ece_user's .bashrc - new feature: when installing the monitoring server profile, you can choose between vanilla Nagios 3 and the icinga fork. New setting: fai_monitoring_nagios_flavour has two supported values, "nagios" and "icinga". Default is icinga. New setting: fai_monitoring_privileged_hosts a string with the hosts which should be allowed to access the monitoring server. - the monitoring server profile now generates a start page for all the monitoring services, accessible on the monitoring host's / resource. - added installation report info about the icinga/nagios interface(s). --- usr/sbin/ece-install | 169 +++++++++++++++++++++++++++++++++---------- 1 file changed, 129 insertions(+), 40 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index ee862508..988df325 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -930,9 +930,6 @@ function install_ece_third_party_packages for el in ant mvn java; do assert_pre_prequesite $el done - - exit 0 - } function get_base_dir_from_bundle() @@ -1641,6 +1638,7 @@ function set_up_user_enviornment() if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" >> ${bashrc} + run chown ${ece_user}:${ece_group} ${bashrc} fi } @@ -2867,7 +2865,7 @@ function install_munin_gatherer() node_list=$user_munin_nodes fi else - node_list=${fai_munin_node_list} + node_list=${fai_monitoring_munin_node_list} fi if [ -z "$node_list" ]; then @@ -2875,7 +2873,7 @@ function install_munin_gatherer() fi for el in $node_list; do - print_and_log "Adding ${el} to the Muning gatherer on ${HOSTNAME} ..." + print_and_log "Adding ${el} to the Munin gatherer on ${HOSTNAME} ..." local file=/etc/munin/munin-conf.d/escenic.conf cat >> $file <:, e.g.: fire:192.168.1.100 -function set_up_icinga_host() +## Sets up the definition file for a given monitoring host. +## +## $1 nagios flavour/vendor +## $2 :, e.g.: fire:192.168.1.100 +function set_up_monitoring_host_def() { - local old_ifs=$IFS - IFS=':' - read host_name ip <<< "$1" local file=/etc/icinga/objects/${host_name}_icinga.cfg + if [[ $1 == $MONITORING_VENDOR_NAGIOS ]]; then + file=/etc/nagios3/conf.d/${host_name}_nagios2.cfg + fi + + local old_ifs=$IFS + IFS='#' + read host_name ip <<< "$2" IFS=$old_ifs - - - if [ $(grep "host_name $host_name" $file | wc -l) -gt 0 ]; then - print "Icinga host" $host_name "already defined, skipping it." + + if [ $(grep "host_name $host_name" $file 2>/dev/null | wc -l) -gt 0 ]; then + print "$1 host" $host_name "already defined, skipping it." return fi @@ -2976,7 +3025,7 @@ EOF } -function install_icinga_client() +function install_nagios_node() { print "Installing an icinga client on $HOSTNAME ..." if [ $on_debian_or_derivative -eq 1 ]; then @@ -2992,6 +3041,7 @@ function install_icinga_client() dont_quote_conf_values=0 run /etc/init.d/nagios-nrpe-server restart + add_next_step "A Nagios NRPE node has been installed on ${HOSTNAME}" } # munin nodes need the IP of the munin gatherer to be escaped. Hence this function. @@ -3057,7 +3107,7 @@ EOF run chmod 755 $file local instance_list=$(get_instance_list) - if [ -z ${instance_list} ]; then + if [ -z "${instance_list}" ]; then print_and_log "No ECE instances found on $HOSTNAME, so I'm not adding" print_and_log "additional Munin configuration" add_next_step "A Munin node has been installed on $HOSTNAME" @@ -3308,12 +3358,18 @@ function restore_from_backup() fi } -function set_up_icinga_host_groups() +## $1 configuration file name +## $2 host group name +## $3 host group alias +## $4..n host group members +function set_up_monitoring_host_group() { - local file=/etc/icinga/objects/hostgroups_icinga.cfg - local host_group_name=$1 - local host_group_alias=$2 - local host_group_member_list=${@:3:$(( $# - 2 ))} + local file=$1 + local host_group_name=$2 + local host_group_alias=$3 + # the remainding arguments passed to the methods is the member + # list members + local host_group_member_list=${@:4:$(( $# - 3 ))} if [ $(grep "hostgroup_name $host_group_name" $file | wc -l) -gt 0 ]; then print "Icinga group member" \ @@ -3337,6 +3393,41 @@ EOF EOF } +## $1 nagios vendor +function create_monitoring_server_overview() +{ + local file=/var/www/index.html + cat > $file < + +

    Welcome to the might monitoring server @ ${HOSTNAME}

    +
      +EOF + if [[ $1 == $MONITORING_VENDOR_NAGIOS ]]; then + echo '
    • Nagios
    • ' \ + else + echo '
    • Icinga (an enhanced Nagios)
    • ' \ + >> $file + fi + cat > $file <Munin +
    + + +EOF + add_next_step "Start page for all monitoring interfaces: http://${HOSTNAME}/" +} + +MONITORING_VENDOR_NAGIOS=nagios +MONITORING_VENDOR_ICINGA=icinga +function install_monitoring_server() +{ + local nagios_flavour=${fai_monitoring_nagios_flavour-$MONITORING_VENDOR_ICINGA} + install_nagios_monitoring_server $nagios_flavour + install_munin_gatherer + create_monitoring_server_overview $nagios_flavour +} + function common_post_install() { if [ $install_profile_number -eq $PROFILE_ALL_IN_ONE -o \ @@ -3352,7 +3443,7 @@ function common_post_install() install_munin_node if [ $install_profile_number -ne $PROFILE_MONITORING_SERVER ]; then - install_icinga_client + install_nagios_node fi fi @@ -3444,9 +3535,7 @@ if [ $fai_enabled -eq 1 ]; then if [ $(get_boolean_conf_value fai_monitoring_install) -eq 1 ]; then install_profile_number=$PROFILE_MONITORING_SERVER - monitoring_install_icinga_server - install_munin_gatherer - # install_web_server 1 + install_monitoring_server no_fai_profile=0 fi From db5e9b41e3d0aac441d4fb987040dc7430be6352 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 15 Feb 2012 18:57:39 +0800 Subject: [PATCH 0291/2585] - when ece cannot find one of its required fields, it will now tell which ece.conf files it has parsed/read instead of just saying "ece.conf" [ece#engine-editor1] You need to specifiy 'tmp_dir' in one of /etc/escenic/ece-editor1.conf /etc/escenic/ece.conf --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index a361bcf3..a7aa3f0c 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -152,7 +152,7 @@ function ensure_that_required_fields_are_set() continue fi - print "You need to specifiy '$el' in your `basename $0`.conf" + print "You need to specifiy '$el' in one of ${ece_conf_files_read}" requirements_failed=1 done From 72d7986c4f74d25e6a7be082420c4ab14e2c3fa8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 15 Feb 2012 18:59:19 +0800 Subject: [PATCH 0292/2585] - small fix: now lists _all_ the parsed files (not just the first one): [ece#engine-dev1] You need to specifiy 'tmp_dir' in one of /etc/escenic/ece-dev1.conf /etc/escenic/ece.conf --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index a7aa3f0c..0febad76 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -152,7 +152,7 @@ function ensure_that_required_fields_are_set() continue fi - print "You need to specifiy '$el' in one of ${ece_conf_files_read}" + print "You need to specifiy '$el' in one of ${ece_conf_files_read[@]}" requirements_failed=1 done From 3fbd45dfdabefd37e9598944e372172deb122e37 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 16 Feb 2012 17:08:57 +0800 Subject: [PATCH 0293/2585] - added dedicated method for getting privileged hosts: Returns the privileged hosts. This will include both the IP(s) the logged in user conduction the ece-install is coming from, as well as any IPs defined in fai_privileged_hosts. - made use of this method in both Munin's Apache configuration and in Varnish' ACL. --- usr/sbin/ece-install | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 988df325..0061b64f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -271,7 +271,7 @@ function download_escenic_components() # # parameters: # $1 : space separated string of package names -install_packages_if_missing() +function install_packages_if_missing() { if [ $on_debian_or_derivative -eq 1 ]; then some_are_missing=0 @@ -1724,6 +1724,27 @@ function un_install_ece() apt-get clean } +## Returns the privileged hosts. This will include both the IP(s) the +## logged in user conduction the ece-install is coming from, as well +## as any IPs defined in fai_privileged_hosts. +function get_privileged_hosts() +{ + local privileged_hosts=${fai_privileged_hosts} + + for ip in $( + w -h | \ + grep pts | \ + sed "s#.*pts/[0-9]*[ ]*\(.*\)#\1#" | \ + cut -d' ' -f1 | \ + sort | \ + uniq + ); do + privileged_hosts=${privileged_hosts}" "${ip} + done + + echo ${privileged_hosts} +} + function set_up_varnish() { print_and_log "Setting up Varnish to match your environment ..." @@ -1744,13 +1765,11 @@ function set_up_varnish() acl staff { "localhost"; EOF - w | cut -d' ' -f7 | sort | uniq | while read l; do - if [ $(echo $l | wc -c) -gt 8 ]; then - cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < Date: Fri, 17 Feb 2012 12:32:19 +0800 Subject: [PATCH 0294/2585] - when using fai__conf_archive, the plugin configuration must now reside with the other configuration files: "If you wish to provide Nursery configuration for the plugins, you simply put them in engine/siteconfig/config-skeleton inside your tarball, together with the other Nursery configuration files." --- usr/share/doc/escenic/ece-install-guide.org | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 5a9dc8d8..a30d35c0 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -419,8 +419,7 @@ $ tar czf /tmp/nursery-skeleton-solr-and-security.tar.gz \ engine/security \ engine/siteconfig/config-skeleton/ \ engine/solr/conf \ - engine/siteconfig/bootstrap-skeleton/ \ - `find -L assemblytool/plugins/ -name siteconfig` + engine/siteconfig/bootstrap-skeleton/` #+END_SRC /tmp/nursery-skeleton-solr-and-security.tar.gz should now have everything @@ -439,6 +438,10 @@ The inclusion of the engine/solr directory makes it easy for users to provide their own, optimised Solr configuration. In this context, also note that a post install hook, set_up_solr.postinst, is available. +If you wish to provide Nursery configuration for the plugins, you +simply put them in engine/siteconfig/config-skeleton inside your +tarball, together with the other Nursery configuration files. + ** Setting up virtual hosts Setting up virtual host definitions in the application server makes a some things easier, such as ECE plugins which set cookies based on From 74f0a947dbefc9806210cd24906a4e13b79b6c23 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 17 Feb 2012 12:48:05 +0800 Subject: [PATCH 0295/2585] - when using conf archives (fai__conf_archive), the plugin information must now be together with the other Nursery configuration you wish to provide/override. See the documentation for exact examples of this. - providing the JAAS (engine/security) configuration in the conf archive is now optional. If it's not provided, ece-install will use the JAAS configuration shipped with ECE. - new method: get_user_home_directory, which will: Method which will try its best to find the home diretory of existing users and probable home directories of new users. (1) On Darwin/Mac OS X, it will return /Users/ (2) On systems using /etc/passwd, it will search there to find the home dir of existing users. (3) For new users, it will check the configuration of adduser (if present). (4) If all of thes above fails, it will return /home/ - it's now possible to have the UNIX users with home directories outside of /home (or /Users on Mac). - the Maven dependency is first required, installed & checked for when it's needed (i.e. when installing WF) --- usr/sbin/ece-install | 82 +++++++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 27 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0061b64f..20808ac0 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -296,7 +296,6 @@ function install_packages_if_missing() fi } - function install_common_os_packages() { print_and_log "Installing 3rd party packages needed by $(basename $0) ..." @@ -413,10 +412,8 @@ name: ${short_name} source-war: ${short_name}.war context-root: ${short_name} EOF - done fi - } function set_up_engine_and_plugins() @@ -609,31 +606,37 @@ function set_up_basic_nursery_configuration() { print_and_log "Setting up the basic Nursery configuration ..." - local escenic_root_dir=$escenic_root_dir - + # we always copy the default plugin configuration (even if there + # is an archive) + for el in $escenic_root_dir/assemblytool/plugins/*; do + if [ ! -d $el/misc/siteconfig/ ]; then + continue + fi + run cp -r $el/misc/siteconfig/* $common_nursery_dir/ + done + + # Then we see if we're using configuration archives, if yes, use + # the JAAS and Nursery configuraiton from here. if [ $(is_using_conf_archive) -eq 1 ]; then print_and_log "Using the supplied Nursery & JAAS configuration from" print_and_log "bundle: $ece_instance_conf_archive" local a_tmp_dir=$(mktemp -d) - escenic_root_dir=$a_tmp_dir + + if [ ! -d ${a_tmp_dir}/engine/security ]; then + print "Archive $ece_instance_conf_archive doesn't have JAAS config," + print "I'll use standard JAAS (engine/security) instead." + run cp -r $escenic_root_dir/engine/security/ $common_nursery_dir/ + fi + run cd $a_tmp_dir run tar xzf $ece_instance_conf_archive + run cp -r engine/siteconfig/config-skeleton/* $common_nursery_dir/ + else + run cp -r $escenic_root_dir/engine/siteconfig/config-skeleton/* \ + $common_nursery_dir/ + run cp -r $escenic_root_dir/engine/security/ $common_nursery_dir/ fi - - run cp -r $escenic_root_dir/engine/siteconfig/config-skeleton/* \ - $common_nursery_dir/ - run cp -r $escenic_root_dir/engine/security/ \ - $common_nursery_dir/ - make_dir $escenic_conf_dir/engine/instance - - for el in $escenic_root_dir/assemblytool/plugins/*; do - if [ ! -d $el/misc/siteconfig/ ]; then - continue - fi - - run cp -r $el/misc/siteconfig/* $common_nursery_dir/ - done if [ -n "${a_tmp_dir}" ]; then run rm -rf ${a_tmp_dir} @@ -920,14 +923,13 @@ function install_ece_third_party_packages libtcnative-1 libmysql-java memcached - maven2 wget" print_and_log "Installing 3rd party packages needed by ECE ..." install_packages_if_missing $packages fi - for el in ant mvn java; do + for el in ant java; do assert_pre_prequesite $el done } @@ -1622,16 +1624,40 @@ EOF echo "TBD" } +## Method which will try its best to find the home diretory of +## existing users and probable home directories of new users. +## +## (1) On Darwin/Mac OS X, it will return /Users/ +## (2) On systems using /etc/passwd, it will search there to find the +## home dir of existing users. +## (3) For new users, it will check the configuration of adduser (if +## present). +## (4) If all of thes above fails, it will return /home/ +## +## Arguments: +## $1 : the user name, can either be an existing user or a non-yet +## created user. +function get_user_home_directory() +{ + if [ $(uname -s) = "Darwin" ]; then + echo /Users/$1 + elif [ $(grep $1 /etc/passwd | wc -l) -gt 0 ]; then + grep $1 /etc/passwd | cut -d':' -f6 + elif [ -r /etc/adduser.conf ]; then + local dir=$(grep DHOME /etc/adduser.conf | grep -v ^# | cut -d'=' -f2) + echo $dir/$1 + else + echo "/home/$1" + fi +} + function set_up_user_enviornment() { print_and_log "Setting up the ${ece_user} user's UNIX environment ..." - local bashrc=/home/${ece_user}/.bashrc - if [ $(uname -s) = "Darwin" ]; then - bashrc=/Users/${ece_user}/.bashrc - fi + local bashrc=$(get_user_home_directory $ece_user)/.bashrc - if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ + if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ wc -l) -lt 1 ]; then echo ". /etc/bash_completion.d/ece" >> ${bashrc} fi @@ -2705,7 +2731,9 @@ EOF run unzip -q -u $download_dir/$(basename $el) done + install_packages_if_missing "maven2" assert_pre_prequesite mvn + export JAVA_HOME=$java_home wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) run cd $wf_maven_dir From 9f1b51f2a48f5ffb4b9f6f364d264de1577cfe68 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 17 Feb 2012 12:59:23 +0800 Subject: [PATCH 0296/2585] - renamed appserver_root_dir -> appserver_parent_dir as that's what it is. --- usr/sbin/ece-install | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 20808ac0..a83e6444 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -35,7 +35,7 @@ escenic_backups_dir=/var/backups/${dir_suffix} escenic_spool_dir=/var/spool/${dir_suffix} escenic_cache_dir=/var/cache/${dir_suffix} escenic_crash_dir=/var/crash/${dir_suffix} -appserver_root_dir=/opt +appserver_parent_dir=/opt # country code for selecting the correct (APT) mirror. mirror_country_suffix=jp @@ -1114,7 +1114,7 @@ function set_up_app_server() run wget $wget_opts $tomcat_download tomcat_dir=$(get_base_dir_from_bundle $tomcat_download) - run cd $appserver_root_dir + run cd $appserver_parent_dir run tar xzf $download_dir/${tomcat_dir}.tar.gz if [ -e tomcat ]; then @@ -1122,11 +1122,11 @@ function set_up_app_server() fi run ln -sf ${tomcat_dir} tomcat - tomcat_home=${appserver_root_dir}/tomcat - tomcat_base=${appserver_root_dir}/tomcat-${instance_name} + tomcat_home=${appserver_parent_dir}/tomcat + tomcat_base=${appserver_parent_dir}/tomcat-${instance_name} make_dir $tomcat_base - run cp -r ${appserver_root_dir}/${tomcat_dir}/conf $tomcat_base + run cp -r ${appserver_parent_dir}/${tomcat_dir}/conf $tomcat_base for el in bin escenic/lib lib work logs temp webapps; do make_dir $tomcat_base/$el done @@ -1715,7 +1715,7 @@ function un_install_ece() print_and_log "Uninstalling ECE ..." # TODO safety, warnings++ rm -rf $escenic_conf_dir/ \ - ${appserver_root_dir}/*tomcat* \ + ${appserver_parent_dir}/*tomcat* \ $escenic_root_dir \ $escenic_data_dir \ $escenic_run_dir/ \ @@ -3369,7 +3369,7 @@ function restore_from_backup() run tar -C / -xf $backup_file opt add_next_step "Successfully restored Escenic & Tomcat binaries" add_next_step "Backup file used: $(basename $backup_file)" - add_next_step "Check ${appserver_root_dir} to verify that they're all there". + add_next_step "Check ${appserver_parent_dir} to verify that they're all there". install_ece_third_party_packages set_up_engine_directories From af988d14bb5583aa23af4beb41267c759b7cf6e2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 17 Feb 2012 12:59:36 +0800 Subject: [PATCH 0297/2585] - renamed appserver_root_dir -> appserver_parent_dir as that's what it is. --- usr/share/doc/escenic/ece-install-guide.org | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index a30d35c0..177290b2 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -674,17 +674,17 @@ ece-install.conf. Here's an example of changing all the paths possible with the same suffix. #+BEGIN_SRC conf -suffix=escenic-parallel - -escenic_root_dir=/opt/$suffix -escenic_conf_dir=/etc/$suffix -escenic_log_dir=/var/log/$suffix -escenic_data_dir=/var/lib/$suffix -escenic_run_dir=/var/run/$suffix -escenic_backups_dir=/var/backups/$suffix -escenic_spool_dir=/var/spool/$suffix -escenic_cache_dir=/var/cache/$suffix -escenic_crash_dir=/var/crash/$suffix +dir_suffix=escenic-parallel +escenic_root_dir=/opt/${dir_suffix} +escenic_conf_dir=/etc/${dir_suffix} +escenic_log_dir=/var/log/${dir_suffix} +escenic_data_dir=/var/lib/${dir_suffix} +escenic_run_dir=/var/run/${dir_suffix} +escenic_backups_dir=/var/backups/${dir_suffix} +escenic_spool_dir=/var/spool/${dir_suffix} +escenic_cache_dir=/var/cache/${dir_suffix} +escenic_crash_dir=/var/crash/${dir_suffix} +appserver_parent_dir=/opt #+END_SRC Note, this is only needed if you are running two completely separate From 5ea12c4165ab15c63351cf6c410fe6f0ac41a2da Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 17 Feb 2012 12:59:55 +0800 Subject: [PATCH 0298/2585] - update HTML --- usr/share/doc/escenic/ece-install-guide.html | 56 +++++++++++++------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index e49f7282..98051c42 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ - + @@ -1128,8 +1128,7 @@

    6.1 Installing from EARs engine/security \ engine/siteconfig/config-skeleton/ \ engine/solr/conf \ - engine/siteconfig/bootstrap-skeleton/ \ - `find -L assemblytool/plugins/ -name siteconfig` + engine/siteconfig/bootstrap-skeleton/`

    @@ -1155,6 +1154,11 @@

    6.1 Installing from EARs provide their own, optimised Solr configuration. In this context, also note that a post install hook, setupsolr.postinst, is available.

    +

    +If you wish to provide Nursery configuration for the plugins, you +simply put them in engine/siteconfig/config-skeleton inside your +tarball, together with the other Nursery configuration files. +

    @@ -1206,8 +1210,23 @@

    6.2 Setting up virtual ho

    -You must of course make sure that these host names are resolvable to -the intended IPs on your system using /etc/hosts or a local DNS. +Furthermore, if these host names are not resolvable to your local +host (neither localhost or the IP of your $HOSTNAME), ece-install will +add entries for these domains in the machine's /etc/hosts: +

    + + +
    # added by ece-install @ Wed Feb  8 19:21:49 CST 2012
    +127.0.1.1 fire.escenic.com
    +
    +# added by ece-install @ Wed Feb  8 19:21:51 CST 2012
    +127.0.1.1 ild.escenic.com
    +
    + + +

    +If you do not want ece-install to touch your /etc/hosts, you can set +fai_keep_off_etc_hosts=1 in your ece-install.conf

    @@ -1259,7 +1278,7 @@

    6.3 Overview of All FAI P fai_publication_use_instancedev1Name of local instance to use for creation fai_publication_war"WF or ECE demo WAR"WAR to base the new publication on fai_publication_war_uri_list""Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, ///tmp/mypub.war and /tmp/mypub.war. -fai_publication_domain_mapping_list""Mapping between publication names and their corresponding domains on the form: one#one.com other#other.com +fai_publication_domain_mapping_list""Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" fai_rmi_install0Install RMI hub profile fai_search_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration fai_search_ear""EAR to use instead of the Escenic binaries @@ -1269,6 +1288,7 @@

    6.3 Overview of All FAI P fai_search_namesearch1Name of the search instance fai_search_port8080HTTP port of the search instance fai_search_shutdown8005Shutdown port of the search instance +fai_keep_off_etc_hosts0Set this to 1 if you don't want ece-install adding entries to /etc/hosts fai_wf_install0Install Widget Framework profile @@ -1510,17 +1530,17 @@

    11 Overriding the Escenic -
    suffix=escenic-parallel
    -
    -escenic_root_dir=/opt/$suffix
    -escenic_conf_dir=/etc/$suffix
    -escenic_log_dir=/var/log/$suffix
    -escenic_data_dir=/var/lib/$suffix
    -escenic_run_dir=/var/run/$suffix
    -escenic_backups_dir=/var/backups/$suffix
    -escenic_spool_dir=/var/spool/$suffix
    -escenic_cache_dir=/var/cache/$suffix
    -escenic_crash_dir=/var/crash/$suffix
    +
    dir_suffix=escenic-parallel
    +escenic_root_dir=/opt/${dir_suffix}
    +escenic_conf_dir=/etc/${dir_suffix}
    +escenic_log_dir=/var/log/${dir_suffix}
    +escenic_data_dir=/var/lib/${dir_suffix}
    +escenic_run_dir=/var/run/${dir_suffix}
    +escenic_backups_dir=/var/backups/${dir_suffix}
    +escenic_spool_dir=/var/spool/${dir_suffix}
    +escenic_cache_dir=/var/cache/${dir_suffix}
    +escenic_crash_dir=/var/crash/${dir_suffix}
    +appserver_parent_dir=/opt
     
    @@ -1734,7 +1754,7 @@

    15 Example Output FAI


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2012-02-07 19:40:37 CST + Date: 2012-02-17 12:59:41 CST

    From 62b8318e01c1f9f5b186c68a364346d96d59aa1b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 17 Feb 2012 18:25:54 +0800 Subject: [PATCH 0299/2585] - added method to get pulse on ongoing tasks, as well as outputting a green "ok" on success or a red "failed" on failure. - made use of the new show_pulse method a few places. --- usr/sbin/ece-install | 429 ++++++++++++++++++++++++------------------- 1 file changed, 235 insertions(+), 194 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a83e6444..17f7572d 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -110,6 +110,11 @@ function print() echo $(get_id) $@ } +function printne() +{ + echo -ne $(get_id) $@ +} + function log() { echo $(get_id) $@ >> $log @@ -244,14 +249,46 @@ function set_up_engine_directories() done } +pulse_bits="-\|/" + +function pulse() +{ + echo -ne "\b${pulse_bits:i++%${#pulse_bits}:1}" +} + +function show_pulse() +{ + printne "$1 ...." + local pulse_pid=$2 + while true; do + kill -0 $pulse_pid 2>/dev/null + if [ $? -eq 0 ]; then + pulse + sleep 0.25 + else + wait $pulse_pid + local exit_code=$? + if [ $exit_code -eq 0 ]; then + echo -e '\b\E[37;32m'" \033[1mok\033[0m" + tput sgr0 + else + echo -e '\b\E[37;31m'" \033[1mfailed\033[0m" + tput sgr0 + fi + + break + fi + done +} + function download_escenic_components() { if [ $ece_software_setup_completed -eq 1 ]; then return fi - + print_and_log "Downloading Escenic software from technet.escenic.com ..." - + cd $download_dir for el in $technet_download_list; do if [ -e $(basename $el) ]; then @@ -263,7 +300,6 @@ function download_escenic_components() --http-password $technet_password \ $el done - } # will install the passed packages if these are not installed from @@ -298,23 +334,24 @@ function install_packages_if_missing() function install_common_os_packages() { - print_and_log "Installing 3rd party packages needed by $(basename $0) ..." - - if [ $on_debian_or_derivative -eq 1 ]; then + $( + if [ $on_debian_or_derivative -eq 1 ]; then # Ubuntu doesn't have git (!) but only git-core. - if [ $on_ubuntu -eq 1 ]; then - git_package=git-core - else - git_package=git - fi + if [ $on_ubuntu -eq 1 ]; then + git_package=git-core + else + git_package=git + fi - packages="curl $git_package wget unzip" - install_packages_if_missing $packages - fi + packages="curl $git_package wget unzip" + install_packages_if_missing $packages + fi - for el in lsb_release curl wget git unzip; do - assert_pre_prequesite $el - done + for el in lsb_release curl wget git unzip; do + assert_pre_prequesite $el + done + ) & + show_pulse "Installing 3rd party packages needed by $(basename $0)" $! } function set_up_assembly_tool() @@ -436,12 +473,12 @@ function set_up_engine_and_plugins() fi done - if [ ! -d ${engine_dir} ]; then + if [ ! -d "${engine_dir}" ]; then run unzip -q -u $download_dir/${engine_file} if [ -h engine ]; then run rm engine fi - + run ln -s ${engine_dir} engine else debug "${engine_dir} is already there, skipping to next step." @@ -465,41 +502,43 @@ function set_up_engine_and_plugins() function set_up_ece_scripts() { - print_and_log "Setting up the ece UNIX scripts ..." - - run cd $download_dir - if [ -d ece-scripts ]; then - ( - run cd ece-scripts - run git pull - ) - else - run git clone $ece_scripts_git_source - fi - - run cp -r ece-scripts/usr/* /usr/ - run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ - run cp -r ece-scripts/etc/bash_completion.d/ece /etc/bash_completion.d/ - run cp -r ece-scripts/etc/init.d/* /etc/init.d/ - run cp -r ece-scripts/etc/default/* /etc/default/ - - local file=${escenic_conf_dir}/ece.conf - set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file - set_conf_file_value backup_dir ${escenic_backups_dir} $file - set_conf_file_value cache_dir ${escenic_cache_dir} ${file} - set_conf_file_value ece_home ${escenic_root_dir}/engine ${file} - set_conf_file_value escenic_conf_dir ${escenic_conf_dir} ${file} - set_conf_file_value heap_dump_dir ${escenic_crash_dir} ${file} - set_conf_file_value log_dir ${escenic_log_dir} ${file} - set_conf_file_value pid_dir ${escenic_run_dir} ${file} - set_conf_file_value rmi_hub_conf ${escenic_conf_dir}/rmi-hub ${file} - set_conf_file_value solr_home ${escenic_data_dir}/solr ${file} - set_conf_file_value ece_security_configuration_dir \ - ${escenic_conf_dir}/engine/common/security \ - ${file} - - run sed -i "s#/etc/escenic#${escenic_conf_dir}#g" /etc/bash_completion.d/ece - run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece + $( + run cd $download_dir + if [ -d ece-scripts ]; then + ( + run cd ece-scripts + run git pull + ) + else + run git clone $ece_scripts_git_source + fi + + run cp -r ece-scripts/usr/* /usr/ + run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ + run cp -r ece-scripts/etc/bash_completion.d/ece /etc/bash_completion.d/ + run cp -r ece-scripts/etc/init.d/* /etc/init.d/ + run cp -r ece-scripts/etc/default/* /etc/default/ + + local file=${escenic_conf_dir}/ece.conf + set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file + set_conf_file_value backup_dir ${escenic_backups_dir} $file + set_conf_file_value cache_dir ${escenic_cache_dir} ${file} + set_conf_file_value ece_home ${escenic_root_dir}/engine ${file} + set_conf_file_value escenic_conf_dir ${escenic_conf_dir} ${file} + set_conf_file_value heap_dump_dir ${escenic_crash_dir} ${file} + set_conf_file_value log_dir ${escenic_log_dir} ${file} + set_conf_file_value pid_dir ${escenic_run_dir} ${file} + set_conf_file_value rmi_hub_conf ${escenic_conf_dir}/rmi-hub ${file} + set_conf_file_value solr_home ${escenic_data_dir}/solr ${file} + set_conf_file_value ece_security_configuration_dir \ + ${escenic_conf_dir}/engine/common/security \ + ${file} + + run sed -i "s#/etc/escenic#${escenic_conf_dir}#g" /etc/bash_completion.d/ece + run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece + ) & + + show_pulse "Setting up the ece UNIX scripts ..." $! } default_db_port=3306 @@ -672,21 +711,21 @@ readConnector=/connector/ReadConnector updateConnector=/connector/UpdateConnector EOF - file=$common_nursery_dir/com/escenic/community/CommunityEngine.properties - if [ -w ${file} ]; then - sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file - exit_on_error "sed on $file" - elif [ ! -e ${file} ]; then - print_and_log "I Could not find an ECOME configuration file," - print_and_log "I assume you have not installed Community Engine." - else - print_and_log "Could not write to ${file}," - print_and_log "Community Engine might not work because of this. " - remove_pid_and_exit_in_error - fi +file=$common_nursery_dir/com/escenic/community/CommunityEngine.properties +if [ -w ${file} ]; then + sed -i 's/jdbc\/ecome/jdbc\/ECE_UPDATE_DS/g' $file + exit_on_error "sed on $file" +elif [ ! -e ${file} ]; then + print_and_log "I Could not find an ECOME configuration file," + print_and_log "I assume you have not installed Community Engine." +else + print_and_log "Could not write to ${file}," + print_and_log "Community Engine might not work because of this. " + remove_pid_and_exit_in_error +fi - file=$common_nursery_dir/com/escenic/webstart/StudioConfig.properties - cat >> $file <> $file < $el/RMI.properties - fi + i=$(( i + 1 )) + if [ $(basename $el) = $instance_name ]; then + rmi_port="8${i}23" + run echo "port=$rmi_port" > $el/RMI.properties + fi done nursery_context=neo/io/managers/HubConnectionManager.properties @@ -925,8 +964,8 @@ function install_ece_third_party_packages memcached wget" - print_and_log "Installing 3rd party packages needed by ECE ..." - install_packages_if_missing $packages + $(install_packages_if_missing $packages) & \ + show_pulse "Installing 3rd party packages needed by ECE" $! fi for el in ant java; do @@ -943,12 +982,12 @@ function get_base_dir_from_bundle() # we'll look inside the archive to determine the base_dir file_name=$( unzip -t ${download_dir}/$file_name | \ - awk '{print $2}' | \ - cut -d'/' -f1 | \ - sort | \ - uniq | \ - grep -v errors | \ - grep [a-z] + awk '{print $2}' | \ + cut -d'/' -f1 | \ + sort | \ + uniq | \ + grep -v errors | \ + grep [a-z] ) else for el in .tar.gz .tar.bz2 .zip; do @@ -1134,7 +1173,7 @@ function set_up_app_server() set_ece_instance_conf tomcat_base $tomcat_base set_ece_instance_conf tomcat_home $tomcat_home set_ece_instance_conf appserver_port $appserver_port - + run cd $tomcat_base/lib make_ln $jdbc_driver @@ -1338,9 +1377,9 @@ EOF WEB-INF/web.xml EOF - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ - $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF - fi +fi - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF - fi +fi - if [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF - elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF - else - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF - fi +fi } function create_user_and_group_if_not_present() { - if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then - print_and_log "Creating UNIX user $ece_user ..." + if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then + print_and_log "Creating UNIX user $ece_user ..." # TODO add support for useradd - run adduser $ece_user \ - --disabled-password \ - --gecos "Escenic-user,Room,Work,Home,Other" - add_next_step "I created a new UNIX user called $ece_user" - add_next_step "and you must set a password using: passwd $ece_user" - fi - - if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then - print_and_log "Creating UNIX group $ece_group ..." - run addgroup $ece_group - fi - } + run adduser $ece_user \ + --disabled-password \ + --gecos "Escenic-user,Room,Work,Home,Other" + add_next_step "I created a new UNIX user called $ece_user" + add_next_step "and you must set a password using: passwd $ece_user" + fi + + if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then + print_and_log "Creating UNIX group $ece_group ..." + run addgroup $ece_group + fi +} # last, give the control back to the ECE user & group function set_correct_permissions() @@ -1558,19 +1597,19 @@ source-war: ${publication_name}.war context-root: ${publication_name} EOF - publication_war=$escenic_root_dir/assemblytool/publications/${publication_name}.war - if [[ $fai_enabled -eq 1 && -n "${fai_publication_war}" ]]; then - print_and_log "Basing ${publication_name}.war on the one specified in $conf_file" - run cp ${fai_publication_war} ${publication_war} - elif [ -d $escenic_root_dir/widget-framework-core-* ]; then - print_and_log "Basing ${publication_name}.war on Widget Framework Demo ..." - run cd $escenic_root_dir/widget-framework-core-*/publications/demo-core - run mvn $maven_opts package - run cp target/demo-core-*.war ${publication_war} - else - print_and_log "Basing your ${publication_name}.war on ECE/demo-clean ..." - run cp $escenic_root_dir/engine/contrib/wars/demo-clean.war ${publication_war} - fi +publication_war=$escenic_root_dir/assemblytool/publications/${publication_name}.war +if [[ $fai_enabled -eq 1 && -n "${fai_publication_war}" ]]; then + print_and_log "Basing ${publication_name}.war on the one specified in $conf_file" + run cp ${fai_publication_war} ${publication_war} +elif [ -d $escenic_root_dir/widget-framework-core-* ]; then + print_and_log "Basing ${publication_name}.war on Widget Framework Demo ..." + run cd $escenic_root_dir/widget-framework-core-*/publications/demo-core + run mvn $maven_opts package + run cp target/demo-core-*.war ${publication_war} +else + print_and_log "Basing your ${publication_name}.war on ECE/demo-clean ..." + run cp $escenic_root_dir/engine/contrib/wars/demo-clean.war ${publication_war} +fi # TODO add support for the community widgets # @@ -1621,7 +1660,7 @@ function install_memcached() service.0.0-memcached-socket-pool=/com/danga/SockIOPool EOF - echo "TBD" +echo "TBD" } ## Method which will try its best to find the home diretory of @@ -1653,19 +1692,20 @@ function get_user_home_directory() function set_up_user_enviornment() { - print_and_log "Setting up the ${ece_user} user's UNIX environment ..." - - local bashrc=$(get_user_home_directory $ece_user)/.bashrc - - if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ - wc -l) -lt 1 ]; then - echo ". /etc/bash_completion.d/ece" >> ${bashrc} - fi + $( + local bashrc=$(get_user_home_directory $ece_user)/.bashrc + + if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ + wc -l) -lt 1 ]; then + echo ". /etc/bash_completion.d/ece" >> ${bashrc} + fi - if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then - echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" >> ${bashrc} - run chown ${ece_user}:${ece_group} ${bashrc} - fi + if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then + echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" >> ${bashrc} + run chown ${ece_user}:${ece_group} ${bashrc} + fi + ) & + show_pulse "Setting up the ${ece_user} user's UNIX environment" $! } function set_up_solr() @@ -1845,7 +1885,7 @@ EOF } EOF - cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < # $2= function install_ece_instance() @@ -2388,9 +2428,9 @@ function install_ece_instance() add_next_step "Admin interface: $admin_uri" add_next_step "View installed versions with:"\ " ece -i $instance_name versions" - add_next_step "Type 'ece help' to see all the options of this script" - add_next_step "Read its guide: /usr/share/doc/escenic/ece-guide.txt" - add_next_step "/etc/default/ece lists all instances started at boot time" +add_next_step "Type 'ece help' to see all the options of this script" +add_next_step "Read its guide: /usr/share/doc/escenic/ece-guide.txt" +add_next_step "/etc/default/ece lists all instances started at boot time" } function get_publication_short_name_list() @@ -2409,7 +2449,7 @@ function get_publication_short_name_list() done echo ${short_name_list} - } +} function get_deploy_white_list() { @@ -2432,7 +2472,7 @@ function get_deploy_white_list() function install_presentation_server() { - print_and_log "Installing a presentation server on $HOSTNAME ..." + print_and_log "Installing a presentation server on $HOSTNAME." type=engine install_ece_instance "web1" $PROFILE_PRESENTATION_SERVER } @@ -2629,11 +2669,11 @@ clientConfiguration=/neo/io/services/HubConnection pingTime=10000 EOF - add_next_step "Restart all your instances to make the hub see them." +add_next_step "Restart all your instances to make the hub see them." - print_and_log "Starting the RMI-hub on $HOSTNAME ..." - ece_command="ece -t rmi-hub restart" - su - $ece_user -c "$ece_command" 1>>$log 2>>$log +print_and_log "Starting the RMI-hub on $HOSTNAME ..." +ece_command="ece -t rmi-hub restart" +su - $ece_user -c "$ece_command" 1>>$log 2>>$log } # reads the value of the desired setting from $conf_file @@ -2720,39 +2760,39 @@ function install_widget_framework() EOF - print_and_log "Downloading Widget Framework from technet.escenic.com ..." - for el in $wf_download_list; do - cd $download_dir - run wget $wget_opts \ - --http-user $technet_user \ - --http-password $technet_password \ - $el - run cd $escenic_root_dir/ - run unzip -q -u $download_dir/$(basename $el) - done +print_and_log "Downloading Widget Framework from technet.escenic.com ..." +for el in $wf_download_list; do + cd $download_dir + run wget $wget_opts \ + --http-user $technet_user \ + --http-password $technet_password \ + $el + run cd $escenic_root_dir/ + run unzip -q -u $download_dir/$(basename $el) +done - install_packages_if_missing "maven2" - assert_pre_prequesite mvn - - export JAVA_HOME=$java_home - wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) - run cd $wf_maven_dir - - print_and_log "Installing Widget Framework into your Maven repository ..." - log "JAVA_HOME=$JAVA_HOME" - - run mvn $maven_opts install +install_packages_if_missing "maven2" +assert_pre_prequesite mvn + +export JAVA_HOME=$java_home +wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) +run cd $wf_maven_dir + +print_and_log "Installing Widget Framework into your Maven repository ..." +log "JAVA_HOME=$JAVA_HOME" + +run mvn $maven_opts install # installing the widget-framework-common as a ECE plugin - wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) - cd $escenic_root_dir/assemblytool/plugins - if [ ! -h $(basename $wf_dist_dir) ]; then - ln -s $wf_dist_dir - fi - - cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ +wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) +cd $escenic_root_dir/assemblytool/plugins +if [ ! -h $(basename $wf_dist_dir) ]; then + ln -s $wf_dist_dir +fi - add_next_step "Widget Framework has been installed into your "\ +cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ + +add_next_step "Widget Framework has been installed into your "\ " Maven repo" } @@ -2763,7 +2803,7 @@ function install_search_server() install_ece_instance "search1" 0 # TODO update instead of append - dir=$common_nursery_dir/com/escenic/framework/search/solr + local dir=$common_nursery_dir/com/escenic/framework/search/solr make_dir $dir echo "solrServerURI=http://${search_host}:${search_port}/solr" \ >> $dir/SolrSearchEngine.properties @@ -3052,7 +3092,8 @@ function set_up_monitoring_host_def() print "$1 host" $host_name "already defined, skipping it." return fi - + + # TODO add more services based on what kind of host it is. cat >> $file <Nagios' \ - else + else echo '
  • Icinga (an enhanced Nagios)
  • ' \ >> $file fi From 836caf0eab8c893456104fc59d934353f1f8b76a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 20 Feb 2012 12:36:27 +0800 Subject: [PATCH 0300/2585] - sun java packages have now been removed from Ubuntu LTS as well (so much for a stable, dependable release right?), hence we now also build local packages for Ubuntu LTS, see https://github.com/skybert/ece-scripts/issues/30 for more background on this issue. - re-factored the pulse handling, introduced pulse methods, suffixed _p so that it's easy to remove their usage if desired. - a few methods have pulse right now, trying it out before adding it all of the place. - NOTE a lot of changed lines due to new indent (basic offset strictly (emacs) speaking) of 2 and braces on first line to bring it into line with Escenic's internal (Java) coding standards. --- usr/sbin/ece-install | 237 +++++++++++++++++++++++-------------------- 1 file changed, 127 insertions(+), 110 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 17f7572d..05a93aa4 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -253,13 +253,13 @@ pulse_bits="-\|/" function pulse() { - echo -ne "\b${pulse_bits:i++%${#pulse_bits}:1}" + echo -ne "\b${pulse_bits: i++ % ${#pulse_bits}: 1}" } function show_pulse() { - printne "$1 ...." - local pulse_pid=$2 + printne "${@: 2: $(( $# - 1 ))} ...." + local pulse_pid=$1 while true; do kill -0 $pulse_pid 2>/dev/null if [ $? -eq 0 ]; then @@ -287,18 +287,16 @@ function download_escenic_components() return fi - print_and_log "Downloading Escenic software from technet.escenic.com ..." - cd $download_dir for el in $technet_download_list; do - if [ -e $(basename $el) ]; then - continue - fi - - run wget $wget_opts \ - --http-user $technet_user \ - --http-password $technet_password \ - $el + if [ -e $(basename $el) ]; then + continue + fi + + run wget $wget_opts \ + --http-user $technet_user \ + --http-password $technet_password \ + $el done } @@ -309,6 +307,8 @@ function download_escenic_components() # $1 : space separated string of package names function install_packages_if_missing() { + log "Installing packages [$1] if they're missing ..." + if [ $on_debian_or_derivative -eq 1 ]; then some_are_missing=0 for el in $@; do @@ -334,25 +334,48 @@ function install_packages_if_missing() function install_common_os_packages() { - $( - if [ $on_debian_or_derivative -eq 1 ]; then + log "Installing common OS packages ..." + + if [ $on_debian_or_derivative -eq 1 ]; then # Ubuntu doesn't have git (!) but only git-core. - if [ $on_ubuntu -eq 1 ]; then - git_package=git-core - else - git_package=git - fi + if [ $on_ubuntu -eq 1 ]; then + git_package=git-core + else + git_package=git + fi + + packages="curl $git_package wget unzip" + install_packages_if_missing $packages + fi + + for el in lsb_release curl wget git unzip; do + assert_pre_prequesite $el + done +} - packages="curl $git_package wget unzip" - install_packages_if_missing $packages - fi +###################################################################### +# +function set_up_user_environment_p() { + $(set_up_user_environment) & + show_pulse $! "Setting up the ${ece_user}'s UNIX environment" +} - for el in lsb_release curl wget git unzip; do - assert_pre_prequesite $el - done - ) & - show_pulse "Installing 3rd party packages needed by $(basename $0)" $! +function set_up_ece_scripts_p() { + $(set_up_ece_scripts) & + show_pulse $! 'Setting up the ece UNIX scripts' +} + +function install_common_os_packages_p() { + $(install_common_os_packages) & + show_pulse $! "Installing 3rd party packages needed by $(basename $0)" +} + +function install_ece_third_party_packages_p() { + $(install_ece_third_party_packages) & + show_pulse $! "Installing 3rd party packages needed by ECE" } +###################################################################### +# function set_up_assembly_tool() { @@ -502,43 +525,39 @@ function set_up_engine_and_plugins() function set_up_ece_scripts() { - $( - run cd $download_dir - if [ -d ece-scripts ]; then - ( - run cd ece-scripts - run git pull - ) - else - run git clone $ece_scripts_git_source - fi - - run cp -r ece-scripts/usr/* /usr/ - run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ - run cp -r ece-scripts/etc/bash_completion.d/ece /etc/bash_completion.d/ - run cp -r ece-scripts/etc/init.d/* /etc/init.d/ - run cp -r ece-scripts/etc/default/* /etc/default/ - - local file=${escenic_conf_dir}/ece.conf - set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file - set_conf_file_value backup_dir ${escenic_backups_dir} $file - set_conf_file_value cache_dir ${escenic_cache_dir} ${file} - set_conf_file_value ece_home ${escenic_root_dir}/engine ${file} - set_conf_file_value escenic_conf_dir ${escenic_conf_dir} ${file} - set_conf_file_value heap_dump_dir ${escenic_crash_dir} ${file} - set_conf_file_value log_dir ${escenic_log_dir} ${file} - set_conf_file_value pid_dir ${escenic_run_dir} ${file} - set_conf_file_value rmi_hub_conf ${escenic_conf_dir}/rmi-hub ${file} - set_conf_file_value solr_home ${escenic_data_dir}/solr ${file} - set_conf_file_value ece_security_configuration_dir \ - ${escenic_conf_dir}/engine/common/security \ - ${file} - - run sed -i "s#/etc/escenic#${escenic_conf_dir}#g" /etc/bash_completion.d/ece - run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece - ) & - - show_pulse "Setting up the ece UNIX scripts ..." $! + run cd $download_dir + if [ -d ece-scripts ]; then + ( + run cd ece-scripts + run git pull + ) + else + run git clone $ece_scripts_git_source + fi + + run cp -r ece-scripts/usr/* /usr/ + run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ + run cp -r ece-scripts/etc/bash_completion.d/ece /etc/bash_completion.d/ + run cp -r ece-scripts/etc/init.d/* /etc/init.d/ + run cp -r ece-scripts/etc/default/* /etc/default/ + + local file=${escenic_conf_dir}/ece.conf + set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file + set_conf_file_value backup_dir ${escenic_backups_dir} $file + set_conf_file_value cache_dir ${escenic_cache_dir} ${file} + set_conf_file_value ece_home ${escenic_root_dir}/engine ${file} + set_conf_file_value escenic_conf_dir ${escenic_conf_dir} ${file} + set_conf_file_value heap_dump_dir ${escenic_crash_dir} ${file} + set_conf_file_value log_dir ${escenic_log_dir} ${file} + set_conf_file_value pid_dir ${escenic_run_dir} ${file} + set_conf_file_value rmi_hub_conf ${escenic_conf_dir}/rmi-hub ${file} + set_conf_file_value solr_home ${escenic_data_dir}/solr ${file} + set_conf_file_value ece_security_configuration_dir \ + ${escenic_conf_dir}/engine/common/security \ + ${file} + + run sed -i "s#/etc/escenic#${escenic_conf_dir}#g" /etc/bash_completion.d/ece + run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece } default_db_port=3306 @@ -917,8 +936,10 @@ function install_ece_third_party_packages if [ $on_debian_or_derivative -eq 1 ]; then if [ $on_ubuntu -eq 1 ]; then - # Sun Java was removed in Ubuntu in 11.10 - local version_needs_local_java_deb=1110 + # Sun Java was removed in Ubuntu in 11.10 and later also + # from LTS 10.04, hence Hardy is the last one with these + # packges now (2012-02-20 11:22) + local version_needs_local_java_deb=1004 local version=$(lsb_release -s -r | sed "s#\.##g") add_apt_source \ @@ -964,8 +985,7 @@ function install_ece_third_party_packages memcached wget" - $(install_packages_if_missing $packages) & \ - show_pulse "Installing 3rd party packages needed by ECE" $! + install_packages_if_missing $packages fi for el in ant java; do @@ -1690,22 +1710,19 @@ function get_user_home_directory() fi } -function set_up_user_enviornment() +function set_up_user_environment() { - $( - local bashrc=$(get_user_home_directory $ece_user)/.bashrc - - if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ - wc -l) -lt 1 ]; then - echo ". /etc/bash_completion.d/ece" >> ${bashrc} - fi - - if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then - echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" >> ${bashrc} - run chown ${ece_user}:${ece_group} ${bashrc} - fi - ) & - show_pulse "Setting up the ${ece_user} user's UNIX environment" $! + local bashrc=$(get_user_home_directory $ece_user)/.bashrc + + if [ $(grep bash_completion.d/ece ${bashrc} 2>/dev/null | \ + wc -l) -lt 1 ]; then + echo ". /etc/bash_completion.d/ece" >> ${bashrc} + fi + + if [ $(grep ECE_CONF_LOCATIONS ${bashrc} | wc -l) -eq 0 ]; then + echo "export ECE_CONF_LOCATIONS=\"$escenic_conf_dir\"" >> ${bashrc} + run chown ${ece_user}:${ece_group} ${bashrc} + fi } function set_up_solr() @@ -2070,9 +2087,9 @@ function common_pre_install() fi make_dir $download_dir - install_common_os_packages + install_common_os_packages_p create_user_and_group_if_not_present - set_up_user_enviornment + set_up_user_environment_p } function assert_pre_prequesite() @@ -2381,7 +2398,7 @@ function install_ece_instance() ask_for_instance_name $1 set_up_engine_directories - set_up_ece_scripts + set_up_ece_scripts_p set_archive_files_depending_on_profile @@ -2760,40 +2777,40 @@ function install_widget_framework() EOF -print_and_log "Downloading Widget Framework from technet.escenic.com ..." -for el in $wf_download_list; do - cd $download_dir - run wget $wget_opts \ + print_and_log "Downloading Widget Framework from technet.escenic.com ..." + for el in $wf_download_list; do + cd $download_dir + run wget $wget_opts \ --http-user $technet_user \ --http-password $technet_password \ $el - run cd $escenic_root_dir/ - run unzip -q -u $download_dir/$(basename $el) -done + run cd $escenic_root_dir/ + run unzip -q -u $download_dir/$(basename $el) + done -install_packages_if_missing "maven2" -assert_pre_prequesite mvn + install_packages_if_missing "maven2" + assert_pre_prequesite mvn -export JAVA_HOME=$java_home -wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) -run cd $wf_maven_dir + export JAVA_HOME=$java_home + wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) + run cd $wf_maven_dir -print_and_log "Installing Widget Framework into your Maven repository ..." -log "JAVA_HOME=$JAVA_HOME" + print_and_log "Installing Widget Framework into your Maven repository ..." + log "JAVA_HOME=$JAVA_HOME" -run mvn $maven_opts install + run mvn $maven_opts install # installing the widget-framework-common as a ECE plugin -wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) -cd $escenic_root_dir/assemblytool/plugins -if [ ! -h $(basename $wf_dist_dir) ]; then - ln -s $wf_dist_dir -fi + wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) + cd $escenic_root_dir/assemblytool/plugins + if [ ! -h $(basename $wf_dist_dir) ]; then + ln -s $wf_dist_dir + fi -cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ + cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ -add_next_step "Widget Framework has been installed into your "\ -" Maven repo" + add_next_step "Widget Framework has been installed into your " \ + " Maven repo" } function install_search_server() From 83524615342d24d2d2c91e26c477fc3e6a537ac0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 20 Feb 2012 12:50:16 +0800 Subject: [PATCH 0301/2585] - fix in ensure_domain_is_known_to_local_host: /etc/hosts should no longer get duplicate entries. --- usr/sbin/ece-install | 62 +++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 05a93aa4..5e96b79d 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1035,43 +1035,39 @@ function get_escaped_bash_string() # # The method will ensure that the passed domain is resolvable by the # host on which ece-install is run. -function ensure_domain_is_known_to_local_host() -{ - if [ -z "$1" ]; then - return 1 - fi - - local localhost_ip=$(ping -c 1 localhost 2>/dev/null | \ - head -1 | \ - cut -d'(' -f2 | \ - cut -d')' -f1) - local hostname_ip=$(ping -c 1 $HOSTNAME 2>/dev/null | \ - head -1 | \ - cut -d'(' -f2 | \ - cut -d')' -f1) - local domain_ip=$(ping -c 1 $1 2>/dev/null | \ - head -1 | \ - cut -d'(' -f2 | \ - cut -d')' -f1) - - local keep_off_etc_hosts=${fai_keep_off_etc_hosts-0} - if [[ $domain_ip != $localhost_ip && \ - $domain_ip != hostname_ip && \ - $keep_off_etc_hosts -ne 1 ]]; then - print_and_log "The domain name ${1} is not resolvable to this host" - print_and_log "I will remedy this by adding it to /etc/hosts" - cat >> /etc/hosts </dev/null | \ + head -1 | \ + cut -d'(' -f2 | \ + cut -d')' -f1) + local domain_ip=$(ping -c 1 $1 2>/dev/null | \ + head -1 | \ + cut -d'(' -f2 | \ + cut -d')' -f1) + + local keep_off_etc_hosts=${fai_keep_off_etc_hosts-0} + if [[ $domain_ip != "127.0.0.1" && \ + $domain_ip != "127.0.1.1" && \ + $domain_ip != $hostname_ip && \ + $keep_off_etc_hosts -ne 1 ]]; then + print_and_log "The domain name ${1} is not resolvable to this host" + print_and_log "I will remedy this by adding it to /etc/hosts" + cat >> /etc/hosts < Date: Mon, 20 Feb 2012 13:00:29 +0800 Subject: [PATCH 0302/2585] - set_up_app_server() : re-wrote the set up of virtual hosts in Tomcat to not use BASH 4.x features (not hash map). This should fix the error when running on BASH < 4.x: ece-install: line 1302: declare: -A: invalid option declare: usage: declare [-afFirtx] [-p] [name[=value] ...] --- usr/sbin/ece-install | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 5e96b79d..8b863032 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1354,25 +1354,20 @@ EOF $install_profile_number == $PROFILE_ALL_IN_ONE) && \ -n "$fai_publication_domain_mapping_list" ]]; then - declare -A publication_domain_map - for el in ${fai_publication_domain_mapping_list}; do local old_ifs=$IFS # the entries in the fai_publication_domain_mapping_list # are on the form: # IFS='#' read publication domain <<< "$el" - publication_domain_map[${publication}]="${domain}" IFS=$old_ifs ensure_domain_is_known_to_local_host ${domain} - done - for el in ${!publication_domain_map[@]}; do cat >> $tomcat_base/conf/server.xml < - + From d2463bd578f7a310b050520abfcee09c22e31f6d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 20 Feb 2012 15:24:02 +0800 Subject: [PATCH 0303/2585] - long overdue modularisation of ece-install. I've resisted the temptation of doing this to keep ece-install as self contained as possible and by that as easy to get started with as possible. However, as the code base is growing, a modularisation became apparent. Furthermore, the code which has been split out in separate common-*.sh files, is the code which is re-usable by other scripts. To include e.g. common-io.sh in your script, you'd do: common_io_is_loaded > /dev/null 2>&1 || source common-io.sh The other libraries have similar functions to avoid re-loading the same library several times. The following common libraries are now available in /usr/share/escenic/ece-scripts: - common-bashing.sh - common-io.sh - common-pulse.sh - common-os.sh See their headers for an explanation of what you can expect to find were. This re-factoring means that you must have ece-install in your path. If you're using standard locations, this is of no problem, but if you're for some reason just running it from somewhere in your home directory, you'd have to do: # export PATH=~/src/ece-scripts/usr/sbin:$PATH # ece-install - checked some spelling mistakes in method names while moving them out to the common library files. --- usr/sbin/ece-install | 397 ++---------------- .../escenic/ece-scripts/common-bashing.sh | 152 +++++++ usr/share/escenic/ece-scripts/common-io.sh | 87 ++++ usr/share/escenic/ece-scripts/common-os.sh | 101 +++++ usr/share/escenic/ece-scripts/common-pulse.sh | 37 ++ 5 files changed, 420 insertions(+), 354 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/common-bashing.sh create mode 100644 usr/share/escenic/ece-scripts/common-io.sh create mode 100644 usr/share/escenic/ece-scripts/common-os.sh create mode 100644 usr/share/escenic/ece-scripts/common-pulse.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8b863032..b1e1785f 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -73,118 +73,10 @@ force_packages=0 # ECE software components. ece_software_setup_completed=0 -NEW_LINE=" -" # the next steps printed when the user has installed his/her # components. next_steps=() -function get_seconds_since_start() -{ - local seconds="n/a" - - if [ -r $pid_file ]; then - now=`date +%s` - started=`stat -c %Y $pid_file` - seconds=$(( now - started )) - fi - - echo "$seconds" -} - -function get_id() -{ - timestamp=$(get_seconds_since_start) - echo "[$(basename $0)-${timestamp}]" -} - -function debug() -{ - if [ $debug -eq 1 ]; then - echo "[$(basename $0)-debug]" "$@" - fi -} - -function print() -{ - echo $(get_id) $@ -} - -function printne() -{ - echo -ne $(get_id) $@ -} - -function log() -{ - echo $(get_id) $@ >> $log -} - -function print_and_log() -{ - print "$@" - log "$@" -} - -function log_call_stack() -{ - log "Call stack (top most is the last one, main is the first):" - - # skipping i=0 as this is log_call_stack itself - for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do - echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:${FUNCNAME[$i]}"()" >> $log - if [ -e ${BASH_SOURCE[$i]} ]; then - echo -n " => " >> $log - sed -n "${BASH_LINENO[$i-1]}p" ${BASH_SOURCE[$i]} | \ - sed "s#^[ \t]*##g" >> $log - else - echo "" >> $log - fi - done -} - -function remove_pid_and_exit_in_error() -{ - if [ -e $pid_file ]; then - rm $pid_file - fi - - log_call_stack - - exit 1 -} - -function exit_on_error() -{ - if [ $? -gt 0 ]; then - print "The command ["$@"] FAILED, exiting :-(" - print "See $log for further details." - remove_pid_and_exit_in_error - fi -} - -function run() -{ - $@ 1>>$log 2>>$log - exit_on_error $@ -} - -# Returns 1 if the passed value is a number, 0 if not -# -# $1 : what you want to test -function is_number() -{ - if [ -z $1 ]; then - echo 0 - elif [ $(echo $1 | grep [a-z] | wc -l) -gt 0 ]; then - echo 0 - elif [ $(echo $1 | grep [A-Z] | wc -l) -gt 0 ]; then - echo 0 - else - echo 1 - fi -} - technet_download_list=" http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.7.0.zip @@ -216,31 +108,14 @@ PROFILE_MONITORING_SERVER=10 PROFILE_RESTORE_FROM_BACKUP=11 PROFILE_ANALYSIS_SERVER=12 -function make_dir() -{ - if [ ! -d $1 ]; then - run mkdir -p $1 - fi -} - -function make_ln() -{ - if [ $2 ]; then - if [ -e $1 -a ! -h $2 ]; then - run ln -s $1 $2 - elif [ ! -e $1 ]; then - print_and_log "Tried to make a symlink to $1, but it doesn't exist" - remove_pid_and_exit_in_error - fi - else - if [ -e $1 -a ! -h $(basename $1) ]; then - run ln -s $1 - elif [ ! -e $1 ]; then - print_and_log "Tried to make a symlink to $1, but it doesn't exist" - remove_pid_and_exit_in_error - fi - fi -} +###################################################################### +# Load common BASH libraries +##################################################################### +common_libraries="common-bashing.sh common-pulse.sh common-io.sh common-os.sh" +for el in $common_libraries; do + source $(dirname $0)/../share/escenic/ece-scripts/${el} || \ + (echo "Could not find $el"; exit 1) +done function set_up_engine_directories() { @@ -249,38 +124,6 @@ function set_up_engine_directories() done } -pulse_bits="-\|/" - -function pulse() -{ - echo -ne "\b${pulse_bits: i++ % ${#pulse_bits}: 1}" -} - -function show_pulse() -{ - printne "${@: 2: $(( $# - 1 ))} ...." - local pulse_pid=$1 - while true; do - kill -0 $pulse_pid 2>/dev/null - if [ $? -eq 0 ]; then - pulse - sleep 0.25 - else - wait $pulse_pid - local exit_code=$? - if [ $exit_code -eq 0 ]; then - echo -e '\b\E[37;32m'" \033[1mok\033[0m" - tput sgr0 - else - echo -e '\b\E[37;31m'" \033[1mfailed\033[0m" - tput sgr0 - fi - - break - fi - done -} - function download_escenic_components() { if [ $ece_software_setup_completed -eq 1 ]; then @@ -349,12 +192,13 @@ function install_common_os_packages() fi for el in lsb_release curl wget git unzip; do - assert_pre_prequesite $el + assert_pre_requisite $el done } ###################################################################### # +###################################################################### function set_up_user_environment_p() { $(set_up_user_environment) & show_pulse $! "Setting up the ${ece_user}'s UNIX environment" @@ -376,6 +220,7 @@ function install_ece_third_party_packages_p() { } ###################################################################### # +###################################################################### function set_up_assembly_tool() { @@ -593,30 +438,30 @@ function set_db_defaults_if_not_set() function set_db_settings_from_fai_conf() { - # Note: the port isn't fully supported. The user must himself - # update the mysql configuration to run it on a non standard port. - - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then - db_port=${fai_db_port-${default_db_port}} - db_host=${fai_db_host-${default_db_host}} - db_user=${fai_db_user-${default_db_user}} - db_password=${fai_db_password-${default_db_password}} - db_schema=${fai_db_schema-${default_db_schema}} - else - db_port=${fai_analysis_db_port-${default_db_port}} - db_host=${fai_analysis_db_host-${default_db_host}} - db_user=${fai_analysis_db_user-${default_db_user}} - db_password=${fai_analysis_db_password-${default_db_password}} - db_schema=${fai_analysis_db_schema-${default_db_schema}} - fi - - if [ -n "${fai_db_drop_old_db_first}" ]; then - drop_db_first=${fai_db_drop_old_db_first} - if [ $fai_db_drop_old_db_first -eq 1 ]; then - print_and_log "WARNING: I hope you know what you're doing!" - print_and_log "WARNING: fai_db_drop_old_db_first is 1 (true)" - fi + # Note: the port isn't fully supported. The user must himself + # update the mysql configuration to run it on a non standard port. + + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + db_port=${fai_db_port-${default_db_port}} + db_host=${fai_db_host-${default_db_host}} + db_user=${fai_db_user-${default_db_user}} + db_password=${fai_db_password-${default_db_password}} + db_schema=${fai_db_schema-${default_db_schema}} + else + db_port=${fai_analysis_db_port-${default_db_port}} + db_host=${fai_analysis_db_host-${default_db_host}} + db_user=${fai_analysis_db_user-${default_db_user}} + db_password=${fai_analysis_db_password-${default_db_password}} + db_schema=${fai_analysis_db_schema-${default_db_schema}} + fi + + if [ -n "${fai_db_drop_old_db_first}" ]; then + drop_db_first=${fai_db_drop_old_db_first} + if [ $fai_db_drop_old_db_first -eq 1 ]; then + print_and_log "$(yellow WARNING): I hope you know what you're doing!" + print_and_log "$(yellow WARNING): fai_db_drop_old_db_first is 1 (true)" fi + fi } function set_up_ecedb() @@ -823,7 +668,6 @@ EOF else run cp $tomcat_home/conf/logging.properties $tomcat_base/conf fi - } # don't quote values when setting conf file values with @@ -831,57 +675,6 @@ EOF # handling of .cfg files. dont_quote_conf_values=0 -# Parameters: -# $1 is the property -# $2 ... to the (last - 1) is the value -# The last argument is the file -# -# e.g.: -# set_conf_file_value \ -# mykey \ -# myvalue1 myvalue2 myvalue3 \ -# /etc/myfile.conf -function set_conf_file_value() -{ - if [ $# -lt 3 ]; then - return - fi - - local file=${@:${#@}} - local key=${@:1:1} - local value_end_index=$(( $# - 2 )) - local value=${@:2:${value_end_index}} - - local parent_dir=$(dirname $file) - if [ ! -d ${parent_dir} ]; then - print ${parent_dir} "doesn't exist, something is wrong :-(" - remove_pid_and_exit_in_error - fi - - if [ -r $file ]; then - if [ $(grep ^$1 $file | wc -l) -gt 0 ]; then - if [ $dont_quote_conf_values -eq 0 ]; then - sed -i "s~$key=.*~$key=\"$value\"~g" $file - else - sed -i "s~$key=.*~$key=$value~g" $file - fi - else - if [ $dont_quote_conf_values -eq 0 ]; then - echo "$key=\"$value\"" >> $file - else - echo "$key=$value" >> $file - fi - - fi - else - if [ $dont_quote_conf_values -eq 0 ]; then - echo "$key=\"$value\"" >> $file - else - echo "$key=$value" >> $file - fi - fi -} - # The function accepts the following parameters: # $1 is the property # $2 is the value @@ -917,16 +710,6 @@ function create_java_deb_packages_and_repo() add_next_step "to your APT system with /etc/apt/sources.list.d/oab.list" } -function has_sun_java_installed() -{ - if [[ -x /usr/lib/jvm/java-6-sun/bin/java || \ - $(java -version 2>&1 > /dev/null | grep HotSpot | wc -l) -gt 0 ]]; then - echo 1 - else - echo 0 - fi -} - # Installs third party packages needed by the ECE (i.e. Java related). # Also see install_common_os_packages for packages common to all # servers in the architecture. @@ -989,7 +772,7 @@ function install_ece_third_party_packages fi for el in ant java; do - assert_pre_prequesite $el + assert_pre_requisite $el done } @@ -1018,19 +801,6 @@ function get_base_dir_from_bundle() echo $file_name } -# Returns an escaped string useful for sed and other BASH commands. -# -# $1 : the string -function get_escaped_bash_string() -{ - local result=$(echo $1 | \ - sed -e 's/\$/\\$/g' \ - -e 's/\*/\\*/g' \ - -e 's#/#\\/#g' \ - -e 's/\./\\./g') - echo $result -} - # $1 the domain # # The method will ensure that the passed domain is resolvable by the @@ -1464,24 +1234,6 @@ EOF fi } -function create_user_and_group_if_not_present() -{ - if [ $(grep $ece_user /etc/passwd | wc -l) -lt 1 ]; then - print_and_log "Creating UNIX user $ece_user ..." - # TODO add support for useradd - run adduser $ece_user \ - --disabled-password \ - --gecos "Escenic-user,Room,Work,Home,Other" - add_next_step "I created a new UNIX user called $ece_user" - add_next_step "and you must set a password using: passwd $ece_user" - fi - - if [ $(grep $ece_group /etc/group | wc -l) -lt 1 ]; then - print_and_log "Creating UNIX group $ece_group ..." - run addgroup $ece_group - fi -} - # last, give the control back to the ECE user & group function set_correct_permissions() { @@ -1547,7 +1299,7 @@ function print_status_and_next_steps() done print $'\n'"Enjoy your time with Escenic Content Engine!"$'\n' - print "-Vizrt Online" + print "-$(red Vizrt) Online" } # Based on Erik Mogensen's work: @@ -1671,34 +1423,7 @@ function install_memcached() service.0.0-memcached-socket-pool=/com/danga/SockIOPool EOF -echo "TBD" -} - -## Method which will try its best to find the home diretory of -## existing users and probable home directories of new users. -## -## (1) On Darwin/Mac OS X, it will return /Users/ -## (2) On systems using /etc/passwd, it will search there to find the -## home dir of existing users. -## (3) For new users, it will check the configuration of adduser (if -## present). -## (4) If all of thes above fails, it will return /home/ -## -## Arguments: -## $1 : the user name, can either be an existing user or a non-yet -## created user. -function get_user_home_directory() -{ - if [ $(uname -s) = "Darwin" ]; then - echo /Users/$1 - elif [ $(grep $1 /etc/passwd | wc -l) -gt 0 ]; then - grep $1 /etc/passwd | cut -d':' -f6 - elif [ -r /etc/adduser.conf ]; then - local dir=$(grep DHOME /etc/adduser.conf | grep -v ^# | cut -d'=' -f2) - echo $dir/$1 - else - echo "/home/$1" - fi + echo "TBD" } function set_up_user_environment() @@ -2079,18 +1804,10 @@ function common_pre_install() make_dir $download_dir install_common_os_packages_p - create_user_and_group_if_not_present + create_user_and_group_if_not_present $ece_user $ece_group set_up_user_environment_p } -function assert_pre_prequesite() -{ - if [ $(which $1 | wc -l) -lt 1 ]; then - print_and_log "Please install $1 and then run $(basename $0) again." - remove_pid_and_exit_in_error - fi -} - function add_apt_source() { escenic_sources=/etc/apt/sources.list.d/escenic.list @@ -2137,7 +1854,7 @@ function install_cache_server() apt-get -y install varnish 1>>$log 2>>$log fi - assert_pre_prequesite varnishd + assert_pre_requisite varnishd if [ $fai_enabled -eq 0 ]; then print "You must now list your backend servers." @@ -2166,7 +1883,7 @@ function add_next_step() return if [ -n "$next_steps" ]; then - next_steps=${next_steps}${NEW_LINE}"[$(basename $0)] "${1} + next_steps=${next_steps}$'\n'"[$(basename $0)] "${1} else next_steps="[$(basename $0)] "${1} fi @@ -2240,7 +1957,7 @@ function install_database_server() fi fi - assert_pre_prequesite mysql + assert_pre_requisite mysql if [ -z "$1" ]; then download_escenic_components @@ -2780,7 +2497,7 @@ EOF done install_packages_if_missing "maven2" - assert_pre_prequesite mvn + assert_pre_requisite mvn export JAVA_HOME=$java_home wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) @@ -2995,23 +2712,6 @@ EOF add_next_step "Make sure all nodes allows its IP to connect to them." } -# will return the IP of the host name. If not found, the host name -# passed to the function will be returned. -# parameters: $1 the host name -function get_ip() -{ - ip=$(ping -c 1 $1 2>/dev/null | \ - grep "bytes from" | \ - cut -d'(' -f2 | \ - cut -d ')' -f1) - - if [ -z "$ip" ]; then - echo $1 - fi - - echo $ip -} - ## Installs the Nagios monitoring server. ## $1 the nagios vendor/falvour, "nagios" and "icinga" are supported. function install_nagios_monitoring_server() @@ -3139,17 +2839,6 @@ function install_nagios_node() add_next_step "A Nagios NRPE node has been installed on ${HOSTNAME}" } -# munin nodes need the IP of the munin gatherer to be escaped. Hence this function. -# $parameters: -# $1 : the IP -function get_perl_escaped() -{ - local escaped_input=$( - echo $1 | sed 's/\./\\./g' - ) - echo "^${escaped_input}$" -} - function install_munin_node() { print_and_log "Installing a Munin node on $HOSTNAME ..." @@ -3434,7 +3123,7 @@ function restore_from_backup() cut -d'=' -f2 | \ sed -e "s/'//g" -e 's/"//g' ) - create_user_and_group_if_not_present + create_user_and_group_if_not_present $ece_user $ece_group if [ -d $escenic_conf_dir -a \ $(ls $escenic_conf_dir/ | grep ^ece- | wc -l) -gt 0 -a \ diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh new file mode 100644 index 00000000..d9052b38 --- /dev/null +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -0,0 +1,152 @@ +#! /usr/bin/env bash + +# Common, useful methods for BASH scripts. The callee is responsible +# for setting the following variables: +# +# * pid_file +# * log +# +# by tkj@vizrt.com + +function common_bashing_is_loaded() { + echo 1 +} + +function get_seconds_since_start() { + local seconds="n/a" + + if [ -r $pid_file ]; then + now=`date +%s` + started=`stat -c %Y $pid_file` + seconds=$(( now - started )) + fi + + echo "$seconds" +} + +function get_id() { + local timestamp=$(get_seconds_since_start) + echo "[$(basename $0)-${timestamp}]" +} + +function debug() { + if [ $debug -eq 1 ]; then + echo "[$(basename $0)-debug]" "$@" + fi +} + +function print() { + echo $(get_id) $@ +} + +function printne() { + echo -ne $(get_id) $@ +} + +function log() { + echo $(get_id) $@ >> $log +} + +function print_and_log() { + print "$@" + log "$@" +} + +function log_call_stack() { + log "Call stack (top most is the last one, main is the first):" + + # skipping i=0 as this is log_call_stack itself + for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do + echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:${FUNCNAME[$i]}"()" >> $log + if [ -e ${BASH_SOURCE[$i]} ]; then + echo -n " => " >> $log + sed -n "${BASH_LINENO[$i-1]}p" ${BASH_SOURCE[$i]} | \ + sed "s#^[ \t]*##g" >> $log + else + echo "" >> $log + fi + done +} + +function remove_pid_and_exit_in_error() { + if [ -e $pid_file ]; then + rm $pid_file + fi + + log_call_stack + + exit 1 +} + +function exit_on_error() { + if [ $? -gt 0 ]; then + print "The command ["$@"] FAILED, exiting :-(" + print "See $log for further details." + remove_pid_and_exit_in_error + fi +} + +function run() { + $@ 1>>$log 2>>$log + exit_on_error $@ +} + +## Returns 1 if the passed argument is a number, 0 if not. +## $1: the value you wish to test. +function is_number() { + for (( i = 0; i < ${#1}; i++ )); do + if [ $(echo ${1:$i:1} | grep [0-9] | wc -l) -lt 1 ]; then + echo 0 + return + fi + done + + echo 1 +} + +## Returns an escaped string useful for sed and other BASH commands. +## +## $1 : the string +function get_escaped_bash_string() { + local result=$(echo $1 | \ + sed -e 's/\$/\\$/g' \ + -e 's/\*/\\*/g' \ + -e 's#/#\\/#g' \ + -e 's/\./\\./g') + echo $result +} + +# Munin nodes need the IP of the munin gatherer to be escaped. Hence +# this function. +# +# Parameters: +# +# $1 : the IP +function get_perl_escaped() { + local escaped_input=$( + echo $1 | sed 's/\./\\./g' + ) + echo "^${escaped_input}$" +} + +## Returns the inputted string(s) as red +## +## $1: input string +function red() { + echo -e "\E[37;31m\033[1m${@}\033[0m" +} + +## Returns the inputted string(s) as green +## +## $1: input string +function green() { + echo -e "\E[37;32m\033[1m${@}\033[0m" +} + +## Returns the inputted string(s) as yellow +## +## $1: input string +function yellow() { + echo -e "\E[37;33m\033[1m${@}\033[0m" +} + diff --git a/usr/share/escenic/ece-scripts/common-io.sh b/usr/share/escenic/ece-scripts/common-io.sh new file mode 100644 index 00000000..0c681e05 --- /dev/null +++ b/usr/share/escenic/ece-scripts/common-io.sh @@ -0,0 +1,87 @@ +#! /usr/bin/env bash + +# by tkj@vizrt.com + +# depends on common-bashing +common_bashing_is_loaded > /dev/null 2>&1 || source common-bashing.sh + +# Can be used like this: +# common_io_is_loaded 2>/dev/null || source common-io.sh +function common_io_is_loaded() { + echo 1 +} + +## Parameters: +## $1 is the property +## $2 ... to the (last - 1) is the value +## $n -1: The last argument is the file +## +## e.g.: +## set_conf_file_value \ +## mykey \ +## myvalue1 myvalue2 myvalue3 \ +## /etc/myfile.conf +function set_conf_file_value() { + if [ $# -lt 3 ]; then + return + fi + + local file=${@:${#@}} + local key=${@:1:1} + local value_end_index=$(( $# - 2 )) + local value=${@:2:${value_end_index}} + + local parent_dir=$(dirname $file) + if [ ! -d ${parent_dir} ]; then + print ${parent_dir} "doesn't exist, something is wrong :-(" + remove_pid_and_exit_in_error + fi + + if [ -r $file ]; then + if [ $(grep ^$1 $file | wc -l) -gt 0 ]; then + if [ $dont_quote_conf_values -eq 0 ]; then + sed -i "s~$key=.*~$key=\"$value\"~g" $file + else + sed -i "s~$key=.*~$key=$value~g" $file + fi + else + if [ $dont_quote_conf_values -eq 0 ]; then + echo "$key=\"$value\"" >> $file + else + echo "$key=$value" >> $file + fi + + fi + else + if [ $dont_quote_conf_values -eq 0 ]; then + echo "$key=\"$value\"" >> $file + else + echo "$key=$value" >> $file + fi + fi +} + +function make_dir() { + if [ ! -d $1 ]; then + run mkdir -p $1 + fi +} + +function make_ln() { + if [ $2 ]; then + if [ -e $1 -a ! -h $2 ]; then + run ln -s $1 $2 + elif [ ! -e $1 ]; then + print_and_log "Tried to make a symlink to $1, but it doesn't exist" + remove_pid_and_exit_in_error + fi + else + if [ -e $1 -a ! -h $(basename $1) ]; then + run ln -s $1 + elif [ ! -e $1 ]; then + print_and_log "Tried to make a symlink to $1, but it doesn't exist" + remove_pid_and_exit_in_error + fi + fi +} + diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh new file mode 100644 index 00000000..736e488a --- /dev/null +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -0,0 +1,101 @@ +#! /usr/bin/env bash + +# Platform/OS specific methods. +# +# by tkj@vizrt.com + +# depends on common-bashing +common_bashing_is_loaded > /dev/null 2>&1 || source common-bashing.sh + +# Can be used like this: +# common_io_os_loaded 2>/dev/null || source common-os.sh +function common_os_is_loaded() { + echo 1 +} + +## Method which will try its best to find the home diretory of +## existing users and probable home directories of new users. +## +## (1) On Darwin/Mac OS X, it will return /Users/ +## (2) On systems using /etc/passwd, it will search there to find the +## home dir of existing users. +## (3) For new users, it will check the configuration of adduser (if +## present). +## (4) If all of thes above fails, it will return /home/ +## +## Arguments: +## $1 : the user name, can either be an existing user or a non-yet +## created user. +function get_user_home_directory() { + if [ $(uname -s) = "Darwin" ]; then + echo /Users/$1 + elif [ $(grep $1 /etc/passwd | wc -l) -gt 0 ]; then + grep $1 /etc/passwd | cut -d':' -f6 + elif [ -r /etc/adduser.conf ]; then + local dir=$(grep DHOME /etc/adduser.conf | grep -v ^# | cut -d'=' -f2) + echo $dir/$1 + else + echo "/home/$1" + fi +} + +function has_sun_java_installed() { + if [[ -x /usr/lib/jvm/java-6-sun/bin/java || \ + $(java -version 2>&1 > /dev/null | grep HotSpot | wc -l) -gt 0 ]]; then + echo 1 + else + echo 0 + fi +} + +## $1 user name +## $2 group name +function create_user_and_group_if_not_present() { + local user=$1 + local group=$1 + + if [ $(grep $user /etc/passwd | wc -l) -lt 1 ]; then + print_and_log "Creating UNIX user $user ..." + # TODO add support for useradd + run adduser $user \ + --disabled-password \ + --gecos "Escenic-user,Room,Work,Home,Other" + add_next_step "I created a new UNIX user called $user" + add_next_step "and you must set a password using: passwd $user" + fi + + if [ $(grep $group /etc/group | wc -l) -lt 1 ]; then + print_and_log "Creating UNIX group $group ..." + run addgroup $group + fi +} + +## Will return the IP of the host name. If not found, the host name +## passed to the function will be returned. +## +## Parameters: $1 the host name +function get_ip() { + local ip=$(ping -c 1 $1 2>/dev/null | \ + grep "bytes from" | \ + cut -d'(' -f2 | \ + cut -d ')' -f1) + + if [ -z "$ip" ]; then + echo $1 + fi + + echo $ip +} + +## Asserts that the passed command/program is indeed accessible in the +## current context. If it is not, the program aborts and removes the +## PID. +## +## $1: the binary/executable/program +function assert_pre_requisite() { + if [ $(which $1 | wc -l) -lt 1 ]; then + print_and_log "Please install $1 and then run $(basename $0) again." + remove_pid_and_exit_in_error + fi +} + diff --git a/usr/share/escenic/ece-scripts/common-pulse.sh b/usr/share/escenic/ece-scripts/common-pulse.sh new file mode 100644 index 00000000..a0f7704c --- /dev/null +++ b/usr/share/escenic/ece-scripts/common-pulse.sh @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +# by tkj@vizrt.com + +# depends on common-bashing +common_bashing_is_loaded > /dev/null 2>&1 || source common-bashing.sh + +pulse_bits="-\|/" + +function pulse() { + echo -ne "\b${pulse_bits: i++ % ${#pulse_bits}: 1}" +} + +function show_pulse() { + printne "${@: 2: $(( $# - 1 ))} ...." + local pulse_pid=$1 + while true; do + kill -0 $pulse_pid 2>/dev/null + if [ $? -eq 0 ]; then + pulse + sleep 0.25 + else + wait $pulse_pid + local exit_code=$? + if [ $exit_code -eq 0 ]; then + echo -e '\b\E[37;32m'" \033[1mok\033[0m" + tput sgr0 + else + echo -e '\b\E[37;31m'" \033[1mfailed\033[0m" + tput sgr0 + fi + + break + fi + done +} + From 7478b9eeef28a1bca27af7ec146a7069a772ed13 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 20 Feb 2012 18:20:00 +0800 Subject: [PATCH 0304/2585] - made explicit mention of BASH version in documentation. --- usr/share/doc/escenic/ece-install-guide.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 177290b2..03823356 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -4,7 +4,7 @@ Welcome to the /usr/sbin/ece-install Guide For the best experience, run this script on Debian Squeeze (or newer), the latest Ubuntu LTS or most other Debian based operating systems, but it should work without any problems on any Unix like system that -has a recent version of BASH. +has BASH version 4.2 or newer. *** Debian based operating systems If running on a Debian based operating system, ece-install will take From 044e2f46b5ff7e7ba4448d5dd1477f40a7cab8c3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 21 Feb 2012 11:59:31 +0800 Subject: [PATCH 0305/2585] - new feature: ece-install will now dynamically figure out the Tomcat downlad URL based on what tomcat.apache.org, which again will be correct for the user's regions. This should hopefully resolve a number of issues users around the world have been having with the default tomcat_download value being wrong for them. This also means that the tomcat_download variable now is obsolete and can be removed from your local ece-install.conf files. - hardening: pulse methods that fail will now terminate the main ece-install BASH process. - hardening: check_for_required_downloads now checks that ZIP archives are not faulty, not only that the required downloads exist in the download folder. - added pulse to more methods - improvement: get_base_dir_from_bundle is now moved to common-bashing and has been generalised so that it's useful for other calees as well as ece-install. --- usr/sbin/ece-install | 145 ++++++++++-------- .../escenic/ece-scripts/common-bashing.sh | 27 ++++ usr/share/escenic/ece-scripts/common-os.sh | 29 ++++ usr/share/escenic/ece-scripts/common-pulse.sh | 14 ++ 4 files changed, 149 insertions(+), 66 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index b1e1785f..e307d601 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -21,7 +21,6 @@ ece_user=escenic ece_group=escenic jdbc_driver=/usr/share/java/mysql.jar debug=0 -tomcat_download=http://ftp.mirror.tw/pub/apache/tomcat/tomcat-6/v6.0.35/bin/apache-tomcat-6.0.35.tar.gz # These variables govern where software is installed and data and run # time files are written. @@ -135,7 +134,8 @@ function download_escenic_components() if [ -e $(basename $el) ]; then continue fi - + + log "Downloading $el ..." run wget $wget_opts \ --http-user $technet_user \ --http-password $technet_password \ @@ -143,7 +143,12 @@ function download_escenic_components() done } -# will install the passed packages if these are not installed from +function download_escenic_components_p() { + $(download_escenic_components) & + show_pulse $! "Downloading Escenic components from Technet" +} + +# WILL install the passed packages if these are not installed from # before. # # parameters: @@ -199,6 +204,31 @@ function install_common_os_packages() ###################################################################### # ###################################################################### +function set_up_instance_specific_nursery_configuration_p() { + $(set_up_instance_specific_nursery_configuration) & + show_pulse $! "Setting up instance specific Nursery configuration" +} + +function set_up_basic_nursery_configuration_p() { + $(set_up_basic_nursery_configuration) & + show_pulse $! "Setting up basic Nursery configuration" +} + +function assemble_deploy_and_restart_type_p() { + echo $(assemble_deploy_and_restart_type) & + show_pulse $! "Assembling, deploying & starting $instance_name" +} + +function set_up_engine_and_plugins_p() { + $(set_up_engine_and_plugins) & + show_pulse $! "Setting up the Escenic Content Engine & its plugins" +} + +function check_for_required_downloads_p() { + $(check_for_required_downloads) & + show_pulse $! "Asserting that required downloads succeeded" +} + function set_up_user_environment_p() { $(set_up_user_environment) & show_pulse $! "Setting up the ${ece_user}'s UNIX environment" @@ -218,13 +248,19 @@ function install_ece_third_party_packages_p() { $(install_ece_third_party_packages) & show_pulse $! "Installing 3rd party packages needed by ECE" } + +function set_up_assembly_tool() { + $(set_up_assembly_tool) & + show_pulse $! "Setting up the Assembly Tool" +} + ###################################################################### # ###################################################################### function set_up_assembly_tool() { - print_and_log "Setting up the Assembly Tool ..." + log "Setting up the Assembly Tool ..." make_dir $escenic_root_dir/assemblytool/ cd $escenic_root_dir/assemblytool/ @@ -327,7 +363,7 @@ function set_up_engine_and_plugins() return fi - print_and_log "Setting up the Escenic Content Engine & its plugins ..." + log "Setting up the Escenic Content Engine & its plugins ..." make_dir $escenic_root_dir cd $escenic_root_dir/ @@ -336,7 +372,7 @@ function set_up_engine_and_plugins() if [ $(basename $el | \ grep -E "^engine-[0-9]|^engine-trunk-SNAPSHOT|^engine-dist" | \ wc -l) -gt 0 ]; then - engine_dir=$(get_base_dir_from_bundle $el) + engine_dir=$(get_base_dir_from_bundle $download_dir/$el) engine_file=$(basename $el) fi done @@ -507,7 +543,7 @@ function set_up_ecedb() function set_up_basic_nursery_configuration() { - print_and_log "Setting up the basic Nursery configuration ..." + log "Setting up basic Nursery configuration ..." # we always copy the default plugin configuration (even if there # is an archive) @@ -776,31 +812,6 @@ function install_ece_third_party_packages done } -function get_base_dir_from_bundle() -{ - local file_name=$(basename $1) - suffix=${file_name##*.} - - if [ ${suffix} = "zip" ]; then - # we'll look inside the archive to determine the base_dir - file_name=$( - unzip -t ${download_dir}/$file_name | \ - awk '{print $2}' | \ - cut -d'/' -f1 | \ - sort | \ - uniq | \ - grep -v errors | \ - grep [a-z] - ) - else - for el in .tar.gz .tar.bz2 .zip; do - file_name=${file_name%$el} - done - fi - - echo $file_name -} - # $1 the domain # # The method will ensure that the passed domain is resolvable by the @@ -934,10 +945,9 @@ function set_up_app_server() search_host=$(echo $user_search | cut -d':' -f1) search_port=$(echo $user_search | cut -d':' -f2) fi - - run cd $download_dir - run wget $wget_opts $tomcat_download - tomcat_dir=$(get_base_dir_from_bundle $tomcat_download) + + download_tomcat_p + tomcat_dir=$(get_base_dir_from_bundle $(get_tomcat_download_url)) run cd $appserver_parent_dir run tar xzf $download_dir/${tomcat_dir}.tar.gz @@ -1237,7 +1247,7 @@ fi # last, give the control back to the ECE user & group function set_correct_permissions() { - print_and_log "Setting correct permissions on ECE related directories ..." + log "Setting correct permissions on ECE related directories ..." for el in $engine_dir_list; do if [ ! -d $el ]; then @@ -1250,8 +1260,8 @@ function set_correct_permissions() -user ${ece_user} | \ wc -l) if [ $correct_permission -gt 0 ]; then - print_and_log "Data directory root, $el," - print_and_log "has correct permissions, skiping sub directories." + log "Data directory root, $el," + log "has correct permissions, skiping sub directories." continue fi fi @@ -1386,25 +1396,28 @@ fi function check_for_required_downloads() { - if [ $ece_software_setup_completed -eq 1 ]; then - return - fi + if [ $ece_software_setup_completed -eq 1 ]; then + return + fi + + log "Asserting that required downloads succeeded ..." + local required_escenic_packages="engine assemblytool" - print_and_log "Asserting that required downloads succeeded ..." - required_escenic_packages="engine assemblytool" - - for el in $required_escenic_packages; do - if [ $(ls $download_dir | \ - grep ${el} | \ - grep -v community | \ - grep -v analysis | \ - grep .zip$ | \ - wc -l) \ - -lt 1 ]; then - print_and_log "Couldn't find $el* in $download_dir" - remove_pid_and_exit_in_error - fi - done + for el in $required_escenic_packages; do + if [ $(ls $download_dir/$el*.zip 2>/dev/null | wc -l) -lt 1 ]; then + log "$el is missing" + else + # want the newest one if there are several + local file=$(ls $download_dir/$el*.zip | tail -1) + unzip -t $file > /dev/null 2>&1 + + if [ $? -ne 0 ]; then + log "$file has been downloaded, but is faulty," + log "remove it and re-run $0" + return 9 + fi + fi + done } function install_memcached() @@ -1960,7 +1973,7 @@ function install_database_server() assert_pre_requisite mysql if [ -z "$1" ]; then - download_escenic_components + download_escenic_components_p set_up_engine_and_plugins set_up_ecedb fi @@ -2113,9 +2126,9 @@ function install_ece_instance() # most likely, the user is _not_ installing from archives (EAR + # configuration bundle), hence the false test goes first. if [ $(is_installing_from_ear) -eq 0 ]; then - download_escenic_components - check_for_required_downloads - set_up_engine_and_plugins + download_escenic_components_p + check_for_required_downloads_p + set_up_engine_and_plugins_p set_up_assembly_tool else verify_that_files_exist_and_are_readable \ @@ -2123,8 +2136,8 @@ function install_ece_instance() $ece_instance_conf_archive fi - set_up_basic_nursery_configuration - set_up_instance_specific_nursery_configuration + set_up_basic_nursery_configuration_p + set_up_instance_specific_nursery_configuration_p set_up_app_server set_up_proper_logging_configuration @@ -2141,7 +2154,7 @@ function install_ece_instance() if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - assemble_deploy_and_restart_type + assemble_deploy_and_restart_type_p fi update_type_instances_to_start_up @@ -2204,7 +2217,7 @@ function install_presentation_server() function assemble_deploy_and_restart_type() { - print_and_log "Assembling, deploying & starting $instance_name ..." + log "Assembling, deploying & starting $instance_name ..." set_correct_permissions @@ -2214,7 +2227,7 @@ function assemble_deploy_and_restart_type() if [ $(is_installing_from_ear) -eq 1 ]; then run cp $ece_instance_ear_file $escenic_cache_dir/engine.ear ece_command="ece -i $instance_name -t $type deploy restart" - print_and_log "Using the supplied EAR instead of running an assembly." + log "Using the supplied EAR instead of running an assembly." fi su - $ece_user -c "$ece_command" 1>>$log 2>>$log diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh index d9052b38..6f2e183a 100644 --- a/usr/share/escenic/ece-scripts/common-bashing.sh +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -150,3 +150,30 @@ function yellow() { echo -e "\E[37;33m\033[1m${@}\033[0m" } + +## $1: full path to the file. +function get_base_dir_from_bundle() +{ + local file_name=$(basename $1) + suffix=${file_name##*.} + + if [ ${suffix} = "zip" ]; then + # we'll look inside the archive to determine the base_dir + file_name=$( + unzip -t $1 2>>$log | \ + awk '{print $2}' | \ + cut -d'/' -f1 | \ + sort | \ + uniq | \ + grep -v errors | \ + grep [a-z] + ) + else + for el in .tar.gz .tar.bz2 .zip; do + file_name=${file_name%$el} + done + fi + + echo $file_name +} + diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh index 736e488a..fb9f6d12 100644 --- a/usr/share/escenic/ece-scripts/common-os.sh +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -6,6 +6,9 @@ # depends on common-bashing common_bashing_is_loaded > /dev/null 2>&1 || source common-bashing.sh +common_pulse_is_loaded > /dev/null 2>&1 || source common-bashing.sh + +wget_opts="--continue --inet4-only --quiet" # Can be used like this: # common_io_os_loaded 2>/dev/null || source common-os.sh @@ -99,3 +102,29 @@ function assert_pre_requisite() { fi } +function get_tomcat_download_url() { + local url=$( + curl -s http://tomcat.apache.org/download-60.cgi | \ + grep tar.gz | \ + head -1 | \ + cut -d'"' -f2 + ) + + echo $url +} + +## Downloads Tomcat from the regional mirror +## +## $1: target directory +function download_tomcat() { + ( + log "Downloading Tomcat from $url ..." + run cd $1 + run wget $wget_opts $(get_tomcat_url) + ) +} + +function download_tomcat_p() { + $(download_tomcat) & + show_pulse $! "Downloading Tomcat from local mirror" +} diff --git a/usr/share/escenic/ece-scripts/common-pulse.sh b/usr/share/escenic/ece-scripts/common-pulse.sh index a0f7704c..bc198d06 100644 --- a/usr/share/escenic/ece-scripts/common-pulse.sh +++ b/usr/share/escenic/ece-scripts/common-pulse.sh @@ -7,12 +7,21 @@ common_bashing_is_loaded > /dev/null 2>&1 || source common-bashing.sh pulse_bits="-\|/" +# Can be used like this: +# common_io_pulse_loaded 2>/dev/null || source common-pulse.sh +function common_pulse_is_loaded() { + echo 1 +} + function pulse() { echo -ne "\b${pulse_bits: i++ % ${#pulse_bits}: 1}" } +## $1: the PID of the process on which you want to view the pulse. +## $2...n: Strings to display while the pulse is running. function show_pulse() { printne "${@: 2: $(( $# - 1 ))} ...." + local pulse_pid=$1 while true; do kill -0 $pulse_pid 2>/dev/null @@ -28,6 +37,11 @@ function show_pulse() { else echo -e '\b\E[37;31m'" \033[1mfailed\033[0m" tput sgr0 + print "See $log for further details" + log_call_stack + + # terminate the main ece script process. + kill $$ 2>/dev/null fi break From 38b982e3f740c549641253b1acfa7287ae3b453f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 21 Feb 2012 12:01:19 +0800 Subject: [PATCH 0306/2585] - various small regression bug fixes (magit took my change set as it was when I first started composing the commit message, and not the updates files. Clever! But also different to all other SCMs I've worked with). --- usr/sbin/ece-install | 6 +++++- usr/share/escenic/ece-scripts/common-bashing.sh | 8 +++++--- usr/share/escenic/ece-scripts/common-os.sh | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index e307d601..0282e1fa 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -237,6 +237,9 @@ function set_up_user_environment_p() { function set_up_ece_scripts_p() { $(set_up_ece_scripts) & show_pulse $! 'Setting up the ece UNIX scripts' + if [ -n "${messages}" ]; then + echo $message + fi } function install_common_os_packages_p() { @@ -372,7 +375,7 @@ function set_up_engine_and_plugins() if [ $(basename $el | \ grep -E "^engine-[0-9]|^engine-trunk-SNAPSHOT|^engine-dist" | \ wc -l) -gt 0 ]; then - engine_dir=$(get_base_dir_from_bundle $download_dir/$el) + engine_dir=$(get_base_dir_from_bundle $download_dir/$(basename $el)) engine_file=$(basename $el) fi done @@ -412,6 +415,7 @@ function set_up_ece_scripts() run cd ece-scripts run git pull ) + if [ $! -eq 1 ]; then echo $!; return 1; fi else run git clone $ece_scripts_git_source fi diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh index 6f2e183a..1e3111cb 100644 --- a/usr/share/escenic/ece-scripts/common-bashing.sh +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -80,8 +80,8 @@ function remove_pid_and_exit_in_error() { function exit_on_error() { if [ $? -gt 0 ]; then - print "The command ["$@"] FAILED, exiting :-(" - print "See $log for further details." + log "The command ["$@"] FAILED, exiting :-(" + log "See $log for further details." remove_pid_and_exit_in_error fi } @@ -173,7 +173,9 @@ function get_base_dir_from_bundle() file_name=${file_name%$el} done fi - + + log "get_base_dir_from_bundle file_name="$file_name $1 + echo $file_name } diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh index fb9f6d12..5dabeef8 100644 --- a/usr/share/escenic/ece-scripts/common-os.sh +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -120,11 +120,11 @@ function download_tomcat() { ( log "Downloading Tomcat from $url ..." run cd $1 - run wget $wget_opts $(get_tomcat_url) + run wget $wget_opts $(get_tomcat_download_url) ) } function download_tomcat_p() { $(download_tomcat) & - show_pulse $! "Downloading Tomcat from local mirror" + show_pulse $! "Downloading Tomcat from nearest mirror" } From 5d15ccac557621a11ac932c184ceae48b6817c82 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 21 Feb 2012 14:01:58 +0800 Subject: [PATCH 0307/2585] - fixed regression in tomcat download (download went into the wrong directory. --- usr/share/escenic/ece-scripts/common-os.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh index 5dabeef8..64e50770 100644 --- a/usr/share/escenic/ece-scripts/common-os.sh +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -125,6 +125,6 @@ function download_tomcat() { } function download_tomcat_p() { - $(download_tomcat) & + $(download_tomcat $1) & show_pulse $! "Downloading Tomcat from nearest mirror" } From 692eaada923e946b60eeaf6be9c7fda6ac8bc76e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 21 Feb 2012 14:03:10 +0800 Subject: [PATCH 0308/2585] - fixed regression in tomcat downloading - added pulse to java deb package generation - fixed status label (ece_user 's ...) --- usr/sbin/ece-install | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0282e1fa..476d6085 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -231,7 +231,7 @@ function check_for_required_downloads_p() { function set_up_user_environment_p() { $(set_up_user_environment) & - show_pulse $! "Setting up the ${ece_user}'s UNIX environment" + show_pulse $! "Setting up the ${ece_user} user's UNIX environment" } function set_up_ece_scripts_p() { @@ -734,16 +734,16 @@ function create_java_deb_packages_and_repo() print $(lsb_release -i | cut -d':' -f2) \ $(lsb_release -r | cut -d':' -f2) \ "doesn't have official Sun/Oracle Java packages," - print "I'm creating packages & local repo for you ..." local tmp_dir=$(mktemp -d) - ( + $( run cd $tmp_dir run git clone https://github.com/flexiondotorg/oab-java6.git run cd oab-java6 run bash oab-java6.sh run rm -rf $tmp_dir - ) + ) & + show_pulse $! "I'm creating packages & local repo for you" add_next_step "Local APT repository with Sun/Oracle Java packages" add_next_step "has been installed at /var/local/oab/deb and added" @@ -950,7 +950,7 @@ function set_up_app_server() search_port=$(echo $user_search | cut -d':' -f2) fi - download_tomcat_p + download_tomcat_p $download_dir tomcat_dir=$(get_base_dir_from_bundle $(get_tomcat_download_url)) run cd $appserver_parent_dir From 57482012e006403060228256592ee88ee2471da3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 21 Feb 2012 17:26:59 +0800 Subject: [PATCH 0309/2585] - moved install_memcached to its own BASH module - first implementation all steps involved in setting up memcached. TODO: test and give user hint on feeding back the changes into the publication source tree --- usr/sbin/ece-install | 19 ---- .../ece-scripts/ece-install.d/memcached.sh | 92 +++++++++++++++++++ 2 files changed, 92 insertions(+), 19 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/memcached.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 476d6085..821e72aa 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1424,25 +1424,6 @@ function check_for_required_downloads() done } -function install_memcached() -{ - if [ $on_debian_or_derivative ]; then - install_packages_if_missing "memcached" - fi - - # 1) download java library - # 2) configure PresenationArticle cache for the publication - # 3) build and re-deploy - - cat >> $escenic_conf_dir/engine/common/Initial.properties </dev/null; do + local publication=$(basename $el .properties) + + if [[ $appserver == "tomcat" ]]; then + dir=$tomcat_base/webapps/$publication + make_dir $dir + memcached_create_publication_nursery_component $dir + fi + done + + # fixing the deployed publications on host + if [[ $appserver == "tomcat" ]]; then + for el in $( + find $tomcat_base/webapps/ \ + -mindepth 1 \ + -maxdepth 1 \ + -type d | \ + egrep -v "solr|webservice|escenic|escenic-admin|indexer-webservice" | \ + egrep -v "indexer-webapp|studio" + ); do + memcached_create_publication_nursery_component $el + done + + fi + + # TODO inform the user that he/she might want to do tihs in the + # publication tree as well. + assemble_deploy_and_restart_type_p +} + +function memcached_set_up_common_nursery() { + local dir=$common_nursery_dir/com/danga + make_dir $dir + cat > $dir/SockIOPool <> $common_nursery_dir/Initial.properties < Date: Tue, 21 Feb 2012 17:34:16 +0800 Subject: [PATCH 0310/2585] - first version of memcached installation (sub) profile. - added README for /usr/share/escenic/ece-scripts structure. --- usr/share/escenic/ece-scripts/README | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/README diff --git a/usr/share/escenic/ece-scripts/README b/usr/share/escenic/ece-scripts/README new file mode 100644 index 00000000..297b649f --- /dev/null +++ b/usr/share/escenic/ece-scripts/README @@ -0,0 +1,14 @@ +############################################################################# +# common-*.sh :: Common libraries +############################################################################# + +The common-*.sh files are considered re-usable BASH libraries, which +can be used by any script, in and outside of the ece-scripts. + +############################################################################# +# ece-install.d :: ece-install modules. +############################################################################# + +The scripts found in ece-install.d are meant for use by ece-install +itself and may not work when used separately as they rely on +ece-install internals. From 53af0201f5a81522b3fba7f6e7f82e2d588f3fa9 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 22 Feb 2012 22:01:54 +0800 Subject: [PATCH 0311/2585] - now uses https instead of git protocol when downloading the ece-scripts. - corrected log message in install_packages_if_missing, it only listed the first package being installed, leading to confusion. - localised some variables to not litter the global naming space - fixed indentation a few places - removed redundant call to git package installation. - removed redundant >> <> $file <> $file <> $file fi - } function set_up_proper_logging_configuration() @@ -674,7 +672,6 @@ EOF make_ln $common_nursery_dir/trace.properties run ln -sf trace.properties log4j.properies - # since the solr webapp otherwise will pollute our logs, we ask # Tomcat specifically to make it log to dedicated logger. if [ $install_profile_number -eq $PROFILE_SEARCH_SERVER -o \ @@ -729,8 +726,6 @@ function set_ece_instance_conf() function create_java_deb_packages_and_repo() { - install_packages_if_missing $git_package - print $(lsb_release -i | cut -d':' -f2) \ $(lsb_release -r | cut -d':' -f2) \ "doesn't have official Sun/Oracle Java packages," @@ -1168,13 +1163,12 @@ EOF cat > $tomcat_base/conf/context.xml < - WEB-INF/web.xml EOF -if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ - $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF -fi + fi -if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF -fi + fi -if [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF -elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF -else - cat >> $tomcat_base/conf/context.xml <> $tomcat_base/conf/context.xml < EOF -fi + fi } # last, give the control back to the ECE user & group @@ -1276,10 +1270,6 @@ function set_correct_permissions() if [ -d "$tomcat_base" ]; then run chown -R ${ece_user}:${ece_group} $tomcat_base fi - - if [ $(ls /tmp | grep ^ece | wc -l) -gt 0 ]; then - run chown -R ${ece_user}:${ece_group} /tmp/ece-* - fi } function print_status_and_next_steps() @@ -1374,19 +1364,19 @@ source-war: ${publication_name}.war context-root: ${publication_name} EOF -publication_war=$escenic_root_dir/assemblytool/publications/${publication_name}.war -if [[ $fai_enabled -eq 1 && -n "${fai_publication_war}" ]]; then - print_and_log "Basing ${publication_name}.war on the one specified in $conf_file" - run cp ${fai_publication_war} ${publication_war} -elif [ -d $escenic_root_dir/widget-framework-core-* ]; then - print_and_log "Basing ${publication_name}.war on Widget Framework Demo ..." - run cd $escenic_root_dir/widget-framework-core-*/publications/demo-core - run mvn $maven_opts package - run cp target/demo-core-*.war ${publication_war} -else - print_and_log "Basing your ${publication_name}.war on ECE/demo-clean ..." - run cp $escenic_root_dir/engine/contrib/wars/demo-clean.war ${publication_war} -fi + publication_war=$escenic_root_dir/assemblytool/publications/${publication_name}.war + if [[ $fai_enabled -eq 1 && -n "${fai_publication_war}" ]]; then + print_and_log "Basing ${publication_name}.war on the one specified in $conf_file" + run cp ${fai_publication_war} ${publication_war} + elif [ -d $escenic_root_dir/widget-framework-core-* ]; then + print_and_log "Basing ${publication_name}.war on Widget Framework Demo ..." + run cd $escenic_root_dir/widget-framework-core-*/publications/demo-core + run mvn $maven_opts package + run cp target/demo-core-*.war ${publication_war} + else + print_and_log "Basing your ${publication_name}.war on ECE/demo-clean ..." + run cp $escenic_root_dir/engine/contrib/wars/demo-clean.war ${publication_war} + fi # TODO add support for the community widgets # @@ -1551,7 +1541,7 @@ function set_up_varnish() # -V for some reason writes to standard error. using_varnish_3=$(varnishd -V 3>&1 1>&2 2>&3 | grep varnish-3 | wc -l) - file=/etc/default/varnish + local file=/etc/default/varnish sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file exit_on_error "sed on $file" @@ -1614,9 +1604,7 @@ EOF cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < -# $2= +## $1= function install_ece_instance() { install_ece_third_party_packages @@ -2149,11 +2136,11 @@ function install_ece_instance() admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin/ add_next_step "New ECE instance $instance_name installed." add_next_step "Admin interface: $admin_uri" - add_next_step "View installed versions with:"\ -" ece -i $instance_name versions" -add_next_step "Type 'ece help' to see all the options of this script" -add_next_step "Read its guide: /usr/share/doc/escenic/ece-guide.txt" -add_next_step "/etc/default/ece lists all instances started at boot time" + add_next_step "View installed versions with:" \ + " ece -i $instance_name versions" + add_next_step "Type 'ece help' to see all the options of this script" + add_next_step "Read its guide: /usr/share/doc/escenic/ece-guide.txt" + add_next_step "/etc/default/ece lists all instances started at boot time" } function get_publication_short_name_list() @@ -2392,11 +2379,11 @@ clientConfiguration=/neo/io/services/HubConnection pingTime=10000 EOF -add_next_step "Restart all your instances to make the hub see them." + add_next_step "Restart all your instances to make the hub see them." -print_and_log "Starting the RMI-hub on $HOSTNAME ..." -ece_command="ece -t rmi-hub restart" -su - $ece_user -c "$ece_command" 1>>$log 2>>$log + print_and_log "Starting the RMI-hub on $HOSTNAME ..." + ece_command="ece -t rmi-hub restart" + su - $ece_user -c "$ece_command" 1>>$log 2>>$log } # reads the value of the desired setting from $conf_file From 4fa45061c5bf56dd110df616be1803f9ac3b13c5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 22 Feb 2012 22:02:23 +0800 Subject: [PATCH 0312/2585] - changed debug statement from log to debug. --- usr/share/escenic/ece-scripts/common-bashing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh index 1e3111cb..4af9492a 100644 --- a/usr/share/escenic/ece-scripts/common-bashing.sh +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -174,7 +174,7 @@ function get_base_dir_from_bundle() done fi - log "get_base_dir_from_bundle file_name="$file_name $1 + debug "get_base_dir_from_bundle file_name="$file_name $1 echo $file_name } From 6f3593804f45225bfdbca9b437ee0db0a3ed074f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 23 Feb 2012 14:51:02 +0800 Subject: [PATCH 0313/2585] - removed set_up_basic_nursery_configuration_p as set_up_basic_nursery_configuration cannot be run in a subshell as it prompts the user for input when run in interactive mode. This resulted in the installer hanging on forever waiting for user input but the user neither knew about the requested input, nor was able to input this because of the i/o redirection when running in a pulse-ed subshell. --- usr/sbin/ece-install | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 79e2193f..98e4d6b1 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -209,11 +209,6 @@ function set_up_instance_specific_nursery_configuration_p() { show_pulse $! "Setting up instance specific Nursery configuration" } -function set_up_basic_nursery_configuration_p() { - $(set_up_basic_nursery_configuration) & - show_pulse $! "Setting up basic Nursery configuration" -} - function assemble_deploy_and_restart_type_p() { echo $(assemble_deploy_and_restart_type) & show_pulse $! "Assembling, deploying & starting $instance_name" @@ -926,8 +921,8 @@ function set_up_app_server() if [ $fai_enabled -eq 0 ]; then print "Last question, I promise!: Where does the search instance run?" print "Press ENTER to accept the default ($HOSTNAME:8080)" + print "or enter: :, e.g.: 'search1:8080'" print "If you're in doubt, just press ENTER :-)" - print "Or enter: :, e.g.: 'search1:8080'" echo -n "Your choice [$HOSTNAME:8080]> " read user_search else @@ -2108,7 +2103,7 @@ function install_ece_instance() $ece_instance_conf_archive fi - set_up_basic_nursery_configuration_p + set_up_basic_nursery_configuration set_up_instance_specific_nursery_configuration_p set_up_app_server From ab1a5ff1983fac1be0aeee7fbc2fe38308bf4dcb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 23 Feb 2012 14:58:39 +0800 Subject: [PATCH 0314/2585] - fix in get_privileged_hosts: it would add :0.0 to the privileged host list when installing e.g. a cache server on a local system. Now, the local :0.0 display address is omitted from the result of the method. --- usr/sbin/ece-install | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 98e4d6b1..36b09848 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1509,22 +1509,22 @@ function un_install_ece() ## Returns the privileged hosts. This will include both the IP(s) the ## logged in user conduction the ece-install is coming from, as well ## as any IPs defined in fai_privileged_hosts. -function get_privileged_hosts() -{ - local privileged_hosts=${fai_privileged_hosts} - - for ip in $( - w -h | \ - grep pts | \ - sed "s#.*pts/[0-9]*[ ]*\(.*\)#\1#" | \ - cut -d' ' -f1 | \ - sort | \ - uniq - ); do - privileged_hosts=${privileged_hosts}" "${ip} - done +function get_privileged_hosts() { + local privileged_hosts=${fai_privileged_hosts} + + for ip in $( + w -h | \ + grep pts | \ + grep -v ":0.0" | \ + sed "s#.*pts/[0-9]*[ ]*\(.*\)#\1#" | \ + cut -d' ' -f1 | \ + sort | \ + uniq + ); do + privileged_hosts=${privileged_hosts}" "${ip} + done - echo ${privileged_hosts} + echo ${privileged_hosts} } function set_up_varnish() @@ -1567,6 +1567,7 @@ backend static { .host = "localhost"; .port = "81"; } + EOF for el in $backend_servers; do appserver_id=$(echo $el | cut -d':' -f1 | sed 's/-/_/g') From f2ec01579f5254eba0c877cf22b60bf151da95e5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 23 Feb 2012 17:40:45 +0800 Subject: [PATCH 0315/2585] - started adding support for RedHat/CentOS (testing with version 6.2). TODO: still a lot. Right now, only the database is done without any errors.. - improved messages in download_tomcat - added fail fast to get_tomcat_download_url --- usr/share/escenic/ece-scripts/common-os.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh index 64e50770..7d602711 100644 --- a/usr/share/escenic/ece-scripts/common-os.sh +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -96,7 +96,7 @@ function get_ip() { ## ## $1: the binary/executable/program function assert_pre_requisite() { - if [ $(which $1 | wc -l) -lt 1 ]; then + if [ $(which $1 2>/dev/null | wc -l) -lt 1 ]; then print_and_log "Please install $1 and then run $(basename $0) again." remove_pid_and_exit_in_error fi @@ -109,6 +109,11 @@ function get_tomcat_download_url() { head -1 | \ cut -d'"' -f2 ) + + if [ -z $url ]; then + print_and_log "Failed to get Tomcat download URL" + kill $$ + fi echo $url } @@ -118,13 +123,10 @@ function get_tomcat_download_url() { ## $1: target directory function download_tomcat() { ( + local url=$(get_tomcat_download_url) log "Downloading Tomcat from $url ..." run cd $1 - run wget $wget_opts $(get_tomcat_download_url) + run wget $wget_opts $url ) } -function download_tomcat_p() { - $(download_tomcat $1) & - show_pulse $! "Downloading Tomcat from nearest mirror" -} From c1870b6a6167cc12e9db2ca3a49c4c8cca569cde Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 23 Feb 2012 17:41:44 +0800 Subject: [PATCH 0316/2585] - started adding support for RedHat/CentOS. TODO still a lot. Right now, the biggest obstacle is that "ece clean" fails on CentOS, whereas running "ant clean" works, funnily enough. --- usr/sbin/ece-install | 124 ++++++++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 50 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 36b09848..799a2790 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -19,7 +19,7 @@ ##################################################################### ece_user=escenic ece_group=escenic -jdbc_driver=/usr/share/java/mysql.jar +jdbc_driver=/usr/share/java/mysql-connector-java.jar debug=0 # These variables govern where software is installed and data and run @@ -63,6 +63,7 @@ ece_install_scripts_dir=$HOME/ece-install.d appserver_host=localhost appserver_port=8080 on_debian_or_derivative=0 +on_redhat_or_derivative=0 on_debian=0 on_ubuntu=0 force_packages=0 @@ -143,59 +144,54 @@ function download_escenic_components() done } -function download_escenic_components_p() { - $(download_escenic_components) & - show_pulse $! "Downloading Escenic components from Technet" -} # Will install the passed packages if these are not installed from # before. # # parameters: # $1 : space separated string of package names -function install_packages_if_missing() -{ +function install_packages_if_missing() { log "Installing package(s) [$@] if missing ..." - if [ $on_debian_or_derivative -eq 1 ]; then - some_are_missing=0 - for el in $@; do + if [ $on_debian_or_derivative -eq 1 ]; then + some_are_missing=0 + for el in $@; do # we don't need to grep away "No packages found matching # ..." since this message from dpkg is written to standard # error. - if [ $(dpkg -l $el 2>/dev/null | grep ^ii | wc -l) -lt 1 ]; then - some_are_missing=1 - fi - done - - if [ $some_are_missing -eq 0 ]; then - return - fi + if [ $(dpkg -l $el 2>/dev/null | grep ^ii | wc -l) -lt 1 ]; then + some_are_missing=1 + fi + done + + if [ $some_are_missing -eq 0 ]; then + return + fi - if [ $force_packages -eq 1 ]; then - run apt-get install $apt_opts --assume-yes --force-yes $@ - else - run apt-get install $apt_opts --assume-yes $@ - fi + if [ $force_packages -eq 1 ]; then + run apt-get install $apt_opts --assume-yes --force-yes $@ + else + run apt-get install $apt_opts --assume-yes $@ fi + elif [ $on_redhat_or_derivative -eq 1 ]; then + run yum install --assumeyes $@ + fi } function install_common_os_packages() { - log "Installing common OS packages ..." - + print_and_log "Installing common OS packages ..." + git_package=git if [ $on_debian_or_derivative -eq 1 ]; then # Ubuntu doesn't have git (!) but only git-core. if [ $on_ubuntu -eq 1 ]; then git_package=git-core - else - git_package=git fi - - local packages="curl $git_package wget unzip" - install_packages_if_missing $packages fi + local packages="curl $git_package wget unzip" + install_packages_if_missing $packages + for el in lsb_release curl wget git unzip; do assert_pre_requisite $el done @@ -204,6 +200,11 @@ function install_common_os_packages() ###################################################################### # ###################################################################### +function download_escenic_components_p() { + $(download_escenic_components) & + show_pulse $! "Downloading Escenic components from Technet" +} + function set_up_instance_specific_nursery_configuration_p() { $(set_up_instance_specific_nursery_configuration) & show_pulse $! "Setting up instance specific Nursery configuration" @@ -797,10 +798,21 @@ function install_ece_third_party_packages libmysql-java memcached wget" - - install_packages_if_missing $packages + elif [ $on_redhat_or_derivative -eq 1 ]; then + packages=" + ant + ant-contrib + ant-nodeps + apr + memcached + mysql-connector-java + wget + " + # TODO no tomcat APR wrappers in official repositories fi + install_packages_if_missing $packages + for el in ant java; do assert_pre_requisite $el done @@ -940,7 +952,7 @@ function set_up_app_server() search_port=$(echo $user_search | cut -d':' -f2) fi - download_tomcat_p $download_dir + download_tomcat $download_dir tomcat_dir=$(get_base_dir_from_bundle $(get_tomcat_download_url)) run cd $appserver_parent_dir @@ -1769,14 +1781,21 @@ function common_pre_install() if [ -e /etc/debian_version -a -x /usr/bin/dpkg ]; then on_debian_or_derivative=1 export DEBIAN_FRONTEND=noninteractive + + # chicken and the egg problem, we need lsb_release to install the + # packages later on, hence as soon as we know we've got a Debian + # based platform, we install lsb-release. Also note, the + # executable, lsb_release, is in the list of required binaries in + # install_common_os_packages. + install_packages_if_missing "lsb-release" + fi + + if [ -e /etc/redhat-release ]; then + on_redhat_or_derivative=1 + install_packages_if_missing "redhat-lsb" fi - # chicken and the egg problem, we need lsb_release to install the - # packages later on, hence as soon as we know we've got a Debian - # based platform, we install lsb-release. Also note, the - # executable, lsb_release, is in the list of required binaries in - # install_common_os_packages. - install_packages_if_missing "lsb-release" + assert_pre_requisite lsb_release if [ $(lsb_release -i | grep Ubuntu | wc -l) -gt 0 ]; then on_ubuntu=1 @@ -1785,9 +1804,9 @@ function common_pre_install() fi make_dir $download_dir - install_common_os_packages_p + install_common_os_packages create_user_and_group_if_not_present $ece_user $ece_group - set_up_user_environment_p + set_up_user_environment } function add_apt_source() @@ -1832,10 +1851,7 @@ function install_cache_server() add_apt_source "deb http://repo.varnish-cache.org/ubuntu/ $(lsb_release -s -c) varnish-3.0" fi - if [ $on_debian_or_derivative -eq 1 ]; then - apt-get -y install varnish 1>>$log 2>>$log - fi - + install_packages_if_missing varnish assert_pre_requisite varnishd if [ $fai_enabled -eq 0 ]; then @@ -1877,7 +1893,7 @@ function install_database_server() { print_and_log "Installing the database server on $HOSTNAME ..." - source /usr/sbin/drop-and-create-ecedb + source $(dirname $0)/drop-and-create-ecedb if [ $on_debian_or_derivative -eq 1 ]; then @@ -1935,8 +1951,16 @@ function install_database_server() print_and_log "I will use vanilla MySQL instead." packages="mysql-server mysql-client" - install_packages_if_missing $packages fi + elif [ $on_redhat_or_derivative -eq 1 ]; then + packages="mysql mysql-server" + fi + + install_packages_if_missing $packages + + if [ $on_redhat_or_derivative -eq 1 ]; then + run chkconfig --level 35 mysqld on + run /etc/init.d/mysqld start fi assert_pre_requisite mysql @@ -2122,7 +2146,7 @@ function install_ece_instance() if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - assemble_deploy_and_restart_type_p + assemble_deploy_and_restart_type fi update_type_instances_to_start_up @@ -2185,7 +2209,7 @@ function install_presentation_server() function assemble_deploy_and_restart_type() { - log "Assembling, deploying & starting $instance_name ..." + print_and_log "Assembling, deploying & starting $instance_name ..." set_correct_permissions @@ -2937,7 +2961,7 @@ function add_server_to_runlevels() "run levels." # TODO add init.d to the default runlevels, for other # distributions too: - # - RedHat/chekcconfig + # - RedHat/chekcconfig chkconfig --level 35 ece on # - Gentoo: rc-update add ece default fi } From c4f9494e23c4dfa9ce924bf34c50840c80c5e513 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 23 Feb 2012 17:48:30 +0800 Subject: [PATCH 0317/2585] - removed one pulse and a return code check which for some reason didn't return a number . --- usr/sbin/ece-install | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 799a2790..07e40592 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -411,7 +411,6 @@ function set_up_ece_scripts() run cd ece-scripts run git pull ) - if [ $! -eq 1 ]; then echo $!; return 1; fi else run git clone $ece_scripts_git_source fi @@ -2111,7 +2110,7 @@ function install_ece_instance() ask_for_instance_name $1 set_up_engine_directories - set_up_ece_scripts_p + set_up_ece_scripts set_archive_files_depending_on_profile From 9696a74c34e37431007075df888d029a77569c49 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 23 Feb 2012 18:10:15 +0800 Subject: [PATCH 0318/2585] - user friendliness: if java_home isn't set, "ece clean" and other commands indirectly using java will fail. For "ece start/stop/restart" this is more visible, but for "ece clean" it is not, here ece will just complain that ant failed. Now ece will complain if the java_home is set to an invalid value: [ece#engine-dev1] java_home /usr/lib/jvm/java-1.6.0. doesn't exist [ece#engine-dev1] check your setting in /etc/escenic/ece-dev1.conf /etc/escenic/ece.conf This should help debugging and installation a *lot* easier :-) --- usr/bin/ece | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 0febad76..66a48c9b 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -464,6 +464,16 @@ function sanity_check() exit 1 fi fi + + # verifies that java_home exists and has the java executable + if [ ! -d $java_home ]; then + print "java_home $java_home doesn't exist" + print "check your setting in ${ece_conf_files_read[@]}" + exit 1 + elif [ ! -x $java_home/bin/java ]; then + print "$java_home/bin/java isn't executable for $ece_user" + exit 1 + fi } function deploy() From 3ced2363365e90013f7cfcfc2cbaf6e6efa0fe73 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 23 Feb 2012 18:16:15 +0800 Subject: [PATCH 0319/2585] - a tad more logical feedback string --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index 66a48c9b..49368ff2 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -468,7 +468,7 @@ function sanity_check() # verifies that java_home exists and has the java executable if [ ! -d $java_home ]; then print "java_home $java_home doesn't exist" - print "check your setting in ${ece_conf_files_read[@]}" + print "check your setting in one of: ${ece_conf_files_read[@]}" exit 1 elif [ ! -x $java_home/bin/java ]; then print "$java_home/bin/java isn't executable for $ece_user" From 6f34a72d9755a1e9619100448e1268de4ad16669 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 12:01:48 +0800 Subject: [PATCH 0320/2585] - fix: ece-install now updates the java_home variable in the global ece.conf, this means that the user can override the java_home variable in his/her ece-install.conf. --- usr/sbin/ece-install | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 07e40592..8c71f57a 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -428,6 +428,7 @@ function set_up_ece_scripts() set_conf_file_value ece_home ${escenic_root_dir}/engine ${file} set_conf_file_value escenic_conf_dir ${escenic_conf_dir} ${file} set_conf_file_value heap_dump_dir ${escenic_crash_dir} ${file} + set_conf_file_value java_home ${java_home} ${file} set_conf_file_value log_dir ${escenic_log_dir} ${file} set_conf_file_value pid_dir ${escenic_run_dir} ${file} set_conf_file_value rmi_hub_conf ${escenic_conf_dir}/rmi-hub ${file} From 41bae8f7ea6cba22fe3449988cf435b2e0e862d6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 14:23:19 +0800 Subject: [PATCH 0321/2585] - new feature: it's possible to say that munin and nagios node/client setup should be skipped with two new FAI parameters: fai_monitoring_node_munin_skip=1 fai_monitoring_node_nagios_skip=1 As the settings suggest, this will apply to all nodes, regardless of installation profile. - non-Debian systems: nagios node will not be installed (or for the time being) nor attempted to configure. The user is simply told that he/she has to do this manually. --- usr/sbin/ece-install | 47 +++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8c71f57a..a4aa2362 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -2827,9 +2827,13 @@ EOF function install_nagios_node() { - print "Installing an icinga client on $HOSTNAME ..." + print "Installing a Nagios client on $HOSTNAME ..." if [ $on_debian_or_derivative -eq 1 ]; then install_packages_if_missing nagios-nrpe-server nagios-plugins + else + print_and_log "Nagios node installation not supported on your system" + print_and_log "You will have to install it manually." + return fi local file=/etc/nagios/nrpe.cfg @@ -3217,29 +3221,32 @@ function install_monitoring_server() create_monitoring_server_overview $nagios_flavour } -function common_post_install() -{ - if [ $install_profile_number -eq $PROFILE_ALL_IN_ONE -o \ - $install_profile_number -eq $PROFILE_PRESENTATION_SERVER -o \ - $install_profile_number -eq $PROFILE_RMI_HUB -o \ - $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - add_server_to_runlevels - fi +function common_post_install() { + if [ $install_profile_number -eq $PROFILE_ALL_IN_ONE -o \ + $install_profile_number -eq $PROFILE_PRESENTATION_SERVER -o \ + $install_profile_number -eq $PROFILE_RMI_HUB -o \ + $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + add_server_to_runlevels + fi - if [ $install_profile_number -ne $PROFILE_WIDGET_FRAMEWORK -a \ - $install_profile_number -ne $PROFILE_CREATE_PUBLICATION -a \ - $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then - install_munin_node + if [ $install_profile_number -ne $PROFILE_WIDGET_FRAMEWORK -a \ + $install_profile_number -ne $PROFILE_CREATE_PUBLICATION -a \ + $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then - if [ $install_profile_number -ne $PROFILE_MONITORING_SERVER ]; then - install_nagios_node - fi + if [[ "${fai_monitoring_node_munin_skip}" != "1" ]]; then + install_munin_node fi - set_correct_permissions - - print_status_and_next_steps - rm $pid_file + if [[ $install_profile_number != $PROFILE_MONITORING_SERVER && + "${fai_monitoring_node_nagios_skip}" != "1" ]]; then + install_nagios_node + fi + fi + + set_correct_permissions + + print_status_and_next_steps + run rm $pid_file } for el in $@; do From e8a9fcfa4178641613c76d89c896602859defc83 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 14:43:31 +0800 Subject: [PATCH 0322/2585] - removed shebang --- usr/share/escenic/ece-scripts/ece-install.d/memcached.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memcached.sh b/usr/share/escenic/ece-scripts/ece-install.d/memcached.sh index 9f4baa51..03da73db 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memcached.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memcached.sh @@ -1,7 +1,3 @@ -#! /usr/bin/env bash - -# by tkj@vizrt.com - memcached_java_lib_url=http://img.whalin.com/memcached/jdk5/log4j/java_memcached-release_2.0.1.tar.gz ## $1: root directory of the publication From 3c4b2283fab98d9e865683e603c165d049fa2882 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 15:15:34 +0800 Subject: [PATCH 0323/2585] - moved install_cache_server and set_up_varnish to ece-install.d/cache-server.sh --- .../ece-scripts/ece-install.d/cache-server.sh | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh new file mode 100644 index 00000000..642e2d6e --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh @@ -0,0 +1,193 @@ +function install_cache_server() +{ + print_and_log "Installing a caching server on $HOSTNAME ..." + + if [[ $on_debian_or_derivative -eq 1 && + $(apt-key list | grep varnish-software.com | wc -l) -eq 0 ]]; then + curl ${curl_opts} \ + http://repo.varnish-cache.org/debian/GPG-key.txt \ + 2>> $log | \ + apt-key add - \ + 1>>$log 2>>$log + run apt-get update + fi + + code_name=$(lsb_release -s -c) + + supported_code_name=0 + # list taken from http://repo.varnish-cache.org/debian/dists/ + supported_list="lenny squeeze hardy lucid" + for el in $supported_list; do + if [ $code_name = $el ]; then + supported_code_name=1 + fi + done + + if [ $supported_code_name -eq 1 -a $on_debian -eq 1 ]; then + add_apt_source "deb http://repo.varnish-cache.org/debian/ $(lsb_release -s -c) varnish-3.0" + elif [ $supported_code_name -eq 1 -a $on_ubuntu -eq 1 ]; then + add_apt_source "deb http://repo.varnish-cache.org/ubuntu/ $(lsb_release -s -c) varnish-3.0" + fi + + install_packages_if_missing varnish + assert_pre_requisite varnishd + + if [ $fai_enabled -eq 0 ]; then + print "You must now list your backend servers." + print "Seperate the entries with a space. e.g.: app1:8080 app2:8080." + print "Press ENTER to accept the default: ${HOSTNAME}:${appserver_port}" + echo -n "Your choice [${HOSTNAME}:${appserver_port}]> " + read backend_servers + else + backend_servers=${fai_cache_backends} + fi + + if [ -z "$backend_servers" ]; then + backend_servers="${HOSTNAME}:${appserver_port}" + fi + + set_up_varnish $backend_servers + + add_next_step "Cache server is up and running at http://${HOSTNAME}:80/" +} + +function set_up_varnish() +{ + print_and_log "Setting up Varnish to match your environment ..." + run /etc/init.d/varnish stop + + # we need to swap standard err and standard out here as varnishd + # -V for some reason writes to standard error. + using_varnish_3=$(varnishd -V 3>&1 1>&2 2>&3 | grep varnish-3 | wc -l) + + local file=/etc/default/varnish + sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file + exit_on_error "sed on $file" + + cat > /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < 0) { +EOF + if [ $using_varnish_3 -gt 0 ]; then + cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < Date: Fri, 24 Feb 2012 15:18:48 +0800 Subject: [PATCH 0324/2585] - moved all cache server related code to ece-install.d/cache-server.sh - added loading of ece-install.d modules and common libraries to new method, init() - clean-up/faster install on Debian based systems: * will not fetch and add the varnish APT key if already exists in the local key ring. * will not add the varnish APT source to escenic.list if already exists in sources.list --- usr/sbin/ece-install | 242 +++++++------------------------------------ 1 file changed, 35 insertions(+), 207 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a4aa2362..ee689c14 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -108,14 +108,33 @@ PROFILE_MONITORING_SERVER=10 PROFILE_RESTORE_FROM_BACKUP=11 PROFILE_ANALYSIS_SERVER=12 -###################################################################### -# Load common BASH libraries -###################################################################### -common_libraries="common-bashing.sh common-pulse.sh common-io.sh common-os.sh" -for el in $common_libraries; do - source $(dirname $0)/../share/escenic/ece-scripts/${el} || \ - (echo "Could not find $el"; exit 1) -done +## Bootstrapping, load files from /usr/share/escenic/ece-scripts The +## method will first try to be smart, in case the user has copied the +## ece-scripts somewhere else., e.g.: moved everything to ~/ece-scrpts +## or /opt/escenic/ece-scripts, this should also work. +function init() { + # first, try to be nice + local dir=$(dirname $0)/../share/escenic/ece-scripts + + # then check the standard location + if [ ! -d $dir ]; then + dir=/usr/share/escenic/ece-scripts + fi + + if [ -d $dir ]; then + # load common librariees + common_libraries="common-bashing.sh common-pulse.sh common-io.sh common-os.sh" + for el in $common_libraries; do + source $dir/${el} + done + + # load ece-install modules + source $dir/ece-install.d/*.sh + else + echo "I cannot find $(basename $0)'s dependencies, exiting :-(" + exit 1 + fi +} function set_up_engine_directories() { @@ -1292,7 +1311,7 @@ function print_status_and_next_steps() seconds_left=$(( seconds_left - $minutes * 60 )) if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then - s="The installation is now complete!" + s="The installation is now $(green complete)!" else s="The restore is now complete!" fi @@ -1539,147 +1558,6 @@ function get_privileged_hosts() { echo ${privileged_hosts} } -function set_up_varnish() -{ - print_and_log "Setting up Varnish to match your environment ..." - run /etc/init.d/varnish stop - - # we need to swap standard err and standard out here as varnishd - # -V for some reason writes to standard error. - using_varnish_3=$(varnishd -V 3>&1 1>&2 2>&3 | grep varnish-3 | wc -l) - - local file=/etc/default/varnish - sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file - exit_on_error "sed on $file" - - cat > /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < 0) { -EOF - if [ $using_varnish_3 -gt 0 ]; then - cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> $escenic_sources run apt-get update elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then @@ -1821,58 +1700,6 @@ function add_apt_source() fi } -function install_cache_server() -{ - print_and_log "Installing a caching server on $HOSTNAME ..." - - if [ $on_debian_or_derivative -eq 1 ]; then - curl ${curl_opts} \ - http://repo.varnish-cache.org/debian/GPG-key.txt \ - 2>> $log | \ - apt-key add - \ - 1>>$log 2>>$log - run apt-get update - fi - - code_name=$(lsb_release -s -c) - - supported_code_name=0 - # list taken from http://repo.varnish-cache.org/debian/dists/ - supported_list="lenny squeeze hardy lucid" - for el in $supported_list; do - if [ $code_name = $el ]; then - supported_code_name=1 - fi - done - - if [ $supported_code_name -eq 1 -a $on_debian -eq 1 ]; then - add_apt_source "deb http://repo.varnish-cache.org/debian/ $(lsb_release -s -c) varnish-3.0" - elif [ $supported_code_name -eq 1 -a $on_ubuntu -eq 1 ]; then - add_apt_source "deb http://repo.varnish-cache.org/ubuntu/ $(lsb_release -s -c) varnish-3.0" - fi - - install_packages_if_missing varnish - assert_pre_requisite varnishd - - if [ $fai_enabled -eq 0 ]; then - print "You must now list your backend servers." - print "Seperate the entries with a space. e.g.: app1:8080 app2:8080." - print "Press ENTER to accept the default: ${HOSTNAME}:${appserver_port}" - echo -n "Your choice [${HOSTNAME}:${appserver_port}]> " - read backend_servers - else - backend_servers=$(get_conf_value fai_cache_backends) - fi - - if [ -z "$backend_servers" ]; then - backend_servers="${HOSTNAME}:${appserver_port}" - fi - - set_up_varnish $backend_servers - - add_next_step "Cache server is up and running at http://${HOSTNAME}:80/" -} - # Parameters: # $1 : your added line function add_next_step() @@ -3089,14 +2916,14 @@ function restore_from_backup() gunzip < $sql_file | mysql $db_schema exit_on_error "restoring from $sql_file" - add_next_step "Successfully restored DB from $(basename $sql_file)" + add_next_step "$(green Successfully) restored DB from $sql_file" fi if [ $restore_data_files -eq 1 -o $restore_all -eq 1 ]; then print_and_log "Restoring the Solr & ECE data files on $HOSTNAME ..." run cd $dir run tar -C / -xf $backup_file var/lib/escenic - add_next_step "Successfully restored Solr & ECE data files" + add_next_step "$(green Successfully) restored Solr & ECE data files" add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check $escenic_data_dir to verify they're all there." fi @@ -3105,7 +2932,7 @@ function restore_from_backup() print_and_log "Restoring the ECE configuration files on $HOSTNAME ..." run cd $dir run tar -C / -xf $backup_file etc - add_next_step "Successfully restored ECE configuration files" + add_next_step "$(green Successfully) restored ECE configuration files" add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check /etc to verify that they're all there." fi @@ -3114,7 +2941,7 @@ function restore_from_backup() print_and_log "Restoring the Escenic & Tomcat binaries on $HOSTNAME ..." run cd $dir run tar -C / -xf $backup_file opt - add_next_step "Successfully restored Escenic & Tomcat binaries" + add_next_step "$(green Successfully) restored Escenic & Tomcat binaries" add_next_step "Backup file used: $(basename $backup_file)" add_next_step "Check ${appserver_parent_dir} to verify that they're all there". install_ece_third_party_packages @@ -3268,6 +3095,7 @@ for el in $@; do fi done +init assert_correct_runtime_environment fai_enabled=$(get_boolean_conf_value fai_enabled) From 5f1780c40d3aab04e9085af33e809823cdda285d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 15:24:03 +0800 Subject: [PATCH 0325/2585] - usability: added more help on giving input to ece-install when installing the cache profile interactively: [ece-install-3] You must now list your backend servers. [ece-install-3] These must be host names (not IPs) and must all be resolvable [ece-install-3] by your cache host (quanah), preferably from /etc/hosts [ece-install-3] Seperate the entries with a space. e.g.: app1:8080 app2:8080. [ece-install-3] Press ENTER to accept the default: quanah:8080 The reason why ece-install wants these as host names and not IPs, is that it'll use these both as the backend host definitions in Varnish, as well as the actual address of the backend. Since Varnish does not support backend names which start which looks like a number, or IP for that matter, we use the host name instead. To make it simple, we encourage having entries for all the other nodes, cache, app servers, db and so on, in the different hosts' /etc/hosts, hence the message above. --- usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh index 642e2d6e..d54319e3 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh @@ -34,6 +34,8 @@ function install_cache_server() if [ $fai_enabled -eq 0 ]; then print "You must now list your backend servers." + print "These must be host names (not IPs) and must all be resolvable" + print "by your cache host ($HOSTNAME), preferably from /etc/hosts" print "Seperate the entries with a space. e.g.: app1:8080 app2:8080." print "Press ENTER to accept the default: ${HOSTNAME}:${appserver_port}" echo -n "Your choice [${HOSTNAME}:${appserver_port}]> " From 2fda9deaed340053bf861cf48e6d04bd5952acf1 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 15:27:52 +0800 Subject: [PATCH 0326/2585] - regression: exists in error where hidden (because of the introduction of pulse) --- usr/share/escenic/ece-scripts/common-bashing.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh index 4af9492a..d963aec0 100644 --- a/usr/share/escenic/ece-scripts/common-bashing.sh +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -80,8 +80,8 @@ function remove_pid_and_exit_in_error() { function exit_on_error() { if [ $? -gt 0 ]; then - log "The command ["$@"] FAILED, exiting :-(" - log "See $log for further details." + print_and_log "The command ["$@"] $(red FAILED), exiting :-(" + print "See $log for further details." remove_pid_and_exit_in_error fi } From 38be3fc41265190a5d7eeb268f5276f8732ab4c5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 15:28:48 +0800 Subject: [PATCH 0327/2585] - renamed memcached.sh -> memory-cache.sh to make it inline with the rest of the modules; using general names for what the components actually are, refraining from getting locked down to vendor/product. --- .../ece-scripts/ece-install.d/memory-cache.sh | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh new file mode 100644 index 00000000..03da73db --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -0,0 +1,88 @@ +memcached_java_lib_url=http://img.whalin.com/memcached/jdk5/log4j/java_memcached-release_2.0.1.tar.gz + +## $1: root directory of the publication +function memcached_create_publication_nursery_component() { + if [ ! -d "$1" ]; then + log $1 "doesn't exist" + return + fi + + log "Adding memcached wrapper to PresenationArticle in $1 ..." + local dir=$1/webapp/WEB-INF/localconfig/neo/xredsys/presentation/cache + make_dir $dir + local file=$dir/PresentationArticleCache.properties + sed -i "s#\$class=.*#\$class=neo.util.cache.Memcached#g" $file +} + +function install_memcached() +{ + if [ $on_debian_or_derivative ]; then + install_packages_if_missing "memcached" + fi + + run $download_dir + run wget $wget_opts $memcached_java_lib_url + local name=$(get_base_dir_from_bundle $memcached_java_lib_url) + run cp $name/$name.jar $assemblytool_home/lib + + memcached_set_up_common_nursery + + log "Configuring all publications for using memcached ..." + for el in $assemblytool_home/publications/*.properties 2>/dev/null; do + local publication=$(basename $el .properties) + + if [[ $appserver == "tomcat" ]]; then + dir=$tomcat_base/webapps/$publication + make_dir $dir + memcached_create_publication_nursery_component $dir + fi + done + + # fixing the deployed publications on host + if [[ $appserver == "tomcat" ]]; then + for el in $( + find $tomcat_base/webapps/ \ + -mindepth 1 \ + -maxdepth 1 \ + -type d | \ + egrep -v "solr|webservice|escenic|escenic-admin|indexer-webservice" | \ + egrep -v "indexer-webapp|studio" + ); do + memcached_create_publication_nursery_component $el + done + + fi + + # TODO inform the user that he/she might want to do tihs in the + # publication tree as well. + assemble_deploy_and_restart_type_p +} + +function memcached_set_up_common_nursery() { + local dir=$common_nursery_dir/com/danga + make_dir $dir + cat > $dir/SockIOPool <> $common_nursery_dir/Initial.properties < Date: Fri, 24 Feb 2012 15:43:14 +0800 Subject: [PATCH 0328/2585] - since some versions of yum use --assume-yes and some use --assumeyes (!) (yes, I know, don't get me started), we just use the short form -y here. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index ee689c14..91bf4791 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -193,7 +193,7 @@ function install_packages_if_missing() { run apt-get install $apt_opts --assume-yes $@ fi elif [ $on_redhat_or_derivative -eq 1 ]; then - run yum install --assumeyes $@ + run yum install -y $@ fi } From 7a3deb692abd688f682c634122d4f1d68a65b79a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 15:51:58 +0800 Subject: [PATCH 0329/2585] - git sometimes, for some reason fails to verify the SSL certifacte of github, we thus turn off this checking. --- usr/sbin/ece-install | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 91bf4791..e0c8dae6 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -193,6 +193,8 @@ function install_packages_if_missing() { run apt-get install $apt_opts --assume-yes $@ fi elif [ $on_redhat_or_derivative -eq 1 ]; then + # since some versions of yum use --assume-yes and some use + # --assumeyes (!), we use the short form, -y, here run yum install -y $@ fi } @@ -1681,6 +1683,10 @@ function common_pre_install() on_debian=1 fi + # git sometimes, for some reason fails to verify the SSL + # certifacte of github. + export GIT_SSL_NO_VERIFY=true + make_dir $download_dir install_common_os_packages create_user_and_group_if_not_present $ece_user $ece_group From ee2ec5d42d606049e16cbc41a51e98775a6a9475 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 16:10:31 +0800 Subject: [PATCH 0330/2585] - removed use of pulses in install_ece_instance - clarified explanation for turning off git SSL checking --- usr/sbin/ece-install | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index e0c8dae6..1740aace 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1683,8 +1683,9 @@ function common_pre_install() on_debian=1 fi - # git sometimes, for some reason fails to verify the SSL - # certifacte of github. + # git will fail if curl doesn't have the right CA certificates + # installed. As this happens on RedHat/CentOS 5.7, we turn it off + # here. export GIT_SSL_NO_VERIFY=true make_dir $download_dir @@ -1693,17 +1694,16 @@ function common_pre_install() set_up_user_environment } -function add_apt_source() -{ - escenic_sources=/etc/apt/sources.list.d/escenic.list - if [[ ! -e $escenic_sources && - $(grep "$@" /etc/apt/sources.list | wc -l) -lt 1 ]]; then - echo "$@" >> $escenic_sources - run apt-get update - elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then - echo "$@" >> $escenic_sources - run apt-get update - fi +function add_apt_source() { + local escenic_sources=/etc/apt/sources.list.d/escenic.list + if [[ ! -e $escenic_sources && + $(grep "$@" /etc/apt/sources.list | wc -l) -lt 1 ]]; then + echo "$@" >> $escenic_sources + run apt-get update + elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then + echo "$@" >> $escenic_sources + run apt-get update + fi } # Parameters: @@ -1951,9 +1951,9 @@ function install_ece_instance() # most likely, the user is _not_ installing from archives (EAR + # configuration bundle), hence the false test goes first. if [ $(is_installing_from_ear) -eq 0 ]; then - download_escenic_components_p - check_for_required_downloads_p - set_up_engine_and_plugins_p + download_escenic_components + check_for_required_downloads + set_up_engine_and_plugins set_up_assembly_tool else verify_that_files_exist_and_are_readable \ @@ -1962,7 +1962,7 @@ function install_ece_instance() fi set_up_basic_nursery_configuration - set_up_instance_specific_nursery_configuration_p + set_up_instance_specific_nursery_configuration set_up_app_server set_up_proper_logging_configuration From 4b4652e874251ab62e052eaec826f7e4d603d77c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 16:31:08 +0800 Subject: [PATCH 0331/2585] - added constants file --- .../escenic/ece-scripts/ece-install.d/constants.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/constants.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/constants.sh b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh new file mode 100644 index 00000000..ec10854b --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh @@ -0,0 +1,13 @@ +PROFILE_ALL_IN_ONE=1 +PROFILE_CACHE_SERVER=5 +PROFILE_DB_SERVER=4 +PROFILE_EDITORIAL_SERVER=2 +PROFILE_PRESENTATION_SERVER=3 +PROFILE_RMI_HUB=6 +PROFILE_SEARCH_SERVER=7 +PROFILE_WIDGET_FRAMEWORK=8 +PROFILE_CREATE_PUBLICATION=9 +PROFILE_MONITORING_SERVER=10 +PROFILE_RESTORE_FROM_BACKUP=11 +PROFILE_ANALYSIS_SERVER=12 + From 872ab81e65b6347be293afae2fb3c8a5d1f3dcf8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 16:49:32 +0800 Subject: [PATCH 0332/2585] - started moving content-engine specific things to its own module --- .../ece-install.d/content-engine.sh | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh new file mode 100644 index 00000000..b99be89f --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -0,0 +1,39 @@ +function get_deploy_white_list() +{ + local white_list="escenic-admin" + + if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER \ + -a -n "${publication_name}" ]; then + white_list="${white_list} ${publication_name} " + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + white_list="${white_list} solr indexer-webapp" + elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + white_list="${white_list} "$(get_publication_short_name_list) + elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + white_list="${white_list} escenic studio indexer-webservice" + white_list="${white_list} "$(get_publication_short_name_list) + fi + + echo ${white_list} +} + +function get_publication_short_name_list() +{ + local short_name_list="" + + local publication_def_dir=${escenic_root_dir}/assemblytool/publications + if [ $(ls ${publication_def_dir} | grep .properties$ | wc -l) -eq 0 ]; then + echo ${short_name_list} + return + fi + + for el in $(find ${publication_def_dir} -name "*.properties"); do + local short_name=$(basename $el .properties) + short_name_list="${short_name_list} ${short_name}" + done + + echo ${short_name_list} +} + + + From f9621da4b241f1377b9e838c9867a165bf768bc0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 16:50:19 +0800 Subject: [PATCH 0333/2585] - started moving content-engine specific things to its own module - moved constants to its own module --- usr/sbin/ece-install | 51 -------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 1740aace..bb256dc7 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -90,24 +90,10 @@ http://technet.escenic.com/downloads/release/53/menu-editor-dist-2.0.6.0.zip http://technet.escenic.com/downloads/release/53/poll-2.1.3.0.zip http://technet.escenic.com/downloads/release/53/xml-editor-dist-2.1.0.0.zip " - wf_download_list=" http://technet.escenic.com/downloads/widget-framework/widget-framework-core-1.10.0.0.zip " -PROFILE_ALL_IN_ONE=1 -PROFILE_CACHE_SERVER=5 -PROFILE_DB_SERVER=4 -PROFILE_EDITORIAL_SERVER=2 -PROFILE_PRESENTATION_SERVER=3 -PROFILE_RMI_HUB=6 -PROFILE_SEARCH_SERVER=7 -PROFILE_WIDGET_FRAMEWORK=8 -PROFILE_CREATE_PUBLICATION=9 -PROFILE_MONITORING_SERVER=10 -PROFILE_RESTORE_FROM_BACKUP=11 -PROFILE_ANALYSIS_SERVER=12 - ## Bootstrapping, load files from /usr/share/escenic/ece-scripts The ## method will first try to be smart, in case the user has copied the ## ece-scripts somewhere else., e.g.: moved everything to ~/ece-scrpts @@ -1996,43 +1982,6 @@ function install_ece_instance() add_next_step "/etc/default/ece lists all instances started at boot time" } -function get_publication_short_name_list() -{ - local short_name_list="" - - local publication_def_dir=${escenic_root_dir}/assemblytool/publications - if [ $(ls ${publication_def_dir} | grep .properties$ | wc -l) -eq 0 ]; then - echo ${short_name_list} - return - fi - - for el in $(find ${publication_def_dir} -name "*.properties"); do - local short_name=$(basename $el .properties) - short_name_list="${short_name_list} ${short_name}" - done - - echo ${short_name_list} -} - -function get_deploy_white_list() -{ - local white_list="escenic-admin" - - if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER \ - -a -n "${publication_name}" ]; then - white_list="${white_list} ${publication_name} " - elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - white_list="${white_list} solr indexer-webapp" - elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - white_list="${white_list} "$(get_publication_short_name_list) - elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - white_list="${white_list} escenic studio indexer-webservice" - white_list="${white_list} "$(get_publication_short_name_list) - fi - - echo ${white_list} -} - function install_presentation_server() { print_and_log "Installing a presentation server on $HOSTNAME." From 38b29a529762ca60e15fd28b6ab3fec4f696a22a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 17:09:24 +0800 Subject: [PATCH 0334/2585] - TDD feels good! Wrote test before fixing get_publication_short_name_list. - fixed bug in get_publication_short_name_list, if you had .properties in sub directories of assemblytool/publications. --- .../test-ece-install-content-engine | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 usr/local/src/unit-tests/test-ece-install-content-engine diff --git a/usr/local/src/unit-tests/test-ece-install-content-engine b/usr/local/src/unit-tests/test-ece-install-content-engine new file mode 100644 index 00000000..6537a253 --- /dev/null +++ b/usr/local/src/unit-tests/test-ece-install-content-engine @@ -0,0 +1,32 @@ +#! /usr/bin/env bash + +# by tkj@vizrt.com + +common_test_is_loaded > /dev/null 2>&1 || source common-test.sh + +# source ../../../share/escenic/ece-scripts/ece-install.d/constants.sh +source ../../../share/escenic/ece-scripts/ece-install.d/content-engine.sh + +function test_get_publication_short_name_list() { + escenic_root_dir=$(mktemp -d) + local pub_dir=$escenic_root_dir/assemblytool/publications + mkdir -p $pub_dir + touch $pub_dir/{pub1,pub2}.properties + + # a sub dir + mkdir -p $pub_dir/sub + touch $pub_dir/sub/{SomeNurseryComponent,SomeOtherNurseryComponent}.properties + + local short_name_list=$(get_publication_short_name_list | sed 's/\ //g') + assert_equals "should not include other .properties" "pub2pub1" $short_name_list + + rm -rf $escenic_root_dir +} + +test_suite=" +test_get_publication_short_name_list +" + +run_test_suite $test_suite + + From 43040d5cf1a6d8824fa164ca3a2f137bd9a2b3d3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 17:09:41 +0800 Subject: [PATCH 0335/2585] - added --- usr/local/src/unit-tests/common-test.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 usr/local/src/unit-tests/common-test.sh diff --git a/usr/local/src/unit-tests/common-test.sh b/usr/local/src/unit-tests/common-test.sh new file mode 100644 index 00000000..3676d432 --- /dev/null +++ b/usr/local/src/unit-tests/common-test.sh @@ -0,0 +1,14 @@ +#! /usr/bin/env bash + +# by tkj@vizrt.com + +if [ ! -e bash_unit ]; then + wget --quiet https://raw.github.com/skybert/my-little-friends/master/bash/bash_unit +fi + +source bash_unit + +function common_test_is_loaded() { + echo 1 +} + From a0c093f22461e105cf4797ed8fb7bda517670794 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 17:11:02 +0800 Subject: [PATCH 0336/2585] - dynamically fetches test & common libraries from my-little-friends --- usr/local/src/unit-tests/common-test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/usr/local/src/unit-tests/common-test.sh b/usr/local/src/unit-tests/common-test.sh index 3676d432..76de94c9 100644 --- a/usr/local/src/unit-tests/common-test.sh +++ b/usr/local/src/unit-tests/common-test.sh @@ -5,6 +5,9 @@ if [ ! -e bash_unit ]; then wget --quiet https://raw.github.com/skybert/my-little-friends/master/bash/bash_unit fi +if [ ! -e alexandria ]; then + wget --quiet https://raw.github.com/skybert/my-little-friends/master/bash/alexandria +fi source bash_unit From 373b1becb315a7d611a4daea6ce0248916a719e1 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 17:13:09 +0800 Subject: [PATCH 0337/2585] - bug fix: fix in get_publication_short_name_list, which again would manifest itself when calling get_deploy_white_list for building up the deploy white list property in ece-.conf The bug was that get_publication_short_name_list would think .properties files in sub folders of assemblytool/publications would be taken for publication definitions. --- usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index b99be89f..28f2c0b2 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -27,7 +27,7 @@ function get_publication_short_name_list() return fi - for el in $(find ${publication_def_dir} -name "*.properties"); do + for el in $(find ${publication_def_dir} -maxdepth 1 -name "*.properties"); do local short_name=$(basename $el .properties) short_name_list="${short_name_list} ${short_name}" done From e8f9e9125ceda28b09a322f3702b64f077b498e5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 17:25:18 +0800 Subject: [PATCH 0338/2585] - fixed bug in memory-cache.sh (2>/dev/null which didn't make any sense, made a source fail.) --- usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh index 03da73db..44bc6a57 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -28,7 +28,7 @@ function install_memcached() memcached_set_up_common_nursery log "Configuring all publications for using memcached ..." - for el in $assemblytool_home/publications/*.properties 2>/dev/null; do + for el in $assemblytool_home/publications/*.properties; do local publication=$(basename $el .properties) if [[ $appserver == "tomcat" ]]; then From b54b5fde7d084026614ed005ab5a0c4e847efe2b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 17:26:26 +0800 Subject: [PATCH 0339/2585] - renamed --- .../ece-scripts/ece-install.d/memcached.sh | 88 ------------------- 1 file changed, 88 deletions(-) delete mode 100644 usr/share/escenic/ece-scripts/ece-install.d/memcached.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memcached.sh b/usr/share/escenic/ece-scripts/ece-install.d/memcached.sh deleted file mode 100644 index 03da73db..00000000 --- a/usr/share/escenic/ece-scripts/ece-install.d/memcached.sh +++ /dev/null @@ -1,88 +0,0 @@ -memcached_java_lib_url=http://img.whalin.com/memcached/jdk5/log4j/java_memcached-release_2.0.1.tar.gz - -## $1: root directory of the publication -function memcached_create_publication_nursery_component() { - if [ ! -d "$1" ]; then - log $1 "doesn't exist" - return - fi - - log "Adding memcached wrapper to PresenationArticle in $1 ..." - local dir=$1/webapp/WEB-INF/localconfig/neo/xredsys/presentation/cache - make_dir $dir - local file=$dir/PresentationArticleCache.properties - sed -i "s#\$class=.*#\$class=neo.util.cache.Memcached#g" $file -} - -function install_memcached() -{ - if [ $on_debian_or_derivative ]; then - install_packages_if_missing "memcached" - fi - - run $download_dir - run wget $wget_opts $memcached_java_lib_url - local name=$(get_base_dir_from_bundle $memcached_java_lib_url) - run cp $name/$name.jar $assemblytool_home/lib - - memcached_set_up_common_nursery - - log "Configuring all publications for using memcached ..." - for el in $assemblytool_home/publications/*.properties 2>/dev/null; do - local publication=$(basename $el .properties) - - if [[ $appserver == "tomcat" ]]; then - dir=$tomcat_base/webapps/$publication - make_dir $dir - memcached_create_publication_nursery_component $dir - fi - done - - # fixing the deployed publications on host - if [[ $appserver == "tomcat" ]]; then - for el in $( - find $tomcat_base/webapps/ \ - -mindepth 1 \ - -maxdepth 1 \ - -type d | \ - egrep -v "solr|webservice|escenic|escenic-admin|indexer-webservice" | \ - egrep -v "indexer-webapp|studio" - ); do - memcached_create_publication_nursery_component $el - done - - fi - - # TODO inform the user that he/she might want to do tihs in the - # publication tree as well. - assemble_deploy_and_restart_type_p -} - -function memcached_set_up_common_nursery() { - local dir=$common_nursery_dir/com/danga - make_dir $dir - cat > $dir/SockIOPool <> $common_nursery_dir/Initial.properties < Date: Fri, 24 Feb 2012 17:27:19 +0800 Subject: [PATCH 0340/2585] - changed the sourcing of modules in a loop to make debugging easier. If one source fails, the others might fail too, obscure the error hunting when just doing source *.sh --- usr/sbin/ece-install | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index bb256dc7..57efe798 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -115,7 +115,10 @@ function init() { done # load ece-install modules - source $dir/ece-install.d/*.sh + for el in $dir/ece-install.d/*.sh; do + print "Loading $(basename $0) module:" $el + source $el + done else echo "I cannot find $(basename $0)'s dependencies, exiting :-(" exit 1 From 35349fcf455407f8667792144c128d4b48e9a0ea Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 24 Feb 2012 17:27:47 +0800 Subject: [PATCH 0341/2585] - changed print to log --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 57efe798..f42f9042 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -116,7 +116,7 @@ function init() { # load ece-install modules for el in $dir/ece-install.d/*.sh; do - print "Loading $(basename $0) module:" $el + log "Loading $(basename $0) module:" $el source $el done else From 2decf48826d5e24360714484abe1747159b25218 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 27 Feb 2012 13:16:23 +0800 Subject: [PATCH 0342/2585] - new feature: installing the Percona database now works on CentOS - moved database specific out into its own ece-install module. --- usr/sbin/ece-install | 86 +-------------- .../ece-scripts/ece-install.d/database.sh | 102 ++++++++++++++++++ 2 files changed, 103 insertions(+), 85 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/database.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index f42f9042..b326c09d 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1388,6 +1388,7 @@ EOF elif [ -d $escenic_root_dir/widget-framework-core-* ]; then print_and_log "Basing ${publication_name}.war on Widget Framework Demo ..." run cd $escenic_root_dir/widget-framework-core-*/publications/demo-core + assert_pre_requisite mvn run mvn $maven_opts package run cp target/demo-core-*.war ${publication_war} else @@ -1709,91 +1710,6 @@ function add_next_step() fi } -## $1: optional parameter, binaries_only. If passed, $1=binaries_only, -## the ECE DB schema is not set up. -function install_database_server() -{ - print_and_log "Installing the database server on $HOSTNAME ..." - - source $(dirname $0)/drop-and-create-ecedb - - if [ $on_debian_or_derivative -eq 1 ]; then - - code_name=$(lsb_release -s -c) - - supported_code_name=0 - supported_list="lenny squeeze hardy lucid maverick" - for el in $supported_list; do - if [ $code_name = $el ]; then - supported_code_name=1 - fi - done - - # some how, this is to install Percona 5.5 - if [ -e /var/lib/mysql/debian-*.flag ]; then - run rm /var/lib/mysql/debian-*.flag - fi - - if [ $supported_code_name -eq 1 ]; then - print_and_log "Installing the Percona database ..." - - if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then - run gpg --keyserver hkp://keys.gnupg.net \ - --recv-keys 1C4CBDCDCD2EFD2A - - # There has been twice now, during six months, that - # the key cannot be retrieved from - # keys.gnupg.net. Therefore, we're checking if it - # failed and if yes, force the package installation. - if [ $? -gt 0 ]; then - s="Failed retrieving the Percona key from keys.gnupg.net" - print_and_log $s - s="Will install the Percona packages without the GPG key" - print_and_log $s - force_packages=1 - else - gpg --armor \ - -a \ - --export 1C4CBDCDCD2EFD2A | \ - apt-key add - \ - 1>>$log 2>>$log - fi - - run apt-get update - fi - - add_apt_source "deb http://repo.percona.com/apt ${code_name} main" - packages="percona-server-server percona-server-client" - install_packages_if_missing $packages - force_packages=0 - else - print_and_log "The Percona APT repsository doesn't have packages" - print_and_log "for your Debian (or derivative) version with code" - print_and_log "name $code_name. " - print_and_log "I will use vanilla MySQL instead." - - packages="mysql-server mysql-client" - fi - elif [ $on_redhat_or_derivative -eq 1 ]; then - packages="mysql mysql-server" - fi - - install_packages_if_missing $packages - - if [ $on_redhat_or_derivative -eq 1 ]; then - run chkconfig --level 35 mysqld on - run /etc/init.d/mysqld start - fi - - assert_pre_requisite mysql - - if [ -z "$1" ]; then - download_escenic_components_p - set_up_engine_and_plugins - set_up_ecedb - fi -} - # $1 is the default instance name, the calee is responsible for # setting this. function ask_for_instance_name() diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh new file mode 100644 index 00000000..a660d074 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -0,0 +1,102 @@ +percona_rpm_release_version=0.0-1 +percona_rpm_release_package_name=percona-release-${percona_rpm_release_version} +percona_rpm_release_url=http://www.percona.com/downloads/percona-release/$percona_rpm_release_package_name.x86_64.rpm +if [[ $(uname -m) != "x86_64" ]]; then + percona_rpm_release_url=http://www.percona.com/downloads/percona-release/$percona_rpm_release_package_name.i386.rpm +fi + +## $1: optional parameter, binaries_only. If passed, $1=binaries_only, +## the ECE DB schema is not set up. +function install_database_server() +{ + print_and_log "Installing the database server on $HOSTNAME ..." + + source $(dirname $0)/drop-and-create-ecedb + + if [ $on_debian_or_derivative -eq 1 ]; then + + code_name=$(lsb_release -s -c) + + supported_code_name=0 + supported_list="lenny squeeze hardy lucid maverick" + for el in $supported_list; do + if [ $code_name = $el ]; then + supported_code_name=1 + fi + done + + # some how, this is to install Percona 5.5 + if [ -e /var/lib/mysql/debian-*.flag ]; then + run rm /var/lib/mysql/debian-*.flag + fi + + if [ $supported_code_name -eq 1 ]; then + print_and_log "Installing the Percona database ..." + + if [ $(apt-key list| grep CD2EFD2A | wc -l) -lt 1 ]; then + run gpg --keyserver hkp://keys.gnupg.net \ + --recv-keys 1C4CBDCDCD2EFD2A + + # There has been twice now, during six months, that + # the key cannot be retrieved from + # keys.gnupg.net. Therefore, we're checking if it + # failed and if yes, force the package installation. + if [ $? -gt 0 ]; then + s="Failed retrieving the Percona key from keys.gnupg.net" + print_and_log $s + s="Will install the Percona packages without the GPG key" + print_and_log $s + force_packages=1 + else + gpg --armor \ + -a \ + --export 1C4CBDCDCD2EFD2A | \ + apt-key add - \ + 1>>$log 2>>$log + fi + + run apt-get update + fi + + add_apt_source "deb http://repo.percona.com/apt ${code_name} main" + packages="percona-server-server percona-server-client" + install_packages_if_missing $packages + force_packages=0 + else + print_and_log "The Percona APT repsository doesn't have packages" + print_and_log "for your Debian (or derivative) version with code" + print_and_log "name $code_name. " + print_and_log "I will use vanilla MySQL instead." + + packages="mysql-server mysql-client" + fi + elif [ $on_redhat_or_derivative -eq 1 ]; then + print_and_log "Installing the Percona database ..." + + if [ $(rpm -qa | grep $percona_rpm_release_package_name | wc -l) -lt 1 ]; then + run rpm -Uhv $percona_rpm_release_url + fi + + packages=" + Percona-Server-shared-compat + Percona-Server-server-55 + Percona-Server-client-55" + fi + + install_packages_if_missing $packages + + if [ $on_redhat_or_derivative -eq 1 ]; then + run chkconfig --level 35 mysql on + run /etc/init.d/mysql start + fi + + assert_pre_requisite mysql + assert_pre_requisite mysqld + + if [ -z "$1" ]; then + download_escenic_components + set_up_engine_and_plugins + set_up_ecedb + fi +} + From a74ba8646021cce088aef46fcd4e6d8076484e3a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 27 Feb 2012 13:23:11 +0800 Subject: [PATCH 0343/2585] - added "tested operating systems" section --- usr/share/doc/escenic/ece-install-guide.org | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 03823356..67a40e95 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -6,12 +6,12 @@ the latest Ubuntu LTS or most other Debian based operating systems, but it should work without any problems on any Unix like system that has BASH version 4.2 or newer. -*** Debian based operating systems +** Debian based operating systems If running on a Debian based operating system, ece-install will take care of installing all required third party software, such as database & caching server. -*** Other GNU/Linux and Unix systems +** Other GNU/Linux and Unix systems If you're installing on another Linux or Unix system, you must first make sure that these 3rd party components are installed. ece-install will tell you which ones it cannot find and tell you to install these, @@ -21,6 +21,14 @@ You may also look inside the ece-install script for all calls to the 'assert\_pre\_prequesite' method and you'll get a list of the binaries ece-install assumes are present. +** Tested Operating Systems +| Operating system | Status | Date updated | +|----------------------+------------------------------------------------------------+------------------| +| Debian 6.0 (squeeze) | Everything is 100% automatic | 2012-02-27 13:18 | +| Ubuntu LTS 10.04 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | +| Ubuntu 11.10 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | +| CentOS 6.2 | Sun Java, Cache server, APR & monitoring are not automatic | 2012-02-27 13:21 | + * A Note on Running ece-install On Non-GNU/Linux Systems Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp & tar. If you're running a system which has a different set of core From cfe932343a311224d5dcc9966f0731d657389d5b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 27 Feb 2012 18:57:52 +0800 Subject: [PATCH 0344/2585] - new feature: automatic installation of Sun/Oracle Java on CentOS - moved installation of java to separate ece-install module --- usr/sbin/ece-install | 28 ++----- .../escenic/ece-scripts/ece-install.d/java.sh | 79 +++++++++++++++++++ 2 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/java.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index b326c09d..165ae5e4 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -84,10 +84,14 @@ http://technet.escenic.com/downloads/release/53/community-engine-3.6.1.0.zip http://technet.escenic.com/downloads/release/53/dashboard-1.0.0.0.zip http://technet.escenic.com/downloads/release/53/engine-5.3.4.7.zip http://technet.escenic.com/downloads/release/53/forum-3.0.0.0.zip +http://technet.escenic.com/downloads/release/53/geocode-dist-2.3.1.0.zip http://technet.escenic.com/downloads/release/53/inpage-1.3.0.0.zip http://technet.escenic.com/downloads/release/53/lucy-dist-4.1.6.0.zip http://technet.escenic.com/downloads/release/53/menu-editor-dist-2.0.6.0.zip +http://technet.escenic.com/downloads/release/53/mobile-expansion-3.0.0.0.zip http://technet.escenic.com/downloads/release/53/poll-2.1.3.0.zip +http://technet.escenic.com/downloads/release/53/revision-history-dist-1.0.0.1.zip +http://technet.escenic.com/downloads/release/53/section-feed-dist-2.0.4.0.zip http://technet.escenic.com/downloads/release/53/xml-editor-dist-2.1.0.0.zip " wf_download_list=" @@ -730,27 +734,6 @@ function set_ece_instance_conf() set_conf_file_value $1 $2 $instance_conf_file } -function create_java_deb_packages_and_repo() -{ - print $(lsb_release -i | cut -d':' -f2) \ - $(lsb_release -r | cut -d':' -f2) \ - "doesn't have official Sun/Oracle Java packages," - - local tmp_dir=$(mktemp -d) - $( - run cd $tmp_dir - run git clone https://github.com/flexiondotorg/oab-java6.git - run cd oab-java6 - run bash oab-java6.sh - run rm -rf $tmp_dir - ) & - show_pulse $! "I'm creating packages & local repo for you" - - add_next_step "Local APT repository with Sun/Oracle Java packages" - add_next_step "has been installed at /var/local/oab/deb and added" - add_next_step "to your APT system with /etc/apt/sources.list.d/oab.list" -} - # Installs third party packages needed by the ECE (i.e. Java related). # Also see install_common_os_packages for packages common to all # servers in the architecture. @@ -818,7 +801,10 @@ function install_ece_third_party_packages mysql-connector-java wget " + # TODO no tomcat APR wrappers in official repositories + + install_sun_java_on_redhat fi install_packages_if_missing $packages diff --git a/usr/share/escenic/ece-scripts/ece-install.d/java.sh b/usr/share/escenic/ece-scripts/ece-install.d/java.sh new file mode 100644 index 00000000..95d15052 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/java.sh @@ -0,0 +1,79 @@ +sun_java_bin_url=http://download.oracle.com/otn-pub/java/jdk/6u31-b04/jdk-6u31-linux-i586.bin +if [[ $(uname -m) == "x86_64" ]]; then + sun_java_bin_url=http://download.oracle.com/otn-pub/java/jdk/6u31-b04/jdk-6u31-linux-x64.bin +fi + +function create_java_deb_packages_and_repo() { + print $(lsb_release -i | cut -d':' -f2) \ + $(lsb_release -r | cut -d':' -f2) \ + "doesn't have official Sun/Oracle Java packages," + + local tmp_dir=$(mktemp -d) + $( + run cd $tmp_dir + run git clone https://github.com/flexiondotorg/oab-java6.git + run cd oab-java6 + run bash oab-java6.sh + run rm -rf $tmp_dir + ) & + show_pulse $! "I'm creating packages & local repo for you" + + add_next_step "Local APT repository with Sun/Oracle Java packages" + add_next_step "has been installed at /var/local/oab/deb and added" + add_next_step "to your APT system with /etc/apt/sources.list.d/oab.list" +} + + +function install_sun_java_on_redhat() { + if [[ $(java -version 2>&1 | grep HotSpot | wc -l) -gt 0 ]]; then + print_and_log "Sun Java is already installed on $HOSTNAME" + return + fi + + print_and_log "Downloading Sun Java from download.oracle.com ..." + run cd $download_dir + run wget $wget_opts $sun_java_bin_url + + # calculating start and stop offset from where to extract the zip + # from the java data blob. calculation taken from + # git://github.com/rraptorr/sun-java6.git + local file_name=$(basename $sun_java_bin_url) + local binsize=$(wc -c $file_name | awk '{print $1}'); + local zipstart=$(unzip -ql $file_name 2>&1 >/dev/null | \ + sed -n -e 's/.* \([0-9][0-9]*\) extra bytes.*/\1/p'); + tail -c $(expr $binsize - $zipstart) $file_name > tmp-jdk.zip + + run cd /opt + run unzip -q -o $download_dir/tmp-jdk.zip + local latest_jdk=$(find . -maxdepth 1 -type d -name "jdk*" | sort -r | head -1) + run rm -f /opt/jdk + run ln -s $latest_jdk jdk + + # generate jar files from the .pack files + for el in $(find /opt/jdk/ -name "*.pack"); do + file_name=$(basename $el .pack) + local dir=$(dirname $el) + run /opt/jdk/bin/unpack200 $el $dir/$file_name.jar + done + + # update RedHat's alternatives system to use Sun Java as its + # default. + for el in java javac jar; do + if [ ! -e /usr/bin/$el ]; then + ln -s /usr/bin/$el /etc/alternatives/$el + fi + alternatives --set $el /opt/jdk/bin/$el + done + + # setting java_home to the newly installed location + java_home=/opt/jdk + + local version=$(java -version 2>&1 | grep version | cut -d'"' -f2) + print_and_log "Sun Java $version is now installed in /opt/jdk" + + add_next_step "By using Sun Java, you must accept this license: " + add_next_step "http://www.oracle.com/technetwork/java/javase/terms/license/" +} + + + From 5841be9935325d17f8f8403f07f68f9b2089ea36 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 11:39:36 +0800 Subject: [PATCH 0345/2585] - updated the download list with the latest from technet.escenic.com. For the curious, this is how I got the list to update: url=http://technet.escenic.com/engine/53/technotes/article5535.ece list=$( curl -s -u user:pass $url | \ grep .zip | \ sed "s#.*. The reason for this, is that CentOS/REHL doesn't add an entry for the host name in /etc/hosts (even when using the admin tools to change the hostname). --- .../ece-scripts/ece-install.d/cache-server.sh | 163 ++++++++++-------- 1 file changed, 88 insertions(+), 75 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh index d54319e3..3fff7e19 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh @@ -1,86 +1,99 @@ +# ece-install module for installing the cache server + +varnish_redhat_rpm_url=http://repo.varnish-cache.org/redhat/varnish-3.0/el5/noarch/varnish-release-3.0-1.noarch.rpm + function install_cache_server() { - print_and_log "Installing a caching server on $HOSTNAME ..." - - if [[ $on_debian_or_derivative -eq 1 && - $(apt-key list | grep varnish-software.com | wc -l) -eq 0 ]]; then - curl ${curl_opts} \ - http://repo.varnish-cache.org/debian/GPG-key.txt \ - 2>> $log | \ - apt-key add - \ - 1>>$log 2>>$log - run apt-get update - fi - - code_name=$(lsb_release -s -c) + print_and_log "Installing a caching server on $HOSTNAME ..." + + if [[ $on_debian_or_derivative -eq 1 && + $(apt-key list | grep varnish-software.com | wc -l) -eq 0 ]]; then + curl ${curl_opts} \ + http://repo.varnish-cache.org/debian/GPG-key.txt \ + 2>> $log | \ + apt-key add - \ + 1>>$log 2>>$log + run apt-get update + code_name=$(lsb_release -s -c) supported_code_name=0 + # list taken from http://repo.varnish-cache.org/debian/dists/ supported_list="lenny squeeze hardy lucid" for el in $supported_list; do - if [ $code_name = $el ]; then - supported_code_name=1 - fi + if [ $code_name = $el ]; then + supported_code_name=1 + fi done if [ $supported_code_name -eq 1 -a $on_debian -eq 1 ]; then - add_apt_source "deb http://repo.varnish-cache.org/debian/ $(lsb_release -s -c) varnish-3.0" + add_apt_source "deb http://repo.varnish-cache.org/debian/ $(lsb_release -s -c) varnish-3.0" elif [ $supported_code_name -eq 1 -a $on_ubuntu -eq 1 ]; then - add_apt_source "deb http://repo.varnish-cache.org/ubuntu/ $(lsb_release -s -c) varnish-3.0" - fi - - install_packages_if_missing varnish - assert_pre_requisite varnishd - - if [ $fai_enabled -eq 0 ]; then - print "You must now list your backend servers." - print "These must be host names (not IPs) and must all be resolvable" - print "by your cache host ($HOSTNAME), preferably from /etc/hosts" - print "Seperate the entries with a space. e.g.: app1:8080 app2:8080." - print "Press ENTER to accept the default: ${HOSTNAME}:${appserver_port}" - echo -n "Your choice [${HOSTNAME}:${appserver_port}]> " - read backend_servers - else - backend_servers=${fai_cache_backends} - fi - - if [ -z "$backend_servers" ]; then - backend_servers="${HOSTNAME}:${appserver_port}" + add_apt_source "deb http://repo.varnish-cache.org/ubuntu/ $(lsb_release -s -c) varnish-3.0" fi - - set_up_varnish $backend_servers - - add_next_step "Cache server is up and running at http://${HOSTNAME}:80/" + + elif [[ $on_redhat_or_derivative -eq 1 && + $(rpm -qa | grep varnish-release | wc -l) -lt 1 ]]; then + print "Installing the Varnish repository RPM" + run rpm --nosignature -i $varnish_redhat_rpm_url + fi + + install_packages_if_missing varnish + assert_pre_requisite varnishd + + if [ $fai_enabled -eq 0 ]; then + print "You must now list your backend servers." + print "These must be host names (not IPs) and must all be resolvable" + print "by your cache host ($HOSTNAME), preferably from /etc/hosts" + print "Seperate the entries with a space. e.g.: app1:8080 app2:8080." + print "Press ENTER to accept the default: ${HOSTNAME}:${appserver_port}" + echo -n "Your choice [${HOSTNAME}:${appserver_port}]> " + read backend_servers + else + backend_servers=${fai_cache_backends} + fi + + if [ -z "$backend_servers" ]; then + backend_servers="localhost:${appserver_port}" + fi + + set_up_varnish $backend_servers + + add_next_step "Cache server is up and running at http://${HOSTNAME}:80/" } function set_up_varnish() { - print_and_log "Setting up Varnish to match your environment ..." - run /etc/init.d/varnish stop + print_and_log "Setting up Varnish to match your environment ..." + run /etc/init.d/varnish stop # we need to swap standard err and standard out here as varnishd # -V for some reason writes to standard error. - using_varnish_3=$(varnishd -V 3>&1 1>&2 2>&3 | grep varnish-3 | wc -l) - - local file=/etc/default/varnish - sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file - exit_on_error "sed on $file" - - cat > /etc/varnish/default.vcl <&1 1>&2 2>&3 | grep varnish-3 | wc -l) + + local file=/etc/default/varnish + if [ $on_redhat_or_derivative -eq 1 ]; then + file=/etc/sysconfig/varnish + fi + + sed -i -e 's/6081/80/g' -e 's/^START=no$/START=yes/' $file + exit_on_error "sed on $file" + + cat > /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < 0) { EOF - if [ $using_varnish_3 -gt 0 ]; then - cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < Date: Tue, 28 Feb 2012 12:26:30 +0800 Subject: [PATCH 0347/2585] - persnickety: indentation of comment --- usr/share/escenic/ece-scripts/common-bashing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh index d963aec0..3e6e4fdf 100644 --- a/usr/share/escenic/ece-scripts/common-bashing.sh +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -55,7 +55,7 @@ function print_and_log() { function log_call_stack() { log "Call stack (top most is the last one, main is the first):" - # skipping i=0 as this is log_call_stack itself + # skipping i=0 as this is log_call_stack itself for ((i = 1; i < ${#FUNCNAME[@]}; i++)); do echo -n ${BASH_SOURCE[$i]}:${BASH_LINENO[$i-1]}:${FUNCNAME[$i]}"()" >> $log if [ -e ${BASH_SOURCE[$i]} ]; then From f35df599cf89d27c3174312e09eaac9502c91351 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 12:27:11 +0800 Subject: [PATCH 0348/2585] - fix: having the JDK download in the $dowload_tmp confused ece-install (issue only on CentOS/REHL) --- usr/sbin/ece-install | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c5097fcb..835ff593 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -143,7 +143,9 @@ function download_escenic_components() return fi - cd $download_dir + print "Downloading software from technet.escenic.com ..." + + run cd $download_dir for el in $technet_download_list; do if [ -e $(basename $el) ]; then continue @@ -410,6 +412,8 @@ function set_up_engine_and_plugins() continue elif [ $(basename $el | grep ^assemblytool-.*.zip | wc -l) -gt 0 ]; then continue + elif [ $(basename $el | grep ^jdk-.*.zip | wc -l) -gt 0 ]; then + continue fi run unzip -q -u $el @@ -2637,25 +2641,25 @@ EOF function add_server_to_runlevels() { - # no need to add init.d scripts to the runlevel(s) for these - # profiles - if [ $install_profile_number -eq $PROFILE_WIDGET_FRAMEWORK -o \ - $install_profile_number -eq $PROFILE_DB_SERVER -o \ - $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then - return - fi + # no need to add init.d scripts to the runlevel(s) for these + # profiles + if [ $install_profile_number -eq $PROFILE_WIDGET_FRAMEWORK -o \ + $install_profile_number -eq $PROFILE_DB_SERVER -o \ + $install_profile_number -eq $PROFILE_CACHE_SERVER ]; then + return + fi - if [ $on_debian_or_derivative -eq 1 ]; then - print_and_log "Adding the ece init.d script to the default run levels ..." - run update-rc.d ece defaults - else - add_next_step "Remember to add /etc/init.d/ece to the desired "\ -"run levels." - # TODO add init.d to the default runlevels, for other - # distributions too: - # - RedHat/chekcconfig chkconfig --level 35 ece on - # - Gentoo: rc-update add ece default - fi + if [ $on_debian_or_derivative -eq 1 ]; then + print_and_log "Adding the ece init.d script to the default run levels ..." + run update-rc.d ece defaults + elif [ $on_redhat_or_derivative -eq 1 ]; then + run chkconfig --level 35 ece on + else + add_next_step "Remember to add /etc/init.d/ece to the desired run levels." + # TODO add init.d to the default runlevels, for other + # distributions too: + # - Gentoo: rc-update add ece default + fi } function restore_from_backup() From 9a8e8168138f77a5430f190fef0aaf582fce957f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 12:27:41 +0800 Subject: [PATCH 0349/2585] - minor: used variable instead of hard coded string for temporary JDK bundle. --- usr/share/escenic/ece-scripts/ece-install.d/java.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/java.sh b/usr/share/escenic/ece-scripts/ece-install.d/java.sh index 95d15052..0048c349 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/java.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/java.sh @@ -37,14 +37,15 @@ function install_sun_java_on_redhat() { # calculating start and stop offset from where to extract the zip # from the java data blob. calculation taken from # git://github.com/rraptorr/sun-java6.git + local tmp_jdk=jdk-tmp.zip local file_name=$(basename $sun_java_bin_url) local binsize=$(wc -c $file_name | awk '{print $1}'); local zipstart=$(unzip -ql $file_name 2>&1 >/dev/null | \ sed -n -e 's/.* \([0-9][0-9]*\) extra bytes.*/\1/p'); - tail -c $(expr $binsize - $zipstart) $file_name > tmp-jdk.zip + tail -c $(expr $binsize - $zipstart) $file_name > $tmp_jdk run cd /opt - run unzip -q -o $download_dir/tmp-jdk.zip + run unzip -q -o $download_dir/$tmp_jdk local latest_jdk=$(find . -maxdepth 1 -type d -name "jdk*" | sort -r | head -1) run rm -f /opt/jdk run ln -s $latest_jdk jdk From e9bb4e56f48c804499d7e10900cfdc876d7bc195 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 12:38:09 +0800 Subject: [PATCH 0350/2585] - updated tested operating systems - generated new markup --- usr/share/doc/escenic/ece-install-guide.html | 46 +++++++++++++++----- usr/share/doc/escenic/ece-install-guide.org | 12 ++--- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index 98051c42..e7ab8faf 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ - + @@ -230,11 +230,9 @@

    Table of Contents

    • 1 Supported Operating Systems
    • 2 A Note on Running ece-install On Non-GNU/Linux Systems
    • @@ -319,14 +317,14 @@

      1 Supported Operating Syste

      For the best experience, run this script on Debian Squeeze (or newer), the latest Ubuntu LTS or most other Debian based operating systems, but it should work without any problems on any Unix like system that -has a recent version of BASH. +has BASH version 4.2 or newer.

      -
      -

      1.1 Debian based operating systems

      -
      +
      +

      1.1 Debian based operating systems

      +

      If running on a Debian based operating system, ece-install will take care of installing all required third party software, such as database @@ -336,9 +334,9 @@

      1.1 Debian based operatin

      -
      -

      1.2 Other GNU/Linux and Unix systems

      -
      +
      +

      1.2 Other GNU/Linux and Unix systems

      +

      If you're installing on another Linux or Unix system, you must first make sure that these 3rd party components are installed. ece-install @@ -350,6 +348,30 @@

      1.2 Other GNU/Linux and U 'assert_pre_prequesite' method and you'll get a list of the binaries ece-install assumes are present.

      +

      + +
      + +
      +

      1.3 Tested Operating Systems

      +
      + + + ++ + + + + + + + + + +
      Operating systemStatusDate updated
      Debian 6.0 (squeeze)Everything is 100% automatic2012-02-27 13:18
      Ubuntu LTS 10.04Everything except Tomcat/APR wrappers are automatic2012-02-27 13:19
      Ubuntu 11.10Everything except Tomcat/APR wrappers are automatic2012-02-27 13:19
      CentOS 6.2APR & monitoring are not automatic2012-02-28 12:37
      + +
      @@ -1754,7 +1776,7 @@

      15 Example Output FAI


      Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2012-02-17 12:59:41 CST + Date: 2012-02-28 12:37:37 CST

      diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 67a40e95..07f0d010 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -22,12 +22,12 @@ You may also look inside the ece-install script for all calls to the ece-install assumes are present. ** Tested Operating Systems -| Operating system | Status | Date updated | -|----------------------+------------------------------------------------------------+------------------| -| Debian 6.0 (squeeze) | Everything is 100% automatic | 2012-02-27 13:18 | -| Ubuntu LTS 10.04 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | -| Ubuntu 11.10 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | -| CentOS 6.2 | Sun Java, Cache server, APR & monitoring are not automatic | 2012-02-27 13:21 | +| Operating system | Status | Date updated | +|----------------------+-----------------------------------------------------+------------------| +| Debian 6.0 (squeeze) | Everything is 100% automatic | 2012-02-27 13:18 | +| Ubuntu LTS 10.04 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | +| Ubuntu 11.10 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | +| CentOS 6.2 | APR & monitoring are not automatic | 2012-02-28 12:37 | * A Note on Running ece-install On Non-GNU/Linux Systems Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp From 8d6d1b0ecc74b3338d446e48d41142aa184f4aa4 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 12:41:30 +0800 Subject: [PATCH 0351/2585] - added section: Supported Operating Systems/RedHat based operating systems. --- usr/share/doc/escenic/ece-install-guide.org | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 07f0d010..e5df4b18 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -11,6 +11,15 @@ If running on a Debian based operating system, ece-install will take care of installing all required third party software, such as database & caching server. +** RedHat based operating systems +ece-install constantly gets better support for automatic installation +on RedHat based operating systems. However, since the public RedHat +repositories are significantly smaller than the Debian ones, there +will often be the case that you'll have to do some manually +installation of 3rd party dependencies prior to running +ece-install. Don't despair though, ece-install will tell you what you +need to do. + ** Other GNU/Linux and Unix systems If you're installing on another Linux or Unix system, you must first make sure that these 3rd party components are installed. ece-install From 710045a92fc1cef30815fd5088a3ed07e6234a7f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 15:45:49 +0800 Subject: [PATCH 0352/2585] - moved RMI hub code to its own module --- .../ece-scripts/ece-install.d/rmi-hub.sh | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/rmi-hub.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/rmi-hub.sh b/usr/share/escenic/ece-scripts/ece-install.d/rmi-hub.sh new file mode 100644 index 00000000..ffaa4cc6 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/rmi-hub.sh @@ -0,0 +1,31 @@ +# ece-install module for installing the RMI hub + +function install_rmi_hub() +{ + make_dir $escenic_conf_dir/rmi-hub + + run cp -r $escenic_root_dir/engine/contrib/rmi-hub/config/* \ + $escenic_conf_dir/rmi-hub/ + + hub_host=$HOSTNAME + file=$common_nursery_dir + file=$file/neo/io/managers/HubConnectionManager.properties + + make_dir $(basename $file) + set_conf_file_value hub \ + "rmi://${hub_host}:1099/hub/Hub" \ + $file + + cat > $common_nursery_dir/io/api/EventManager.properties <>$log 2>>$log +} + + From 4910132fb486ec3b510be37c6dbff7fdfb659f4b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 15:51:29 +0800 Subject: [PATCH 0353/2585] - added /etc/sysconfig to the list of directories potentially holding the init.d script's configuration. --- etc/init.d/ece | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/init.d/ece b/etc/init.d/ece index f4cce807..979bd279 100755 --- a/etc/init.d/ece +++ b/etc/init.d/ece @@ -46,11 +46,12 @@ ece_script=/usr/bin/ece # The values above may be overidden a file named the same as this # init.d script. This init.d configuration must also hold the # variables controlling which ECE instances to start. The list of -# locations per default caters (at least) for Debian & Gentoo based -# systems: +# locations per default caters (at least) for Debian, RedHat & Gentoo +# based systems: conf_file_location_list=" /etc/default /etc/conf.d +/etc/sysconfig " function read_conf_file() From 77bf552a00ed0757d42ac6da78082368319ff732 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 15:55:23 +0800 Subject: [PATCH 0354/2585] - moved EAE specific code to analysis-engine.sh - moved more ECE specific code to content-engine.sh - moved Icinga, Nagios & Munin specific code to monitoring.sh - moved solr & indexer specific code to search-server.sh --- usr/sbin/ece-install | 544 +----------------- .../ece-install.d/analysis-engine.sh | 67 +++ .../ece-install.d/content-engine.sh | 116 +++- .../ece-scripts/ece-install.d/memory-cache.sh | 12 +- .../ece-scripts/ece-install.d/monitoring.sh | 290 ++++++++++ .../ece-install.d/search-server.sh | 71 +++ 6 files changed, 540 insertions(+), 560 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/analysis-engine.sh create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/search-server.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 835ff593..8b1e1ee9 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -562,7 +562,7 @@ function set_up_ecedb() function set_up_basic_nursery_configuration() { - log "Setting up basic Nursery configuration ..." + print_and_log "Setting up basic Nursery configuration ..." # we always copy the default plugin configuration (even if there # is an archive) @@ -654,22 +654,24 @@ EOF function set_up_instance_specific_nursery_configuration() { - for el in $escenic_conf_dir/engine/instance/*; do - i=$(( i + 1 )) - if [ $(basename $el) = $instance_name ]; then - rmi_port="8${i}23" - run echo "port=$rmi_port" > $el/RMI.properties - fi - done + print_and_log "Setting up instance specific Nursery configuration ..." + + for el in $escenic_conf_dir/engine/instance/*; do + i=$(( i + 1 )) + if [ $(basename $el) = $instance_name ]; then + rmi_port="8${i}23" + run echo "port=$rmi_port" > $el/RMI.properties + fi + done - nursery_context=neo/io/managers/HubConnectionManager.properties - file=$escenic_conf_dir/engine/instance/$instance_name/$nursery_context - make_dir $(dirname $file) - + nursery_context=neo/io/managers/HubConnectionManager.properties + file=$escenic_conf_dir/engine/instance/$instance_name/$nursery_context + make_dir $(dirname $file) + # we don't touch it if the file already exists. - if [ ! -e $file ]; then - run echo "hostname=$HOSTNAME" >> $file - fi + if [ ! -e $file ]; then + run echo "hostname=$HOSTNAME" >> $file + fi } function set_up_proper_logging_configuration() @@ -1438,45 +1440,6 @@ function set_up_user_environment() fi } -function set_up_solr() -{ - run_hook set_up_solr.preinst - - print_and_log "Setting up solr ..." - if [ ! -d $escenic_conf_dir/solr ]; then - if [ $(is_using_conf_archive) -eq 1 ]; then - print_and_log "Using the supplied Solr configuration from" - print_and_log "bundle: $ece_instance_conf_archive" - local a_tmp_dir=$(mktemp -d) - run cd $a_tmp_dir - run tar xzf $ece_instance_conf_archive engine/solr/conf - run cp -r engine/solr/conf $escenic_conf_dir/solr - run rm -r $a_tmp_dir - else - print_and_log "Installing default Solr conf shipped with ECE ..." - run cp -r $escenic_root_dir/engine/solr/conf $escenic_conf_dir/solr - fi - else - print_and_log "$escenic_conf_dir/solr already exists, not touching it." - fi - - make_dir $escenic_data_dir/solr/ - run cd $escenic_data_dir/solr/ - if [ ! -h conf ]; then - run ln -s $escenic_conf_dir/solr conf - fi - - local editorial_search_instance=${fai_search_for_editor-0} - local file=$escenic_conf_dir/solr/solrconfig.xml - if [ $editorial_search_instance -eq 1 ]; then - run sed -i "s#[0-9]*#5000#g" $file - else - run sed -i "s#[0-9]*#60000#g" $file - fi - - run_hook set_up_solr.postinst -} - # So far, I've used this method for copy/past-ing it into the shell # before running ece-insatll anew. It might be useful for its own # command later, though. @@ -1785,7 +1748,6 @@ function set_archive_files_depending_on_profile() fi } - # Returns 1 if we're installing the ECE instances from a provided EAR # file function is_installing_from_ear() @@ -1833,65 +1795,6 @@ function verify_that_files_exist_and_are_readable() done } -## $1= -function install_ece_instance() -{ - install_ece_third_party_packages - - ask_for_instance_name $1 - set_up_engine_directories - set_up_ece_scripts - - set_archive_files_depending_on_profile - - # most likely, the user is _not_ installing from archives (EAR + - # configuration bundle), hence the false test goes first. - if [ $(is_installing_from_ear) -eq 0 ]; then - download_escenic_components - check_for_required_downloads - set_up_engine_and_plugins - set_up_assembly_tool - else - verify_that_files_exist_and_are_readable \ - $ece_instance_ear_file \ - $ece_instance_conf_archive - fi - - set_up_basic_nursery_configuration - set_up_instance_specific_nursery_configuration - - set_up_app_server - set_up_proper_logging_configuration - - # We set a WAR white list for all profiles except all in one - if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE ]; then - file=$escenic_conf_dir/ece-${instance_name}.conf - print_and_log "Creating deployment white list in $file ..." - set_conf_file_value \ - deploy_webapp_white_list \ - $(get_deploy_white_list) \ - $file - fi - - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ - $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - assemble_deploy_and_restart_type - fi - - update_type_instances_to_start_up - set_conf_file_value ece_unix_user $ece_user /etc/default/ece - set_conf_file_value ece_unix_group $ece_group /etc/default/ece - - admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin/ - add_next_step "New ECE instance $instance_name installed." - add_next_step "Admin interface: $admin_uri" - add_next_step "View installed versions with:" \ - " ece -i $instance_name versions" - add_next_step "Type 'ece help' to see all the options of this script" - add_next_step "Read its guide: /usr/share/doc/escenic/ece-guide.txt" - add_next_step "/etc/default/ece lists all instances started at boot time" -} - function install_presentation_server() { print_and_log "Installing a presentation server on $HOSTNAME." @@ -2004,100 +1907,6 @@ function run_hook() fi } -function install_analysis_server() -{ - run_hook install_analysis_server.preinst - - print_and_log "Installing an analysis server on $HOSTNAME ..." - type=analysis - - install_ece_instance "analysis1" - - run cp ${escenic_root_dir}/analysis-engine-*/wars/*.war \ - ${tomcat_base}/webapps - - if [ -n "${fai_analysis_host}" ]; then - appserver_host=${fai_analysis_host} - fi - if [ -n "${fai_analysis_port}" ]; then - appserver_port=${fai_analysis_port} - fi - - # deploy the EAE WARs - run cp ${escenic_root_dir}/analysis-engine-*/wars/*.war \ - ${tomcat_base}/webapps - - set_correct_permissions - - local ece_command="ece -i ${instance_name} -t ${type} start" - su - $ece_user -c "$ece_command" 1>>$log 2>>$log - exit_on_error "su - $ece_user -c \"$ece_command\"" - - local seconds=3 - print_and_log "Waiting ${seconds} seconds for EAE to come up ..." - sleep 10 - - # EAE cannot handle .cfg files with quotes values (!) - dont_quote_conf_values=1 - - print_and_log "Configuring EAE Reports ..." - local file=${tomcat_base}/webapps/analysis-reports/WEB-INF/config/reports.cfg - set_conf_file_value queryServiceUrl \ - http://${appserver_host}:${appserver_port}/analysis-qs/QueryService \ - ${file} - - print_and_log "Configuring EAE Logger ..." - local file=${tomcat_base}/webapps/analysis-logger/WEB-INF/config/logger.cfg - set_conf_file_value databaseQueueManager.reinsertcount \ - 6 \ - ${file} - set_conf_file_value pageview.maintenance.cron.expr '0 0 4 * * ? *' $file - set_conf_file_value pageview.aggr.hour.cron.expr '0 10 * * * ?' $file - set_conf_file_value imageResponse false $file - set_conf_file_value pageview.maintenance.older.than.months 6 $file - set_conf_file_value pageview.aggr.day.older.than.days 7 $file - set_conf_file_value pageview.aggr.day.cron.expr '0 0 5 * * ? *' $file - set_conf_file_value pageview.aggr.hour.older.than.hours 2 $file - set_conf_file_value pageview.maintenance.older.than.days 0 $file - - # important to turn this off here, it's only for the EAE .cfg - # files, see above. - dont_quote_conf_values=0 - - # touching web.xml to trigger a re-deploy of the EAE Reports - # application. - run touch ${tomcat_base}/webapps/analysis-reports/WEB-INF/web.xml - run_hook install_analysis_server.postinst -} - -function install_rmi_hub() -{ - make_dir $escenic_conf_dir/rmi-hub - - run cp -r $escenic_root_dir/engine/contrib/rmi-hub/config/* \ - $escenic_conf_dir/rmi-hub/ - - hub_host=$HOSTNAME - file=$common_nursery_dir - file=$file/neo/io/managers/HubConnectionManager.properties - - make_dir $(basename $file) - set_conf_file_value hub \ - "rmi://${hub_host}:1099/hub/Hub" \ - $file - - cat > $common_nursery_dir/io/api/EventManager.properties <>$log 2>>$log -} - # reads the value of the desired setting from $conf_file # # parameters: $1 : the conf key, see ece-install-guide.txt for an @@ -2218,37 +2027,6 @@ EOF " Maven repo" } -function install_search_server() -{ - print_and_log "Installing a search server on $HOSTNAME ..." - type=search - install_ece_instance "search1" 0 - - # TODO update instead of append - local dir=$common_nursery_dir/com/escenic/framework/search/solr - make_dir $dir - echo "solrServerURI=http://${search_host}:${search_port}/solr" \ - >> $dir/SolrSearchEngine.properties - - dir=$common_nursery_dir/com/escenic/webservice/search - make_dir $dir - echo "solrURI=http://${search_host}:${search_port}/solr/select" \ - >> $dir/DelegatingSearchEngine.properties - - dir=$common_nursery_dir/com/escenic/lucy - make_dir $dir - echo "solrURI=http://${search_host}:${search_port}/solr" \ - >> $dir/LucySearchEngine.properties - - dir=$common_nursery_dir/com/escenic/forum/search/lucy - make_dir $dir - echo "solrURI=http://${search_host}:${search_port}/solr" \ - >> $dir/SearchEngine.properties - - set_up_solr - assemble_deploy_and_restart_type -} - # useful for development and test environments. function install_all_in_one_environment() { @@ -2351,294 +2129,6 @@ EOF fi } -function install_munin_gatherer() -{ - print_and_log "Installing a Munin gatherer on $HOSTNAME ..." - - if [ $on_debian_or_derivative -eq 1 ]; then - packages="munin" - install_packages_if_missing $packages - else - print_and_log "Munin gatherer installation not supported on your" - print_and_log "system :-( You will have to install it manually." - return - fi - - if [ $fai_enabled -eq 0 ]; then - print "Which nodes shall this Munin monitor gather from?" - print "Separate your hosts with a space, e.g.: 'editor01 db01 web01'" - echo -n "Your choice> " - read user_munin_nodes - - if [ -n "$user_munin_nodes" ]; then - node_list=$user_munin_nodes - fi - else - node_list=${fai_monitoring_munin_node_list} - fi - - if [ -z "$node_list" ]; then - return - fi - - for el in $node_list; do - print_and_log "Adding ${el} to the Munin gatherer on ${HOSTNAME} ..." - local file=/etc/munin/munin-conf.d/escenic.conf - cat >> $file <:, e.g.: fire:192.168.1.100 -function set_up_monitoring_host_def() -{ - local file=/etc/icinga/objects/${host_name}_icinga.cfg - if [[ $1 == $MONITORING_VENDOR_NAGIOS ]]; then - file=/etc/nagios3/conf.d/${host_name}_nagios2.cfg - fi - - local old_ifs=$IFS - IFS='#' - read host_name ip <<< "$2" - IFS=$old_ifs - - if [ $(grep "host_name $host_name" $file 2>/dev/null | wc -l) -gt 0 ]; then - print "$1 host" $host_name "already defined, skipping it." - return - fi - - # TODO add more services based on what kind of host it is. - cat >> $file < " - read user_monitoring_server - - if [ -n "$user_monitoring_server" ]; then - monitoring_server_ip=$user_monitoring_server - fi - fi - - if [ -z "$monitoring_server_ip" ]; then - monitoring_server_ip=$default_ip - fi - - if [ $on_debian_or_derivative -eq 1 ]; then - packages="munin-node munin-plugins-extra munin-java-plugins" - install_packages_if_missing $packages - else - print_and_log "Munin node installation not supported on your system" - print_and_log "You will have to install it manually." - return - fi - - if [ -n "$monitoring_server_ip" ]; then - escaped_munin_gather_ip=$(get_perl_escaped ${monitoring_server_ip}) - file=/etc/munin/munin-node.conf - cat >> $file <.pid. It's - # now -.pid - file=$escenic_run_dir/$type-${instance_name}.pid - if [ ! -e $file ]; then - run touch $file - fi - - # enabling the instance specific munin entries: - for el in /usr/share/munin/plugins/escenic_jstat_[a-z]*; do - run cd /etc/munin/plugins - make_ln $el - done - done - - # TODO in which version(s) of munin is this directory called - # client-conf.d? - file=/etc/munin/plugin-conf.d/munin-node - if [ -e $file ]; then - cat >> $file <>$log 2>>$log + exit_on_error "su - $ece_user -c \"$ece_command\"" + + local seconds=3 + print_and_log "Waiting ${seconds} seconds for EAE to come up ..." + sleep 10 + + # EAE cannot handle .cfg files with quotes values (!) + dont_quote_conf_values=1 + + print_and_log "Configuring EAE Reports ..." + local file=${tomcat_base}/webapps/analysis-reports/WEB-INF/config/reports.cfg + set_conf_file_value queryServiceUrl \ + http://${appserver_host}:${appserver_port}/analysis-qs/QueryService \ + ${file} + + print_and_log "Configuring EAE Logger ..." + local file=${tomcat_base}/webapps/analysis-logger/WEB-INF/config/logger.cfg + set_conf_file_value databaseQueueManager.reinsertcount \ + 6 \ + ${file} + set_conf_file_value pageview.maintenance.cron.expr '0 0 4 * * ? *' $file + set_conf_file_value pageview.aggr.hour.cron.expr '0 10 * * * ?' $file + set_conf_file_value imageResponse false $file + set_conf_file_value pageview.maintenance.older.than.months 6 $file + set_conf_file_value pageview.aggr.day.older.than.days 7 $file + set_conf_file_value pageview.aggr.day.cron.expr '0 0 5 * * ? *' $file + set_conf_file_value pageview.aggr.hour.older.than.hours 2 $file + set_conf_file_value pageview.maintenance.older.than.days 0 $file + + # important to turn this off here, it's only for the EAE .cfg + # files, see above. + dont_quote_conf_values=0 + + # touching web.xml to trigger a re-deploy of the EAE Reports + # application. + run touch ${tomcat_base}/webapps/analysis-reports/WEB-INF/web.xml + run_hook install_analysis_server.postinst +} diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index 28f2c0b2..16062922 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -1,39 +1,99 @@ +# ece-install module Content Engine specific code. + function get_deploy_white_list() { - local white_list="escenic-admin" - - if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER \ - -a -n "${publication_name}" ]; then - white_list="${white_list} ${publication_name} " - elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - white_list="${white_list} solr indexer-webapp" - elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - white_list="${white_list} "$(get_publication_short_name_list) - elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - white_list="${white_list} escenic studio indexer-webservice" - white_list="${white_list} "$(get_publication_short_name_list) - fi - - echo ${white_list} + local white_list="escenic-admin" + + if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER \ + -a -n "${publication_name}" ]; then + white_list="${white_list} ${publication_name} " + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + white_list="${white_list} solr indexer-webapp" + elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + white_list="${white_list} "$(get_publication_short_name_list) + elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + white_list="${white_list} escenic studio indexer-webservice" + white_list="${white_list} "$(get_publication_short_name_list) + fi + + echo ${white_list} } function get_publication_short_name_list() { - local short_name_list="" - - local publication_def_dir=${escenic_root_dir}/assemblytool/publications - if [ $(ls ${publication_def_dir} | grep .properties$ | wc -l) -eq 0 ]; then - echo ${short_name_list} - return - fi - - for el in $(find ${publication_def_dir} -maxdepth 1 -name "*.properties"); do - local short_name=$(basename $el .properties) - short_name_list="${short_name_list} ${short_name}" - done - + local short_name_list="" + + local publication_def_dir=${escenic_root_dir}/assemblytool/publications + if [ $(ls ${publication_def_dir} | grep .properties$ | wc -l) -eq 0 ]; then echo ${short_name_list} + return + fi + + for el in $(find ${publication_def_dir} -maxdepth 1 -name "*.properties"); do + local short_name=$(basename $el .properties) + short_name_list="${short_name_list} ${short_name}" + done + + echo ${short_name_list} } +## $1= +function install_ece_instance() +{ + install_ece_third_party_packages + + ask_for_instance_name $1 + set_up_engine_directories + set_up_ece_scripts + + set_archive_files_depending_on_profile + + # most likely, the user is _not_ installing from archives (EAR + + # configuration bundle), hence the false test goes first. + if [ $(is_installing_from_ear) -eq 0 ]; then + download_escenic_components + check_for_required_downloads + set_up_engine_and_plugins + set_up_assembly_tool + else + verify_that_files_exist_and_are_readable \ + $ece_instance_ear_file \ + $ece_instance_conf_archive + fi + + set_up_basic_nursery_configuration + set_up_instance_specific_nursery_configuration + + set_up_app_server + set_up_proper_logging_configuration + + # We set a WAR white list for all profiles except all in one + if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE ]; then + file=$escenic_conf_dir/ece-${instance_name}.conf + print_and_log "Creating deployment white list in $file ..." + set_conf_file_value \ + deploy_webapp_white_list \ + $(get_deploy_white_list) \ + $file + fi + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ + $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then + assemble_deploy_and_restart_type + install_memory_cache + fi + + update_type_instances_to_start_up + set_conf_file_value ece_unix_user $ece_user /etc/default/ece + set_conf_file_value ece_unix_group $ece_group /etc/default/ece + + admin_uri=http://$HOSTNAME:${appserver_port}/escenic-admin/ + add_next_step "New ECE instance $instance_name installed." + add_next_step "Admin interface: $admin_uri" + add_next_step "View installed versions with:" \ + " ece -i $instance_name versions" + add_next_step "Type 'ece help' to see all the options of this script" + add_next_step "Read its guide: /usr/share/doc/escenic/ece-guide.txt" + add_next_step "/etc/default/ece lists all instances started at boot time" +} diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh index 44bc6a57..2ff196cf 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -1,3 +1,5 @@ +# ece-install module for installing the memory cache + memcached_java_lib_url=http://img.whalin.com/memcached/jdk5/log4j/java_memcached-release_2.0.1.tar.gz ## $1: root directory of the publication @@ -12,15 +14,15 @@ function memcached_create_publication_nursery_component() { make_dir $dir local file=$dir/PresentationArticleCache.properties sed -i "s#\$class=.*#\$class=neo.util.cache.Memcached#g" $file + exit_on_error "sed on $file" } -function install_memcached() +function install_memory_cache() { - if [ $on_debian_or_derivative ]; then - install_packages_if_missing "memcached" - fi + install_packages_if_missing "memcached" + assert_pre_requisite memcached - run $download_dir + run cd $download_dir run wget $wget_opts $memcached_java_lib_url local name=$(get_base_dir_from_bundle $memcached_java_lib_url) run cp $name/$name.jar $assemblytool_home/lib diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh new file mode 100644 index 00000000..9fc01959 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -0,0 +1,290 @@ +# module for installing monitoring software, both server and client side. + +## Installs the Nagios monitoring server. +## $1 the nagios vendor/falvour, "nagios" and "icinga" are supported. +function install_nagios_monitoring_server() +{ + print "Installing an $1 server on $HOSTNAME ..." + local monitoring_vendor=$1 + + if [ $on_debian_or_derivative -eq 1 ]; then + if [[ $monitoring_vendor == $MONITORING_VENDOR_NAGIOS ]]; then + install_packages_if_missing apache2 nagios3 nagios-nrpe-plugin + else + install_packages_if_missing \ + apache2 icinga nagios-nrpe-plugin icinga-doc + fi + fi + + if [[ $monitoring_vendor == $MONITORING_VENDOR_ICINGA ]]; then + print "Setting user/pass for icinga admin ..." + local file=/etc/icinga/htpasswd.users + run htpasswd -b -c $file icingaadmin \ + ${fai_monitoring_admin_password-admin} + fi + + # enable remote commands + if [[ $monitoring_vendor == $MONITORING_VENDOR_NAGIOS ]]; then + local file=/etc/nagios3/nagios.cfg + else + local file=/etc/icinga/icinga.cfg + fi + + dont_quote_conf_values=1 + set_conf_file_value check_external_commands 1 $file + dont_quote_conf_values=0 + + for el in $fai_monitoring_host_list; do + set_up_monitoring_host_def $monitoring_vendor $el + done + + if [[ $monitoring_vendor == $MONITORING_VENDOR_NAGIOS ]]; then + file=/etc/nagios3/conf.d/hostgroups_nagios2.cfg + else + file=/etc/icinga/objects/hostgroups_icinga.cfg + fi + + set_up_monitoring_host_group \ + $file \ + "ece-hosts" \ + 'Hosts running one or more ECE' \ + ${fai_monitoring_ece_host_list} + + set_up_monitoring_host_group \ + $file \ + "search-hosts" \ + 'Hosts running search instance(s) (Solr + indexer)' \ + ${fai_monitoring_search_host_list} + + if [[ $monitoring_vendor == $MONITORING_VENDOR_NAGIOS ]]; then + run /etc/init.d/nagios3 restart + add_next_step "Icinga monitoring interface: http://${HOSTNAME}/nagios3" + else + run /etc/init.d/icinga restart + add_next_step "Icinga monitoring interface: http://${HOSTNAME}/icinga" + fi + + run /etc/init.d/apache2 reload + +} + +## Sets up the definition file for a given monitoring host. +## +## $1 nagios flavour/vendor +## $2 :, e.g.: fire:192.168.1.100 +function set_up_monitoring_host_def() +{ + local file=/etc/icinga/objects/${host_name}_icinga.cfg + if [[ $1 == $MONITORING_VENDOR_NAGIOS ]]; then + file=/etc/nagios3/conf.d/${host_name}_nagios2.cfg + fi + + local old_ifs=$IFS + IFS='#' + read host_name ip <<< "$2" + IFS=$old_ifs + + if [ $(grep "host_name $host_name" $file 2>/dev/null | wc -l) -gt 0 ]; then + print "$1 host" $host_name "already defined, skipping it." + return + fi + + # TODO add more services based on what kind of host it is. + cat >> $file < " + read user_monitoring_server + + if [ -n "$user_monitoring_server" ]; then + monitoring_server_ip=$user_monitoring_server + fi + fi + + if [ -z "$monitoring_server_ip" ]; then + monitoring_server_ip=$default_ip + fi + + if [ $on_debian_or_derivative -eq 1 ]; then + packages="munin-node munin-plugins-extra munin-java-plugins" + install_packages_if_missing $packages + else + print_and_log "Munin node installation not supported on your system" + print_and_log "You will have to install it manually." + return + fi + + if [ -n "$monitoring_server_ip" ]; then + escaped_munin_gather_ip=$(get_perl_escaped ${monitoring_server_ip}) + file=/etc/munin/munin-node.conf + cat >> $file <.pid. It's + # now -.pid + file=$escenic_run_dir/$type-${instance_name}.pid + if [ ! -e $file ]; then + run touch $file + fi + + # enabling the instance specific munin entries: + for el in /usr/share/munin/plugins/escenic_jstat_[a-z]*; do + run cd /etc/munin/plugins + make_ln $el + done + done + + # TODO in which version(s) of munin is this directory called + # client-conf.d? + file=/etc/munin/plugin-conf.d/munin-node + if [ -e $file ]; then + cat >> $file < " + read user_munin_nodes + + if [ -n "$user_munin_nodes" ]; then + node_list=$user_munin_nodes + fi + else + node_list=${fai_monitoring_munin_node_list} + fi + + if [ -z "$node_list" ]; then + return + fi + + for el in $node_list; do + print_and_log "Adding ${el} to the Munin gatherer on ${HOSTNAME} ..." + local file=/etc/munin/munin-conf.d/escenic.conf + cat >> $file <> $dir/SolrSearchEngine.properties + + dir=$common_nursery_dir/com/escenic/webservice/search + make_dir $dir + echo "solrURI=http://${search_host}:${search_port}/solr/select" \ + >> $dir/DelegatingSearchEngine.properties + + dir=$common_nursery_dir/com/escenic/lucy + make_dir $dir + echo "solrURI=http://${search_host}:${search_port}/solr" \ + >> $dir/LucySearchEngine.properties + + dir=$common_nursery_dir/com/escenic/forum/search/lucy + make_dir $dir + echo "solrURI=http://${search_host}:${search_port}/solr" \ + >> $dir/SearchEngine.properties + + set_up_solr + assemble_deploy_and_restart_type +} + +function set_up_solr() +{ + run_hook set_up_solr.preinst + + print_and_log "Setting up solr ..." + if [ ! -d $escenic_conf_dir/solr ]; then + if [ $(is_using_conf_archive) -eq 1 ]; then + print_and_log "Using the supplied Solr configuration from" + print_and_log "bundle: $ece_instance_conf_archive" + local a_tmp_dir=$(mktemp -d) + run cd $a_tmp_dir + run tar xzf $ece_instance_conf_archive engine/solr/conf + run cp -r engine/solr/conf $escenic_conf_dir/solr + run rm -r $a_tmp_dir + else + print_and_log "Installing default Solr conf shipped with ECE ..." + run cp -r $escenic_root_dir/engine/solr/conf $escenic_conf_dir/solr + fi + else + print_and_log "$escenic_conf_dir/solr already exists, not touching it." + fi + + make_dir $escenic_data_dir/solr/ + run cd $escenic_data_dir/solr/ + if [ ! -h conf ]; then + run ln -s $escenic_conf_dir/solr conf + fi + + local editorial_search_instance=${fai_search_for_editor-0} + local file=$escenic_conf_dir/solr/solrconfig.xml + if [ $editorial_search_instance -eq 1 ]; then + run sed -i "s#[0-9]*#5000#g" $file + else + run sed -i "s#[0-9]*#60000#g" $file + fi + + run_hook set_up_solr.postinst +} From 3da4a9fdc02a83d25aeaac455275de0eda4dc2c3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 18:55:43 +0800 Subject: [PATCH 0355/2585] - added documentation on the profile=analysis --- usr/share/doc/escenic/ece-install-guide.org | 99 ++++++++++++--------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index e5df4b18..46a7b633 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -293,6 +293,11 @@ what's available on the host where it's run): - ECE, cache and web server configuration - Escenic software +** Profile - Analysis Server +This profile will install the Escenic Analysis Engine and configure it +for production use with sane defaults. Be sure to use a different DB +than you use for ECE. + ** Running interactively *** Start ece-install and choose the Option, "Restore from backup" #+BEGIN_SRC text @@ -512,51 +517,59 @@ The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| Parameter | Default | Description | +| Parameter | Default | Description | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_db\_schema | ece5db | Useful for DB installation profile | -| fai\_db\_user | ece5user | Useful for DB installation profile | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | -| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | +| fai\_all\_install | 0 | Install all components on your server. | +| fai\_analysis\_db\_port | 3306 | For the EAE DB (different from ECE's) | +| fai\_analysis\_db\_host | localhost | For the EAE DB (different from ECE's) | +| fai\_analysis\_db\_user | ece5user | For the EAE DB (different from ECE's) | +| fai\_analysis\_db\_password | read-the-source-luke | For the EAE DB (different from ECE's) | +| fai\_analysis\_db\_schema | ece5db | For the EAE DB (different from ECE's) | +| fai\_analysis\_port | 8080 | Port of the EAE | +| fai\_analysis\_shutdown | 8005 | Shutdown port for the EAE app server | +| fai\_analysis\_name | analysis1 | EAE instance name | +| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai\_cache\_install | 0 | Install cache server profile | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_install | 0 | Install db profile | +| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | +| fai\_db\_port | 3306 | Useful for editor & presentation profiles | +| fai\_db\_schema | ece5db | Useful for DB installation profile | +| fai\_db\_user | ece5user | Useful for DB installation profile | +| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_editor\_install | 0 | Install the editorial profile | +| fai\_editor\_name | editor1 | Name of the editor instance | +| fai\_editor\_port | 8080 | HTTP port of the editor instance | +| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | +| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | +| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_presentation\_install | 0 | Install the presentation server profile | +| fai\_presentation\_name | web1 | Name of the presentation server instance | +| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | +| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | +| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | +| fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_name | mypub | Name of the publication | +| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | +| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | +| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | +| fai\_rmi\_install | 0 | Install RMI hub profile | +| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | +| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | | fai\_search\_indexer\_ws\_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | -| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_keep\_off\_etc\_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | -| fai\_wf\_install | 0 | Install Widget Framework profile | +| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | +| fai\_search\_name | search1 | Name of the search instance | +| fai\_search\_port | 8080 | HTTP port of the search instance | +| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | +| fai\_keep\_off\_etc\_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | +| fai\_wf\_install | 0 | Install Widget Framework profile | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| #+TBLFM: $2=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ 0 From 5fc11d0f07c69a47ae233a7158686b10d0bba14a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 18:56:01 +0800 Subject: [PATCH 0356/2585] - update markup --- usr/share/doc/escenic/ece-install-guide.html | 106 +++++++++++++------ 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index e7ab8faf..b03ed535 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ - + @@ -231,8 +231,9 @@

      Table of Contents

    • 1 Supported Operating Systems
    • 2 A Note on Running ece-install On Non-GNU/Linux Systems
    • @@ -261,16 +262,17 @@

      Table of Contents

    • 5.9 Profile - Create Publication
    • 5.10 Profile - Monitoring Server
    • 5.11 Profile - Restoring from backup
    • -
    • 5.12 Running interactively +
    • 5.12 Profile - Analysis Server
    • +
    • 5.13 Running interactively
    • -
    • 5.13 Running in FAI mode
    • -
    • 5.14 Data security
    • +
    • 5.14 Running in FAI mode
    • +
    • 5.15 Data security
  • 6 Full Automatic Install (FAI) @@ -335,9 +337,25 @@

    1.1 Debian based operatin
    -

    1.2 Other GNU/Linux and Unix systems

    +

    1.2 RedHat based operating systems

    +

    ece-install constantly gets better support for automatic installation +on RedHat based operating systems. However, since the public RedHat +repositories are significantly smaller than the Debian ones, there +will often be the case that you'll have to do some manually +installation of 3rd party dependencies prior to running +ece-install. Don't despair though, ece-install will tell you what you +need to do. +

    +
    + +
    + +
    +

    1.3 Other GNU/Linux and Unix systems

    +
    +

    If you're installing on another Linux or Unix system, you must first make sure that these 3rd party components are installed. ece-install will tell you which ones it cannot find and tell you to install these, @@ -352,9 +370,9 @@

    1.2 Other GNU/Linux and U

    -
    -

    1.3 Tested Operating Systems

    -
    +
    +

    1.4 Tested Operating Systems

    +
    @@ -885,15 +903,27 @@

    5.11 Profile - Restoring
    -

    5.12 Running interactively

    +

    5.12 Profile - Analysis Server

    +

    This profile will install the Escenic Analysis Engine and configure it +for production use with sane defaults. Be sure to use a different DB +than you use for ECE. +

    +
    + +
    + +
    +

    5.13 Running interactively

    +
    +
    -
    -

    5.12.1 Start ece-install and choose the Option, "Restore from backup"

    -
    +
    +

    5.13.1 Start ece-install and choose the Option, "Restore from backup"

    +
    @@ -906,9 +936,9 @@

    5.12.1 Start ece-insta

    -
    -

    5.12.2 Select Which Backup to Restore

    -
    +
    +

    5.13.2 Select Which Backup to Restore

    +
    @@ -937,9 +967,9 @@

    5.12.2 Select Which Ba

    -
    -

    5.12.3 Choose What to Restore

    -
    +
    +

    5.13.3 Choose What to Restore

    +
    @@ -957,9 +987,9 @@

    5.12.3 Choose What to

    -
    -

    5.12.4 Sit Back and Watch ece-install Restore the Data for You

    -
    +
    +

    5.13.4 Sit Back and Watch ece-install Restore the Data for You

    +
    @@ -977,9 +1007,9 @@

    5.12.4 Sit Back and Wa

    -
    -

    5.13 Running in FAI mode

    -
    +
    +

    5.14 Running in FAI mode

    +

    If you're running in FAI mode, you can choose between these settings to decide what to restore and where to find the backup file to @@ -1021,9 +1051,9 @@

    5.13 Running in FAI mode

    -
    -

    5.14 Data security

    -
    +
    +

    5.15 Data security

    +

    You should take heed when running restore, so that you're not restoring a system which you didn't want to change (yes, this mishap @@ -1270,6 +1300,14 @@

    6.3 Overview of All FAI P

    + + + + + + + + @@ -1776,7 +1814,7 @@

    15 Example Output FAI


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2012-02-28 12:37:37 CST + Date: 2012-02-28 18:55:47 CST

    From 04611781870e85c4f86fbc8726a58e8f35dee750 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 18:56:24 +0800 Subject: [PATCH 0357/2585] - moved the assemble_deploy_and_restart_type to after setting up the memory cache. --- usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index 16062922..a20a5653 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -79,8 +79,8 @@ function install_ece_instance() if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - assemble_deploy_and_restart_type install_memory_cache + assemble_deploy_and_restart_type fi update_type_instances_to_start_up From 7e5f4c63a687f0f3bdc20b3663033e606e75b146 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 28 Feb 2012 18:57:02 +0800 Subject: [PATCH 0358/2585] - first working version of the automatic installation of memcached with per publication Nursery configuration. --- .../ece-scripts/ece-install.d/memory-cache.sh | 61 +++++++++++-------- 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh index 2ff196cf..89a23a27 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -9,55 +9,62 @@ function memcached_create_publication_nursery_component() { return fi - log "Adding memcached wrapper to PresenationArticle in $1 ..." - local dir=$1/webapp/WEB-INF/localconfig/neo/xredsys/presentation/cache + print_and_log "Adding memcached wrapper to PresenationArticle in $1 ..." + local dir=$1/WEB-INF/localconfig/neo/xredsys/presentation/cache make_dir $dir local file=$dir/PresentationArticleCache.properties - sed -i "s#\$class=.*#\$class=neo.util.cache.Memcached#g" $file + + if [[ -e $file && $(grep "\$class" $file | wc -l) -gt 0 ]]; then + sed -i "s#\$class=.*#\$class=neo.util.cache.Memcached#g" $file + else + echo "\$class=neo.util.cache.Memcached" >> $file + fi + exit_on_error "sed on $file" } function install_memory_cache() { + print "Installing a distributed memory cache on $HOSTNAME ..." + install_packages_if_missing "memcached" assert_pre_requisite memcached run cd $download_dir run wget $wget_opts $memcached_java_lib_url + + local tmp_dir=$(mktemp -d) + run cd $tmp_dir + run tar xzf $download_dir/$(basename $memcached_java_lib_url) local name=$(get_base_dir_from_bundle $memcached_java_lib_url) - run cp $name/$name.jar $assemblytool_home/lib + run cp $name/$name.jar ${escenic_root_dir}/assemblytool/lib + run rm -rf $tmp_dir memcached_set_up_common_nursery - log "Configuring all publications for using memcached ..." - for el in $assemblytool_home/publications/*.properties; do + print_and_log "Configuring all publications for using memcached ..." + for el in $(ls $escenic_root_dir/assemblytool/publications/*.properties \ + 2>/dev/null); do local publication=$(basename $el .properties) - - if [[ $appserver == "tomcat" ]]; then - dir=$tomcat_base/webapps/$publication - make_dir $dir - memcached_create_publication_nursery_component $dir - fi + dir=$tomcat_base/webapps/$publication + make_dir $dir + memcached_create_publication_nursery_component $dir done # fixing the deployed publications on host - if [[ $appserver == "tomcat" ]]; then - for el in $( - find $tomcat_base/webapps/ \ - -mindepth 1 \ - -maxdepth 1 \ - -type d | \ - egrep -v "solr|webservice|escenic|escenic-admin|indexer-webservice" | \ - egrep -v "indexer-webapp|studio" - ); do - memcached_create_publication_nursery_component $el - done - - fi + for el in $( + find $tomcat_base/webapps/ \ + -mindepth 1 \ + -maxdepth 1 \ + -type d | \ + egrep -v "solr|webservice|escenic|escenic-admin|indexer-webservice" | \ + egrep -v "indexer-webapp|studio" + ); do + memcached_create_publication_nursery_component $el + done # TODO inform the user that he/she might want to do tihs in the # publication tree as well. - assemble_deploy_and_restart_type_p } function memcached_set_up_common_nursery() { @@ -84,7 +91,7 @@ EOF cat >> $common_nursery_dir/Initial.properties < Date: Wed, 29 Feb 2012 11:42:02 +0800 Subject: [PATCH 0359/2585] - hardening: now fails fast if one of the required technet downloads are missing, before it would first actually fail and exit when one of these artifacts were missing. Now, ece-install will fail as follows: [ece-install-7] Asserting that required downloads succeeded ... [ece-install-7] -> engine is missing [ece-install-7] -> assemblytool is missing [ece-install-8] Add these download archives to the technet_download_list [ece-install-8] and re-run ece-install --- usr/sbin/ece-install | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8b1e1ee9..c71919ec 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1405,12 +1405,14 @@ function check_for_required_downloads() return fi - log "Asserting that required downloads succeeded ..." + print_and_log "Asserting that required downloads succeeded ..." local required_escenic_packages="engine assemblytool" - + local some_is_missing=0 + for el in $required_escenic_packages; do if [ $(ls $download_dir/$el*.zip 2>/dev/null | wc -l) -lt 1 ]; then - log "$el is missing" + print_and_log "-> $el is missing" + some_is_missing=1 else # want the newest one if there are several local file=$(ls $download_dir/$el*.zip | tail -1) @@ -1423,6 +1425,12 @@ function check_for_required_downloads() fi fi done + + if [ $some_is_missing -eq 1 ]; then + print_and_log "Add these download archives to the technet_download_list" + print_and_log "and re-run $(basename $0)" + exit 1 + fi } function set_up_user_environment() From 820032904c2e292f23dbc55f3a1cdb73810263cc Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 29 Feb 2012 14:03:51 +0800 Subject: [PATCH 0360/2585] - new feature: new setting fai_analysis_db_install=1 for installing the EAE DB. This is just to add consistency with the optional fai_analysis_db_* keys. You can still use the regular fai_db_* parameters for installing the EAE DB, this is, as a7f864a09bf1cf3263d5b19081fafd56b04902d3 explains, only necessary to use when you're installing multiple profiles from the same ece-install.conf - or if you really like the idea that all EAE parameters have the same fai_analysis_* prefix, including the db options. - moved all database related code to database.sh --- usr/sbin/ece-install | 103 +--------------- .../ece-scripts/ece-install.d/database.sh | 114 +++++++++++++++++- 2 files changed, 115 insertions(+), 102 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c71919ec..a51070c7 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -460,106 +460,6 @@ function set_up_ece_scripts() run sed -i "s#/opt/escenic#${escenic_root_dir}#g" /etc/bash_completion.d/ece } -default_db_port=3306 -default_db_host=localhost -default_db_user=ece5user -default_db_password=ece5password -default_db_schema=ece5db - -# Method used both from interactive mode to set any missing values -# (defaults) -function set_db_defaults_if_not_set() -{ - if [ -z "$db_host" ]; then - db_host=${default_db_host} - fi - - if [ -z "$db_port" ]; then - db_port=${default_db_port} - fi - - if [ -z "$db_user" ]; then - db_user=${default_db_user} - fi - - if [ -z "$db_password" ]; then - db_password=${default_db_password} - fi - - if [ -z "$db_schema" ]; then - db_schema=${default_db_schema} - fi -} - -function set_db_settings_from_fai_conf() -{ - # Note: the port isn't fully supported. The user must himself - # update the mysql configuration to run it on a non standard port. - - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then - db_port=${fai_db_port-${default_db_port}} - db_host=${fai_db_host-${default_db_host}} - db_user=${fai_db_user-${default_db_user}} - db_password=${fai_db_password-${default_db_password}} - db_schema=${fai_db_schema-${default_db_schema}} - else - db_port=${fai_analysis_db_port-${default_db_port}} - db_host=${fai_analysis_db_host-${default_db_host}} - db_user=${fai_analysis_db_user-${default_db_user}} - db_password=${fai_analysis_db_password-${default_db_password}} - db_schema=${fai_analysis_db_schema-${default_db_schema}} - fi - - if [ -n "${fai_db_drop_old_db_first}" ]; then - drop_db_first=${fai_db_drop_old_db_first} - if [ $fai_db_drop_old_db_first -eq 1 ]; then - print_and_log "$(yellow WARNING): I hope you know what you're doing!" - print_and_log "$(yellow WARNING): fai_db_drop_old_db_first is 1 (true)" - fi - fi -} - -function set_up_ecedb() -{ - print_and_log "Setting up the ECE database schema ..." - - make_dir $escenic_root_dir/engine/plugins - run cd $escenic_root_dir/engine/plugins - - find ../../ -maxdepth 1 -type d | \ - grep -v assemblytool | \ - while read directory; do - if [ $directory = "../../" ]; then - continue - fi - - # nuisance to get the community engine, but not the engine - if [ $(echo $directory | grep engine | wc -l) -gt 0 ]; then - if [ $(echo $directory | grep community | wc -l) -lt 1 ]; then - continue - fi - fi - - if [ ! -h $(basename $directory) ]; then - run ln -s $directory - fi - done - - # the user may override standard DB settings in ece-install.conf - set_db_settings_from_fai_conf - set_db_defaults_if_not_set - - # the methods in drop-and-create-ecedb needs ece_home to be set - ece_home=${escenic_root_dir}/engine - pre_install_new_ecedb - create_ecedb - - cd ~/ - run rm -rf $escenic_root_dir/engine/plugins - - add_next_step "DB is now set up on ${db_host}:${db_port}" -} - function set_up_basic_nursery_configuration() { print_and_log "Setting up basic Nursery configuration ..." @@ -2477,7 +2377,8 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi - if [ $(get_boolean_conf_value fai_db_install) -eq 1 ]; then + if [[ $(get_boolean_conf_value fai_db_install) -eq 1 || + $(get_boolean_conf_value fai_analysis_db_install) -eq 1 ]]; then install_profile_number=$PROFILE_DB_SERVER install_database_server no_fai_profile=0 diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index a660d074..c699f0f2 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -1,3 +1,5 @@ +# ece-install module for installing the database. + percona_rpm_release_version=0.0-1 percona_rpm_release_package_name=percona-release-${percona_rpm_release_version} percona_rpm_release_url=http://www.percona.com/downloads/percona-release/$percona_rpm_release_package_name.x86_64.rpm @@ -5,6 +7,12 @@ if [[ $(uname -m) != "x86_64" ]]; then percona_rpm_release_url=http://www.percona.com/downloads/percona-release/$percona_rpm_release_package_name.i386.rpm fi +default_db_port=3306 +default_db_host=localhost +default_db_user=ece5user +default_db_password=ece5password +default_db_schema=ece5db + ## $1: optional parameter, binaries_only. If passed, $1=binaries_only, ## the ECE DB schema is not set up. function install_database_server() @@ -87,7 +95,7 @@ function install_database_server() if [ $on_redhat_or_derivative -eq 1 ]; then run chkconfig --level 35 mysql on - run /etc/init.d/mysql start + run /etc/init.d/mysql restart fi assert_pre_requisite mysql @@ -100,3 +108,107 @@ function install_database_server() fi } +function set_up_ecedb() +{ + print_and_log "Setting up the ECE database schema ..." + + make_dir $escenic_root_dir/engine/plugins + run cd $escenic_root_dir/engine/plugins + + find ../../ -maxdepth 1 -type d | \ + grep -v assemblytool | \ + while read directory; do + if [ $directory = "../../" ]; then + continue + fi + + # nuisance to get the community engine, but not the engine + if [ $(echo $directory | grep engine | wc -l) -gt 0 ]; then + if [ $(echo $directory | grep community | wc -l) -lt 1 ]; then + continue + fi + fi + + if [ ! -h $(basename $directory) ]; then + run ln -s $directory + fi + done + + # the user may override standard DB settings in ece-install.conf + set_db_settings_from_fai_conf + set_db_defaults_if_not_set + + # the methods in drop-and-create-ecedb needs ece_home to be set + ece_home=${escenic_root_dir}/engine + pre_install_new_ecedb + create_ecedb + + cd ~/ + run rm -rf $escenic_root_dir/engine/plugins + + add_next_step "DB is now set up on ${db_host}:${db_port}" +} + +function set_db_settings_from_fai_conf() +{ + # Note: the port isn't fully supported. The user must himself + # update the mysql configuration to run it on a non standard port. + + # if either the profile=analysis or the profile=db && + # fai_analysis_db_install is set, we try first to use the + # fai_analysis_db_* variables. + if [[ ( $install_profile_number -eq $PROFILE_DB_SERVER && + $(get_boolean_conf_value fai_analysis_db_install) -eq 1 ) || + $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]]; then + + # order of precedence: + # 1) fai_analysis_db_* + # 2) fai_db_* + # 3) default_db_* + db_port=${fai_analysis_db_port-${fai_db_port-${default_db_port}}} + db_host=${fai_analysis_db_host-${fai_db_host-${default_db_host}}} + db_user=${fai_analysis_db_user-${fai_db_user-${default_db_user}}} + db_password=${fai_analysis_db_password-${fai_db_password-${default_db_password}}} + db_schema=${fai_analysis_db_schema-${fai_db_schema-${default_db_schema}}} + else + db_port=${fai_db_port-${default_db_port}} + db_host=${fai_db_host-${default_db_host}} + db_user=${fai_db_user-${default_db_user}} + db_password=${fai_db_password-${default_db_password}} + db_schema=${fai_db_schema-${default_db_schema}} + fi + + if [ -n "${fai_db_drop_old_db_first}" ]; then + drop_db_first=${fai_db_drop_old_db_first} + if [ $fai_db_drop_old_db_first -eq 1 ]; then + print_and_log "$(yellow WARNING): I hope you know what you're doing!" + print_and_log "$(yellow WARNING): fai_db_drop_old_db_first is 1 (true)" + fi + fi +} + +# Method used both from interactive mode to set any missing values +# (defaults) +function set_db_defaults_if_not_set() +{ + if [ -z "$db_host" ]; then + db_host=${default_db_host} + fi + + if [ -z "$db_port" ]; then + db_port=${default_db_port} + fi + + if [ -z "$db_user" ]; then + db_user=${default_db_user} + fi + + if [ -z "$db_password" ]; then + db_password=${default_db_password} + fi + + if [ -z "$db_schema" ]; then + db_schema=${default_db_schema} + fi +} + From dd9a9d98e1f04ab8b70797271db18324df2aa4e9 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 29 Feb 2012 14:12:36 +0800 Subject: [PATCH 0361/2585] - updated doc; use https instead of git protocol for getting the scripts. --- usr/share/doc/escenic/ece-install-guide.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 46a7b633..7d4558bd 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -50,7 +50,7 @@ The ece-install script can be downloaded from: https://raw.github.com/skybert/ece-scripts/master/usr/sbin/ece-install or be downloaded together with the other ece-scripts using git: #+BEGIN_SRC conf -$ git clone git@github.com:skybert/ece-scripts.git +$ git clone https://github.com/skybert/ece-scripts.git #+END_SRC * Running the ece-install Script From 539c0126a08d0305297f4b513a0c08f8794f9f7c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 29 Feb 2012 14:19:56 +0800 Subject: [PATCH 0362/2585] - elaborated about the ece-install log file. - included -f/--file in the documentation - added fai_analysis_db_install to FAI config overview - sorted fai config overview list --- usr/share/doc/escenic/ece-install-guide.org | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 7d4558bd..cfa461c3 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -99,9 +99,12 @@ below. With the configuration file in place, you're now ready to run it: #+BEGIN_SRC conf -# bash ece-install [-v|--verbose] +# bash ece-install [-v|--verbose|<-f|--file> ] #+END_SRC +It will output everything from internal and external commands to its +log file located at /var/log/ece-install.log. + ece-install will try to "fail fast", exiting as soon as it detects an error during its execution: #+BEGIN_SRC conf @@ -520,18 +523,19 @@ $HOME/ece-install.conf file of the root user: | Parameter | Default | Description | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| | fai\_all\_install | 0 | Install all components on your server. | -| fai\_analysis\_db\_port | 3306 | For the EAE DB (different from ECE's) | | fai\_analysis\_db\_host | localhost | For the EAE DB (different from ECE's) | -| fai\_analysis\_db\_user | ece5user | For the EAE DB (different from ECE's) | +| fai\_analysis\_db\_install | 0 | Install db profile | | fai\_analysis\_db\_password | read-the-source-luke | For the EAE DB (different from ECE's) | +| fai\_analysis\_db\_port | 3306 | For the EAE DB (different from ECE's) | | fai\_analysis\_db\_schema | ece5db | For the EAE DB (different from ECE's) | +| fai\_analysis\_db\_user | ece5user | For the EAE DB (different from ECE's) | +| fai\_analysis\_name | analysis1 | EAE instance name | | fai\_analysis\_port | 8080 | Port of the EAE | | fai\_analysis\_shutdown | 8005 | Shutdown port for the EAE app server | -| fai\_analysis\_name | analysis1 | EAE instance name | | fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | | fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | | fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | +| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | | fai\_db\_install | 0 | Install db profile | | fai\_db\_password | read-the-source-luke | Useful for DB installation profile | | fai\_db\_port | 3306 | Useful for editor & presentation profiles | @@ -544,6 +548,7 @@ $HOME/ece-install.conf file of the root user: | fai\_editor\_port | 8080 | HTTP port of the editor instance | | fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | | fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai\_keep\_off\_etc\_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | | fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | | fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | | fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | @@ -554,11 +559,11 @@ $HOME/ece-install.conf file of the root user: | fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | | fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | | fai\_publication\_create | 0 | Create a new publication | +| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | | fai\_publication\_name | mypub | Name of the publication | | fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | | fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | | fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | | fai\_rmi\_install | 0 | Install RMI hub profile | | fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | | fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | @@ -568,7 +573,6 @@ $HOME/ece-install.conf file of the root user: | fai\_search\_name | search1 | Name of the search instance | | fai\_search\_port | 8080 | HTTP port of the search instance | | fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_keep\_off\_etc\_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | | fai\_wf\_install | 0 | Install Widget Framework profile | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| #+TBLFM: $2=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ 0 From b9801fdbdd3a9e22407c087f485d293b59aacfd7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 29 Feb 2012 18:16:37 +0800 Subject: [PATCH 0363/2585] - hardening: re-implemented update_type_instances_to_start_up, it now should work with non-standard instance names. The benefit of this, is that newly installed instances always are added to the instances started by the init.d script. - improved user feedback: don't call sub shells in sub shells as it makes reporting had, as with get_tomcat_download_url instide get_base_idr. - hardening: set_up_ece_scripts now checks if /etc/ files already exists, and if they do, it will not overwrite those. It reports everything it skips in both log and standard out. --- usr/sbin/ece-install | 80 +++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a51070c7..c4bed8db 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -424,6 +424,8 @@ function set_up_engine_and_plugins() function set_up_ece_scripts() { + print_and_log 'Setting up the ece UNIX scripts ...' + run cd $download_dir if [ -d ece-scripts ]; then ( @@ -435,10 +437,26 @@ function set_up_ece_scripts() fi run cp -r ece-scripts/usr/* /usr/ - run cp -r ece-scripts/etc/escenic/* ${escenic_conf_dir}/ run cp -r ece-scripts/etc/bash_completion.d/ece /etc/bash_completion.d/ run cp -r ece-scripts/etc/init.d/* /etc/init.d/ - run cp -r ece-scripts/etc/default/* /etc/default/ + + for el in ece-scripts/etc/default/*; do + local file=/etc/default/$(basename $el) + if [ -e $file ]; then + print_and_log "$file already exists, not overwriting it" + continue + fi + run cp $el /etc/default/ + done + + for el in ece-scripts/etc/escenic/*; do + local file=/etc/escenic/$(basename $el) + if [ -e $file ]; then + print_and_log "$file already exists, not overwriting it" + continue + fi + run cp $el /etc/escenic/ + done local file=${escenic_conf_dir}/ece.conf set_conf_file_value assemblytool_home ${escenic_root_dir}/assemblytool $file @@ -856,7 +874,8 @@ function set_up_app_server() fi download_tomcat $download_dir - tomcat_dir=$(get_base_dir_from_bundle $(get_tomcat_download_url)) + local tomcat_download_url=$(get_tomcat_download_url) + tomcat_dir=$(get_base_dir_from_bundle $tomcat_download_url) run cd $appserver_parent_dir run tar xzf $download_dir/${tomcat_dir}.tar.gz @@ -1604,39 +1623,30 @@ function ask_for_instance_name() # search and analysis instances. function update_type_instances_to_start_up() { - if [ $type = "engine" ]; then - for el in $escenic_conf_dir/ece-*.conf; do - if [ $(echo el | grep analysis | wc -l) -gt 0 ]; then - continue - elif [ $(echo el | grep search | wc -l) -gt 0 ]; then - continue - fi - - current_instance=$(basename ${el} | \ - sed -e 's/ece-//g' -e 's/engine-//g' | \ - cut -d'.' -f1) - all_instances="$all_instances $current_instance" - - set_conf_file_value engine_instance_list \ - "$all_instances" \ - /etc/default/ece - done - elif [ $type = "search" ]; then - for el in $escenic_conf_dir/ece-*.conf; do - if [ $(echo el | grep search | wc -l) -lt 1 ]; then - continue - fi - - current_instance=$(basename ${el} | \ - sed -e 's/ece-//g' -e 's/search-//g' | \ - cut -d'.' -f1) - all_instances="$all_instances $current_instance" - - set_conf_file_value search_instance_list \ - "$all_instances" \ - /etc/default/ece - done + run source /etc/default/ece + + if [ $type = "engine" ]; then + if [[ "$engine_instance_list" != *"${instance_name}"* ]]; then + print_and_log "Adding $instance_name instance to the init.d configuration" + set_conf_file_value engine_instance_list \ + "$engine_instance_list $instance_name" \ + /etc/default/ece + fi + elif [ $type = "search" ]; then + if [[ "$search_instance_list" != *"${instance_name}"* ]]; then + print_and_log "Adding $instance_name instance to the init.d configuration" + set_conf_file_value search_instance_list \ + "$search_instance_list $instance_name" \ + /etc/default/ece + fi + elif [ $type = "analysis" ]; then + if [[ "$analysis_instance_list" != *"${instance_name}"* ]]; then + print_and_log "Adding $instance_name instance to the init.d configuration" + set_conf_file_value analysis_instance_list \ + "$analysis_instance_list $instance_name" \ + /etc/default/ece fi + fi } ece_instance_ear_file="" From 44678cd51ba6cd91f13c828b0bb2ac187878fb9e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 29 Feb 2012 18:17:14 +0800 Subject: [PATCH 0364/2585] - fix: create the assemblytool/lib if it doesn't exist. Needed to put the memcached java library in there. --- usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh index 89a23a27..a1b88a5a 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -37,6 +37,7 @@ function install_memory_cache() run cd $tmp_dir run tar xzf $download_dir/$(basename $memcached_java_lib_url) local name=$(get_base_dir_from_bundle $memcached_java_lib_url) + make_dir ${escenic_root_dir}/assemblytool/lib run cp $name/$name.jar ${escenic_root_dir}/assemblytool/lib run rm -rf $tmp_dir From 340dd2b1c6cac618569b2586f09824abe1dcadaa Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 29 Feb 2012 18:31:29 +0800 Subject: [PATCH 0365/2585] - hardening: we need to do one apt-get update to be sure the package list is fresh. However, we don't want to do this in a common method, since it *might* be that all pre-requisite packages already are present. Hence, ece-install will only do a (one time) apt-get update when installing the first non installed, required package. --- usr/sbin/ece-install | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c4bed8db..25c1ecd3 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -159,6 +159,10 @@ function download_escenic_components() done } +# we need to do one apt-get update to be sure the package list is +# fresh. However, we don't want to do this in a common method, since +# it *might* be that all pre-requisite packages already are present. +one_time_apt_update_done=0 # Will install the passed packages if these are not installed from # before. @@ -181,6 +185,10 @@ function install_packages_if_missing() { if [ $some_are_missing -eq 0 ]; then return + elif [ $one_time_apt_update_done -eq 0 ]; then + log "Running one time APT update to ensure package list is up to date" + run apt-get update + one_time_apt_update_done=1 fi if [ $force_packages -eq 1 ]; then From a00c7417f2a6300a6cf1d9bff658d0ddb97b2974 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Mar 2012 12:21:42 +0800 Subject: [PATCH 0366/2585] - now running the alternatives command inside the run wrapper --- usr/share/escenic/ece-scripts/ece-install.d/java.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/java.sh b/usr/share/escenic/ece-scripts/ece-install.d/java.sh index 0048c349..a05c7d32 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/java.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/java.sh @@ -63,7 +63,7 @@ function install_sun_java_on_redhat() { if [ ! -e /usr/bin/$el ]; then ln -s /usr/bin/$el /etc/alternatives/$el fi - alternatives --set $el /opt/jdk/bin/$el + run alternatives --set $el /opt/jdk/bin/$el done # setting java_home to the newly installed location From 5a587acf88322fa4fcbd23f5e2e098e50b7abc4b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Mar 2012 12:22:15 +0800 Subject: [PATCH 0367/2585] - re-added user feedback when installing 3rd party software. --- usr/sbin/ece-install | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 25c1ecd3..cfb553c4 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -673,6 +673,7 @@ function set_ece_instance_conf() function install_ece_third_party_packages { run_hook install_ece_third_party_packages.preinst + print_and_log "Installing 3rd party packages needed by $type instances" if [ $on_debian_or_derivative -eq 1 ]; then if [ $on_ubuntu -eq 1 ]; then From 32da47e9a3ca48d5ebec33d2705a0dcf15296e59 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Mar 2012 13:53:00 +0800 Subject: [PATCH 0368/2585] - new feature: the HTTPS/SSL/redirect port can now be overridden: fai_analysis_redirect fai_editor_redirect fai_presentation_redirect fai_search_redirect and in interactive mode (of course). - speed: if Java in already installed and this is in a non default location, ece-install still picks it up if the java_home is passed and will hence not attempt to (re) install Java, either from APT repo, build deb packages or install via oracle.com bundles. If you're installing on a non-Debian based system, chances are that the default java_home path of /usr/lib/jvm/java-6-sun will not apply to you. Here, set java_home in your ece-install.conf to speed up the installation (if you've already installed java, that is, if not just let ece-install do the work for you). - hardening: if determining the tomcat mirror URL fails, this is handled in the right place, rather than propagating the error. - moved application server specific code to its own mnodule, app-server.sh --- usr/sbin/ece-install | 393 ----------------- usr/share/escenic/ece-scripts/common-os.sh | 22 +- .../ece-scripts/ece-install.d/app-server.sh | 401 ++++++++++++++++++ .../escenic/ece-scripts/ece-install.d/java.sh | 8 +- 4 files changed, 417 insertions(+), 407 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/app-server.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index cfb553c4..3d124516 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -787,399 +787,6 @@ EOF fi } -function set_up_app_server() -{ - print_and_log "Setting up the application server ..." - - if [ $fai_enabled -eq 0 ]; then - print "On which ports do you wish to run the app server on?" - print "Press ENTER to accept port 8080 and shutdown port 8005" - print "Or enter: , e.g.: '8180 8105'" - echo -n "Your choice [8080 8005]> " - read user_ports - - if [ -n "$user_ports" ]; then - appserver_port=$(echo $user_ports | cut -d' ' -f1) - shutdown_port=$(echo $user_ports | cut -d' ' -f2) - fi - else - if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - appserver_port=${fai_editor_port-8080} - shutdown_port=${fai_editor_shutdown-8005} - elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - appserver_port=${fai_presentation_port-8080} - shutdown_port=${fai_presentation_shutdown-8005} - elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - appserver_port=${fai_search_port-8080} - shutdown_port=${fai_search_shutdown-8005} - elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then - appserver_port=${fai_analysis_port-8080} - shutdown_port=${fai_analysis_shutdown-8005} - fi - fi - - if [ -z "$appserver_port" ]; then - appserver_port=8080 - elif [ -z "$appserver_shutdown" ]; then - appserver_shutdown=8005 - fi - - debug "appserver_port=$appserver_port shutdown_port=$shutdown_port" - - if [ $fai_enabled -eq 0 ]; then - print "Another question: Where does the database run?" - print "Press ENTER to accept the default " \ - "($HOSTNAME:${default_db_port}:${default_db_schema})" - print "Or enter: ::, e.g.: 'db1:${default_db_port}:mydb'" - echo -n "Your choice [$HOSTNAME:${default_db_port}:schema]> " - read user_database - - db_host=$(echo $user_database | cut -d':' -f1) - db_port=$(echo $user_database | cut -d':' -f2) - db_schema=$(echo $user_database | cut -d':' -f3) - else - set_db_settings_from_fai_conf - fi - - set_db_defaults_if_not_set - - if [ $fai_enabled -eq 0 ]; then - print "Awfully sorry to bug you with so many questions, but:" - print "What's the URI to the indexer-webservice? (this is typically" - print "something like http://editor1/indexer-webservice/index/)" - echo -n "Your choice [http://${HOSTNAME}:8080/indexer-webservice/index/]> " - read user_indexer_ws_uri - else - user_indexer_ws_uri=${fai_search_indexer_ws_uri} - fi - - if [ -n "$user_indexer_ws_uri" ]; then - indexer_ws_uri=$user_indexer_ws_uri - else - indexer_ws_uri=http://${HOSTNAME}:${appserver_port}/indexer-webservice/index/ - fi - - - if [ $fai_enabled -eq 0 ]; then - print "Last question, I promise!: Where does the search instance run?" - print "Press ENTER to accept the default ($HOSTNAME:8080)" - print "or enter: :, e.g.: 'search1:8080'" - print "If you're in doubt, just press ENTER :-)" - echo -n "Your choice [$HOSTNAME:8080]> " - read user_search - else - user_search=$(get_conf_value fai_search_host) - if [ -n "${user_search}" ]; then - user_search=${user_search}":"$(get_conf_value fai_search_port) - fi - fi - - if [ -z "$user_search" ]; then - search_host=$HOSTNAME - search_port=8080 - else - search_host=$(echo $user_search | cut -d':' -f1) - search_port=$(echo $user_search | cut -d':' -f2) - fi - - download_tomcat $download_dir - local tomcat_download_url=$(get_tomcat_download_url) - tomcat_dir=$(get_base_dir_from_bundle $tomcat_download_url) - - run cd $appserver_parent_dir - run tar xzf $download_dir/${tomcat_dir}.tar.gz - - if [ -e tomcat ]; then - run rm tomcat - fi - run ln -sf ${tomcat_dir} tomcat - - tomcat_home=${appserver_parent_dir}/tomcat - tomcat_base=${appserver_parent_dir}/tomcat-${instance_name} - make_dir $tomcat_base - - run cp -r ${appserver_parent_dir}/${tomcat_dir}/conf $tomcat_base - for el in bin escenic/lib lib work logs temp webapps; do - make_dir $tomcat_base/$el - done - - set_ece_instance_conf tomcat_base $tomcat_base - set_ece_instance_conf tomcat_home $tomcat_home - set_ece_instance_conf appserver_port $appserver_port - - run cd $tomcat_base/lib - make_ln $jdbc_driver - - # it's important to append (and not pre-pend) the ECE libraries so - # that things put it in the standard loader behaves as expected, - # such as log4j configuration put in ${tomcat.base}/lib. - file=$tomcat_base/conf/catalina.properties - common_loader=$(grep ^common.loader $file) - escaped_common_loader=$(get_escaped_bash_string ${common_loader}) - escenic_loader=",$\{catalina.base\}/escenic/lib/*.jar" - old=$(get_escaped_bash_string ${common_loader}) - new=${escaped_common_loader}$(get_escaped_bash_string ${escenic_loader}) - run sed -i "s#${old}#${new}#g" $file - - cat > $tomcat_base/conf/server.xml < - - - - - - - - - -EOF - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER \ - -a $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - cat >> $tomcat_base/conf/server.xml < - -EOF - elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then - cat >> $tomcat_base/conf/server.xml < - -EOF - fi - - cat >> $tomcat_base/conf/server.xml < - - - - - - - - -EOF - if [[ ($install_profile_number == $PROFILE_EDITORIAL_SERVER || \ - $install_profile_number == $PROFILE_PRESENTATION_SERVER || \ - $install_profile_number == $PROFILE_ALL_IN_ONE) && \ - -n "$fai_publication_domain_mapping_list" ]]; then - - for el in ${fai_publication_domain_mapping_list}; do - local old_ifs=$IFS - # the entries in the fai_publication_domain_mapping_list - # are on the form: # - IFS='#' - read publication domain <<< "$el" - IFS=$old_ifs - - ensure_domain_is_known_to_local_host ${domain} - - cat >> $tomcat_base/conf/server.xml < - - -EOF - done - fi - - cat >> $tomcat_base/conf/server.xml < - - - -EOF - cat > $tomcat_base/conf/context.xml < - - WEB-INF/web.xml -EOF - - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ - $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then - cat >> $tomcat_base/conf/context.xml < - -EOF - fi - - if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then - cat >> $tomcat_base/conf/context.xml < -EOF - fi - - if [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - cat >> $tomcat_base/conf/context.xml < - - - - -EOF - elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then - cat >> $tomcat_base/conf/context.xml < - - -EOF - else - cat >> $tomcat_base/conf/context.xml < -EOF - fi -} - # last, give the control back to the ECE user & group function set_correct_permissions() { diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh index 7d602711..5af8b48f 100644 --- a/usr/share/escenic/ece-scripts/common-os.sh +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -110,11 +110,6 @@ function get_tomcat_download_url() { cut -d'"' -f2 ) - if [ -z $url ]; then - print_and_log "Failed to get Tomcat download URL" - kill $$ - fi - echo $url } @@ -122,11 +117,16 @@ function get_tomcat_download_url() { ## ## $1: target directory function download_tomcat() { - ( - local url=$(get_tomcat_download_url) - log "Downloading Tomcat from $url ..." - run cd $1 - run wget $wget_opts $url - ) + local url=$(get_tomcat_download_url) + + if [ -z $url ]; then + print_and_log "Failed to get Tomcat download URL" + kill $$ + fi + + print "Downloading Tomcat ..." + log "Downloading Tomcat from $url ..." + run cd $1 + run wget $wget_opts $url } diff --git a/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh new file mode 100644 index 00000000..9eceddb2 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh @@ -0,0 +1,401 @@ +function set_up_app_server() +{ + print_and_log "Setting up the application server ..." + + if [ $fai_enabled -eq 0 ]; then + print "On which ports do you wish to run the app server on?" + print "Press ENTER to accept the default ports" + print "or enter: :" + echo -n "Your choice [8080 8005 8443]> " + read user_ports + + if [ -n "$user_ports" ]; then + read appserver_port shutdown_port redirect_port <<< $user_ports + fi + else + if [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + appserver_port=${fai_editor_port-8080} + shutdown_port=${fai_editor_shutdown-8005} + redirect_port=${fai_editor_redirect-8443} + elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + appserver_port=${fai_presentation_port-8080} + shutdown_port=${fai_presentation_shutdown-8005} + redirect_port=${fai_presentation_redirect-8443} + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + appserver_port=${fai_search_port-8080} + shutdown_port=${fai_search_shutdown-8005} + redirect_port=${fai_search_redirect-8443} + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then + appserver_port=${fai_analysis_port-8080} + shutdown_port=${fai_analysis_shutdown-8005} + redirect_port=${fai_analysis_redirect-8443} + fi + fi + + if [ -z "$appserver_port" ]; then + appserver_port=8080 + fi + if [ -z "$shutdown_port" ]; then + shutdown_port=8005 + fi + if [ -z "$redirect_port" ]; then + redirect_port=8443 + fi + + if [ $fai_enabled -eq 0 ]; then + print "Another question: Where does the database run?" + print "Press ENTER to accept the default " \ + "($HOSTNAME:${default_db_port}:${default_db_schema})" + print "Or enter: ::, e.g.: 'db1:${default_db_port}:mydb'" + echo -n "Your choice [$HOSTNAME:${default_db_port}:schema]> " + read user_database + + db_host=$(echo $user_database | cut -d':' -f1) + db_port=$(echo $user_database | cut -d':' -f2) + db_schema=$(echo $user_database | cut -d':' -f3) + else + set_db_settings_from_fai_conf + fi + + set_db_defaults_if_not_set + + if [ $fai_enabled -eq 0 ]; then + print "Awfully sorry to bug you with so many questions, but:" + print "What's the URI to the indexer-webservice? (this is typically" + print "something like http://editor1/indexer-webservice/index/)" + echo -n "Your choice [http://${HOSTNAME}:8080/indexer-webservice/index/]> " + read user_indexer_ws_uri + else + user_indexer_ws_uri=${fai_search_indexer_ws_uri} + fi + + if [ -n "$user_indexer_ws_uri" ]; then + indexer_ws_uri=$user_indexer_ws_uri + else + indexer_ws_uri=http://${HOSTNAME}:${appserver_port}/indexer-webservice/index/ + fi + + + if [ $fai_enabled -eq 0 ]; then + print "Last question, I promise!: Where does the search instance run?" + print "Press ENTER to accept the default ($HOSTNAME:8080)" + print "or enter: :, e.g.: 'search1:8080'" + print "If you're in doubt, just press ENTER :-)" + echo -n "Your choice [$HOSTNAME:8080]> " + read user_search + else + user_search=$(get_conf_value fai_search_host) + if [ -n "${user_search}" ]; then + user_search=${user_search}":"$(get_conf_value fai_search_port) + fi + fi + + if [ -z "$user_search" ]; then + search_host=$HOSTNAME + search_port=8080 + else + search_host=$(echo $user_search | cut -d':' -f1) + search_port=$(echo $user_search | cut -d':' -f2) + fi + + download_tomcat $download_dir + local tomcat_archive=$( + find $download_dir \ + -name "apache-tomcat*.tar.gz" | \ + tail -1 + ) + tomcat_dir=$(get_base_dir_from_bundle $tomcat_archive) + + run cd $appserver_parent_dir + run tar xzf $download_dir/${tomcat_dir}.tar.gz + + if [ -e tomcat ]; then + run rm tomcat + fi + run ln -sf ${tomcat_dir} tomcat + + tomcat_home=${appserver_parent_dir}/tomcat + tomcat_base=${appserver_parent_dir}/tomcat-${instance_name} + make_dir $tomcat_base + + run cp -r ${appserver_parent_dir}/${tomcat_dir}/conf $tomcat_base + for el in bin escenic/lib lib work logs temp webapps; do + make_dir $tomcat_base/$el + done + + set_ece_instance_conf tomcat_base $tomcat_base + set_ece_instance_conf tomcat_home $tomcat_home + set_ece_instance_conf appserver_port $appserver_port + + run cd $tomcat_base/lib + make_ln $jdbc_driver + + # it's important to append (and not pre-pend) the ECE libraries so + # that things put it in the standard loader behaves as expected, + # such as log4j configuration put in ${tomcat.base}/lib. + file=$tomcat_base/conf/catalina.properties + common_loader=$(grep ^common.loader $file) + escaped_common_loader=$(get_escaped_bash_string ${common_loader}) + escenic_loader=",$\{catalina.base\}/escenic/lib/*.jar" + old=$(get_escaped_bash_string ${common_loader}) + new=${escaped_common_loader}$(get_escaped_bash_string ${escenic_loader}) + run sed -i "s#${old}#${new}#g" $file + + cat > $tomcat_base/conf/server.xml < + + + + + + + + + +EOF + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER \ + -a $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then + cat >> $tomcat_base/conf/server.xml < + +EOF + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then + cat >> $tomcat_base/conf/server.xml < + +EOF + fi + + cat >> $tomcat_base/conf/server.xml < + + + + + + + + +EOF + if [[ ($install_profile_number == $PROFILE_EDITORIAL_SERVER || \ + $install_profile_number == $PROFILE_PRESENTATION_SERVER || \ + $install_profile_number == $PROFILE_ALL_IN_ONE) && \ + -n "$fai_publication_domain_mapping_list" ]]; then + + for el in ${fai_publication_domain_mapping_list}; do + local old_ifs=$IFS + # the entries in the fai_publication_domain_mapping_list + # are on the form: # + IFS='#' + read publication domain <<< "$el" + IFS=$old_ifs + + ensure_domain_is_known_to_local_host ${domain} + + cat >> $tomcat_base/conf/server.xml < + + +EOF + done + fi + + cat >> $tomcat_base/conf/server.xml < + + + +EOF + cat > $tomcat_base/conf/context.xml < + + WEB-INF/web.xml +EOF + + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER -a \ + $install_profile_number -ne $PROFILE_SEARCH_SERVER ]; then + cat >> $tomcat_base/conf/context.xml < + +EOF + fi + + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + cat >> $tomcat_base/conf/context.xml < +EOF + fi + + if [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + cat >> $tomcat_base/conf/context.xml < + + + + +EOF + elif [ $install_profile_number -eq $PROFILE_ANALYSIS_SERVER ]; then + cat >> $tomcat_base/conf/context.xml < + + +EOF + else + cat >> $tomcat_base/conf/context.xml < +EOF + fi +} diff --git a/usr/share/escenic/ece-scripts/ece-install.d/java.sh b/usr/share/escenic/ece-scripts/ece-install.d/java.sh index a05c7d32..b8603aee 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/java.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/java.sh @@ -25,11 +25,12 @@ function create_java_deb_packages_and_repo() { function install_sun_java_on_redhat() { - if [[ $(java -version 2>&1 | grep HotSpot | wc -l) -gt 0 ]]; then + if [[ $(${java_home}/bin/java -version 2>&1 | \ + grep HotSpot | wc -l) -gt 0 ]]; then print_and_log "Sun Java is already installed on $HOSTNAME" return fi - + print_and_log "Downloading Sun Java from download.oracle.com ..." run cd $download_dir run wget $wget_opts $sun_java_bin_url @@ -63,7 +64,8 @@ function install_sun_java_on_redhat() { if [ ! -e /usr/bin/$el ]; then ln -s /usr/bin/$el /etc/alternatives/$el fi - run alternatives --set $el /opt/jdk/bin/$el + # doesn't seem to like running inside the run wrapper + alternatives --set $el /opt/jdk/bin/$el 1>>$log 2>>$log done # setting java_home to the newly installed location From 482f388d137219e078a7ba69566fa3c8f7f405d8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Mar 2012 14:19:58 +0800 Subject: [PATCH 0369/2585] - fix: memcached component, SockIOPool was falsely names. - resilience: if fai_memcached_node_list is not set, ece-install will use the default of localhost:11211. This means that at the very least, all ECE instances will get their own memcached instance configured. - when running on REHL, a newly instqalled memcached is not installed automatically. Hence, we have to (re) start it ourselves. --- .../escenic/ece-scripts/ece-install.d/memory-cache.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh index a1b88a5a..cb597bb8 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -28,6 +28,10 @@ function install_memory_cache() print "Installing a distributed memory cache on $HOSTNAME ..." install_packages_if_missing "memcached" + if [ $on_redhat_or_derivative -eq 1 ]; then + run /etc/init.d/memcached restart + fi + assert_pre_requisite memcached run cd $download_dir @@ -71,10 +75,10 @@ function install_memory_cache() function memcached_set_up_common_nursery() { local dir=$common_nursery_dir/com/danga make_dir $dir - cat > $dir/SockIOPool < $dir/SockIOPool.properties < Date: Thu, 1 Mar 2012 14:20:17 +0800 Subject: [PATCH 0370/2585] - pretty: re-positioned comments --- usr/share/escenic/ece-scripts/ece-install.d/app-server.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh index 9eceddb2..a34ff16e 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh @@ -130,9 +130,9 @@ function set_up_app_server() run cd $tomcat_base/lib make_ln $jdbc_driver - # it's important to append (and not pre-pend) the ECE libraries so - # that things put it in the standard loader behaves as expected, - # such as log4j configuration put in ${tomcat.base}/lib. + # it's important to append (and not pre-pend) the ECE libraries so + # that things put it in the standard loader behaves as expected, + # such as log4j configuration put in ${tomcat.base}/lib. file=$tomcat_base/conf/catalina.properties common_loader=$(grep ^common.loader $file) escaped_common_loader=$(get_escaped_bash_string ${common_loader}) From 9571db476cbdeccfa4267ecd8c3c8a7dbd135b60 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Mar 2012 15:17:01 +0800 Subject: [PATCH 0371/2585] - resilience: now using default_schema instead of "schema" string when addressing the app server database in interactive mode. --- usr/share/escenic/ece-scripts/ece-install.d/app-server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh index a34ff16e..70d902c4 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/app-server.sh @@ -47,7 +47,7 @@ function set_up_app_server() print "Press ENTER to accept the default " \ "($HOSTNAME:${default_db_port}:${default_db_schema})" print "Or enter: ::, e.g.: 'db1:${default_db_port}:mydb'" - echo -n "Your choice [$HOSTNAME:${default_db_port}:schema]> " + echo -n "Your choice [$HOSTNAME:${default_db_port}:${default_db_schema}]> " read user_database db_host=$(echo $user_database | cut -d':' -f1) From 265b960c701ad64f25e51a95bb63a6ae9447d09d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Mar 2012 15:18:42 +0800 Subject: [PATCH 0372/2585] - hardening: introduced default tomcat archive download URL, fallback_tomcat_url. Only used if the Tomcat download mirror couldn't be resolved. --- usr/share/escenic/ece-scripts/common-os.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh index 5af8b48f..2f34d0e3 100644 --- a/usr/share/escenic/ece-scripts/common-os.sh +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -8,6 +8,9 @@ common_bashing_is_loaded > /dev/null 2>&1 || source common-bashing.sh common_pulse_is_loaded > /dev/null 2>&1 || source common-bashing.sh +## Only used if the Tomcat download mirror couldn't be determined +fallback_tomcat_url="http://mirrorservice.nomedia.no/apache.org/tomcat/tomcat-6/v6.0.35/bin/apache-tomcat-6.0.35.tar.gz" + wget_opts="--continue --inet4-only --quiet" # Can be used like this: @@ -110,20 +113,21 @@ function get_tomcat_download_url() { cut -d'"' -f2 ) + if [ -z $url ]; then + url=$fallback_tomcat_url + log "Failed to get Tomcat mirror URL, will use fallback URL $url" + fi + echo $url } + ## Downloads Tomcat from the regional mirror ## ## $1: target directory function download_tomcat() { local url=$(get_tomcat_download_url) - if [ -z $url ]; then - print_and_log "Failed to get Tomcat download URL" - kill $$ - fi - print "Downloading Tomcat ..." log "Downloading Tomcat from $url ..." run cd $1 From 71fb798687bfbff9f6bb1c775e22555803456287 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 1 Mar 2012 15:22:20 +0800 Subject: [PATCH 0373/2585] - mode the setting of publication specific Nursery configuration to "ece deploy". This can be turned off with do_not_add_memcached_support=1 in ece[-].conf. It's pretty harmless as the regular LRU cache works if the memcached libraries or daemons are not present. - fixed the class declaration of the memcached Nursery component. - this means that fully automatic installation and set up of memcached is working. TODO: test this thoroughly :-) --- usr/bin/ece | 39 ++++++++++++++- .../ece-scripts/ece-install.d/memory-cache.sh | 49 ++----------------- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 49368ff2..5dbd0549 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -107,7 +107,7 @@ function print() function log() { - echo "$id" `date` $@ 2>/dev/null >> $log_file + echo $(date) "$id" $@ 2>/dev/null >> $log_file } function print_and_log() @@ -576,6 +576,7 @@ function deploy() jar xf $war \ 1>>$log_file \ 2>>$log_file) + add_memcached_support $tomcat_base/webapps/$name exit_on_error "extracting $war to $tomcat_base/webapps/" done ;; @@ -598,6 +599,42 @@ function deploy() run rm -rf ${dir} } +## $1 : dir of the webapp +function add_memcached_support() { + if [[ "$do_not_add_memcached_support" == "1" ]]; then + return + fi + + if [ ! -d "$1" ]; then + log $1 "doesn't exist" + return + fi + + local exempt_from_memcached_list=" + escenic escenic-admin indexer-webservice indexer-webapp studio + inpage-ws dashboard + " + + for el in $exempt_from_memcached_list; do + if [[ $(basename $1) == "$el" ]]; then + return + fi + done + + log "Adding memcached support in $1 ..." + local dir=$1/WEB-INF/localconfig/neo/xredsys/presentation/cache + run mkdir -p $dir + local file=$dir/PresentationArticleCache.properties + + if [[ -e $file && $(grep "\$class" $file | wc -l) -gt 0 ]]; then + sed -i "s#\$class=.*#\$class=neo.util.cache.Memcached#g" $file + else + echo "\$class=neo.util.cache.Memcached" >> $file + fi + + exit_on_error "sed on $file" +} + # Returns the file (can be a directory) passed to the function only # if it's the actual file/directory and not a link to it. If the # passed file is a link, the link target is returned instead. diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh index cb597bb8..f3a71e89 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -2,27 +2,6 @@ memcached_java_lib_url=http://img.whalin.com/memcached/jdk5/log4j/java_memcached-release_2.0.1.tar.gz -## $1: root directory of the publication -function memcached_create_publication_nursery_component() { - if [ ! -d "$1" ]; then - log $1 "doesn't exist" - return - fi - - print_and_log "Adding memcached wrapper to PresenationArticle in $1 ..." - local dir=$1/WEB-INF/localconfig/neo/xredsys/presentation/cache - make_dir $dir - local file=$dir/PresentationArticleCache.properties - - if [[ -e $file && $(grep "\$class" $file | wc -l) -gt 0 ]]; then - sed -i "s#\$class=.*#\$class=neo.util.cache.Memcached#g" $file - else - echo "\$class=neo.util.cache.Memcached" >> $file - fi - - exit_on_error "sed on $file" -} - function install_memory_cache() { print "Installing a distributed memory cache on $HOSTNAME ..." @@ -46,29 +25,11 @@ function install_memory_cache() run rm -rf $tmp_dir memcached_set_up_common_nursery + + # ece deploy will set up the necessary in-publication Nursery + # configuration, if needed. - print_and_log "Configuring all publications for using memcached ..." - for el in $(ls $escenic_root_dir/assemblytool/publications/*.properties \ - 2>/dev/null); do - local publication=$(basename $el .properties) - dir=$tomcat_base/webapps/$publication - make_dir $dir - memcached_create_publication_nursery_component $dir - done - - # fixing the deployed publications on host - for el in $( - find $tomcat_base/webapps/ \ - -mindepth 1 \ - -maxdepth 1 \ - -type d | \ - egrep -v "solr|webservice|escenic|escenic-admin|indexer-webservice" | \ - egrep -v "indexer-webapp|studio" - ); do - memcached_create_publication_nursery_component $el - done - - # TODO inform the user that he/she might want to do tihs in the + # TODO inform the user that he/she might want to do this in the # publication tree as well. } @@ -76,7 +37,7 @@ function memcached_set_up_common_nursery() { local dir=$common_nursery_dir/com/danga make_dir $dir cat > $dir/SockIOPool.properties < Date: Sun, 5 Feb 2012 16:40:11 +0100 Subject: [PATCH 0374/2585] Added basic initial example. --- usr/lib/vosa/install-engine.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 usr/lib/vosa/install-engine.sh diff --git a/usr/lib/vosa/install-engine.sh b/usr/lib/vosa/install-engine.sh new file mode 100644 index 00000000..e864c4dc --- /dev/null +++ b/usr/lib/vosa/install-engine.sh @@ -0,0 +1,19 @@ +# Only argument is the "vm name" as provided on the command line, +# which is a directory path of a VOSA vm definition: +# +# e.g. /etc/vosa/available.d/vm03 + +# The VM is assumed to be booted and ready for SSH using the passwordless +# key in the vm03 directory, as the XXX user (ubuntu?) + +cat > /dev/null < Date: Mon, 6 Feb 2012 08:28:26 +0100 Subject: [PATCH 0375/2585] Added a developer-keys --- .../vosa/post-install-hooks/install-engine.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 usr/share/vizrt/vosa/post-install-hooks/install-engine.sh diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh new file mode 100644 index 00000000..e864c4dc --- /dev/null +++ b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh @@ -0,0 +1,19 @@ +# Only argument is the "vm name" as provided on the command line, +# which is a directory path of a VOSA vm definition: +# +# e.g. /etc/vosa/available.d/vm03 + +# The VM is assumed to be booted and ready for SSH using the passwordless +# key in the vm03 directory, as the XXX user (ubuntu?) + +cat > /dev/null < Date: Mon, 6 Feb 2012 17:39:26 +0100 Subject: [PATCH 0376/2585] skeleton of vosa command --- usr/bin/vosa | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 usr/bin/vosa diff --git a/usr/bin/vosa b/usr/bin/vosa new file mode 100644 index 00000000..b9f7a1f6 --- /dev/null +++ b/usr/bin/vosa @@ -0,0 +1,12 @@ +#!/bin/bash + +# /usr/bin/vosa --- command to manage vizrt on-line system-administration +# managed instances of virtual machines. + +# vosa list --- list all available virtual machines +# vosa -i /etc/vizrt/vosa/available.d/vm03 enable --- enable a specific vm +# vosa -i /etc/vizrt/vosa/available.d/vm03 disable --- disable a specific vm +# vosa -i /etc/vizrt/vosa/enabled.d/vm03 disable --- disable a vm +# vosa -i ... restart +# vosa -i ... reinstall + From fb5b57ca1e1412e56abb36b73bcc52a7a667fbfb Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 00:19:03 +0100 Subject: [PATCH 0377/2585] Much more fleshed out vosa command --- usr/bin/vosa | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) mode change 100644 => 100755 usr/bin/vosa diff --git a/usr/bin/vosa b/usr/bin/vosa old mode 100644 new mode 100755 index b9f7a1f6..b2685958 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -10,3 +10,103 @@ # vosa -i ... restart # vosa -i ... reinstall +instance_dir= +usage=0 + +### To add an option, add it to the optstring, as defined in getopts, in alphabetical order. +### Also add a function "option-x" where x is the option. $1 will be the option value, if any. +### set any variables needed to default values as globals first. +OPTSTRING=":i:h" + +function option-h() { + usage=1 +} + +function option-i() { + instance_dir="${1}" +} + + +function unknown-option() { + echo "Unknown option $@" + usage=1 +} + +### To add a command, simply define a function with a do_ prefix. +function do_list() { + if [ ! -z $instance_dir ] ; then + echo "Instance cannot be specified" + usage=1 + return + fi + availabledir=$(dirname $0)/../../etc/vizrt/vosa/available.d + availabledir=$(readlink -f ${availabledir}) + ls -d ${availabledir}/* | grep "/[0-9a-z][-0-9a-z]*$" +} + +### To add a command, simply define a function with a do_ prefix. +function do_install() { + if [ -z $instance_dir ] ; then + echo "Instance is required" + usage=1 + return + fi + echo "Starting $instance_dir" +} + +function unknown-argument() { + echo "Unknown argument $@" + usage=1 +} + +LASTOPTIND=0 + +function parseopts() { +local OPTARG +local OPTIND +local opt +local currentconfigdirectory="${@:1:1}" +shift; +while getopts "${OPTSTRING}" opt; do + case $opt in + \?) + unknown-option "-$OPTARG" + ;; + *) + option-$opt "${OPTARG}" + ;; + esac +done + +LASTOPTIND=$OPTIND + +} + +parseopts "$PWD" "${@}" + +# get rid of all parsed parameters from command line, leaving real arguments +shift $((LASTOPTIND-1)) + +if [ $usage -eq 0 -a "${#@}" -gt 1 ] ; then + echo "Only one argument is allowed" + usage=1 +fi + +if [ $usage -eq 0 -a "${#@}" -lt 1 ] ; then + echo "A command must be specified." + usage=1 +fi + + +# Check if the command +fn="do_$1" +declare > /dev/null -f "$fn" || unknown-argument "$1" +declare > /dev/null -f "$fn" && "$fn" + +if [ $usage -eq 1 ] ; then + echo "Usage: $0 -i instance-dir " + exit 1; +fi + + + From 11e18a0aa2f2b28a2eed6544ce4a9cb44a393369 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 00:28:28 +0100 Subject: [PATCH 0378/2585] Added vosa enabled which shows all enabled VMs on this host --- usr/bin/vosa | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index b2685958..339bd766 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -33,6 +33,7 @@ function unknown-option() { } ### To add a command, simply define a function with a do_ prefix. + function do_list() { if [ ! -z $instance_dir ] ; then echo "Instance cannot be specified" @@ -44,7 +45,17 @@ function do_list() { ls -d ${availabledir}/* | grep "/[0-9a-z][-0-9a-z]*$" } -### To add a command, simply define a function with a do_ prefix. +function do_enabled() { + if [ ! -z $instance_dir ] ; then + echo "Instance cannot be specified" + usage=1 + return + fi + availabledir=$(dirname $0)/../../etc/vizrt/vosa/enabled.d + availabledir=$(readlink -f ${availabledir}) + ls -d ${availabledir}/* | grep "/[0-9a-z][-0-9a-z]*$" +} + function do_install() { if [ -z $instance_dir ] ; then echo "Instance is required" From daa2ff3e4b3d17784f984dcc1d9b7e401097f472 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 00:42:00 +0100 Subject: [PATCH 0379/2585] Added "enable" command to enable a VM by making a symlink. With error checks --- usr/bin/vosa | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 339bd766..1301827f 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -33,16 +33,18 @@ function unknown-option() { } ### To add a command, simply define a function with a do_ prefix. +available_dir=$(dirname $0)/../../etc/vizrt/vosa/available.d +available_dir=$(readlink -f ${available_dir}) +enabled_dir=$(dirname $0)/../../etc/vizrt/vosa/enabled.d +enabled_dir=$(readlink -f ${enabled_dir}) -function do_list() { +function do_available() { if [ ! -z $instance_dir ] ; then echo "Instance cannot be specified" usage=1 return fi - availabledir=$(dirname $0)/../../etc/vizrt/vosa/available.d - availabledir=$(readlink -f ${availabledir}) - ls -d ${availabledir}/* | grep "/[0-9a-z][-0-9a-z]*$" + ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" } function do_enabled() { @@ -51,11 +53,35 @@ function do_enabled() { usage=1 return fi - availabledir=$(dirname $0)/../../etc/vizrt/vosa/enabled.d - availabledir=$(readlink -f ${availabledir}) - ls -d ${availabledir}/* | grep "/[0-9a-z][-0-9a-z]*$" + ls -d ${enabled_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" } +function do_enable() { + if [ -z $instance_dir ] ; then + echo "Instance must be specified" + usage=1 + return + fi + # check if the instance isn't enabled already + # check if the instance isn't forcibly disabled + # check if the instance is available + # make a symbolic link + if [ -L ${enabled_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir is already enabled" + exit 1 + fi + if [ -r ${enabled_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir has been forcibly disabled by the presence of the file ${enabled_dir}/$(basename $instance_dir)" + exit 1 + fi + if [ ! -d ${available_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir is not an available instance" + exit 1 + fi + ln -v -s "../available.d/$(basename $instance_dir)" "$enabled_dir" || exit 1 +} + + function do_install() { if [ -z $instance_dir ] ; then echo "Instance is required" From fa9a79ffe6a29c7fb8e78cc06ed1e33658ff8ec5 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 00:47:57 +0100 Subject: [PATCH 0380/2585] Doodling around the "install" command. Here is where most of my full-image.sh comes in --- usr/bin/vosa | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 1301827f..1f1a2df0 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -88,7 +88,11 @@ function do_install() { usage=1 return fi - echo "Starting $instance_dir" + echo "Installing $instance_dir from scratch... This takes a few minutes" + echo "Archiving old image ..." + echo "Extracting a new fresh image" + # TODO should this just be in + # $(dirname $0)/../../usr/share/vizrt/vosa/commands/install.sh "$instance_dir" } function unknown-argument() { From 6f397820951fc217277b8212e6da8a52847e8241 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 14:51:07 +0100 Subject: [PATCH 0381/2585] Skeleton of config file parser, common to many functions in vosa, perhaps --- usr/bin/vosa | 10 +-- usr/share/vizrt/vosa/commands/functions | 17 +++++ usr/share/vizrt/vosa/commands/install.sh | 52 ++++++++++++++++ .../vizrt/vosa/commands/install_config_parser | 62 +++++++++++++++++++ 4 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 usr/share/vizrt/vosa/commands/functions create mode 100755 usr/share/vizrt/vosa/commands/install.sh create mode 100644 usr/share/vizrt/vosa/commands/install_config_parser diff --git a/usr/bin/vosa b/usr/bin/vosa index 1f1a2df0..6903321a 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -88,11 +88,11 @@ function do_install() { usage=1 return fi - echo "Installing $instance_dir from scratch... This takes a few minutes" - echo "Archiving old image ..." - echo "Extracting a new fresh image" - # TODO should this just be in - # $(dirname $0)/../../usr/share/vizrt/vosa/commands/install.sh "$instance_dir" + # todo: + # Error checking: if the instance is not enabled, abort + # If the instance exists and is running, stop it (gracefully?) + # If the instance exists, take a tar.gz backup of the entire image directory + $(dirname $0)/../share/vizrt/vosa/commands/install.sh "$instance_dir" } function unknown-argument() { diff --git a/usr/share/vizrt/vosa/commands/functions b/usr/share/vizrt/vosa/commands/functions new file mode 100644 index 00000000..70a387e5 --- /dev/null +++ b/usr/share/vizrt/vosa/commands/functions @@ -0,0 +1,17 @@ +# bash functions common to the vosa commands +# e.g. parsing a config file + + +# $1 == config file name +# $2 == prefix of functions to call +function parse_config_file() { + local line + local prefix=$2 + while read line; do + if [ "${line:0:1}" == "#" ] ; then continue; fi + if [ -z "${line}" ] ; then continue; fi + local line2=(${line}) + local name="${line2[0]}" + declare > /dev/null -f ${prefix}${name} && ${prefix}${name} "${line2[@]:1}" + done < "${1}" +} diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh new file mode 100755 index 00000000..bbcc1e36 --- /dev/null +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Performs a full installation of a vm + +# Expects a single argument, namely a directory or symlink to a directory which +# contains a "vosa" configuration, and a second directory, which should not exist +# which is where the installation will be held. + +# This command will create the second directory and build a virtual machine image from +# scratch and configure it as described by relevant files in the first directory. + +# Usually this command is executed from "/usr/bin/vosa -i somevm install" +# or similar. + +source $(dirname $0)/functions +source $(dirname $0)/install_config_parser + +config=$1 +image=$2 + +if [ -z "$image" -o -z "$config" ] ; then + echo "You need to specify a config directory (e.g. /etc/vizrt/vosa/enabled.d/foo) " + echo "and a place to hold the installation (e.g. /usr/lib/vizrt/vosa/images/foo)" + echo "the former MUST exist and contain vosa configuration files" + echo "the latter MUST NOT exist, and will be created as a result of this command" + exit 1 +fi + +# basic error checking +if [ ! -d "$config" ] ; then + echo "Config directory $config isn't a directory." + exit 1 +fi + +if [ -d "$image" -o -r "$image" ] ; then + echo "image directory $image already exist. Can't continue." + exit 1 +fi + +# make the holding area. +mkdir $image + +# Parse all install config items +parse_config_file $config/install.conf install_config_ + + +echo $ip_address + + + + + diff --git a/usr/share/vizrt/vosa/commands/install_config_parser b/usr/share/vizrt/vosa/commands/install_config_parser new file mode 100644 index 00000000..b0be6f32 --- /dev/null +++ b/usr/share/vizrt/vosa/commands/install_config_parser @@ -0,0 +1,62 @@ +# bash functions common to the vosa commands +# e.g. parsing a config file + +ip_address= +original_image= +kernel= +initial_disk_size= +macaddr= +netmask= +gateway= +timezone= + + +function install_config_ip_address() { + ip_address=$1 +} + +function install_config_original_image() { + original_image=$1 +} + +function install_config_kernel() { + kernel=$1 +} + +function install_config_initial_disk_size() { + initial_disk_size=$1 +} + +function install_config_macaddr() { + macaddr=$1 +} + +function install_config_netmask() { + netmask=$1 +} + +function install_config_gateway() { + gateway=$1 +} + +function install_config_timezone() { + timezone=$1 +} + +# Handle arrays + +declare -a ssh_keys +declare -a overlay +declare -a postinstall + +function install_config_ssh_keys() { + ssh_keys=( "${ssh_keys[@]}" "$1" ) +} + +function install_config_overlay() { + overlay=( "${overlay[@]}" "$1" ) +} + +function install_config_postinstall() { + postinstall=( "${postinstall[@]}" "$1" ) +} From 317801e9941f006213ea291fdd95ab5427b0ba3a Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 16:05:41 +0100 Subject: [PATCH 0382/2585] boot file parsed, and the install script is getting underway... --- .../vizrt/vosa/commands/boot_config_parser | 28 +++ usr/share/vizrt/vosa/commands/install.sh | 171 +++++++++++++++++- .../vizrt/vosa/commands/install_config_parser | 44 ++--- 3 files changed, 214 insertions(+), 29 deletions(-) create mode 100644 usr/share/vizrt/vosa/commands/boot_config_parser diff --git a/usr/share/vizrt/vosa/commands/boot_config_parser b/usr/share/vizrt/vosa/commands/boot_config_parser new file mode 100644 index 00000000..e4c46b49 --- /dev/null +++ b/usr/share/vizrt/vosa/commands/boot_config_parser @@ -0,0 +1,28 @@ +# bash functions common to the vosa commands +# specifically functions to aid parsing boot.conf files + +boot_config_memory= +boot_config_processor_affinity= +boot_config_vnc_port= + + +function boot_config_memory() { + boot_config_memory=$1 +} + +function boot_config_processor_affinity() { + boot_config_processor_affinity=$1 +} + +function boot_config_vnc_port() { + boot_config_vnc_port=$1 +} + +# Handle arrays + +# declare -a ssh_keys + +#function install_config_ssh_keys() { +# ssh_keys=( "${ssh_keys[@]}" "$1" ) +#} + diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index bbcc1e36..1b38984f 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -12,8 +12,25 @@ # Usually this command is executed from "/usr/bin/vosa -i somevm install" # or similar. +debug=3 + +function decho() { + if [ $debug -ge $1 ] ; then + echo "${@:2}" + fi +} + +function exitonerror() { + rc=$1 + if [ "$rc" != "0" ] ; then + echo "$2 (rc=$rc)" + exit 2 + fi +} + source $(dirname $0)/functions source $(dirname $0)/install_config_parser +source $(dirname $0)/boot_config_parser config=$1 image=$2 @@ -37,16 +54,156 @@ if [ -d "$image" -o -r "$image" ] ; then exit 1 fi +if [ "$(basename "$image")" != "$(basename "$config")" ] ; then + echo "$image and $config appear to try to name different VMs. Try using $(basename $config) instead." + exit 2 +fi + # make the holding area. mkdir $image # Parse all install config items parse_config_file $config/install.conf install_config_ - - -echo $ip_address - - - - +parse_config_file $config/boot.conf boot_config_ + +function make_temp_dir() { + tempdir=/tmp/some-rundir + mkdir "${tempdir}" +} + +function cleanup_temp_dir() { + if [ ! -z "${tempdir}" -a -d "${tempdir}" ] ; then + rm -rf "${tempdir}" + fi +} + +make_run_dir() { + rundir=/var/run/vizrt/vosa/ + if [ "$(id -u)" != "0" ] ; then + rundir=$HOME/.vizrt/vosa/ + fi + decho 1 "Making var-run directory $rundir" + mkdir -p $rundir || exitonerror $? "Unable to make the place where pidfiles are stored ($rundir)" +} + +function copy_original_image() { + decho 1 "Copying original image file to image file" + if [ -z "$install_config_original_image" -o ! -r "$install_config_original_image" ] ; then + echo "Original image file $install_config_original_image does not exist. exiting." + exit 2; + fi + img="$image/disk.img" + kernel="$image/vmlinuz" + cp "$install_config_original_image" "$img" + cp "$install_config_kernel" "$kernel" +} + +function resize_original_image() { + decho 1 "Resizing 'disk.img' to ${install_config_initial_disk_size}Gb" + fsck.ext4 > /dev/null -p -f ${img}; exitonerror $? "fsck.ext4 failed before resize" + resize2fs > /dev/null ${img} ${install_config_initial_disk_size}G; exitonerror $? "resize2fs failed" + fsck.ext4 > /dev/null -n -f ${img}; exitonerror $? "fsck.ext4 failed after resize" +} + +function generate_ssh_key() { + decho 1 "generating SSH key" + ssh-keygen -t dsa -N "" -f ${image}/id_dsa -b 1024 -C "kvm-one-time-key:$name" > /dev/null; exitonerror $? "Unable to generate ssh key" +} + +function create_user_data_file() { + cat >> ${tempdir}/user_data.txt < /dev/null 2>/dev/null + if [ $? -ne 0 ] ; then + cat < Date: Wed, 8 Feb 2012 16:09:54 +0100 Subject: [PATCH 0383/2585] Don't resize disk if we don't need to --- usr/share/vizrt/vosa/commands/install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 1b38984f..49055edb 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -54,6 +54,8 @@ if [ -d "$image" -o -r "$image" ] ; then exit 1 fi +# todo: verify that basename and image don't end with slashes... + if [ "$(basename "$image")" != "$(basename "$config")" ] ; then echo "$image and $config appear to try to name different VMs. Try using $(basename $config) instead." exit 2 @@ -61,6 +63,7 @@ fi # make the holding area. mkdir $image +hostname=$(basename "$image") # Parse all install config items parse_config_file $config/install.conf install_config_ @@ -99,6 +102,7 @@ function copy_original_image() { } function resize_original_image() { + if [ -z "${install_config_initial_disk_size}" ] ; then return; fi decho 1 "Resizing 'disk.img' to ${install_config_initial_disk_size}Gb" fsck.ext4 > /dev/null -p -f ${img}; exitonerror $? "fsck.ext4 failed before resize" resize2fs > /dev/null ${img} ${install_config_initial_disk_size}G; exitonerror $? "resize2fs failed" @@ -147,7 +151,7 @@ EOF } function touch_pid_file() { - pidfile=$rundir/$(basename $config).pid + pidfile=$rundir/$hostname.pid touch "${pidfile}" } @@ -192,7 +196,6 @@ resize_original_image generate_ssh_key make_run_dir configure_vnc_option -touch_pid_file # should maybe be part of boot process? dunno. ### functions below require tempdir @@ -203,6 +206,7 @@ create_user_data_file +touch_pid_file # should maybe be part of boot process? dunno. boot_kvm cleanup_temp_dir From 0b4b7dd5639ac7ce002a3f8fff70959c2826facf Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 16:36:43 +0100 Subject: [PATCH 0384/2585] Forked boot portion into its own script --- usr/share/vizrt/vosa/commands/boot.sh | 153 +++++++++++++++++++++++ usr/share/vizrt/vosa/commands/install.sh | 85 ++----------- usr/share/vizrt/vosa/commands/qemu-ifup | 5 + 3 files changed, 168 insertions(+), 75 deletions(-) create mode 100755 usr/share/vizrt/vosa/commands/boot.sh create mode 100755 usr/share/vizrt/vosa/commands/qemu-ifup diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh new file mode 100755 index 00000000..94478286 --- /dev/null +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# Boots a kvm instance. + +# Expects two arguments, namely a directory or symlink to a directory which +# contains a "vosa" configuration, and a second directory, +# which is where the installation disks are held. + +# Usually this command is executed from "/usr/bin/vosa -i somevm install" +# or similar. + +debug=3 + +function decho() { + if [ $debug -ge $1 ] ; then + echo "${@:2}" + fi +} + +function exitonerror() { + rc=$1 + if [ "$rc" != "0" ] ; then + echo "$2 (rc=$rc)" + exit 2 + fi +} + +source $(dirname $0)/functions +source $(dirname $0)/install_config_parser +source $(dirname $0)/boot_config_parser + +config=$1 +image=$2 + +if [ -z "$image" -o -z "$config" ] ; then + echo "You need to specify a config directory (e.g. /etc/vizrt/vosa/enabled.d/foo) " + echo "and a place to hold the installation (e.g. /usr/lib/vizrt/vosa/images/foo)" + echo "the former MUST exist and contain vosa configuration files" + echo "the latter MUST also exist, and contain disk images and kernels etc" + exit 1 +fi + +# basic error checking +if [ ! -d "$config" ] ; then + echo "Config directory $config isn't a directory." + exit 1 +fi + +if [ ! -d "$image" ] ; then + echo "image directory $image does not exist. Can't continue." + exit 1 +fi + +# todo: verify that basename and image don't end with slashes... + +if [ "$(basename "$image")" != "$(basename "$config")" ] ; then + echo "$image and $config appear to try to name different VMs. Try using $(basename $config) instead." + exit 2 +fi + +# make the holding area. +hostname=$(basename "$image") + +# Parse all install config items +parse_config_file $config/install.conf install_config_ +parse_config_file $config/boot.conf boot_config_ + +make_run_dir() { + rundir=/var/run/vizrt/vosa/ + if [ "$(id -u)" != "0" ] ; then + rundir=$HOME/.vizrt/vosa/ + fi + decho 1 "Making var-run directory $rundir" + mkdir -p $rundir || exitonerror $? "Unable to make the place where pidfiles are stored ($rundir)" +} + +function check_sudo() { + sudo='' + runas='' + if [ "$(id -u)" != "0" ] ; then + decho 1 "Checking if we have sudo access to kvm" + sudo -n kvm --help > /dev/null 2>/dev/null + if [ $? -ne 0 ] ; then + cat < /dev/null 2>/dev/null - if [ $? -ne 0 ] ; then - cat < Date: Wed, 8 Feb 2012 17:05:20 +0100 Subject: [PATCH 0385/2585] Now boot.sh can boot vms the first time and subsequent times --- usr/bin/vosa | 2 +- usr/share/vizrt/vosa/commands/boot.sh | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 6903321a..c9fa1e8f 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -92,7 +92,7 @@ function do_install() { # Error checking: if the instance is not enabled, abort # If the instance exists and is running, stop it (gracefully?) # If the instance exists, take a tar.gz backup of the entire image directory - $(dirname $0)/../share/vizrt/vosa/commands/install.sh "$instance_dir" + $(dirname $0)/../share/vizrt/vosa/commands/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" } function unknown-argument() { diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 94478286..9d54785e 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -6,7 +6,7 @@ # contains a "vosa" configuration, and a second directory, # which is where the installation disks are held. -# Usually this command is executed from "/usr/bin/vosa -i somevm install" +# Usually this command is executed from "/usr/bin/vosa -i somevm start" # or similar. debug=3 @@ -104,6 +104,11 @@ function touch_pid_file() { touch "${pidfile}" } +function touch_state_file() { + statefile=$rundir/$hostname.state + echo 'running' > $statefile +} + function configure_vnc_option() { if [ -z "$boot_config_vnc_port" -o "$boot_config_vnc_port" == "none" ] ; then vncoption="-vnc none" @@ -116,6 +121,7 @@ function configure_vnc_option() { function boot_kvm() { kernel=${image}/vmlinuz img=${image}/disk.img + cloud_param="nocloud;h=${hostname}" # should _maybe_ be put in some other script? Needed by e.g. vosa start too. decho 1 "Starting the machine" @@ -134,20 +140,24 @@ function boot_kvm() { -net "nic,model=virtio,macaddr=${install_config_macaddr}" -net "tap,script=$(dirname $0)/qemu-ifup") + updates=${image}/updates.iso + if [ -r "${updates}" ] ; then + startupcmd=("${startupcmd[@]}" -drive "file=${updates},if=virtio") + xupdate="xupdate=vdb:mnt" + fi - #-drive "file=updates.iso,if=virtio" \ - - firstboot=("${startupcmd[@]}" -append "root=/dev/vda ro init=/usr/lib/cloud-init/uncloud-init ds=${cloud_param} ubuntu-pass=random xupdate=vdb:mnt" ) + startupcmd=("${startupcmd[@]}" -append "root=/dev/vda ro init=/usr/lib/cloud-init/uncloud-init ds=${cloud_param} ubuntu-pass=random $xupdate" ) # actually execute kvm - echo "${firstboot[@]}" - "${firstboot[@]}"; exitonerror $? "Unable to start kvm :-/" + echo "${startupcmd[@]}" + "${startupcmd[@]}"; exitonerror $? "Unable to start kvm :-/" } + check_sudo make_run_dir configure_vnc_option touch_pid_file # should maybe be part of boot process? dunno. boot_kvm - +touch_state_file From 1b08f82d7ed76fe2d358dd6198888c1440539144 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 17:08:54 +0100 Subject: [PATCH 0386/2585] Make "start" an official command. no "stop" just yet, though --- usr/bin/vosa | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index c9fa1e8f..008bf631 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -82,6 +82,9 @@ function do_enable() { } +commands="$(dirname $0)/../share/vizrt/vosa/commands" + + function do_install() { if [ -z $instance_dir ] ; then echo "Instance is required" @@ -90,11 +93,26 @@ function do_install() { fi # todo: # Error checking: if the instance is not enabled, abort - # If the instance exists and is running, stop it (gracefully?) + # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. + # If the instance exists, take a tar.gz backup of the entire image directory + ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" +} + +function do_start() { + if [ -z $instance_dir ] ; then + echo "Instance is required" + usage=1 + return + fi + # todo: + # Error checking: if the instance is not enabled, abort + # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. # If the instance exists, take a tar.gz backup of the entire image directory - $(dirname $0)/../share/vizrt/vosa/commands/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" + ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" } + + function unknown-argument() { echo "Unknown argument $@" usage=1 From 41a1bbd2f0867c55ac9abf11ef90fbfcf535b699 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 17:26:40 +0100 Subject: [PATCH 0387/2585] Added most of the overlay. It's created with the wrong file name, so it won't work just yet, but it's mostly done. Nearly there. --- usr/share/vizrt/vosa/commands/install.sh | 64 +++++++++++++++++++++--- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 253bc2a7..12243eb7 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -30,7 +30,6 @@ function exitonerror() { source $(dirname $0)/functions source $(dirname $0)/install_config_parser -source $(dirname $0)/boot_config_parser config=$1 image=$2 @@ -67,7 +66,6 @@ hostname=$(basename "$image") # Parse all install config items parse_config_file $config/install.conf install_config_ -parse_config_file $config/boot.conf boot_config_ function make_temp_dir() { tempdir=/tmp/some-rundir @@ -116,8 +114,19 @@ function generate_ssh_key() { ssh-keygen -t dsa -N "" -f ${image}/id_dsa -b 1024 -C "kvm-one-time-key:$name" > /dev/null; exitonerror $? "Unable to generate ssh key" } -function create_user_data_file() { - cat >> ${tempdir}/user_data.txt <> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/user-data <> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/meta-data <> ${tempdir}/overlay/updates/home/ubuntu/.ssh/authorized_keys; exitonerror $? "Unable to add authorized key $o" + done + + chmod 600 ${tempdir}/overlay/updates/home/ubuntu/.ssh/authorized_keys; exitonerror $? "Unable to chmod 600 authorized_keys" + # todo: inject another key to go from one server to another??? + + cat > ${tempdir}/overlay/updates.script <> /etc/network/interfaces < Date: Wed, 8 Feb 2012 21:50:09 +0100 Subject: [PATCH 0388/2585] Added "status" to show the state of one or all vms --- usr/bin/vosa | 176 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 124 insertions(+), 52 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 008bf631..2c29d47c 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -38,47 +38,124 @@ available_dir=$(readlink -f ${available_dir}) enabled_dir=$(dirname $0)/../../etc/vizrt/vosa/enabled.d enabled_dir=$(readlink -f ${enabled_dir}) -function do_available() { - if [ ! -z $instance_dir ] ; then - echo "Instance cannot be specified" + +function requires_instance_dir() { + if [ -z "$instance_dir" ] ; then + echo "Instance is required" usage=1 - return + return 1 fi - ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" } - -function do_enabled() { - if [ ! -z $instance_dir ] ; then +function prohibits_instance_dir() { + if [ ! -z "$instance_dir" ] ; then echo "Instance cannot be specified" usage=1 - return + return 1 fi - ls -d ${enabled_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" } -function do_enable() { - if [ -z $instance_dir ] ; then - echo "Instance must be specified" - usage=1 - return - fi - # check if the instance isn't enabled already - # check if the instance isn't forcibly disabled - # check if the instance is available - # make a symbolic link - if [ -L ${enabled_dir}/$(basename $instance_dir) ] ; then - echo "$instance_dir is already enabled" - exit 1 - fi - if [ -r ${enabled_dir}/$(basename $instance_dir) ] ; then - echo "$instance_dir has been forcibly disabled by the presence of the file ${enabled_dir}/$(basename $instance_dir)" - exit 1 +function do_available() { + prohibits_instance_dir && { + ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" + } +} + +function do_enabled() { + prohibits_instance_dir && { + ls -d ${enabled_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" + } +} + +function do_help() { + if [ -z "$instance_dir" ] ; then + cat < /dev/null $(<$pidfile) ; then + output=( ${output[@]} dead ); + else + output=( ${output[@]} alive "$( ps -p "$(<$pidfile)" -o "etime=" )" ); + fi + fi + fi + fi + fi + fi + fi + echo ${output[@]} +} + +function do_enable() { + requires_instance_dir && { + # check if the instance is available + # check if the instance isn't enabled already + # check if the instance isn't forcibly disabled + # make a symbolic link + if [ ! -d ${available_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir is not an available instance" + exit 1 + fi + if [ -L ${enabled_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir is already enabled" + exit 1 + fi + if [ -r ${enabled_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir has been forcibly disabled by the presence of the file ${enabled_dir}/$(basename $instance_dir)" + exit 1 + fi + ln -v -s "../available.d/$(basename $instance_dir)" "$enabled_dir" || exit 1 + } } @@ -86,29 +163,23 @@ commands="$(dirname $0)/../share/vizrt/vosa/commands" function do_install() { - if [ -z $instance_dir ] ; then - echo "Instance is required" - usage=1 - return - fi - # todo: - # Error checking: if the instance is not enabled, abort - # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. - # If the instance exists, take a tar.gz backup of the entire image directory - ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" + requires_instance_dir && { + # todo: + # Error checking: if the instance is not enabled, abort + # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. + # If the instance exists, take a tar.gz backup of the entire image directory + ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" + } } function do_start() { - if [ -z $instance_dir ] ; then - echo "Instance is required" - usage=1 - return - fi - # todo: - # Error checking: if the instance is not enabled, abort - # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. - # If the instance exists, take a tar.gz backup of the entire image directory - ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" + requires_instance_dir && { + # todo: + # Error checking: if the instance is not enabled, abort + # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. + # If the instance exists, take a tar.gz backup of the entire image directory + ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" + } } @@ -163,7 +234,8 @@ declare > /dev/null -f "$fn" || unknown-argument "$1" declare > /dev/null -f "$fn" && "$fn" if [ $usage -eq 1 ] ; then - echo "Usage: $0 -i instance-dir " + echo "Usage: $0 [-i ] " + echo "$0 available to get a list of instances" exit 1; fi From 77406e707ef599e8bfec2bad603d34fccbf97002 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 22:07:15 +0100 Subject: [PATCH 0389/2585] removed typos --- usr/share/vizrt/vosa/commands/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 12243eb7..4755fc12 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -120,7 +120,7 @@ function create_overlay() { # local copy of overlay mkdir -p ${tempdir}/overlay/updates; exitonerror $? "Unable to make overlay directory" - for o in "${install_config_overlay[@]}" ; do **** + for o in "${install_config_overlay[@]}" ; do if [ "${o}" == "" ] ; then continue; fi cp -rp --dereference $o/* ${tempdir}/overlay/updates/; exitonerror $? "Unable to copy overlay $o" done From bb720ef059c9e88427fbcf83d250fa26132e0c4d Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 22:25:02 +0100 Subject: [PATCH 0390/2585] added uninstall command. fixed status to not depend on vm02... --- usr/bin/vosa | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 2c29d47c..516d177d 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -108,7 +108,7 @@ function really_do_status () { output=( ${output[@]} disabled ); else output=( ${output[@]} enabled ); - local vmdir=$(readlink -f ${avdir}/../../../../../var/lib/vizrt/vosa/images/vm02/) + local vmdir=$(readlink -f ${avdir}/../../../../../var/lib/vizrt/vosa/images/$(basename $1)) if [ ! -d "$vmdir" ] ; then output=( ${output[@]} uninstalled ); else @@ -158,6 +158,31 @@ function do_enable() { } } +function do_uninstall() { + requires_instance_dir && { + # check if the instance is available + # check if the instance isn't enabled already + # check if the instance isn't forcibly disabled + # make a symbolic link + if [ ! -d ${available_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir is not an available instance" + exit 1 + fi + if [ ! -L ${enabled_dir}/$(basename $instance_dir) ] ; then + echo "$instance_dir is not enabled" + exit 1 + fi + local imgdir=$(readlink -f ${instance_dir}/../../../../../var/lib/vizrt/vosa/images/$(basename $instance_dir)/) + local backupdir="/var/backup/vizrt/vosa/$(basename $instance_dir)" + local backupparent="/var/backup/vizrt/vosa" + if [ ! -z "$imgdir" -a -d "$imgdir" -a -w "$backupparent" ] ; then + # todo: rotate a few times? + rm -rf "$backupdir" + mv "$imgdir" "$backupparent" + fi + } +} + commands="$(dirname $0)/../share/vizrt/vosa/commands" From e8d53a7fa970020025e76e22221f66a9d5b31d7e Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 22:33:28 +0100 Subject: [PATCH 0391/2585] Fixed backup to only backup if backups directory exist. one generation only. Add system generated ssh key to image --- usr/bin/vosa | 19 +++++++++++++------ usr/share/vizrt/vosa/commands/install.sh | 1 + 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 516d177d..c1e0c1f4 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -7,8 +7,10 @@ # vosa -i /etc/vizrt/vosa/available.d/vm03 enable --- enable a specific vm # vosa -i /etc/vizrt/vosa/available.d/vm03 disable --- disable a specific vm # vosa -i /etc/vizrt/vosa/enabled.d/vm03 disable --- disable a vm -# vosa -i ... restart -# vosa -i ... reinstall +# vosa -i /etc/vizrt/vosa/enabled.d/vm03 install --- creates a new disk image etc. +# vosa -i /etc/vizrt/vosa/enabled.d/vm03 uninstall --- removes the disk image etc. +# vosa -i ... start +# vosa -i ... status --- tells you about the VM, if it's enabled, running, alive, its uptime. instance_dir= usage=0 @@ -24,6 +26,7 @@ function option-h() { function option-i() { instance_dir="${1}" + # TODO remove trailing "/" since it's supposed to be a directory. This script depends on it! } @@ -175,10 +178,14 @@ function do_uninstall() { local imgdir=$(readlink -f ${instance_dir}/../../../../../var/lib/vizrt/vosa/images/$(basename $instance_dir)/) local backupdir="/var/backup/vizrt/vosa/$(basename $instance_dir)" local backupparent="/var/backup/vizrt/vosa" - if [ ! -z "$imgdir" -a -d "$imgdir" -a -w "$backupparent" ] ; then - # todo: rotate a few times? - rm -rf "$backupdir" - mv "$imgdir" "$backupparent" + if [ ! -z "$imgdir" -a -d "$imgdir" ] ; then + if [ -w "$backupparent" ] ; then + # todo: rotate a few times? + rm -rf "$backupdir" + mv "$imgdir" "$backupparent" + else + rm -rf "$imgdir" + fi fi } } diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 4755fc12..4f697df3 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -140,6 +140,7 @@ EOF mkdir -p ${tempdir}/overlay/updates/home/ubuntu/.ssh chmod 700 ${tempdir}/overlay/updates/home/ubuntu/.ssh + cp "${image}/id_dsa.pub" "${tempdir}/overlay/updates/home/ubuntu/.ssh/authorized_keys"; exitonerror $? "Unable to install id_dsa.pub" for o in "${install_config_ssh_keys[@]}" ; do cat $o >> ${tempdir}/overlay/updates/home/ubuntu/.ssh/authorized_keys; exitonerror $? "Unable to add authorized key $o" done From 566c658c26c5c7ee5308d075b0243ced85c45a21 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 22:45:12 +0100 Subject: [PATCH 0392/2585] uninstall now checks if the vm is alive or in the "running" state, and aborts --- usr/bin/vosa | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/usr/bin/vosa b/usr/bin/vosa index c1e0c1f4..d41bc1f8 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -175,6 +175,15 @@ function do_uninstall() { echo "$instance_dir is not enabled" exit 1 fi + local status=( $(really_do_status $instance_dir) ) + if [ "${status[6]}" == "alive" ] ; then + echo "$instance_dir is alive. Stop it first. exiting" + exit 1 + fi + if [ "${status[4]}" == "running" ] ; then + echo "$instance_dir is supposed to be running. Stop it first. Exiting!" + exit 1 + fi local imgdir=$(readlink -f ${instance_dir}/../../../../../var/lib/vizrt/vosa/images/$(basename $instance_dir)/) local backupdir="/var/backup/vizrt/vosa/$(basename $instance_dir)" local backupparent="/var/backup/vizrt/vosa" From 6f05b0e85a5bb8291b849f3009b87a69b421c137 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 23:04:15 +0100 Subject: [PATCH 0393/2585] Added "stop" command. Added checks for start and install to not destroy a running system --- usr/bin/vosa | 59 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index d41bc1f8..f8f9027f 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -161,6 +161,21 @@ function do_enable() { } } +function help_uninstall() { + cat </ if it exists. + +If the VM is in the running state (i.e. it _should_ be +running, this command does nothing. + +If the VM is alive, i.e. actually is running, this +command does nothing. +EOF +} + function do_uninstall() { requires_instance_dir && { # check if the instance is available @@ -205,24 +220,51 @@ commands="$(dirname $0)/../share/vizrt/vosa/commands" function do_install() { requires_instance_dir && { - # todo: - # Error checking: if the instance is not enabled, abort - # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. - # If the instance exists, take a tar.gz backup of the entire image directory + local status=( $(really_do_status $instance_dir) ) + if [ "${status[3]}" == "installed" ] ; then + echo "$instance_dir is already installed. Uninstall before installing." + exit 1 + fi + if [ "${status[2]}" != "enabled" ] ; then + echo "$instance_dir is not enabled, enable it on this host first" + exit 1 + fi ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" } } function do_start() { requires_instance_dir && { - # todo: - # Error checking: if the instance is not enabled, abort - # If the instance exists and is running, abort. its 'statefile' must be set to 'stopped' for a reinstall to happen. - # If the instance exists, take a tar.gz backup of the entire image directory + local status=( $(really_do_status $instance_dir) ) + if [ "${status[6]}" == "alive" ] ; then + echo "$instance_dir is alive. No need to start it." + exit 1 + fi + if [ "${status[3]}" != "installed" ] ; then + echo "$instance_dir is not installed. I can't start it." + exit 1 + fi ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" } } +function do_stop() { + requires_instance_dir && { + local status=( $(really_do_status $instance_dir) ) + if [ "${status[3]}" != "installed" ] ; then + echo "$instance_dir is not installed. I can't stop it." + exit 1 + fi + if [ "${status[6]}" == "alive" ] ; then + echo "Killing kvm process ${status[5]}" + kill "${status[5]}" + # should now be "dead" if we asked it. + fi + # set state to "stopped" + echo "stopped" > "$(dirname $0)/../../var/run/vizrt/vosa/$(basename $instance_dir).state" + } +} + function unknown-argument() { @@ -260,6 +302,7 @@ shift $((LASTOPTIND-1)) if [ $usage -eq 0 -a "${#@}" -gt 1 ] ; then echo "Only one argument is allowed" + # TODO parse each command one at a time... usage=1 fi From 36d2b0b670393cb0c13d145c6749626f064e5cb2 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 23:09:30 +0100 Subject: [PATCH 0394/2585] Made it possible to do many commands after one another (but it doesn't quite work) --- usr/bin/vosa | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index f8f9027f..286c9037 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -295,33 +295,33 @@ LASTOPTIND=$OPTIND } +function verify_usage() { + if [ $usage -eq 1 ] ; then + echo "Usage: $0 [-i ] " + echo "$0 available to get a list of instances" + exit 1; + fi +} + parseopts "$PWD" "${@}" # get rid of all parsed parameters from command line, leaving real arguments shift $((LASTOPTIND-1)) -if [ $usage -eq 0 -a "${#@}" -gt 1 ] ; then - echo "Only one argument is allowed" - # TODO parse each command one at a time... - usage=1 -fi - if [ $usage -eq 0 -a "${#@}" -lt 1 ] ; then echo "A command must be specified." usage=1 fi +verify_usage -# Check if the command -fn="do_$1" -declare > /dev/null -f "$fn" || unknown-argument "$1" -declare > /dev/null -f "$fn" && "$fn" - -if [ $usage -eq 1 ] ; then - echo "Usage: $0 [-i ] " - echo "$0 available to get a list of instances" - exit 1; -fi +for cmd in "${@}" ; do + # Check if the command + fn="do_${cmd}" + declare > /dev/null -f "$fn" || unknown-argument "$1" + declare > /dev/null -f "$fn" && "$fn" + verify_usage +done From 38b1d488a6ed4328306e08a3ff46c9917a6b640d Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 23:12:53 +0100 Subject: [PATCH 0395/2585] local variable --- usr/bin/vosa | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/bin/vosa b/usr/bin/vosa index 286c9037..95c3ba3b 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -88,6 +88,7 @@ EOF function do_status() { if [ -z "$instance_dir" ] ; then + local a for a in $( ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" ) ; do From a48cf9d075caa953a4ba0b0fbddbcd190f8600a8 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 23:25:51 +0100 Subject: [PATCH 0396/2585] Fixed install so that it can actually boot a working machine, with IP and timezone, etc --- usr/share/vizrt/vosa/commands/install.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 4f697df3..aade1f3c 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -129,10 +129,10 @@ function create_overlay() { cat >> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/user-data <> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/meta-data < ${tempdir}/overlay/updates.script <> /etc/network/interfaces < Date: Wed, 8 Feb 2012 23:34:48 +0100 Subject: [PATCH 0397/2585] Actually try to run the postinstall scripts... --- usr/share/vizrt/vosa/commands/install.sh | 12 ++++++++++-- .../vizrt/vosa/post-install-hooks/install-engine.sh | 0 2 files changed, 10 insertions(+), 2 deletions(-) mode change 100644 => 100755 usr/share/vizrt/vosa/post-install-hooks/install-engine.sh diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index aade1f3c..2a856bfb 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -177,8 +177,16 @@ function delete_overlay() { } function postinstall() { - echo "running all the postinstall scripts..." - sleep 4; + for o in "${install_config_postinstall[@]}" ; do + echo "Executing postinstall $o" + local cmd="$(dirname $0)/../post-install-hooks/$o" + if [ ! -x "$cmd" ] ; then + echo "Unable to execute non-executable post-install hook: $cmd" + exit 1 + fi + cmd=$(readlink -f "$cmd") + $cmd "$config" "$image" + done } copy_original_image diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh old mode 100644 new mode 100755 From d8849a2b816ea42d3c5ba78616b4cdec56856ae3 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 23:36:24 +0100 Subject: [PATCH 0398/2585] Handle all values of arrays --- usr/share/vizrt/vosa/commands/install_config_parser | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/install_config_parser b/usr/share/vizrt/vosa/commands/install_config_parser index 1748d1b4..6eedd346 100644 --- a/usr/share/vizrt/vosa/commands/install_config_parser +++ b/usr/share/vizrt/vosa/commands/install_config_parser @@ -50,13 +50,13 @@ declare -a install_config_overlay declare -a install_config_postinstall function install_config_ssh_keys() { - install_config_ssh_keys=( "${ssh_keys[@]}" "$1" ) + install_config_ssh_keys=( "${install_config_ssh_keys[@]}" "$1" ) } function install_config_overlay() { - install_config_overlay=( "${overlay[@]}" "$1" ) + install_config_overlay=( "${install_config_overlay[@]}" "$1" ) } function install_config_postinstall() { - install_config_postinstall=( "${postinstall[@]}" "$1" ) + install_config_postinstall=( "${install_config_postinstall[@]}" "$1" ) } From fe68d84ac8340bedddbc2e2a1939b0fe09f650ae Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 8 Feb 2012 23:54:27 +0100 Subject: [PATCH 0399/2585] Working wait-for-ssh, working ssh.conf, can now ssh from host to guest as root too. --- usr/share/vizrt/vosa/commands/install.sh | 16 ++++++++++++++++ .../vizrt/vosa/post-install-hooks/make-a-motd.sh | 3 +++ .../vosa/post-install-hooks/wait-for-ssh.sh | 12 ++++++++++++ 3 files changed, 31 insertions(+) create mode 100755 usr/share/vizrt/vosa/post-install-hooks/make-a-motd.sh create mode 100755 usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 2a856bfb..9b7fda8f 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -114,6 +114,17 @@ function generate_ssh_key() { ssh-keygen -t dsa -N "" -f ${image}/id_dsa -b 1024 -C "kvm-one-time-key:$name" > /dev/null; exitonerror $? "Unable to generate ssh key" } +function make_ssh_conf() { + cat > ${image}/ssh.conf <> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/meta-data <> ${tempdir}/overlay/updates/home/ubuntu/.ssh/authorized_keys; exitonerror $? "Unable to add authorized key $o" done + chmod 600 ${tempdir}/overlay/updates/root/.ssh/authorized_keys; exitonerror $? "Unable to chmod 600 authorized_keys" chmod 600 ${tempdir}/overlay/updates/home/ubuntu/.ssh/authorized_keys; exitonerror $? "Unable to chmod 600 authorized_keys" cat > ${tempdir}/overlay/updates.script < /dev/null + if [ $? == 0 ] ; then + exit 0 + fi + sleep 1; +done + +exit 1 + From a5b99508559a8eb54717b067addc3e1972117ca3 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 9 Feb 2012 00:02:43 +0100 Subject: [PATCH 0400/2585] Made ssh.conf specifically call the guest for "guest" to make things more homogenous --- usr/share/vizrt/vosa/commands/install.sh | 13 +++++++------ .../vizrt/vosa/post-install-hooks/make-a-motd.sh | 3 ++- .../vizrt/vosa/post-install-hooks/wait-for-ssh.sh | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 9b7fda8f..c5a877b8 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -116,12 +116,13 @@ function generate_ssh_key() { function make_ssh_conf() { cat > ${image}/ssh.conf < /dev/null + ssh -q -F $2/ssh.conf root@guest id > /dev/null if [ $? == 0 ] ; then exit 0 fi From 611ddb3f162f989469c9086485f9533316eb1f29 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 9 Feb 2012 00:14:57 +0100 Subject: [PATCH 0401/2585] now actually calls ece-install as postinst job --- .../vosa/post-install-hooks/install-engine.sh | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh index e864c4dc..171c0143 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh @@ -1,19 +1,34 @@ -# Only argument is the "vm name" as provided on the command line, +#!/bin/bash + +# Two arguments are passed: +# the "vm name" as provided on the command line, # which is a directory path of a VOSA vm definition: # # e.g. /etc/vosa/available.d/vm03 +# +# and the image directory, which holds generated files +# which stick across reboots, but not reinstalls. +# +# e.g. /var/lib/vosa/image/vm03 # The VM is assumed to be booted and ready for SSH using the passwordless -# key in the vm03 directory, as the XXX user (ubuntu?) +# key referenced by the $2/ssh.conf file, as the ubuntu or root users. + +# ensure we have an ece-install image. +if [ ! -r $2/ece-install ] ; then + if [ -r $1/ece-install ] ; then + cp $1/ece-install $2 + else + wget "https://raw.github.com/skybert/ece-scripts/master/usr/sbin/ece-install" -O $2/ece-install + fi +fi +chmod +x $2/ece-install +if [ ! -r $1/ece-install.conf ] ; then + echo "No ece-install.conf file present in $1" + exit 1; +fi + +scp -F $2/ssh.conf $2/ece-install $1/ece-install.conf root@guest: -cat > /dev/null < Date: Thu, 9 Feb 2012 12:22:51 +0800 Subject: [PATCH 0402/2585] - added listing of available commands to "vosa help" --- usr/bin/vosa | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/bin/vosa b/usr/bin/vosa index 95c3ba3b..81abb3ef 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -78,6 +78,12 @@ the regime of Vizrt Online System Administration. "$0 status" gives you a quick overview of the VMs running on this host. EOF + echo "$0 supports these commands:" + declare | grep ^"do_" | cut -d' ' -f1 | cut -d'_' -f2 | sort | \ + while read f; do + echo " *" $f + done + exit 0; else echo "\"$0 -i $instance_dir status\" to see the status of this instance" From 27b4f8eebf7c1de0934ce0e7cc450aa21c24c442 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 9 Feb 2012 22:59:57 +0100 Subject: [PATCH 0403/2585] Added "long help" by using awk to parse the shell script itself, looking for preceding ## comments --- usr/bin/vosa | 115 +++++++++++++++++++++---- usr/share/vizrt/vosa/commands/help.awk | 37 ++++++++ 2 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 usr/share/vizrt/vosa/commands/help.awk diff --git a/usr/bin/vosa b/usr/bin/vosa index 81abb3ef..dd18c353 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -57,18 +57,31 @@ function prohibits_instance_dir() { fi } +## Lists the names of all the instance definitions known to vosa. +## +## An instance known to vosa is an instance which has a directory +## /etc/vizrt/vosa/available.d// +## +## An available instance function do_available() { prohibits_instance_dir && { ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" } } +## Lists the names of all enabled instances +## +## An enabled instance is an instance with a symlink from +## /etc/vizrt/vosa/enabled.d/ -> ../available.d/ +## +## An enabled instance may be installed function do_enabled() { prohibits_instance_dir && { ls -d ${enabled_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" } } +## Provides help... :-) function do_help() { if [ -z "$instance_dir" ] ; then cat < status +## Provides status information on a single VM +## +## Status information consists of up to 8 columns +## - instance name +## The first column is the instance name itself. +## An instance may be unavailable or available. +## - unavailable or available +## An unavailable VM isn't known to the system. +## An available VM is known to the system, in that it contains +## a directory in /etc/vizrt/vosa/available.d// +## An available VM can be enabled or disabled. +## - enabled or disabled +## A disabled VM is not "supposed" to run on this physical machine +## An enabled VM *is* supposed to run on this machine. +## An enabled VM may be installed or uninstalled. +## - installed or uninstalled +## An uninstalled VM does not have an image directory +## An installed VM *does* have an image directory +## An installed VM may be running or stopped +## rest of help excluded because of pending changes to vosa command) function do_status() { if [ -z "$instance_dir" ] ; then local a @@ -146,6 +187,16 @@ function really_do_status () { echo ${output[@]} } +## Tells vosa that it is allowed to install and run an instance. +## +## vosa -i enable +## Enables on this host. +## +## vosa will create a symlink in /etc/vizrt/vosa/enabled.d/ +## pointing to "../available.d/". This is all that is +## required to enable an instance. +## +## An enabled instance can be installed. function do_enable() { requires_instance_dir && { # check if the instance is available @@ -168,21 +219,23 @@ function do_enable() { } } -function help_uninstall() { - cat </ if it exists. - -If the VM is in the running state (i.e. it _should_ be -running, this command does nothing. - -If the VM is alive, i.e. actually is running, this -command does nothing. -EOF -} +## Removes the installed image of an instance +## +## This deletes the contents of the directory +## /var/lib/vizrt/vosa/images// if it exists. +## +## Removing the installed image of an instance makes it impossible to +## start the instance unless it is (re)installed later. Uninstalling +## an instance is destructive, but if the directory +## /var/backup/vizrt/vosa/ +## exists then ONE generation of backups will be kept in that directory, +## and it will be possible to restore an old backup by manually moving +## the backup directory back into /var/lib/vizrt/vosa/images/ +## +## It is not possible to uninstall an instance that is running, or +## that _should_ be running (i.e. if a .pid file exists). function do_uninstall() { requires_instance_dir && { # check if the instance is available @@ -225,6 +278,18 @@ function do_uninstall() { commands="$(dirname $0)/../share/vizrt/vosa/commands" +## Installs an image of an instance +## +## This creates a disk image and supporting files in the directory +## /var/lib/vizrt/vosa/images//, starts a virtual machine +## and configures it as described in the instance configuration +## directory /etc/vizrt/vosa/available.d//. +## +## Installing an instance usually ends up in a functional virtual +## machine. +## +## It is not possible to install an instance that has already been +## installed. function do_install() { requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) @@ -240,7 +305,17 @@ function do_install() { } } +## Starts a stopped instance. +## +## Starts an installed instance that has been stopped or has died +## (or has been shut down from within the guest). A pidfile will be +## created, with the pid of the kvm process, which also serves as a +## flag that the "desired state" of the VM is to be started. +## +## It is not possible to start an instance that has not been installed, +## or that is running. function do_start() { + # TODO: let's make this command wait for SSH access before returning? requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) if [ "${status[6]}" == "alive" ] ; then @@ -255,7 +330,17 @@ function do_start() { } } + +## Stops a running instance. +## +## Stops an instance that has previously been been started. If the instance +## has died, the pidfile will be removed to indicate that the desired state +## is to be stopped. +## +## It is not possible to start an instance that has not been installed, +## or that is running. function do_stop() { + # TODO: let's make this command try to do a graceful shutdown (by ssh'ing to the guest and performing a shutdown, and eventually killing, and wait for the kvm process to die, eventually killing it. requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) if [ "${status[3]}" != "installed" ] ; then diff --git a/usr/share/vizrt/vosa/commands/help.awk b/usr/share/vizrt/vosa/commands/help.awk new file mode 100644 index 00000000..77a0c184 --- /dev/null +++ b/usr/share/vizrt/vosa/commands/help.awk @@ -0,0 +1,37 @@ +#!/usr/bin/awk -f + +BEGIN { + linecount++; +} + +# main +match($0, /^##(.*)$/, a) { + # Store the line for safe keeping. + lines[linecount++] = a[1] + next; +} + + +match($0, /^function +([a-zA-Z0-9_]*)[^a-zA-Z0-9_]/, a) { + if (("fn="a[1]) == ARGV[ARGC-1]) { + for (i = 0; i < linecount; i++) { + if (i == 0) { + print lines[i] + } + else { + print " " lines[i] + } + } + } + next; +} + +{ + # ignore all non-matching lines... + for (i in lines) + delete lines[i] + linecount=0 + next; +} + + From f00c9125d899b944ccac15b7606b5a591743b1e4 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 9 Feb 2012 23:15:52 +0100 Subject: [PATCH 0404/2585] Made vosa completion use the new and concise vosa commands instead of vosa help. --- usr/bin/vosa | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index dd18c353..15038179 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -83,28 +83,29 @@ function do_enabled() { ## Provides help... :-) function do_help() { - if [ -z "$instance_dir" ] ; then - cat < Date: Thu, 9 Feb 2012 23:48:50 +0100 Subject: [PATCH 0405/2585] Made most commands not use the instance name, and print names, instead of full paths... --- usr/bin/vosa | 72 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 15038179..0d961de5 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -13,6 +13,7 @@ # vosa -i ... status --- tells you about the VM, if it's enabled, running, alive, its uptime. instance_dir= +instance_name= usage=0 ### To add an option, add it to the optstring, as defined in getopts, in alphabetical order. @@ -25,8 +26,18 @@ function option-h() { } function option-i() { - instance_dir="${1}" - # TODO remove trailing "/" since it's supposed to be a directory. This script depends on it! + if [[ "${1}" =~ ^[-a-z0-9]*$ ]] ; then + instance_dir=${available_dir}/$1 + else + instance_dir="${1}" + fi + if [ ! -r $instance_dir ] ; then + echo "Instance $1 does not exist. Exiting" + exit 1 + else + instance_dir=$(readlink -f "$instance_dir") + instance_name=$(basename $instance_dir) + fi } @@ -36,21 +47,21 @@ function unknown-option() { } ### To add a command, simply define a function with a do_ prefix. -available_dir=$(dirname $0)/../../etc/vizrt/vosa/available.d +available_dir=$(dirname $(readlink -f $0))/../../etc/vizrt/vosa/available.d available_dir=$(readlink -f ${available_dir}) -enabled_dir=$(dirname $0)/../../etc/vizrt/vosa/enabled.d +enabled_dir=$(dirname $(readlink -f $0))/../../etc/vizrt/vosa/enabled.d enabled_dir=$(readlink -f ${enabled_dir}) function requires_instance_dir() { - if [ -z "$instance_dir" ] ; then + if [ -z "$instance_dir" -o -z "${instance_name}" ] ; then echo "Instance is required" usage=1 return 1 fi } function prohibits_instance_dir() { - if [ ! -z "$instance_dir" ] ; then + if [ ! -z "$instance_dir" -o -z "${instance_name}" ] ; then echo "Instance cannot be specified" usage=1 return 1 @@ -151,7 +162,7 @@ function really_do_status () { local output local avdir="${available_dir}/$(basename $1)" local endir="${enabled_dir}/$(basename $1)" - output=( $avdir ) + output=( $(basename $1) ) if [ ! -d "$avdir" ] ; then output=( ${output[@]} unavailable ); else @@ -204,19 +215,19 @@ function do_enable() { # check if the instance isn't enabled already # check if the instance isn't forcibly disabled # make a symbolic link - if [ ! -d ${available_dir}/$(basename $instance_dir) ] ; then - echo "$instance_dir is not an available instance" + if [ ! -d ${available_dir}/$instance_name ] ; then + echo "$instance_name is not an available instance" exit 1 fi - if [ -L ${enabled_dir}/$(basename $instance_dir) ] ; then - echo "$instance_dir is already enabled" + if [ -L ${enabled_dir}/$instance_name ] ; then + echo "$instance_name is already enabled" exit 1 fi - if [ -r ${enabled_dir}/$(basename $instance_dir) ] ; then - echo "$instance_dir has been forcibly disabled by the presence of the file ${enabled_dir}/$(basename $instance_dir)" + if [ -r ${enabled_dir}/$instance_name ] ; then + echo "$instance_name has been forcibly disabled by the presence of the file ${enabled_dir}/$instance_name" exit 1 fi - ln -v -s "../available.d/$(basename $instance_dir)" "$enabled_dir" || exit 1 + ln -v -s "../available.d/$instance_name" "$enabled_dir" || exit 1 } } @@ -243,25 +254,25 @@ function do_uninstall() { # check if the instance isn't enabled already # check if the instance isn't forcibly disabled # make a symbolic link - if [ ! -d ${available_dir}/$(basename $instance_dir) ] ; then - echo "$instance_dir is not an available instance" + if [ ! -d ${available_dir}/$instance_name ] ; then + echo "$instance_name is not an available instance" exit 1 fi - if [ ! -L ${enabled_dir}/$(basename $instance_dir) ] ; then - echo "$instance_dir is not enabled" + if [ ! -L ${enabled_dir}/$instance_name ] ; then + echo "$instance_name is not enabled" exit 1 fi local status=( $(really_do_status $instance_dir) ) if [ "${status[6]}" == "alive" ] ; then - echo "$instance_dir is alive. Stop it first. exiting" + echo "$instance_name is alive. Stop it first. exiting" exit 1 fi if [ "${status[4]}" == "running" ] ; then - echo "$instance_dir is supposed to be running. Stop it first. Exiting!" + echo "$instance_name is supposed to be running. Stop it first. Exiting!" exit 1 fi - local imgdir=$(readlink -f ${instance_dir}/../../../../../var/lib/vizrt/vosa/images/$(basename $instance_dir)/) - local backupdir="/var/backup/vizrt/vosa/$(basename $instance_dir)" + local imgdir=$(readlink -f ${instance_dir}/../../../../../var/lib/vizrt/vosa/images/$instance_name/) + local backupdir="/var/backup/vizrt/vosa/$instance_name" local backupparent="/var/backup/vizrt/vosa" if [ ! -z "$imgdir" -a -d "$imgdir" ] ; then if [ -w "$backupparent" ] ; then @@ -295,14 +306,14 @@ function do_install() { requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) if [ "${status[3]}" == "installed" ] ; then - echo "$instance_dir is already installed. Uninstall before installing." + echo "$instance_name is already installed. Uninstall before installing." exit 1 fi if [ "${status[2]}" != "enabled" ] ; then - echo "$instance_dir is not enabled, enable it on this host first" + echo "$instance_name is not enabled, enable it on this host first" exit 1 fi - ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" + ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" } } @@ -320,14 +331,14 @@ function do_start() { requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) if [ "${status[6]}" == "alive" ] ; then - echo "$instance_dir is alive. No need to start it." + echo "$instance_name is alive. No need to start it." exit 1 fi if [ "${status[3]}" != "installed" ] ; then - echo "$instance_dir is not installed. I can't start it." + echo "$instance_name is not installed. I can't start it." exit 1 fi - ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$(basename $instance_dir)" + ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" } } @@ -345,16 +356,17 @@ function do_stop() { requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) if [ "${status[3]}" != "installed" ] ; then - echo "$instance_dir is not installed. I can't stop it." + echo "$instance_name is not installed. I can't stop it." exit 1 fi if [ "${status[6]}" == "alive" ] ; then echo "Killing kvm process ${status[5]}" kill "${status[5]}" # should now be "dead" if we asked it. + # TODO: remove PIDfile instead of keeping the state file fi # set state to "stopped" - echo "stopped" > "$(dirname $0)/../../var/run/vizrt/vosa/$(basename $instance_dir).state" + echo "stopped" > "$(dirname $0)/../../var/run/vizrt/vosa/$instance_name.state" } } From b742b50423c9f8893770553965159b8e419638e0 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 10 Feb 2012 00:17:34 +0100 Subject: [PATCH 0406/2585] Simplified status. No longer "probably_dead" and "running" and "stopped" states. Now only running, and dead, both indicating that it _should_ be running --- usr/bin/vosa | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 0d961de5..167412e5 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -173,24 +173,17 @@ function really_do_status () { output=( ${output[@]} enabled ); local vmdir=$(readlink -f ${avdir}/../../../../../var/lib/vizrt/vosa/images/$(basename $1)) if [ ! -d "$vmdir" ] ; then - output=( ${output[@]} uninstalled ); + output=( ${output[@]} uninstalled ); else output=( ${output[@]} installed ); - local statefile=$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/$(basename $1).state) - if [ ! -r "$statefile" ] ; then - output=( ${output[@]} probably_stopped ); - else - output=( ${output[@]} $(<$statefile) ); - if [ ! -z "$(<$statefile)" -a $(<$statefile) == "running" ] ; then - local pidfile="$(dirname $statefile)/$(basename $1).pid" - if [ -r "$pidfile" ] ; then - output=( ${output[@]} $(<$pidfile) ); - if ! ps > /dev/null $(<$pidfile) ; then - output=( ${output[@]} dead ); - else - output=( ${output[@]} alive "$( ps -p "$(<$pidfile)" -o "etime=" )" ); - fi - fi + local pidfile=$(echo_pidfile_of $(basename $1)) + if [ -r "$pidfile" ] ; then + if ! ps > /dev/null $(<$pidfile) ; then + output=( ${output[@]} dead ); + else + output=( ${output[@]} alive ) + output=( ${output[@]} $(<$pidfile) ); + output=( ${output[@]} "$( ps -p "$(<$pidfile)" -o "etime=" )" ); fi fi fi @@ -199,6 +192,11 @@ function really_do_status () { echo ${output[@]} } +function echo_pidfile_of() { + local avdir="${available_dir}/$(basename $1)" + echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/$1.pid)" +} + ## Tells vosa that it is allowed to install and run an instance. ## ## vosa -i enable @@ -263,12 +261,12 @@ function do_uninstall() { exit 1 fi local status=( $(really_do_status $instance_dir) ) - if [ "${status[6]}" == "alive" ] ; then + if [ "${status[4]}" == "alive" ] ; then echo "$instance_name is alive. Stop it first. exiting" exit 1 fi - if [ "${status[4]}" == "running" ] ; then - echo "$instance_name is supposed to be running. Stop it first. Exiting!" + if [ "${status[4]}" == "dead" ] ; then + echo "$instance_name is dead, but supposed to be running. Stop it first. Exiting!" exit 1 fi local imgdir=$(readlink -f ${instance_dir}/../../../../../var/lib/vizrt/vosa/images/$instance_name/) @@ -313,7 +311,7 @@ function do_install() { echo "$instance_name is not enabled, enable it on this host first" exit 1 fi - ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" + ${commands}/install.sh "$instance_dir" "$(dirname $(readlink -f $0))/../../var/lib/vizrt/vosa/images/$instance_name" } } @@ -330,7 +328,7 @@ function do_start() { # TODO: let's make this command wait for SSH access before returning? requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) - if [ "${status[6]}" == "alive" ] ; then + if [ "${status[4]}" == "alive" ] ; then echo "$instance_name is alive. No need to start it." exit 1 fi @@ -338,7 +336,7 @@ function do_start() { echo "$instance_name is not installed. I can't start it." exit 1 fi - ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" + ${commands}/boot.sh "$instance_dir" "$(dirname $(readlink -f $0))/../../var/lib/vizrt/vosa/images/$instance_name" } } @@ -359,14 +357,16 @@ function do_stop() { echo "$instance_name is not installed. I can't stop it." exit 1 fi - if [ "${status[6]}" == "alive" ] ; then + if [ "${status[4]}" == "alive" ] ; then echo "Killing kvm process ${status[5]}" kill "${status[5]}" # should now be "dead" if we asked it. - # TODO: remove PIDfile instead of keeping the state file fi - # set state to "stopped" - echo "stopped" > "$(dirname $0)/../../var/run/vizrt/vosa/$instance_name.state" + # regardless of this, we _want_ it to die. remove pidfile. + # TODO: continue trying to kill it more forcibly, and + # remove the pidfile when it's done. + local pidfile=$(echo_pidfile_of ${status[0]}) + rm $pidfile } } From cbcd16ab120435e03f1d223e36d74250cf113544 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 10 Feb 2012 00:42:32 +0100 Subject: [PATCH 0407/2585] Removed dirname resolution, to allow symlinks in production environment --- usr/bin/vosa | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 167412e5..8b83d171 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -47,9 +47,9 @@ function unknown-option() { } ### To add a command, simply define a function with a do_ prefix. -available_dir=$(dirname $(readlink -f $0))/../../etc/vizrt/vosa/available.d +available_dir=$(dirname $0)/../../etc/vizrt/vosa/available.d available_dir=$(readlink -f ${available_dir}) -enabled_dir=$(dirname $(readlink -f $0))/../../etc/vizrt/vosa/enabled.d +enabled_dir=$(dirname $0)/../../etc/vizrt/vosa/enabled.d enabled_dir=$(readlink -f ${enabled_dir}) @@ -61,7 +61,7 @@ function requires_instance_dir() { fi } function prohibits_instance_dir() { - if [ ! -z "$instance_dir" -o -z "${instance_name}" ] ; then + if [ ! -z "$instance_dir" -o ! -z "${instance_name}" ] ; then echo "Instance cannot be specified" usage=1 return 1 @@ -76,7 +76,7 @@ function prohibits_instance_dir() { ## An available instance function do_available() { prohibits_instance_dir && { - ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" + ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" | sed s,.*/,, } } @@ -88,7 +88,7 @@ function do_available() { ## An enabled instance may be installed function do_enabled() { prohibits_instance_dir && { - ls -d ${enabled_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" + ls -d ${enabled_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" | sed s,.*/,, } } @@ -285,7 +285,7 @@ function do_uninstall() { } -commands="$(readlink -f "$(dirname $(readlink -f $0))")/../share/vizrt/vosa/commands" +commands="$(readlink -f "$(dirname $0)")/../share/vizrt/vosa/commands" ## Installs an image of an instance @@ -311,7 +311,7 @@ function do_install() { echo "$instance_name is not enabled, enable it on this host first" exit 1 fi - ${commands}/install.sh "$instance_dir" "$(dirname $(readlink -f $0))/../../var/lib/vizrt/vosa/images/$instance_name" + ${commands}/install.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" } } @@ -336,7 +336,7 @@ function do_start() { echo "$instance_name is not installed. I can't start it." exit 1 fi - ${commands}/boot.sh "$instance_dir" "$(dirname $(readlink -f $0))/../../var/lib/vizrt/vosa/images/$instance_name" + ${commands}/boot.sh "$instance_dir" "$(dirname $0)/../../var/lib/vizrt/vosa/images/$instance_name" } } From 235aa370339ecb4744b5012205ed7ffd4f42fb63 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 10 Feb 2012 01:27:46 +0100 Subject: [PATCH 0408/2585] Changed status output to be even more terse. Less to describe, less to be confused about. Added lots more documentation. Removed "enabled" and "available" command, they are no longer needed. --- usr/bin/vosa | 151 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 89 insertions(+), 62 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 8b83d171..5fb5d287 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -68,38 +68,45 @@ function prohibits_instance_dir() { fi } -## Lists the names of all the instance definitions known to vosa. -## -## An instance known to vosa is an instance which has a directory -## /etc/vizrt/vosa/available.d// -## -## An available instance -function do_available() { - prohibits_instance_dir && { - ls -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" | sed s,.*/,, - } -} +## Provides instant gratification. +function do_help() { + cat < -> ../available.d/ -## -## An enabled instance may be installed -function do_enabled() { - prohibits_instance_dir && { - ls -d ${enabled_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" | sed s,.*/,, - } +Useful starting points: + - $(basename $0) commands Shows what commands vosa supports + - $(basename $0) status Shows the current status + - $(basename $0) longhelp Shows a long help text +EOF } -## Provides help... :-) -function do_help() { +## Provides even more help. +function do_longhelp() { cat </ + +Each instance needs an 'install.conf' and 'boot.conf' which +describes installation and boot parameters, like ip address, +memory and disk space, and so on. + +An instance may be "enabled" if it is desirable that an instance +definition actually be used on this installation of vosa. + +The idea behind available and enabled instances is that a vosa +cluster will consist of many vosa hosts. For example, 15 different +guests might be running on four different hosts. All 15 definitions +should be present as "available" on all four hosts, but only +the three or four that _should be working_ should be "enabled" for +a host. + EOF echo "$0 supports these commands:" declare -F | grep ^"declare -f do_" | cut -d ' ' -f3 | cut -d'_' -f2 | sort | \ @@ -127,24 +134,46 @@ function do_commands() { ## vosa -i status ## Provides status information on a single VM ## -## Status information consists of up to 8 columns +## Example output: +## vm01 +## vm02 enabled +## vm03 enabled installed +## vm04 enabled installed alive 31374 4-12:45:06 +## vm05 enabled installed dead +## +## The example output shows all possible status. +## +## An instance is defined to be the existsence of a +## directory /etc/vizrt/vosa/available.d// with +## configuration information about the instance. +## +## Status information consists of up to 6 columns ## - instance name ## The first column is the instance name itself. -## An instance may be unavailable or available. -## - unavailable or available -## An unavailable VM isn't known to the system. -## An available VM is known to the system, in that it contains -## a directory in /etc/vizrt/vosa/available.d// -## An available VM can be enabled or disabled. -## - enabled or disabled -## A disabled VM is not "supposed" to run on this physical machine -## An enabled VM *is* supposed to run on this machine. -## An enabled VM may be installed or uninstalled. -## - installed or uninstalled -## An uninstalled VM does not have an image directory -## An installed VM *does* have an image directory -## An installed VM may be running or stopped -## rest of help excluded because of pending changes to vosa command) +## An instance may be enabled (see "vosa enable") +## - enabled +## Only enabled instances are "supposed" to run on this +## physical machine. +## An enabled instance *is* supposed to run on this machine. +## An enabled instance may later be be installed. (see +## "vosa install") +## - installed +## An installed instance has an image directory, created by +## "vosa install" +## An installed instance may be started (see "vosa start"). +## An installed instance may be uninstalled (see "vosa +## uninstall"). +## - alive or dead (actual run state) +## If any of the words alive or dead are present in the fourth +## column then that means that the instance _should_ be running. +## An instance that has been started will probably be alive, but +## if the instance is killed or dies for whatever reason other +## than the "vosa stop" command, it will be labeled as "dead". +## - pid (of an alive instance) +## - uptime (of an alive instance) +## Only shown for instances that are "alive". +## For informational purposes, the pid and the uptime of the pid +## are shown in dd-hh:mm:ss format. function do_status() { if [ -z "$instance_dir" ] ; then local a @@ -163,18 +192,13 @@ function really_do_status () { local avdir="${available_dir}/$(basename $1)" local endir="${enabled_dir}/$(basename $1)" output=( $(basename $1) ) - if [ ! -d "$avdir" ] ; then - output=( ${output[@]} unavailable ); - else - output=( ${output[@]} available ); + if [ -d "$avdir" ] ; then if [ ! -L "$endir" ] ; then output=( ${output[@]} disabled ); else output=( ${output[@]} enabled ); local vmdir=$(readlink -f ${avdir}/../../../../../var/lib/vizrt/vosa/images/$(basename $1)) - if [ ! -d "$vmdir" ] ; then - output=( ${output[@]} uninstalled ); - else + if [ -d "$vmdir" ] ; then output=( ${output[@]} installed ); local pidfile=$(echo_pidfile_of $(basename $1)) if [ -r "$pidfile" ] ; then @@ -246,6 +270,8 @@ function do_enable() { ## ## It is not possible to uninstall an instance that is running, or ## that _should_ be running (i.e. if a .pid file exists). +## +## Please be careful with uninstall. function do_uninstall() { requires_instance_dir && { # check if the instance is available @@ -261,11 +287,11 @@ function do_uninstall() { exit 1 fi local status=( $(really_do_status $instance_dir) ) - if [ "${status[4]}" == "alive" ] ; then + if [ "${status[3]}" == "alive" ] ; then echo "$instance_name is alive. Stop it first. exiting" exit 1 fi - if [ "${status[4]}" == "dead" ] ; then + if [ "${status[3]}" == "dead" ] ; then echo "$instance_name is dead, but supposed to be running. Stop it first. Exiting!" exit 1 fi @@ -303,11 +329,11 @@ commands="$(readlink -f "$(dirname $0)")/../share/vizrt/vosa/commands" function do_install() { requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) - if [ "${status[3]}" == "installed" ] ; then + if [ "${status[2]}" == "installed" ] ; then echo "$instance_name is already installed. Uninstall before installing." exit 1 fi - if [ "${status[2]}" != "enabled" ] ; then + if [ "${status[1]}" != "enabled" ] ; then echo "$instance_name is not enabled, enable it on this host first" exit 1 fi @@ -323,16 +349,17 @@ function do_install() { ## flag that the "desired state" of the VM is to be started. ## ## It is not possible to start an instance that has not been installed, -## or that is running. +## or that is already running and alive. It _is_ possible to (re)start +## a dead instance. function do_start() { # TODO: let's make this command wait for SSH access before returning? requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) - if [ "${status[4]}" == "alive" ] ; then + if [ "${status[3]}" == "alive" ] ; then echo "$instance_name is alive. No need to start it." exit 1 fi - if [ "${status[3]}" != "installed" ] ; then + if [ "${status[2]}" != "installed" ] ; then echo "$instance_name is not installed. I can't start it." exit 1 fi @@ -347,26 +374,26 @@ function do_start() { ## has died, the pidfile will be removed to indicate that the desired state ## is to be stopped. ## -## It is not possible to start an instance that has not been installed, -## or that is running. +## It is not possible to stop an instance that has not been installed. Stopping +## an already stopped instance has no effect. function do_stop() { # TODO: let's make this command try to do a graceful shutdown (by ssh'ing to the guest and performing a shutdown, and eventually killing, and wait for the kvm process to die, eventually killing it. requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) - if [ "${status[3]}" != "installed" ] ; then + if [ "${status[2]}" != "installed" ] ; then echo "$instance_name is not installed. I can't stop it." exit 1 fi - if [ "${status[4]}" == "alive" ] ; then - echo "Killing kvm process ${status[5]}" - kill "${status[5]}" + if [ "${status[3]}" == "alive" ] ; then + echo "Killing kvm process ${status[4]}" + kill "${status[4]}" # should now be "dead" if we asked it. fi # regardless of this, we _want_ it to die. remove pidfile. # TODO: continue trying to kill it more forcibly, and # remove the pidfile when it's done. local pidfile=$(echo_pidfile_of ${status[0]}) - rm $pidfile + rm -f $pidfile } } From 8b1cd0f857d3d3ac52939bd6c7807146b866c240 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 10 Feb 2012 02:20:21 +0100 Subject: [PATCH 0409/2585] Delete overlay, added comments on monitor... --- usr/share/vizrt/vosa/commands/boot.sh | 4 ++++ usr/share/vizrt/vosa/commands/install.sh | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 9d54785e..a94b193a 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -140,6 +140,10 @@ function boot_kvm() { -net "nic,model=virtio,macaddr=${install_config_macaddr}" -net "tap,script=$(dirname $0)/qemu-ifup") +# http://dwdwwebcache.googleusercontent.com/search?q=cache:mEAjcA2zHosJ:kerneltrap.org/mailarchive/linux-kvm/2010/1/26/6257297/thread+qemu-kvm+acpi+shutdown&cd=1&hl=no&ct=clnk&gl=no +# http://kerneltrap.org/mailarchive/linux-kvm/2010/1/26/6257297/thread +# provide a monitor socket to talk to kvm. /var/run? + updates=${image}/updates.iso if [ -r "${updates}" ] ; then startupcmd=("${startupcmd[@]}" -drive "file=${updates},if=virtio") diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index c5a877b8..96950908 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -190,6 +190,7 @@ EOF function delete_overlay() { # ... decho 1 'Deleting initial overlay file' + rm -f ${image}/updates.iso } function postinstall() { @@ -198,10 +199,11 @@ function postinstall() { local cmd="$(dirname $0)/../post-install-hooks/$o" if [ ! -x "$cmd" ] ; then echo "Unable to execute non-executable post-install hook: $cmd" + delete_overlay exit 1 fi cmd=$(readlink -f "$cmd") - $cmd "$config" "$image" + $cmd "$config" "$image" || exitonerror $? "Postinstall script $cmd exited nonzero return code." done } From fab1766e03e2f8c8612e205799baee0884d86db9 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 10 Feb 2012 14:09:04 +0100 Subject: [PATCH 0410/2585] Added monitor socket, to be able to communicate with the guest hardware --- usr/share/vizrt/vosa/commands/boot.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index a94b193a..fa6987fb 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -134,6 +134,7 @@ function boot_kvm() { -m "${boot_config_memory}" $runas -enable-kvm + -monitor unix:${image}/monitor.sock,server,nowait -balloon "virtio" -drive "file=${img},if=virtio,cache=none" -kernel ${kernel} From a59e032bf8bd89f5f6ef555da6ce242428ea2e6f Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 10 Feb 2012 15:45:34 +0100 Subject: [PATCH 0411/2585] Moved socket to rundir per FHS. Added graceful and escalated shutdown. Requires acpi in guest, though. --- usr/bin/vosa | 36 ++++++++++++++++++++++----- usr/share/vizrt/vosa/commands/boot.sh | 2 +- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 5fb5d287..624ba8dd 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -200,7 +200,7 @@ function really_do_status () { local vmdir=$(readlink -f ${avdir}/../../../../../var/lib/vizrt/vosa/images/$(basename $1)) if [ -d "$vmdir" ] ; then output=( ${output[@]} installed ); - local pidfile=$(echo_pidfile_of $(basename $1)) + local pidfile=$(echo_rundir_of $(basename $1))/$(basename $1).pid if [ -r "$pidfile" ] ; then if ! ps > /dev/null $(<$pidfile) ; then output=( ${output[@]} dead ); @@ -216,9 +216,9 @@ function really_do_status () { echo ${output[@]} } -function echo_pidfile_of() { +function echo_rundir_of() { local avdir="${available_dir}/$(basename $1)" - echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/$1.pid)" + echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/)" } ## Tells vosa that it is allowed to install and run an instance. @@ -385,20 +385,44 @@ function do_stop() { exit 1 fi if [ "${status[3]}" == "alive" ] ; then - echo "Killing kvm process ${status[4]}" - kill "${status[4]}" + shutdown "${status[0]}" # should now be "dead" if we asked it. fi # regardless of this, we _want_ it to die. remove pidfile. # TODO: continue trying to kill it more forcibly, and # remove the pidfile when it's done. - local pidfile=$(echo_pidfile_of ${status[0]}) + local pidfile=$(echo_rundir_of ${status[0]})/${status[0]}.pid rm -f $pidfile } } +# Performs a graceful shutdown of the guest +function shutdown() { + local pid=$(<$(echo_rundir_of $1)/$1.pid) + echo "Sending ACPI shutdown signal to guest ($pid) ($(echo_rundir_of $1)/$1.monitor)" + echo 'system_powerdown' | nc -U $(echo_rundir_of $1)/$1.monitor + local a; + for a in $(seq 1 10) ; do + sleep 1 + ps > /dev/null $pid || return 0 + done + ps > /dev/null $pid && { + echo "guest did not shut down. Killing its power." + kill "${status[4]}" + } + for a in $(seq 1 10) ; do + sleep 1 + ps > /dev/null $pid || return 0 + done + ps > /dev/null $pid && { + echo "guest did not respond to kill. Killing with -9." + kill -9 "${status[4]}" + } +} + + function unknown-argument() { echo "Unknown argument $@" usage=1 diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index fa6987fb..effc8672 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -134,7 +134,7 @@ function boot_kvm() { -m "${boot_config_memory}" $runas -enable-kvm - -monitor unix:${image}/monitor.sock,server,nowait + -monitor unix:${rundir}/${hostname}.monitor,server,nowait -balloon "virtio" -drive "file=${img},if=virtio,cache=none" -kernel ${kernel} From d400caeb8f674a351485b5882204cedd8ed094d5 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Tue, 14 Feb 2012 17:29:29 +0100 Subject: [PATCH 0412/2585] Added sanity check for existence of enabled/available directories --- usr/bin/vosa | 51 +++++++++++---------------- usr/share/vizrt/vosa/commands/boot.sh | 1 - 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 624ba8dd..0b038802 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -81,6 +81,11 @@ Useful starting points: EOF } +## Initializes the etc directory with a skeleton of a virtual machine +function do_init() { + echo "not implemented." +} + ## Provides even more help. function do_longhelp() { cat < /dev/null $(<$pidfile) ; then output=( ${output[@]} dead ); @@ -216,9 +221,9 @@ function really_do_status () { echo ${output[@]} } -function echo_rundir_of() { +function echo_pidfile_of() { local avdir="${available_dir}/$(basename $1)" - echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/)" + echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/$1.pid)" } ## Tells vosa that it is allowed to install and run an instance. @@ -385,44 +390,20 @@ function do_stop() { exit 1 fi if [ "${status[3]}" == "alive" ] ; then - shutdown "${status[0]}" + echo "Killing kvm process ${status[4]}" + kill "${status[4]}" # should now be "dead" if we asked it. fi # regardless of this, we _want_ it to die. remove pidfile. # TODO: continue trying to kill it more forcibly, and # remove the pidfile when it's done. - local pidfile=$(echo_rundir_of ${status[0]})/${status[0]}.pid + local pidfile=$(echo_pidfile_of ${status[0]}) rm -f $pidfile } } -# Performs a graceful shutdown of the guest -function shutdown() { - local pid=$(<$(echo_rundir_of $1)/$1.pid) - echo "Sending ACPI shutdown signal to guest ($pid) ($(echo_rundir_of $1)/$1.monitor)" - echo 'system_powerdown' | nc -U $(echo_rundir_of $1)/$1.monitor - local a; - for a in $(seq 1 10) ; do - sleep 1 - ps > /dev/null $pid || return 0 - done - ps > /dev/null $pid && { - echo "guest did not shut down. Killing its power." - kill "${status[4]}" - } - for a in $(seq 1 10) ; do - sleep 1 - ps > /dev/null $pid || return 0 - done - ps > /dev/null $pid && { - echo "guest did not respond to kill. Killing with -9." - kill -9 "${status[4]}" - } -} - - function unknown-argument() { echo "Unknown argument $@" usage=1 @@ -471,6 +452,16 @@ fi verify_usage +if [ -z "$available_dir" -o ! -r "$available_dir" ] ; then + echo "Not initialized! RTFM! Exiting. ($available_dir)" + exit 2 +fi + +if [ -z "$enabled_dir" -o ! -r "$enabled_dir" ] ; then + echo "Not initialized! RTFM! Exiting. ($enabled_dir)" + exit 2 +fi + for cmd in "${@}" ; do # Check if the command fn="do_${cmd}" diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index effc8672..a94b193a 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -134,7 +134,6 @@ function boot_kvm() { -m "${boot_config_memory}" $runas -enable-kvm - -monitor unix:${rundir}/${hostname}.monitor,server,nowait -balloon "virtio" -drive "file=${img},if=virtio,cache=none" -kernel ${kernel} From 8938394582172008206d094a5a1acad67b73df90 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Tue, 14 Feb 2012 17:30:14 +0100 Subject: [PATCH 0413/2585] Merge --- usr/bin/vosa | 36 ++++++++++++++++++++++----- usr/share/vizrt/vosa/commands/boot.sh | 1 + 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index 0b038802..dcd9020c 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -205,7 +205,7 @@ function really_do_status () { local vmdir=$(readlink -f ${avdir}/../../../../../var/lib/vizrt/vosa/images/$(basename $1)) if [ -d "$vmdir" ] ; then output=( ${output[@]} installed ); - local pidfile=$(echo_pidfile_of $(basename $1)) + local pidfile=$(echo_rundir_of $(basename $1))/$(basename $1).pid if [ -r "$pidfile" ] ; then if ! ps > /dev/null $(<$pidfile) ; then output=( ${output[@]} dead ); @@ -221,9 +221,9 @@ function really_do_status () { echo ${output[@]} } -function echo_pidfile_of() { +function echo_rundir_of() { local avdir="${available_dir}/$(basename $1)" - echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/$1.pid)" + echo "$(readlink -f ${avdir}/../../../../../var/run/vizrt/vosa/)" } ## Tells vosa that it is allowed to install and run an instance. @@ -390,20 +390,44 @@ function do_stop() { exit 1 fi if [ "${status[3]}" == "alive" ] ; then - echo "Killing kvm process ${status[4]}" - kill "${status[4]}" + shutdown "${status[0]}" # should now be "dead" if we asked it. fi # regardless of this, we _want_ it to die. remove pidfile. # TODO: continue trying to kill it more forcibly, and # remove the pidfile when it's done. - local pidfile=$(echo_pidfile_of ${status[0]}) + local pidfile=$(echo_rundir_of ${status[0]})/${status[0]}.pid rm -f $pidfile } } +# Performs a graceful shutdown of the guest +function shutdown() { + local pid=$(<$(echo_rundir_of $1)/$1.pid) + echo "Sending ACPI shutdown signal to guest ($pid) ($(echo_rundir_of $1)/$1.monitor)" + echo 'system_powerdown' | nc -U $(echo_rundir_of $1)/$1.monitor + local a; + for a in $(seq 1 10) ; do + sleep 1 + ps > /dev/null $pid || return 0 + done + ps > /dev/null $pid && { + echo "guest did not shut down. Killing its power." + kill "${status[4]}" + } + for a in $(seq 1 10) ; do + sleep 1 + ps > /dev/null $pid || return 0 + done + ps > /dev/null $pid && { + echo "guest did not respond to kill. Killing with -9." + kill -9 "${status[4]}" + } +} + + function unknown-argument() { echo "Unknown argument $@" usage=1 diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index a94b193a..effc8672 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -134,6 +134,7 @@ function boot_kvm() { -m "${boot_config_memory}" $runas -enable-kvm + -monitor unix:${rundir}/${hostname}.monitor,server,nowait -balloon "virtio" -drive "file=${img},if=virtio,cache=none" -kernel ${kernel} From bb047cb1fca0d45b726c7a7688972e29af53166c Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 17 Feb 2012 13:15:23 +0100 Subject: [PATCH 0414/2585] Added support for selecting kvm interface at boot time. --- usr/share/vizrt/vosa/commands/boot.sh | 62 +++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index effc8672..9f93039b 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -65,7 +65,7 @@ hostname=$(basename "$image") parse_config_file $config/install.conf install_config_ parse_config_file $config/boot.conf boot_config_ -make_run_dir() { +function make_run_dir() { rundir=/var/run/vizrt/vosa/ if [ "$(id -u)" != "0" ] ; then rundir=$HOME/.vizrt/vosa/ @@ -74,6 +74,52 @@ make_run_dir() { mkdir -p $rundir || exitonerror $? "Unable to make the place where pidfiles are stored ($rundir)" } +## Select a tap interface +## For now it selects a single tap interface at random, of the ones not +## listed in use by any other machines. +function select_tap_interface() { + if [ -r $rundir/$hostname.network ] ; then + rm -f $rundir/$hostname.network + fi + for try in 1 2 3 4 ; do + # Read the list of available networks we have + declare -A availablenetworks + local network + for network in $rundir/*.availablenetwork ; do + [[ -f $network ]] || continue + # $network == tap4.availablenetwork. Contains the name(s) of vm04's networks + local tap + tap=$(basename $network .availablenetwork) + availablenetworks["$tap"]=1 + done + + echo "Available tap interfaces: ${!availablenetworks[@]}" + + # Read the list of used networks we have + declare -A usednetworks + for network in $rundir/*.network ; do + [[ -f $network ]] || continue + # $network == /.../vm04.network. File contains a list of taps used by vm04 + local vm # "vm04" + vm=$(basename $network .network) + local i + for i in $(<$network) ; do + usednetworks["$i"]=$vm # e.g. usednetworks[tap3]=vm04 + done + done + + echo "tap interfaces already in use : ${!usednetworks[@]}" + + for tap in ${!availablenetworks[@]} ; do + if [ -z "${usednetworks["$tap"]}" ] ; then + tapinterface=$tap + # TODO: only exit when the desired number of network interfaces has been reached... + echo $tap >> "${rundir}/${hostname}.network" && return + fi + done + done +} + function check_sudo() { sudo='' runas='' @@ -104,11 +150,6 @@ function touch_pid_file() { touch "${pidfile}" } -function touch_state_file() { - statefile=$rundir/$hostname.state - echo 'running' > $statefile -} - function configure_vnc_option() { if [ -z "$boot_config_vnc_port" -o "$boot_config_vnc_port" == "none" ] ; then vncoption="-vnc none" @@ -119,6 +160,9 @@ function configure_vnc_option() { function boot_kvm() { + if [ -z "$tapinterface" ] ; then + echo "No tap interface available..." + fi kernel=${image}/vmlinuz img=${image}/disk.img cloud_param="nocloud;h=${hostname}" @@ -139,7 +183,7 @@ function boot_kvm() { -drive "file=${img},if=virtio,cache=none" -kernel ${kernel} -net "nic,model=virtio,macaddr=${install_config_macaddr}" - -net "tap,script=$(dirname $0)/qemu-ifup") + -net "tap,ifname=$tapinterface,script=no,downscript=no") # http://dwdwwebcache.googleusercontent.com/search?q=cache:mEAjcA2zHosJ:kerneltrap.org/mailarchive/linux-kvm/2010/1/26/6257297/thread+qemu-kvm+acpi+shutdown&cd=1&hl=no&ct=clnk&gl=no # http://kerneltrap.org/mailarchive/linux-kvm/2010/1/26/6257297/thread @@ -162,7 +206,7 @@ function boot_kvm() { check_sudo make_run_dir configure_vnc_option +select_tap_interface -touch_pid_file # should maybe be part of boot process? dunno. +touch_pid_file boot_kvm -touch_state_file From be119369543de15676f21898369e5f66cb88a3f6 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 17 Feb 2012 13:30:28 +0100 Subject: [PATCH 0415/2585] Added optional ability to pass user-data to vosa install --- usr/share/vizrt/vosa/commands/install.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index 96950908..db19e43c 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -138,7 +138,10 @@ function create_overlay() { done mkdir -p ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/ - cat >> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/user-data <> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/user-data <> ${tempdir}/overlay/updates/var/lib/cloud/data/cache/nocloud/meta-data < Date: Fri, 17 Feb 2012 13:33:29 +0100 Subject: [PATCH 0416/2585] Made it possible to _configure_ the user data too --- usr/share/vizrt/vosa/commands/install_config_parser | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/share/vizrt/vosa/commands/install_config_parser b/usr/share/vizrt/vosa/commands/install_config_parser index 6eedd346..e44795f3 100644 --- a/usr/share/vizrt/vosa/commands/install_config_parser +++ b/usr/share/vizrt/vosa/commands/install_config_parser @@ -9,6 +9,7 @@ install_config_macaddr= install_config_netmask= install_config_gateway= install_config_timezone= +install_config_user_data= function install_config_ip_address() { @@ -43,6 +44,10 @@ function install_config_timezone() { install_config_timezone=$1 } +function install_config_user_data() { + install_config_user_data=$1 +} + # Handle arrays declare -a install_config_ssh_keys From 823a9ebf4b552abfb7e6ce812c6675f971799236 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 17 Feb 2012 14:52:17 +0100 Subject: [PATCH 0417/2585] Added seed information to actually use the user-data file on boot! --- usr/share/vizrt/vosa/commands/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 9f93039b..e6a21e07 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -165,7 +165,7 @@ function boot_kvm() { fi kernel=${image}/vmlinuz img=${image}/disk.img - cloud_param="nocloud;h=${hostname}" + cloud_param="nocloud;h=${hostname};s=file:///var/lib/cloud/data/cache/nocloud/" # should _maybe_ be put in some other script? Needed by e.g. vosa start too. decho 1 "Starting the machine" From b8941875501cc599a353ea76dbf7132141c60456 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Sun, 19 Feb 2012 12:53:48 +0100 Subject: [PATCH 0418/2585] Made KVM binary not always be "kvm" in path, but look for other obvious locations --- usr/share/vizrt/vosa/commands/boot.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index e6a21e07..5a306c4b 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -9,6 +9,17 @@ # Usually this command is executed from "/usr/bin/vosa -i somevm start" # or similar. +if [ -z "$KVM_BINARY" ] ; then + KVM_BINARY=$(which 2>/dev/null kvm) || + KVM_BINARY=$(which 2>/dev/null qemu-kvm) || + KVM_BINARY=/usr/libexec/qemu-kvm +fi + +if [ -z "$KVM_BINARY" -o ! -x $KVM_BINARY ] ; then + echo Unable to figure out where kvm is installed... export KVM_BINARY to make it work. +fi + + debug=3 function decho() { @@ -132,7 +143,7 @@ $0: It seems I am not able to run kvm as root. I will not be able to run kvm with bridged networking. To fix this, add the file /etc/sudoers.d/kvm with the line - $(id -un) ALL=(ALL) NOPASSWD: /usr/bin/kvm + $(id -un) ALL=(ALL) NOPASSWD: $KVM_BINARY Exiting. EOF @@ -165,11 +176,11 @@ function boot_kvm() { fi kernel=${image}/vmlinuz img=${image}/disk.img - cloud_param="nocloud;h=${hostname};s=file:///var/lib/cloud/data/cache/nocloud/" + cloud_param="nocloud;h=${hostname}" # should _maybe_ be put in some other script? Needed by e.g. vosa start too. decho 1 "Starting the machine" - startupcmd=($sudo kvm + startupcmd=($sudo $KVM_BINARY -daemonize ${vncoption} -name "${hostname}" From 10d1bda310e82010536328ea28f51f2618959484 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Sun, 19 Feb 2012 12:54:07 +0100 Subject: [PATCH 0419/2585] Merged heads --- usr/share/vizrt/vosa/commands/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 5a306c4b..1e6676b6 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -176,7 +176,7 @@ function boot_kvm() { fi kernel=${image}/vmlinuz img=${image}/disk.img - cloud_param="nocloud;h=${hostname}" + cloud_param="nocloud;h=${hostname};s=file:///var/lib/cloud/data/cache/nocloud/" # should _maybe_ be put in some other script? Needed by e.g. vosa start too. decho 1 "Starting the machine" From 47c3e2a5c3743e09800f2d743b4f75b70b191e54 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Sun, 19 Feb 2012 22:09:46 +0100 Subject: [PATCH 0420/2585] Removed pidfile if unable to start kvm --- usr/share/vizrt/vosa/commands/boot.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 1e6676b6..46da8fc3 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -210,7 +210,13 @@ function boot_kvm() { # actually execute kvm echo "${startupcmd[@]}" - "${startupcmd[@]}"; exitonerror $? "Unable to start kvm :-/" + "${startupcmd[@]}"; + local rc + rc=$? + if [ $rc != 0 ] ; + rm $pidfile + exitonerror $rc "Unable to start kvm :-/" + fi } From 5a81332aa3f9a4c7a8675aeabc17b5255ff7b917 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Mon, 20 Feb 2012 12:52:26 +0100 Subject: [PATCH 0421/2585] Typo in boot. Now vosa start respects the processor affinity in boot.conf. --- usr/share/vizrt/vosa/commands/boot.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 46da8fc3..25c6778f 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -174,17 +174,24 @@ function boot_kvm() { if [ -z "$tapinterface" ] ; then echo "No tap interface available..." fi + + if [ ! -z "$boot_config_processor_affinity" ] ; then + echo "Applying processor restriction to $boot_config_processor_affinity" + taskset="taskset -c $boot_config_processor_affinity" + smpoption="-smp cores=2" + fi kernel=${image}/vmlinuz img=${image}/disk.img cloud_param="nocloud;h=${hostname};s=file:///var/lib/cloud/data/cache/nocloud/" # should _maybe_ be put in some other script? Needed by e.g. vosa start too. decho 1 "Starting the machine" - startupcmd=($sudo $KVM_BINARY + startupcmd=($sudo $taskset $KVM_BINARY -daemonize ${vncoption} -name "${hostname}" -cpu "host" + $smpoption -pidfile "${pidfile}" -m "${boot_config_memory}" $runas @@ -213,7 +220,7 @@ function boot_kvm() { "${startupcmd[@]}"; local rc rc=$? - if [ $rc != 0 ] ; + if [ $rc != 0 ] ; then rm $pidfile exitonerror $rc "Unable to start kvm :-/" fi From 52d1fcae1d2463e668fd39e2980ec83b02d87b6c Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Mon, 20 Feb 2012 14:28:20 +0100 Subject: [PATCH 0422/2585] Made number of CPU cores that the guest emulates identical to the number of pinned processors --- usr/share/vizrt/vosa/commands/boot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 25c6778f..4ab03268 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -178,7 +178,8 @@ function boot_kvm() { if [ ! -z "$boot_config_processor_affinity" ] ; then echo "Applying processor restriction to $boot_config_processor_affinity" taskset="taskset -c $boot_config_processor_affinity" - smpoption="-smp cores=2" + cpulist=( "${boot_config_processor_affinity//,/ }" ) + smpoption="-smp cores=${#cpulist[@]}" fi kernel=${image}/vmlinuz img=${image}/disk.img From b2f89ad66dce2a0739172107b7793e98096952e7 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Mon, 20 Feb 2012 14:42:54 +0100 Subject: [PATCH 0423/2585] Now top on host shows the name of the guest: "kvm/vm03" --- usr/share/vizrt/vosa/commands/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 4ab03268..465594d7 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -190,7 +190,7 @@ function boot_kvm() { startupcmd=($sudo $taskset $KVM_BINARY -daemonize ${vncoption} - -name "${hostname}" + -name "${hostname}",process=kvm/${hostname} -cpu "host" $smpoption -pidfile "${pidfile}" From 4d591a1c63331ca72a1ef774ceb1e3cd44d58261 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Mon, 20 Feb 2012 16:58:17 +0100 Subject: [PATCH 0424/2585] typo --- usr/share/vizrt/vosa/commands/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 465594d7..727c3f27 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -178,7 +178,7 @@ function boot_kvm() { if [ ! -z "$boot_config_processor_affinity" ] ; then echo "Applying processor restriction to $boot_config_processor_affinity" taskset="taskset -c $boot_config_processor_affinity" - cpulist=( "${boot_config_processor_affinity//,/ }" ) + cpulist=( ${boot_config_processor_affinity//,/ } ) smpoption="-smp cores=${#cpulist[@]}" fi kernel=${image}/vmlinuz From 3c4f53dac22516eeb57debe91ce9b4b435f67102 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 23 Feb 2012 14:55:23 +0100 Subject: [PATCH 0425/2585] Update to make it possible to run the latest ece-install scripts --- .../vizrt/vosa/post-install-hooks/install-engine.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh index 171c0143..1f6d2851 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh @@ -19,16 +19,15 @@ if [ ! -r $2/ece-install ] ; then if [ -r $1/ece-install ] ; then cp $1/ece-install $2 else - wget "https://raw.github.com/skybert/ece-scripts/master/usr/sbin/ece-install" -O $2/ece-install + wget "https://github.com/skybert/ece-scripts/tarball/master" -O $2/ece-install.tar.gz fi fi -chmod +x $2/ece-install if [ ! -r $1/ece-install.conf ] ; then echo "No ece-install.conf file present in $1" exit 1; fi -scp -F $2/ssh.conf $2/ece-install $1/ece-install.conf root@guest: - -ssh -F $2/ssh.conf root@guest ./ece-install +scp -F $2/ssh.conf $2/ece-install.tar.gz $1/ece-install.conf root@guest: +ssh -F $2/ssh.conf root@guest tar xfz ece-install.tar.gz +ssh -F $2/ssh.conf root@guest bash *-ece-scripts-*/usr/sbin/ece-install From 7ebf804fb18dd6db90f22d13ceae0b3a62545789 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 23 Feb 2012 14:55:41 +0100 Subject: [PATCH 0426/2585] Forced rtl8139 to avoid networking issues --- usr/share/vizrt/vosa/commands/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 727c3f27..19e564bb 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -201,7 +201,7 @@ function boot_kvm() { -balloon "virtio" -drive "file=${img},if=virtio,cache=none" -kernel ${kernel} - -net "nic,model=virtio,macaddr=${install_config_macaddr}" + -net "nic,model=rtl8139,macaddr=${install_config_macaddr}" -net "tap,ifname=$tapinterface,script=no,downscript=no") # http://dwdwwebcache.googleusercontent.com/search?q=cache:mEAjcA2zHosJ:kerneltrap.org/mailarchive/linux-kvm/2010/1/26/6257297/thread+qemu-kvm+acpi+shutdown&cd=1&hl=no&ct=clnk&gl=no From bb73b1e723e0f24f3a0ba9c0076fd70748121d4f Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 23 Feb 2012 15:30:24 +0100 Subject: [PATCH 0427/2585] Made "vosa init" which sets up the /etc directory if anything is missing. Various small typos --- usr/bin/vosa | 241 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 227 insertions(+), 14 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index dcd9020c..b8f22c2d 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -54,6 +54,7 @@ enabled_dir=$(readlink -f ${enabled_dir}) function requires_instance_dir() { + if [ -z "$instance_dir" -o -z "${instance_name}" ] ; then echo "Instance is required" usage=1 @@ -81,9 +82,218 @@ Useful starting points: EOF } -## Initializes the etc directory with a skeleton of a virtual machine +## Initializes the etc directory with the required directory structure +## +## Creates /etc/vizrt/vosa/available.d, enabled.d and a skeleton directory +## with a documented set of files. The files and directories are not +## overwritten if they already exist. function do_init() { - echo "not implemented." + mkdir -p /etc/vizrt/vosa/available.d || exit 2 + mkdir -p /etc/vizrt/vosa/enabled.d || exit 2 + mkdir -p /etc/vizrt/vosa/skeleton || exit 2 + if [ ! -r /etc/vizrt/vosa/skeleton/install.conf ] ; then + cat > /etc/vizrt/vosa/skeleton/install.conf < /etc/vizrt/vosa/skeleton/boot.conf < /etc/vizrt/vosa/user-data/puppetmaster.pp < /tmp/auth.patch </dev/null -d ${available_dir}/* | grep "/[0-9a-z][-0-9a-z]*$" ) ; do really_do_status $a done @@ -459,7 +682,7 @@ LASTOPTIND=$OPTIND function verify_usage() { if [ $usage -eq 1 ] ; then echo "Usage: $0 [-i ] " - echo "$0 available to get a list of instances" + echo "$0 help for more help" exit 1; fi } @@ -476,16 +699,6 @@ fi verify_usage -if [ -z "$available_dir" -o ! -r "$available_dir" ] ; then - echo "Not initialized! RTFM! Exiting. ($available_dir)" - exit 2 -fi - -if [ -z "$enabled_dir" -o ! -r "$enabled_dir" ] ; then - echo "Not initialized! RTFM! Exiting. ($enabled_dir)" - exit 2 -fi - for cmd in "${@}" ; do # Check if the command fn="do_${cmd}" From 2f71b437fdbbebf4355e85e1f2964c442a424e80 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 23 Feb 2012 15:31:37 +0100 Subject: [PATCH 0428/2585] Full support of specifying a mirror --- usr/share/vizrt/vosa/commands/install_config_parser | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/share/vizrt/vosa/commands/install_config_parser b/usr/share/vizrt/vosa/commands/install_config_parser index e44795f3..8c7eea47 100644 --- a/usr/share/vizrt/vosa/commands/install_config_parser +++ b/usr/share/vizrt/vosa/commands/install_config_parser @@ -10,6 +10,7 @@ install_config_netmask= install_config_gateway= install_config_timezone= install_config_user_data= +install_config_mirror= function install_config_ip_address() { @@ -48,6 +49,10 @@ function install_config_user_data() { install_config_user_data=$1 } +function install_config_mirror() { + install_config_mirror=$1 +} + # Handle arrays declare -a install_config_ssh_keys From d9f749f50b8912163f1873458485ce17b3384a4f Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 23 Feb 2012 18:42:40 +0100 Subject: [PATCH 0429/2585] Added "vosa -v oneiric download" which downloads the latest UEC image to /var/lib --- usr/bin/vosa | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index b8f22c2d..a42f7c60 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -19,7 +19,7 @@ usage=0 ### To add an option, add it to the optstring, as defined in getopts, in alphabetical order. ### Also add a function "option-x" where x is the option. $1 will be the option value, if any. ### set any variables needed to default values as globals first. -OPTSTRING=":i:h" +OPTSTRING=":i:v:h" function option-h() { usage=1 @@ -40,6 +40,10 @@ function option-i() { fi } +function option-v() { + uec_version="${1}" +} + function unknown-option() { echo "Unknown option $@" @@ -296,6 +300,40 @@ EOF fi } +## Download an Ubuntu Enterprise Cloud server image +## +## Downloads a tar.gz from Ubuntu Enterprise Cloud and makes it +## available under /var/lib/vizrt/vosa/images// +## unless it already exists. +## +## Required option -v naming the distribution, e.g. oneiric or lucid. +## +## Also changes the symlink of "current" to point to this image. +function do_download() { + if [ -z "$uec_version" ] ; then + echo "-v option is required when downloading." + exit 2 + fi + date=$(date --iso) + local dir + dir=/var/lib/vizrt/vosa/images/$uec_version-$date/ + if [ -d $dir ] ; then + echo "$uec_version-$date already exists, skipping download." + echo "Remove $dir if you want to re-download this image." + else + mkdir /var/lib/vizrt/vosa/images/$uec_version-$date/ || exit 2 + wget http://uec-images.ubuntu.com/server/$uec_version/current/$uec_version-server-cloudimg-amd64.tar.gz -O $dir/$uec_version-server-cloudimg-amd64.tar.gz || exit 2 + tar -x -C $dir -v -f $dir/$uec_version-server-cloudimg-amd64.tar.gz | tee $dir/contents > /dev/null || exit 2 + local kernel + kernel=$(cd $dir && echo ${base}-vmlinuz-*) + fi + if [ -L /var/lib/vizrt/vosa/images/current ] ; then + rm /var/lib/vizrt/vosa/images/current + fi + echo "Marking $uec_version-$date as"' "current"' + ln -s $uec_version-$date /var/lib/vizrt/vosa/images/current +} + ## Provides even more help. function do_longhelp() { cat < Date: Thu, 23 Feb 2012 19:43:52 +0100 Subject: [PATCH 0430/2585] Made some additional checks to see if the image file or the process name are running / in use. Moved the default location of UEC images to /var/lib/vizrt/vosa/uec-images, and fixed the names to be correct (disk.img and vmlinuz) --- usr/bin/vosa | 21 ++++++++++++--------- usr/share/vizrt/vosa/commands/boot.sh | 13 +++++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index a42f7c60..cf1b635a 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -106,8 +106,8 @@ function do_init() { # Which image to use to install the sucker -original_image /var/lib/vizrt/vosa/original-images/current/disk.img -kernel /var/lib/vizrt/vosa/original-images/current/vmlinuz +original_image /var/lib/vizrt/vosa/uec-images/current/disk.img +kernel /var/lib/vizrt/vosa/uec-images/current/vmlinuz # Initial disk size for its own storage, in Gigabytes: Disk will be # reized to this size on installation. @@ -316,22 +316,25 @@ function do_download() { fi date=$(date --iso) local dir - dir=/var/lib/vizrt/vosa/images/$uec_version-$date/ + dir=/var/lib/vizrt/vosa/uec-images/$uec_version-$date/ if [ -d $dir ] ; then echo "$uec_version-$date already exists, skipping download." echo "Remove $dir if you want to re-download this image." else - mkdir /var/lib/vizrt/vosa/images/$uec_version-$date/ || exit 2 + mkdir -p $dir || exit 2 wget http://uec-images.ubuntu.com/server/$uec_version/current/$uec_version-server-cloudimg-amd64.tar.gz -O $dir/$uec_version-server-cloudimg-amd64.tar.gz || exit 2 - tar -x -C $dir -v -f $dir/$uec_version-server-cloudimg-amd64.tar.gz | tee $dir/contents > /dev/null || exit 2 + tar -x --no-same-owner --no-same-permissions -C $dir -v -f $dir/$uec_version-server-cloudimg-amd64.tar.gz | tee $dir/contents > /dev/null || exit 2 local kernel - kernel=$(cd $dir && echo ${base}-vmlinuz-*) + kernel=$(cd $dir && echo *-vmlinuz-*) + image=$(cd $dir && echo *.img) + mv $dir/$kernel $dir/vmlinuz + mv $dir/$image $dir/disk.img fi - if [ -L /var/lib/vizrt/vosa/images/current ] ; then - rm /var/lib/vizrt/vosa/images/current + if [ -L /var/lib/vizrt/vosa/uec-images/current ] ; then + rm /var/lib/vizrt/vosa/uec-images/current fi echo "Marking $uec_version-$date as"' "current"' - ln -s $uec_version-$date /var/lib/vizrt/vosa/images/current + ln -s $uec_version-$date /var/lib/vizrt/vosa/uec-images/current } ## Provides even more help. diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 19e564bb..689c724a 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -216,7 +216,20 @@ function boot_kvm() { startupcmd=("${startupcmd[@]}" -append "root=/dev/vda ro init=/usr/lib/cloud-init/uncloud-init ds=${cloud_param} ubuntu-pass=random $xupdate" ) + ps > /dev/null -fC "kvm/${hostname}" && { + echo "This instance of KVM is already running. Just see here:" + ps -fC "kvm/${hostname}" + exit 2 + } + + lsof > /dev/null ${img} && { + echo "This instance of KVM seems to be in use by another process. Just see here:" + lsof "${img}" + exit 2 + } + # actually execute kvm + echo "Command to star this KVM:" echo "${startupcmd[@]}" "${startupcmd[@]}"; local rc From 7440b766d76cac4a14824221c96db2c2280c090b Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 24 Feb 2012 02:48:34 +0100 Subject: [PATCH 0431/2585] Cleaned up the "shutdown" procedure. Now uses "quit" kvm command, which is possibly more guest friendly. --- usr/bin/vosa | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/usr/bin/vosa b/usr/bin/vosa index cf1b635a..7214f8f3 100755 --- a/usr/bin/vosa +++ b/usr/bin/vosa @@ -646,7 +646,6 @@ function do_start() { ## It is not possible to stop an instance that has not been installed. Stopping ## an already stopped instance has no effect. function do_stop() { - # TODO: let's make this command try to do a graceful shutdown (by ssh'ing to the guest and performing a shutdown, and eventually killing, and wait for the kvm process to die, eventually killing it. requires_instance_dir && { local status=( $(really_do_status $instance_dir) ) if [ "${status[2]}" != "installed" ] ; then @@ -670,20 +669,50 @@ function do_stop() { # Performs a graceful shutdown of the guest function shutdown() { local pid=$(<$(echo_rundir_of $1)/$1.pid) - echo "Sending ACPI shutdown signal to guest ($pid) ($(echo_rundir_of $1)/$1.monitor)" - echo 'system_powerdown' | nc -U $(echo_rundir_of $1)/$1.monitor + if [ -z "$pid" ] ; then + return 0 + fi + ps > /dev/null $pid && { + echo "Sending ACPI shutdown signal to guest ($pid) ($(echo_rundir_of $1)/$1.monitor)" + echo 'system_powerdown' | nc -U $(echo_rundir_of $1)/$1.monitor + echo + echo -n "Waiting 15 seconds for it to shut down" + } local a; - for a in $(seq 1 10) ; do + for a in $(seq 1 15) ; do + echo -n '.' + sleep 1 + ps > /dev/null $pid || { + echo; + return 0; + } + done + ps > /dev/null $pid && { + echo; + echo "Sending quit signal to kvm ($(echo_rundir_of $1)/$1.monitor)" + echo 'quit' | nc -U $(echo_rundir_of $1)/$1.monitor + echo -n "Waiting 30 seconds for it to stop" + } + local a; + for a in $(seq 1 30) ; do + echo -n '.' sleep 1 - ps > /dev/null $pid || return 0 + ps > /dev/null $pid || { + echo; + return 0; + } done ps > /dev/null $pid && { - echo "guest did not shut down. Killing its power." + echo "guest did not shut down. Killing (like pulling power cord)." kill "${status[4]}" + echo -n "Waiting 10 seconds for it to die" } for a in $(seq 1 10) ; do sleep 1 - ps > /dev/null $pid || return 0 + ps > /dev/null $pid || { + echo; + return 0; + } done ps > /dev/null $pid && { echo "guest did not respond to kill. Killing with -9." From b87b6fd565fc6eddd3b9bed1c5f8b027f3d74dc0 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 29 Feb 2012 15:16:37 +0100 Subject: [PATCH 0432/2585] Initial stab at installation of puppet master script. not tested yet. --- .../install-puppet-master.sh | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh new file mode 100644 index 00000000..0b1b378b --- /dev/null +++ b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# A post installation script to set up a fully functional puppet master with +# the ability to prime puppet hosts. + +# First get newly installed packages to not auto-start on installation: +ssh -F $2/ssh.conf root@guest tee > /dev/null /usr/sbin/policy-rc.d < Date: Wed, 29 Feb 2012 19:35:29 +0100 Subject: [PATCH 0433/2585] Setting up the puppet master and generating a key, extract it and provide it to vosa to create clients --- .../install-puppet-master.sh | 77 ++++++++++++++++++- .../vosa/puppet/client-postinst-script.tmpl | 10 +++ 2 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh index 0b1b378b..38b1fca0 100644 --- a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh @@ -3,6 +3,8 @@ # A post installation script to set up a fully functional puppet master with # the ability to prime puppet hosts. +hostname=$(basename $2) + # First get newly installed packages to not auto-start on installation: ssh -F $2/ssh.conf root@guest tee > /dev/null /usr/sbin/policy-rc.d < /etc/vizrt/vosa/puppet/$hostname-client.sh + +/etc/vizrt/vosa/puppet/$hostname-client.sh < /etc/vizrt/vosa/puppet/$hostname-client.sh < /var/lib/puppet/ssl/certs/generic-puppetmaster-client.pem < Date: Wed, 29 Feb 2012 20:53:25 +0100 Subject: [PATCH 0434/2585] Seemingly working puppet-master-from-scratch --- usr/share/vizrt/vosa/commands/install.sh | 2 +- .../vizrt/vosa/commands/install_config_parser | 6 +- .../install-puppet-master.sh | 70 +++++++------------ .../vosa/puppet/client-postinst-script.tmpl | 35 +++++++++- 4 files changed, 61 insertions(+), 52 deletions(-) mode change 100644 => 100755 usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index db19e43c..df179051 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -145,7 +145,7 @@ function create_overlay() { #cloud-config manage_etc_hosts: true timezone: ${install_config_timezone} -apt_update: false +apt_update: true apt_upgrade: false apt_mirror: ${install_config_mirror} EOF diff --git a/usr/share/vizrt/vosa/commands/install_config_parser b/usr/share/vizrt/vosa/commands/install_config_parser index 8c7eea47..d2794de3 100644 --- a/usr/share/vizrt/vosa/commands/install_config_parser +++ b/usr/share/vizrt/vosa/commands/install_config_parser @@ -2,15 +2,15 @@ # e.g. parsing a config file install_config_ip_address= -install_config_original_image= -install_config_kernel= +install_config_original_image=/var/lib/vizrt/vosa/uec-images/current/disk.img +install_config_kernel=/var/lib/vizrt/vosa/uec-images/current/vmlinuz install_config_initial_disk_size= install_config_macaddr= install_config_netmask= install_config_gateway= install_config_timezone= install_config_user_data= -install_config_mirror= +install_config_mirror=mirror://mirrors.ubuntu.com/mirrors.txt function install_config_ip_address() { diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh old mode 100644 new mode 100755 index 38b1fca0..f879d23e --- a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh @@ -45,58 +45,36 @@ ssh -F $2/ssh.conf root@guest rm -f /usr/sbin/policy-rc.d || exit $2 ### Set up the host to be able to spawn puppet master clients: -# 1. Get the private key of the puppetmaster's certificate. Store it to $2/puppet-$hostname.pem. +# 1. Get the private key and signed certficate of the +# newly created puppetmaster's generic client. +# Store it inside $2 -# 1. store a user-data-file in the /etc directory as "$hstname-client.sh" +scp -F $2/ssh.conf \ + root@guest:/var/lib/puppet/ssl/ca/signed/generic-$hostname-client.pem \ + $2/generic-$hostname-client-certificate.pem || exit 2 + +scp -F $2/ssh.conf \ + root@guest:/var/lib/puppet/ssl/private_keys/generic-$hostname-client.pem \ + $2/generic-$hostname-client-private.pem || exit 2 + +# 2. store a user-data-file in the /etc directory as "$hostname-client.sh" # Note: # This needs to be propagated to other vosa servers that want to spawn clients from # this puppet master. This might not be possible to handle using puppet, since puppet # requires this file (certificates) in order to work... # This is designed to run as a postinst hook. +mkdir -p /etc/vizrt/vosa/puppet/ awk ' BEGIN { RS="" } - FILENAME==ARGV[1] { r=$0 } - FILENAME==ARGV[2] { sub("@@PRIVATE_KEY@@",r) ; print } - ' $2/puppet-$hostname.pem /usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl > /etc/vizrt/vosa/puppet/$hostname-client.sh - -/etc/vizrt/vosa/puppet/$hostname-client.sh < /etc/vizrt/vosa/puppet/$hostname-client.sh < /var/lib/puppet/ssl/certs/generic-puppetmaster-client.pem < /etc/vizrt/vosa/puppet/$hostname-client.sh || exit 2 + +ssh -F $2/ssh.conf root@guest /etc/init.d/puppetmaster restart || exit 2 diff --git a/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl b/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl index 2b51e5f5..306635d2 100644 --- a/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl +++ b/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl @@ -2,9 +2,40 @@ ## This file is generated from /var/lib/vizrt/vosa/puppet/client-postinst-script.tmpl!!! -## THE @@ stuff below is replaced on the fly. -cat < /dev/null /usr/sbin/policy-rc.d < /dev/null /etc/puppet/puppet.conf < /dev/null /var/lib/puppet/ssl/private_keys/generic-@@HOSTNAME@@-client.pem < /dev/null /var/lib/puppet/ssl/certs/generic-@@HOSTNAME@@-client.pem < Date: Wed, 29 Feb 2012 21:05:39 +0100 Subject: [PATCH 0435/2585] Allow postinstall scripts to be absolute paths. --- usr/share/vizrt/vosa/commands/install.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/install.sh b/usr/share/vizrt/vosa/commands/install.sh index df179051..6e96e475 100755 --- a/usr/share/vizrt/vosa/commands/install.sh +++ b/usr/share/vizrt/vosa/commands/install.sh @@ -200,7 +200,11 @@ function delete_overlay() { function postinstall() { for o in "${install_config_postinstall[@]}" ; do echo "Executing postinstall $o" - local cmd="$(dirname $0)/../post-install-hooks/$o" + if [ "${o:0:1}" == "/" ] ; then + local cmd=$o + else + local cmd="$(dirname $0)/../post-install-hooks/$o" + fi if [ ! -x "$cmd" ] ; then echo "Unable to execute non-executable post-install hook: $cmd" delete_overlay From 66c2a5a7b52a145bb2f308427d473a025d03b45e Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 29 Feb 2012 21:08:16 +0100 Subject: [PATCH 0436/2585] Made the "hostname-client.sh" postinstall script executable and non-world-readable. --- .../vizrt/vosa/post-install-hooks/install-puppet-master.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh index f879d23e..869a5ae0 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh @@ -77,4 +77,9 @@ awk ' BEGIN { RS="" } | sed s/'@@HOSTNAME@@'/"$hostname"/g \ > /etc/vizrt/vosa/puppet/$hostname-client.sh || exit 2 +# Make private and executable to root only (since it contains the private keys of +# the SSL certificate. +chmod 0500 /etc/vizrt/vosa/puppet/$hostname-client.sh || exit 2 + ssh -F $2/ssh.conf root@guest /etc/init.d/puppetmaster restart || exit 2 + From fe958b3338e6eb7b0cb006d4c7594cd2a03f1232 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Wed, 29 Feb 2012 21:30:05 +0100 Subject: [PATCH 0437/2585] Full puppet config on the client. Make directories before copying them in. Change permisions to match puppet expectations --- .../vosa/puppet/client-postinst-script.tmpl | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl b/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl index 306635d2..074b28ae 100644 --- a/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl +++ b/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl @@ -11,9 +11,23 @@ EOF ssh -F $2/ssh.conf root@guest chmod +x /usr/sbin/policy-rc.d -# Install the puppet master package on the guest, and patch the configuration file +# Install the puppet master package on the guest, and overwrite the configuration file ssh -F $2/ssh.conf root@guest apt-get -y install puppet || exit 2 ssh -F $2/ssh.conf root@guest tee > /dev/null /etc/puppet/puppet.conf < /dev/null /var/lib/puppet/ssl/private_keys/generic-@@HOSTNAME@@-client.pem < /dev/null /var/lib/puppet/ssl/certs/generic-@@HOSTNAME@@-client.pem < Date: Wed, 29 Feb 2012 21:43:51 +0100 Subject: [PATCH 0438/2585] Fixed omission in generifying the scripts --- .../vizrt/vosa/post-install-hooks/install-puppet-master.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh index 869a5ae0..f0ec3662 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh @@ -25,7 +25,7 @@ ssh -F $2/ssh.conf root@guest patch /etc/puppet/auth.conf < Date: Wed, 29 Feb 2012 21:48:47 +0100 Subject: [PATCH 0439/2585] Overwrite the puppet master configuration with one that has a fixed certname for the server. --- .../install-puppet-master.sh | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh index f0ec3662..53a84912 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh @@ -31,6 +31,32 @@ ssh -F $2/ssh.conf root@guest patch /etc/puppet/auth.conf < /dev/null /etc/puppet/puppet.conf < Date: Wed, 29 Feb 2012 21:54:14 +0100 Subject: [PATCH 0440/2585] Feed the IP address of the puppet master to the slaves before installing the puppet client --- .../vizrt/vosa/post-install-hooks/install-puppet-master.sh | 2 ++ usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl | 3 +++ 2 files changed, 5 insertions(+) diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh index 53a84912..27bc6e4b 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-puppet-master.sh @@ -4,6 +4,7 @@ # the ability to prime puppet hosts. hostname=$(basename $2) +ipaddress=$(ssh -F $2/ssh.conf root@guest hostname -I | cut -f 1 -d ' ') # First get newly installed packages to not auto-start on installation: ssh -F $2/ssh.conf root@guest tee > /dev/null /usr/sbin/policy-rc.d < /etc/vizrt/vosa/puppet/$hostname-client.sh || exit 2 # Make private and executable to root only (since it contains the private keys of diff --git a/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl b/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl index 074b28ae..0033b5d6 100644 --- a/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl +++ b/usr/share/vizrt/vosa/puppet/client-postinst-script.tmpl @@ -2,6 +2,9 @@ ## This file is generated from /var/lib/vizrt/vosa/puppet/client-postinst-script.tmpl!!! +ssh -F $2/ssh.conf root@guest tee > /dev/null -a /etc/hosts < /dev/null /usr/sbin/policy-rc.d < Date: Wed, 29 Feb 2012 23:30:27 +0100 Subject: [PATCH 0441/2585] Increased wait-for-ssh timeout to 90 seconds. Also made wait-for-ssh wait for any cloud-init apt-get calls to complete, to avoid race conditions when installing on fast hardware, or when apt-get update is comparatively slow, or apt-get is busy installing lots of packages. Maybe a better way to handle this, by checking some cloud-init status. --- usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh b/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh index 5b52ab44..7f293685 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh @@ -1,8 +1,15 @@ #!/bin/bash -for a in $(seq 1 20) ; do +for a in $(seq 1 90) ; do ssh -q -F $2/ssh.conf root@guest id > /dev/null if [ $? == 0 ] ; then + pid=1 + # Wait for existing apt-gets to complete. + while [ ! -z "$pid" ] ; + do + pid=$(ssh -F $2/ssh.conf root@guest pidof apt-get) + sleep 1; + done exit 0 fi sleep 1; From cf0708ea13467f76a38779891f87f925cc45ff10 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 1 Mar 2012 14:46:03 +0100 Subject: [PATCH 0442/2585] Removed unused file --- usr/lib/vosa/install-engine.sh | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 usr/lib/vosa/install-engine.sh diff --git a/usr/lib/vosa/install-engine.sh b/usr/lib/vosa/install-engine.sh deleted file mode 100644 index e864c4dc..00000000 --- a/usr/lib/vosa/install-engine.sh +++ /dev/null @@ -1,19 +0,0 @@ -# Only argument is the "vm name" as provided on the command line, -# which is a directory path of a VOSA vm definition: -# -# e.g. /etc/vosa/available.d/vm03 - -# The VM is assumed to be booted and ready for SSH using the passwordless -# key in the vm03 directory, as the XXX user (ubuntu?) - -cat > /dev/null < Date: Thu, 1 Mar 2012 14:51:46 +0100 Subject: [PATCH 0443/2585] Updated initial README file, and initial documentation of /usr/bin/vosa --- usr/share/doc/vizrt/vosa-guide.org | 63 ++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 usr/share/doc/vizrt/vosa-guide.org diff --git a/usr/share/doc/vizrt/vosa-guide.org b/usr/share/doc/vizrt/vosa-guide.org new file mode 100644 index 00000000..ff31004b --- /dev/null +++ b/usr/share/doc/vizrt/vosa-guide.org @@ -0,0 +1,63 @@ +Installation of VOSA package +============================ + + +Requirements: +--------------- +- kvm or qemu-kvm +- genisoimage +- nc +- tunctl +- sudo access to kvm (or run vosa as root...) +- a bridged network + +This installation will: + +- download an UEC image and kernel +- create 10 tap interfaces (or less) (see below) +- create ten "tap*.availablenetwork" fileis (where * is a number between 0 and 10) + in /var/run/vizrt/vosa/ signifying the names of the tap interfaces that can be used. +- write (exclusively?) access to /var/run/vizrt/vosa/ + + +Setup +----- + + mkdir -p /etc/vizrt/vosa/available.d/vm-1 + mkdir -p /etc/vizrt/vosa/enabled.d + mkdir -p /var/run/vizrt/vosa/ + mkdir -p /var/lib/vizrt/vosa/images/ + mkdir -p /var/lib/vizrt/vosa/original-images/ + +Install the vosa command itself and its required libraries + + ln -s $PWD/usr/bin/vosa /usr/bin/vosa + ln -s $PWD/usr/share/vizrt/ /usr/share/vizrt + +Set up an image + + cd /var/lib/vizrt/vosa/original-images/ + mkdir oneiric-2012-01-29 + ln -s oneiric-2012-01-29 latest + cd latest + tar xfz ~/mogsie-autovm/downloads/oneiric-server-cloudimg-amd64.tar.gz + mv oneiric-server-cloudimg-amd64.img disk.img + mv oneiric-server-cloudimg-amd64-vmlinuz-virtual vmlinuz + rm oneiric-server-cloudimg-amd64-* + rm README.files + +Make a bridge, br0 or something (outside the scope of this document) + +make 10 tap interfaces and make them known to vosa (this probably needs to happen every boot... + + for i in $(seq 1 10) ; do + tap=$(tunctl -b) + touch /var/run/vizrt/vosa/$tap.availablenetwork + brctl addif br0 $tap + ifconfig $tap up 0.0.0.0 + done + + + + + From 726bb0531ca97ceb1f46d141a95daed70df04d83 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 1 Mar 2012 14:52:42 +0100 Subject: [PATCH 0444/2585] Updated initial README file, and initial documentation of /usr/bin/vosa --- README | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README b/README index 7a85bc50..33f7381d 100644 --- a/README +++ b/README @@ -28,6 +28,11 @@ Features of the /usr/bin/system-info script =========================================== https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/system-info-guide.org +==================================== +Features of the /usr/bin/vosa script +==================================== +https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/vizrt/vosa-guide.org + ======== Feedback ======== From c20082ae1ea99ad9ccbecc443bcdec6b137b154c Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 1 Mar 2012 15:01:25 +0100 Subject: [PATCH 0445/2585] updated documentation, now closer to reality. migrated to orgmode. --- usr/share/doc/vizrt/vosa-guide.org | 52 ++++++++++++++++-------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/usr/share/doc/vizrt/vosa-guide.org b/usr/share/doc/vizrt/vosa-guide.org index ff31004b..63f9bf67 100644 --- a/usr/share/doc/vizrt/vosa-guide.org +++ b/usr/share/doc/vizrt/vosa-guide.org @@ -1,14 +1,12 @@ -Installation of VOSA package -============================ +* Installation of VOSA package -Requirements: ---------------- +** Requirements: - kvm or qemu-kvm - genisoimage - nc - tunctl -- sudo access to kvm (or run vosa as root...) +- sudo access to kvm (or the more usual, which is to run vosa as root...) - a bridged network This installation will: @@ -20,31 +18,19 @@ This installation will: - write (exclusively?) access to /var/run/vizrt/vosa/ -Setup ------ +** Setup +Install the vosa command itself and its required libraries - mkdir -p /etc/vizrt/vosa/available.d/vm-1 - mkdir -p /etc/vizrt/vosa/enabled.d - mkdir -p /var/run/vizrt/vosa/ - mkdir -p /var/lib/vizrt/vosa/images/ - mkdir -p /var/lib/vizrt/vosa/original-images/ + : ln -s $PWD/usr/bin/vosa /usr/bin/vosa + : ln -s $PWD/usr/share/vizrt/ /usr/share/vizrt -Install the vosa command itself and its required libraries +Initialize it: - ln -s $PWD/usr/bin/vosa /usr/bin/vosa - ln -s $PWD/usr/share/vizrt/ /usr/share/vizrt + : vosa init -Set up an image +Download an Ubuntu Enterprise Cloud image to use as the base OS - cd /var/lib/vizrt/vosa/original-images/ - mkdir oneiric-2012-01-29 - ln -s oneiric-2012-01-29 latest - cd latest - tar xfz ~/mogsie-autovm/downloads/oneiric-server-cloudimg-amd64.tar.gz - mv oneiric-server-cloudimg-amd64.img disk.img - mv oneiric-server-cloudimg-amd64-vmlinuz-virtual vmlinuz - rm oneiric-server-cloudimg-amd64-* - rm README.files + vosa -v oneiric download Make a bridge, br0 or something (outside the scope of this document) @@ -57,7 +43,23 @@ make 10 tap interfaces and make them known to vosa (this probably needs to happe ifconfig $tap up 0.0.0.0 done +Define a virtual machine: + + mkdir /etc/vizrt/vosa/available.d/my-first-vm && + cp /etc/vizrt/vosa/skeleton/* /etc/vizrt/vosa/available.d/my-first-vm/ + +Configure these files (they're well documented) + + vi /etc/vizrt/vosa/available.d/my-first-vm/* + +Make sure your IP and MAC addresses are unique + +Finally, enable and install your virtual machine: + + vosa -i my-first-vm enable install +When it's done you can SSH into the system: + ssh -F /var/lib/vizrt/vosa/my-first-vm/ssh.conf root@guest From d084f8d424b4e858999e3707ba47bf5ce5e5308c Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 1 Mar 2012 15:02:29 +0100 Subject: [PATCH 0446/2585] more orgmode --- usr/share/doc/vizrt/vosa-guide.org | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/usr/share/doc/vizrt/vosa-guide.org b/usr/share/doc/vizrt/vosa-guide.org index 63f9bf67..49acb0b3 100644 --- a/usr/share/doc/vizrt/vosa-guide.org +++ b/usr/share/doc/vizrt/vosa-guide.org @@ -30,36 +30,36 @@ Initialize it: Download an Ubuntu Enterprise Cloud image to use as the base OS - vosa -v oneiric download + : vosa -v oneiric download Make a bridge, br0 or something (outside the scope of this document) make 10 tap interfaces and make them known to vosa (this probably needs to happen every boot... - for i in $(seq 1 10) ; do - tap=$(tunctl -b) - touch /var/run/vizrt/vosa/$tap.availablenetwork - brctl addif br0 $tap - ifconfig $tap up 0.0.0.0 - done + : for i in $(seq 1 10) ; do + : tap=$(tunctl -b) + : touch /var/run/vizrt/vosa/$tap.availablenetwork + : brctl addif br0 $tap + : ifconfig $tap up 0.0.0.0 + : done Define a virtual machine: - mkdir /etc/vizrt/vosa/available.d/my-first-vm && - cp /etc/vizrt/vosa/skeleton/* /etc/vizrt/vosa/available.d/my-first-vm/ + : mkdir /etc/vizrt/vosa/available.d/my-first-vm && + : cp /etc/vizrt/vosa/skeleton/* /etc/vizrt/vosa/available.d/my-first-vm/ Configure these files (they're well documented) - vi /etc/vizrt/vosa/available.d/my-first-vm/* + : vi /etc/vizrt/vosa/available.d/my-first-vm/* Make sure your IP and MAC addresses are unique Finally, enable and install your virtual machine: - vosa -i my-first-vm enable install + : vosa -i my-first-vm enable install When it's done you can SSH into the system: - ssh -F /var/lib/vizrt/vosa/my-first-vm/ssh.conf root@guest + : ssh -F /var/lib/vizrt/vosa/my-first-vm/ssh.conf root@guest From cbfd24c1fbc9b095187667c270a5878ec9570d23 Mon Sep 17 00:00:00 2001 From: "Erik Mogensen (@mogsie)" Date: Thu, 1 Mar 2012 22:32:36 +0100 Subject: [PATCH 0447/2585] Updated vosa documentation --- usr/share/doc/vizrt/vosa-guide.org | 210 ++++++++++++++++++++++++++--- 1 file changed, 192 insertions(+), 18 deletions(-) diff --git a/usr/share/doc/vizrt/vosa-guide.org b/usr/share/doc/vizrt/vosa-guide.org index 49acb0b3..0c271d2d 100644 --- a/usr/share/doc/vizrt/vosa-guide.org +++ b/usr/share/doc/vizrt/vosa-guide.org @@ -1,7 +1,88 @@ +* What is VOSA? + +Vosa is short for Vizrt Online System Administration, a set of tools +created by Vizrt to ease various system administration chores. + +It facilitates the set up and installation of virtual machines, +booting them up and installing software on them. It also includes +steps to set up a puppet master with a generic client certificate, +and tools to install puppets that do what that puppet master says. + +All of this is possible manually by apt-get'ing and configuring +manualy; the value of these scripts is that all of this is +made possible without intervention. + +There are various post-installation hooks available which perform +various tasks, for simple things like creating an /etc/motd file, +to more advanced things like setting up a puppet master or +installing a production ready Escenic Content Engine using +ece-install. + +** Who is it for? + +It is aimed at anyone who needs to install lots of virtual machines, +and particularly at those who agree that it is valuable to be able +to completely reinstall a virtual machine from scratch without +intervention, and then put that virtual machine in production. + +** Why should I use it? + +The normal way of virtualizing is to sit someone in front of a +fresh VM and start to install software on it. The software is +configured, and more software piles on until it's production ready. +Some final tweaks are made and then it might go into production. +All of this installation, configuration and tweaking is /valuable/ +in the sense that doing it all over again actually costs money since +someone needs to sit down and do it. Most of the time, the people +who did the original installation are no longer around, so doing it +all from scratch would also revert some tweaks. + +A different way of looking at virtualization is to ascribe /no +value/ to the virtual machine itself, or at least to the the disk +image that constitutes the virtual machine. In order to do so, you +need to make sure that it is possible to create a fully functional +virtual machine at any time. One that is production ready, with all +the last-minute-tweaks in place. + +The Vosa scripts do just that. + +A short definition file tells vosa what base image to start with; +this is an Ubuntu Enterprise Cloud image, but in theory other image +types could be supported. The file also provides it an IP address +and the machine's time zone, and so on. Most importantly, these +files also tell vosa what to do with the virtual machine once it +has booted, so-called "post install hooks". Each of these are +executed when the "first boot" has completed. These hooks run +only the first time the virtual machine has booted, and should be +written in such a way that they end up with a production ready +system that can go straight into production. + +The machines can be stopped, started, and so on, and since vosa +relies on simple kvm, you can access the kvm monitor or vnc if you +want to. Vosa also sets up throw-away ssh keys that the host uses +to interact with the guest during the first boot. + +** What can I do with this? + +Probably the most advanced thing you could do with this is to +automate the setup of a complete data center, with database, nfs, +a virtualized server park, and various virtual machines, tomcats, +varnishes and so on, all from a single set of configuration files, +without intervention. + +But you could also use it to just easily re-install a VM using the +latest and greatest Ubuntu release, instead of (as is usual) not even +daring to run apt-get upgrade. + * Installation of VOSA package +This is a terse description of what it takes to get a system to run +vosa. ** Requirements: + +In order to benefit from vosa, you need the following: + - kvm or qemu-kvm - genisoimage - nc @@ -9,32 +90,58 @@ - sudo access to kvm (or the more usual, which is to run vosa as root...) - a bridged network -This installation will: +In theory, it should be possible to script this too, or to get e.g. +puppet to do this for us. Watch this space! -- download an UEC image and kernel -- create 10 tap interfaces (or less) (see below) -- create ten "tap*.availablenetwork" fileis (where * is a number between 0 and 10) - in /var/run/vizrt/vosa/ signifying the names of the tap interfaces that can be used. -- write (exclusively?) access to /var/run/vizrt/vosa/ +** Setup +Setting up vosa entails a few manual installation steps. Among other +things it will +- provide the "vosa" command and its required libraries +- download an image of an Ubuntu Enterprise Cloud +- create 10 tap interfaces (an arbitrary number, really, see below + for an explanation). +- create the same number of "tap*.availablenetwork" files in + /var/run/vizrt/vosa/ each one signifying the names of the tap + interfaces that can be used. -** Setup -Install the vosa command itself and its required libraries +So without further ado, let's get started. First of all, we need to +onstall the vosa command itself and its required libraries: : ln -s $PWD/usr/bin/vosa /usr/bin/vosa : ln -s $PWD/usr/share/vizrt/ /usr/share/vizrt -Initialize it: +The vosa command should now work (to a certain extent) + + : vosa help + : vosa commands + : vosa longhelp | less + +To make vosa usable, you need to initialize it: : vosa init -Download an Ubuntu Enterprise Cloud image to use as the base OS +This will create the /etc/vizrt/vosa directory structure, and a +skeleton of a virtual machine definition. + +Let's download an Ubuntu Enterprise Cloud (UEC) image to use as the +base OS. vosa does this for you: : vosa -v oneiric download -Make a bridge, br0 or something (outside the scope of this document) +** Networking + +For this to be useful, your machines need to be accessible directly +on the local network. The scripts have only been tested on a bridged +network. So make a bridge, call it br0 or something. How this is +done is, however outside the scope of this document. -make 10 tap interfaces and make them known to vosa (this probably needs to happen every boot... +Once you have a bridged network, you need to create tap interfaces +for each of your virtual machines. Let's make 10 to start with. +The reason these need to be pre-allocated is that we've seen that +doing this temporarily (ca 10 seconds) makes the network go +completely dark, and so shouldn't happen whenever any virtual +machines are running. : for i in $(seq 1 10) ; do : tap=$(tunctl -b) @@ -43,23 +150,90 @@ make 10 tap interfaces and make them known to vosa (this probably needs to happe : ifconfig $tap up 0.0.0.0 : done -Define a virtual machine: +tunctl creates the tap interface, and we create a file with the name +of the created tap interface in a directory. This little snippet +needs to run every time the host machine boots. + +** Defining a virtual machine + +Defining a virtual machine is a bit different than when using virsh +or VMware of VirtualBox. Vosa exploits the fact that the UEC images +are pre-seeded with cloud-init, and so have a hook to execute code +during the first boot. This means we don't need to make any changes +to the image file itself, but can boot the unmodified UEC image. + +A big benefit of this is that the exact same UEC images are available +in Amazon EC2, and also in a Eucalyptus private cloud. This means +that vosa will be able to control Amazon EC2 images in the same way. + +Defining a virtual machine means creating two files (boot.conf and +install.conf) in a directory. "vosa init" has already created a +documented skeleton which you can customize as you see fit. : mkdir /etc/vizrt/vosa/available.d/my-first-vm && : cp /etc/vizrt/vosa/skeleton/* /etc/vizrt/vosa/available.d/my-first-vm/ + : vi /etc/vizrt/vosa/available.d/my-first-vm/* -Configure these files (they're well documented) +Note that the name you choose ("my-first-vm") must be a valid +internet host name with no domain part. I.e. only lowercase +alphanumerics and hyphens. The name you choose will become the +virtual machine's host name. - : vi /etc/vizrt/vosa/available.d/my-first-vm/* +When you're happy with them you should of course track these in a +version control system, so you don't lose them. Over time, these +will become more valuable than the virtual machine images themselves. + +Make sure your IP and MAC addresses are unique, or make a script to +randomize them. + +Now, enable your virtual machine: + + : vosa -i my-first-vm enable -Make sure your IP and MAC addresses are unique +This creates a symlink from available.d/my-first-vm to enabled.d, it +serves no other purpose than to differentiate between a possibly long +list of virtual machine definitions (in available.d), and the ones +you have decided to actually run on this machine. -Finally, enable and install your virtual machine: +To install the machine, just issue the "install" command: - : vosa -i my-first-vm enable install + : vosa -i my-first-vm install + +This will copy the disk image to /var/lib/vizrt/vosa/images/my-first-vm/ +and put some more files in there (like the SSH private key), and +finally it will boot up the image and use the UEC's cloud-init +support to prime the image and execute any post-installation hooks +you defined. + +The host name (as the machine sees it, at least) will be the same as +the name of the virtual machine; in this case "my-first-vm" When it's done you can SSH into the system: : ssh -F /var/lib/vizrt/vosa/my-first-vm/ssh.conf root@guest +Not that you should need to do that, of course. + +* Puppet Master + +Setting up a puppet master is also an important piece of vosa. + +To make this possible, vosa supplies a post-install hook. This hook: + +- installs the puppet master from the apt repositories, +- configures the puppet master to use hostnames instead of its DNS + name for certificates +- configures a self signed certificate for all guests (mainly to avoid + having to sign or auto-sign the puppets, since that would be + problematic when a puppet happens to be re-installed) +- creates vosa post-installation hook to set up a pre-authenticated + puppet in /etc/vizrt/vosa/puppet/-client.sh + +This makes it possible to define more virtual machines that +automatically dance to the puppet master's tune. + +Making this useful of course means pushing your puppet configuration +into the puppet master, but that's outside the scope of this +document. + From 0906f4ef84af300f2e7ac4ce17653950b09e5070 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 11:48:51 +0800 Subject: [PATCH 0448/2585] - moved WF code to its own module --- usr/sbin/ece-install | 86 ------------------- .../ece-install.d/widget-framework.sh | 86 +++++++++++++++++++ 2 files changed, 86 insertions(+), 86 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/widget-framework.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 3d124516..6065a250 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1474,92 +1474,6 @@ function get_boolean_conf_value() fi } -function install_widget_framework() -{ - print_and_log "Installing Widget Framework on $HOSTNAME ..." - # TODO java.lang.NoClassDefFoundError: - # Lcom/escenic/framework/captcha/ReCaptchaConfig; - - wf_user=$(get_conf_value wf_user) - wf_password=$(get_conf_value wf_password) - - if [ -z "$wf_user" -o -z "$wf_password" ]; then - print_and_log "Missing wf_user and wf_password in ${conf_file}. If you" - print_and_log "don't have these, please contact support@escenic.com" - remove_pid_and_exit_in_error - fi - - print_and_log "Creating a Maven settings file: $HOME/.m2/settings.xml ..." - make_dir $HOME/.m2 - cat > $HOME/.m2/settings.xml < - - - - escenic-repo - ${wf_user} - ${wf_password} - - - - - - escenic-profile - - - escenic-repo - Repository for EWF libraries - http://repo.escenic.com/ - default - - - - - - - escenic-profile - - -EOF - - print_and_log "Downloading Widget Framework from technet.escenic.com ..." - for el in $wf_download_list; do - cd $download_dir - run wget $wget_opts \ - --http-user $technet_user \ - --http-password $technet_password \ - $el - run cd $escenic_root_dir/ - run unzip -q -u $download_dir/$(basename $el) - done - - install_packages_if_missing "maven2" - assert_pre_requisite mvn - - export JAVA_HOME=$java_home - wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) - run cd $wf_maven_dir - - print_and_log "Installing Widget Framework into your Maven repository ..." - log "JAVA_HOME=$JAVA_HOME" - - run mvn $maven_opts install - - # installing the widget-framework-common as a ECE plugin - wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) - cd $escenic_root_dir/assemblytool/plugins - if [ ! -h $(basename $wf_dist_dir) ]; then - ln -s $wf_dist_dir - fi - - cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ - - add_next_step "Widget Framework has been installed into your " \ - " Maven repo" -} # useful for development and test environments. function install_all_in_one_environment() diff --git a/usr/share/escenic/ece-scripts/ece-install.d/widget-framework.sh b/usr/share/escenic/ece-scripts/ece-install.d/widget-framework.sh new file mode 100644 index 00000000..00ed8075 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/widget-framework.sh @@ -0,0 +1,86 @@ +function install_widget_framework() +{ + print_and_log "Installing Widget Framework on $HOSTNAME ..." + # TODO java.lang.NoClassDefFoundError: + # Lcom/escenic/framework/captcha/ReCaptchaConfig; + + local wf_user=$(get_conf_value wf_user) + local wf_password=$(get_conf_value wf_password) + + if [ -z "$wf_user" -o -z "$wf_password" ]; then + print_and_log "Missing wf_user and wf_password in ${conf_file}. If you" + print_and_log "don't have these, please contact support@escenic.com" + remove_pid_and_exit_in_error + fi + + print_and_log "Creating a Maven settings file: $HOME/.m2/settings.xml ..." + make_dir $HOME/.m2 + cat > $HOME/.m2/settings.xml < + + + + escenic-repo + ${wf_user} + ${wf_password} + + + + + + escenic-profile + + + escenic-repo + Repository for EWF libraries + http://repo.escenic.com/ + default + + + + + + + escenic-profile + + +EOF + + print_and_log "Downloading Widget Framework from technet.escenic.com ..." + for el in $wf_download_list; do + cd $download_dir + run wget $wget_opts \ + --http-user $technet_user \ + --http-password $technet_password \ + $el + run cd $escenic_root_dir/ + run unzip -q -u $download_dir/$(basename $el) + done + + install_packages_if_missing "maven2" + assert_pre_requisite mvn + + export JAVA_HOME=$java_home + wf_maven_dir=$(echo $escenic_root_dir/widget-framework-core-*/maven) + run cd $wf_maven_dir + + print_and_log "Installing Widget Framework into your Maven repository ..." + log "JAVA_HOME=$JAVA_HOME" + + run mvn $maven_opts install + + # installing the widget-framework-common as a ECE plugin + wf_dist_dir=$(echo $wf_maven_dir/widget-framework-common/target/widget-framework-common-*-dist/widget-framework-common-*) + cd $escenic_root_dir/assemblytool/plugins + if [ ! -h $(basename $wf_dist_dir) ]; then + ln -s $wf_dist_dir + fi + + cp -r $wf_dist_dir/misc/siteconfig/* $common_nursery_dir/ + + add_next_step "Widget Framework has been installed into your " \ + " Maven repo" +} From 1d823a7d9cef88bbc11f56d37852b31077df879b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 11:53:03 +0800 Subject: [PATCH 0449/2585] - moved restore from backup into its own module --- usr/sbin/ece-install | 181 ------------------ .../ece-install.d/restore-from-backup.sh | 179 +++++++++++++++++ 2 files changed, 179 insertions(+), 181 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/restore-from-backup.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 6065a250..a8ed7edf 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1600,187 +1600,6 @@ function add_server_to_runlevels() fi } -function restore_from_backup() -{ - # locals - restore_all=0 - restore_db=0 - restore_binaries=0 - restore_data_files=0 - restore_conf=0 - backup_file="" - backup_dir=$escenic_backups_dir - - if [ $(get_boolean_conf_value fai_enabled) -eq 1 -a \ - $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then - backup_file=${fai_restore_from_file} - - if [ -z "$backup_file" ]; then - print_and_log "You must specifify fai_restore_from_file" - remove_pid_and_exit_in_error - fi - - if [ $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then - restore_all=1 - elif [ $(get_boolean_conf_value fai_restore_db) -eq 1 ]; then - restore_db=1 - elif [ $(get_boolean_conf_value fai_restore_data_files) -eq 1 ]; then - restore_data_files=1 - elif [ $(get_boolean_conf_value fai_restore_software_binaries) -eq 1 ] - then - restore_binaries=1 - elif [ $(get_boolean_conf_value fai_restore_configuration) -eq 1 ]; then - restore_conf=1 - fi - elif [ $(get_boolean_conf_value fai_enabled) -eq 0 ]; then - print "From which dataset do you wish to restore?" - if [ ! -d $backup_dir ]; then - print_and_log "Directory $backup_dir doesn't exist or isn't readable" - remove_pid_and_exit_in_error - fi - - if [ $(ls $backup_dir | grep ".tar$" | wc -l) -lt 1 ]; then - print_and_log "No backup files (.tar) found in $backup_dir, exiting." - exit 0 - fi - - tarball_array=($(ls -t $backup_dir/*.tar)) - - for (( i = 0; i <${#tarball_array[@]}; i++ )); do - echo " " $(( ${i} + 1 )) "-" $(basename ${tarball_array[$i]}) - done - - print "Enter the number next to the tarball, from 1 to $i" - echo -n "Your choice [1]> " - read user_tarball - - if [ -z "$user_tarball" ]; then - user_tarball=1 - fi - - backup_file=${tarball_array[$(( ${user_tarball} - 1 ))]} - - print "Which part of the system do you wish to restore?" - restore_profiles=( - "The database" - "The Solr and ECE data files (multimedia archive)" - "The ECE configuration files" - "The Escenic and Tomcat software binaries + publication templates" - "Restore everything of the above" - ) - for (( i = 0; i <${#restore_profiles[@]}; i++ )); do - echo " " $(( ${i} + 1 )) "-" ${restore_profiles[$i]} - done - - print "Enter the number next to the tarball, from 1 to $i" - echo -n "Your choice [1]> " - read user_restore_profile - - if [ -z "$user_restore_profile" ]; then - user_restore_profile=1 - fi - - if [ $user_restore_profile -eq 1 ]; then - restore_db=1 - elif [ $user_restore_profile -eq 2 ]; then - restore_data_files=1 - elif [ $user_restore_profile -eq 3 ]; then - restore_conf=1 - elif [ $user_restore_profile -eq 4 ]; then - restore_binaries=1 - elif [ $user_restore_profile -eq 5 ]; then - restore_all=1 - fi - fi - - if [ ! -r "$backup_file" ]; then - print_and_log "$backup_file either doesn't exist or cannot be read." - print_and_log "I cannot restore from it :-(" - remove_pid_and_exit_in_error - fi - - dir=$(mktemp -d) - - if [ $restore_db -eq 1 -o $restore_all -eq 1 ]; then - install_database_server "binaries_only" - print_and_log "Restoring the database contents on $HOSTNAME ..." - run cd $dir - run tar xf $backup_file --wildcards var/backups/escenic/*.sql.gz - # picking the DB backup file to restore - sql_file=$(ls -tra var/backups/escenic/*.sql.gz | tail -1) - print_and_log "Selecting database dump: $(basename $sql_file)" - - # methods in drop-and-create-ecedb to set up the database - # schema & user - pre_install_new_ecedb - create_schema - - # db_schema is defined in drop-and-create-ecedb - gunzip < $sql_file | mysql $db_schema - exit_on_error "restoring from $sql_file" - - add_next_step "$(green Successfully) restored DB from $sql_file" - fi - - if [ $restore_data_files -eq 1 -o $restore_all -eq 1 ]; then - print_and_log "Restoring the Solr & ECE data files on $HOSTNAME ..." - run cd $dir - run tar -C / -xf $backup_file var/lib/escenic - add_next_step "$(green Successfully) restored Solr & ECE data files" - add_next_step "Backup file used: $(basename $backup_file)" - add_next_step "Check $escenic_data_dir to verify they're all there." - fi - - if [ $restore_conf -eq 1 -o $restore_all -eq 1 ]; then - print_and_log "Restoring the ECE configuration files on $HOSTNAME ..." - run cd $dir - run tar -C / -xf $backup_file etc - add_next_step "$(green Successfully) restored ECE configuration files" - add_next_step "Backup file used: $(basename $backup_file)" - add_next_step "Check /etc to verify that they're all there." - fi - - if [ $restore_binaries -eq 1 -o $restore_all -eq 1 ]; then - print_and_log "Restoring the Escenic & Tomcat binaries on $HOSTNAME ..." - run cd $dir - run tar -C / -xf $backup_file opt - add_next_step "$(green Successfully) restored Escenic & Tomcat binaries" - add_next_step "Backup file used: $(basename $backup_file)" - add_next_step "Check ${appserver_parent_dir} to verify that they're all there". - install_ece_third_party_packages - set_up_engine_directories - - # doing some educated guessing on which tomcat_base/home as - # well as UNIX user/group to use for this. - file=/etc/default/ece - if [ -r $file ]; then - ece_user=$(grep ^ece_unix_user $file | \ - cut -d'=' -f2 | \ - sed -e "s/'//g" -e 's/"//g' - ) - ece_group=$(grep ^ece_unix_group $file | \ - cut -d'=' -f2 | \ - sed -e "s/'//g" -e 's/"//g' - ) - create_user_and_group_if_not_present $ece_user $ece_group - - if [ -d $escenic_conf_dir -a \ - $(ls $escenic_conf_dir/ | grep ^ece- | wc -l) -gt 0 -a \ - $(grep ^tomcat_base $escenic_conf_dir/ece*.conf | wc -l) -gt 0 ]; then - directories=$(grep ^tomcat_base $escenic_conf_dir/ece*.conf | \ - cut -d'=' -f2 | \ - sed -e "s/'//g" -e 's/"//g' - ) - s="Setting file permissions according to /etc/default/ece" - print_and_log $s - s="and $escenic_conf_dir/ece*.conf" - print_and_log $s - run chown -R ${ece_user}:${ece_group} ${directories} - fi - fi - fi -} - ## $1 configuration file name ## $2 host group name ## $3 host group alias diff --git a/usr/share/escenic/ece-scripts/ece-install.d/restore-from-backup.sh b/usr/share/escenic/ece-scripts/ece-install.d/restore-from-backup.sh new file mode 100644 index 00000000..3313da65 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/restore-from-backup.sh @@ -0,0 +1,179 @@ +function restore_from_backup() +{ + local restore_all=0 + local restore_db=0 + local restore_binaries=0 + local restore_data_files=0 + local restore_conf=0 + local backup_file="" + local backup_dir=$escenic_backups_dir + + if [ $(get_boolean_conf_value fai_enabled) -eq 1 -a \ + $(get_boolean_conf_value fai_restore_from_backup) -eq 1 ]; then + backup_file=${fai_restore_from_file} + + if [ -z "$backup_file" ]; then + print_and_log "You must specifify fai_restore_from_file" + remove_pid_and_exit_in_error + fi + + if [ $(get_boolean_conf_value fai_restore_all) -eq 1 ]; then + restore_all=1 + elif [ $(get_boolean_conf_value fai_restore_db) -eq 1 ]; then + restore_db=1 + elif [ $(get_boolean_conf_value fai_restore_data_files) -eq 1 ]; then + restore_data_files=1 + elif [ $(get_boolean_conf_value fai_restore_software_binaries) -eq 1 ] + then + restore_binaries=1 + elif [ $(get_boolean_conf_value fai_restore_configuration) -eq 1 ]; then + restore_conf=1 + fi + elif [ $(get_boolean_conf_value fai_enabled) -eq 0 ]; then + print "From which dataset do you wish to restore?" + if [ ! -d $backup_dir ]; then + print_and_log "Directory $backup_dir doesn't exist or isn't readable" + remove_pid_and_exit_in_error + fi + + if [ $(ls $backup_dir | grep ".tar$" | wc -l) -lt 1 ]; then + print_and_log "No backup files (.tar) found in $backup_dir, exiting." + exit 0 + fi + + local tarball_array=($(ls -t $backup_dir/*.tar)) + + for (( i = 0; i <${#tarball_array[@]}; i++ )); do + echo " " $(( ${i} + 1 )) "-" $(basename ${tarball_array[$i]}) + done + + print "Enter the number next to the tarball, from 1 to $i" + echo -n "Your choice [1]> " + read user_tarball + + if [ -z "$user_tarball" ]; then + user_tarball=1 + fi + + backup_file=${tarball_array[$(( ${user_tarball} - 1 ))]} + + print "Which part of the system do you wish to restore?" + restore_profiles=( + "The database" + "The Solr and ECE data files (multimedia archive)" + "The ECE configuration files" + "The Escenic and Tomcat software binaries + publication templates" + "Restore everything of the above" + ) + for (( i = 0; i <${#restore_profiles[@]}; i++ )); do + echo " " $(( ${i} + 1 )) "-" ${restore_profiles[$i]} + done + + print "Enter the number next to the tarball, from 1 to $i" + echo -n "Your choice [1]> " + read user_restore_profile + + if [ -z "$user_restore_profile" ]; then + user_restore_profile=1 + fi + + if [ $user_restore_profile -eq 1 ]; then + restore_db=1 + elif [ $user_restore_profile -eq 2 ]; then + restore_data_files=1 + elif [ $user_restore_profile -eq 3 ]; then + restore_conf=1 + elif [ $user_restore_profile -eq 4 ]; then + restore_binaries=1 + elif [ $user_restore_profile -eq 5 ]; then + restore_all=1 + fi + fi + + if [ ! -r "$backup_file" ]; then + print_and_log "$backup_file either doesn't exist or cannot be read." + print_and_log "I cannot restore from it :-(" + remove_pid_and_exit_in_error + fi + + local dir=$(mktemp -d) + + if [ $restore_db -eq 1 -o $restore_all -eq 1 ]; then + install_database_server "binaries_only" + print_and_log "Restoring the database contents on $HOSTNAME ..." + run cd $dir + run tar xf $backup_file --wildcards var/backups/escenic/*.sql.gz + # picking the DB backup file to restore + sql_file=$(ls -tra var/backups/escenic/*.sql.gz | tail -1) + print_and_log "Selecting database dump: $(basename $sql_file)" + + # methods in drop-and-create-ecedb to set up the database + # schema & user + pre_install_new_ecedb + create_schema + + # db_schema is defined in drop-and-create-ecedb + gunzip < $sql_file | mysql $db_schema + exit_on_error "restoring from $sql_file" + + add_next_step "$(green Successfully) restored DB from $sql_file" + fi + + if [ $restore_data_files -eq 1 -o $restore_all -eq 1 ]; then + print_and_log "Restoring the Solr & ECE data files on $HOSTNAME ..." + run cd $dir + run tar -C / -xf $backup_file var/lib/escenic + add_next_step "$(green Successfully) restored Solr & ECE data files" + add_next_step "Backup file used: $(basename $backup_file)" + add_next_step "Check $escenic_data_dir to verify they're all there." + fi + + if [ $restore_conf -eq 1 -o $restore_all -eq 1 ]; then + print_and_log "Restoring the ECE configuration files on $HOSTNAME ..." + run cd $dir + run tar -C / -xf $backup_file etc + add_next_step "$(green Successfully) restored ECE configuration files" + add_next_step "Backup file used: $(basename $backup_file)" + add_next_step "Check /etc to verify that they're all there." + fi + + if [ $restore_binaries -eq 1 -o $restore_all -eq 1 ]; then + print_and_log "Restoring the Escenic & Tomcat binaries on $HOSTNAME ..." + run cd $dir + run tar -C / -xf $backup_file opt + add_next_step "$(green Successfully) restored Escenic & Tomcat binaries" + add_next_step "Backup file used: $(basename $backup_file)" + add_next_step "Check ${appserver_parent_dir} to verify that they're all there". + install_ece_third_party_packages + set_up_engine_directories + + # doing some educated guessing on which tomcat_base/home as well + # as UNIX user/group to use for this. + file=/etc/default/ece + if [ -r $file ]; then + ece_user=$(grep ^ece_unix_user $file | \ + cut -d'=' -f2 | \ + sed -e "s/'//g" -e 's/"//g' + ) + ece_group=$(grep ^ece_unix_group $file | \ + cut -d'=' -f2 | \ + sed -e "s/'//g" -e 's/"//g' + ) + create_user_and_group_if_not_present $ece_user $ece_group + + if [ -d $escenic_conf_dir -a \ + $(ls $escenic_conf_dir/ | grep ^ece- | wc -l) -gt 0 -a \ + $(grep ^tomcat_base $escenic_conf_dir/ece*.conf | wc -l) -gt 0 ]; then + directories=$(grep ^tomcat_base $escenic_conf_dir/ece*.conf | \ + cut -d'=' -f2 | \ + sed -e "s/'//g" -e 's/"//g' + ) + s="Setting file permissions according to /etc/default/ece" + print_and_log $s + s="and $escenic_conf_dir/ece*.conf" + print_and_log $s + run chown -R ${ece_user}:${ece_group} ${directories} + fi + fi + fi +} From bb647f14e337fd02a09410409bcec46f214e0241 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 13:23:17 +0800 Subject: [PATCH 0450/2585] - hardening: better error reporting if the creation of java packages & repo fails (was wrapped in a pulse). --- .../escenic/ece-scripts/ece-install.d/java.sh | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/java.sh b/usr/share/escenic/ece-scripts/ece-install.d/java.sh index b8603aee..48544acb 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/java.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/java.sh @@ -7,17 +7,15 @@ function create_java_deb_packages_and_repo() { print $(lsb_release -i | cut -d':' -f2) \ $(lsb_release -r | cut -d':' -f2) \ "doesn't have official Sun/Oracle Java packages," + print "creating packages & local repo for you ..." local tmp_dir=$(mktemp -d) - $( - run cd $tmp_dir - run git clone https://github.com/flexiondotorg/oab-java6.git - run cd oab-java6 - run bash oab-java6.sh - run rm -rf $tmp_dir - ) & - show_pulse $! "I'm creating packages & local repo for you" - + run cd $tmp_dir + run git clone https://github.com/flexiondotorg/oab-java6.git + run cd oab-java6 + run bash oab-java6.sh + run rm -rf $tmp_dir + add_next_step "Local APT repository with Sun/Oracle Java packages" add_next_step "has been installed at /var/local/oab/deb and added" add_next_step "to your APT system with /etc/apt/sources.list.d/oab.list" From 29106588644def77d330dd68322dd60d7fe6155c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 13:27:18 +0800 Subject: [PATCH 0451/2585] - removed redundant call to install_packages_if_missing inside install_database_server --- usr/share/escenic/ece-scripts/ece-install.d/database.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index c699f0f2..95a6d6f3 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -68,7 +68,6 @@ function install_database_server() add_apt_source "deb http://repo.percona.com/apt ${code_name} main" packages="percona-server-server percona-server-client" - install_packages_if_missing $packages force_packages=0 else print_and_log "The Percona APT repsository doesn't have packages" From 0acb4e07147ed2c36c6d227e88fbe5e3deab3036 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 13:31:11 +0800 Subject: [PATCH 0452/2585] - constants are just that, constant, this implies they're also read-only. -> declare -r --- .../ece-scripts/ece-install.d/constants.sh | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/constants.sh b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh index ec10854b..c743494d 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/constants.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh @@ -1,13 +1,17 @@ -PROFILE_ALL_IN_ONE=1 -PROFILE_CACHE_SERVER=5 -PROFILE_DB_SERVER=4 -PROFILE_EDITORIAL_SERVER=2 -PROFILE_PRESENTATION_SERVER=3 -PROFILE_RMI_HUB=6 -PROFILE_SEARCH_SERVER=7 -PROFILE_WIDGET_FRAMEWORK=8 -PROFILE_CREATE_PUBLICATION=9 -PROFILE_MONITORING_SERVER=10 -PROFILE_RESTORE_FROM_BACKUP=11 -PROFILE_ANALYSIS_SERVER=12 +# constants used in several/all the ece-install modules. + +# constants are just that, constant, this implies they're also +# read-only. +declare -r PROFILE_ALL_IN_ONE=1 +declare -r PROFILE_CACHE_SERVER=5 +declare -r PROFILE_DB_SERVER=4 +declare -r PROFILE_EDITORIAL_SERVER=2 +declare -r PROFILE_PRESENTATION_SERVER=3 +declare -r PROFILE_RMI_HUB=6 +declare -r PROFILE_SEARCH_SERVER=7 +declare -r PROFILE_WIDGET_FRAMEWORK=8 +declare -r PROFILE_CREATE_PUBLICATION=9 +declare -r PROFILE_MONITORING_SERVER=10 +declare -r PROFILE_RESTORE_FROM_BACKUP=11 +declare -r PROFILE_ANALYSIS_SERVER=12 From 30d9a0b0ac7f1d329c65a9b2217494af94feddf6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 14:02:40 +0800 Subject: [PATCH 0453/2585] - moved more monitoring code to the monitoring.sh module --- usr/sbin/ece-install | 36 ------------------- .../ece-scripts/ece-install.d/monitoring.sh | 36 +++++++++++++++++++ 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a8ed7edf..c8833d89 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1635,41 +1635,6 @@ EOF EOF } -## $1 nagios vendor -function create_monitoring_server_overview() -{ - local file=/var/www/index.html - cat > $file < - -

    Welcome to the might monitoring server @ ${HOSTNAME}

    -
      -EOF - if [[ $1 == $MONITORING_VENDOR_NAGIOS ]]; then - echo '
    • Nagios
    • ' \ - else - echo '
    • Icinga (an enhanced Nagios)
    • ' \ - >> $file - fi - cat > $file <Munin -
    - - -EOF - add_next_step "Start page for all monitoring interfaces: http://${HOSTNAME}/" -} - -MONITORING_VENDOR_NAGIOS=nagios -MONITORING_VENDOR_ICINGA=icinga -function install_monitoring_server() -{ - local nagios_flavour=${fai_monitoring_nagios_flavour-$MONITORING_VENDOR_ICINGA} - install_nagios_monitoring_server $nagios_flavour - install_munin_gatherer - create_monitoring_server_overview $nagios_flavour -} - function common_post_install() { if [ $install_profile_number -eq $PROFILE_ALL_IN_ONE -o \ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER -o \ @@ -1719,7 +1684,6 @@ done init assert_correct_runtime_environment - fai_enabled=$(get_boolean_conf_value fai_enabled) if [ $fai_enabled -eq 1 ]; then diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 9fc01959..6b545ddd 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -1,5 +1,8 @@ # module for installing monitoring software, both server and client side. +MONITORING_VENDOR_NAGIOS=nagios +MONITORING_VENDOR_ICINGA=icinga + ## Installs the Nagios monitoring server. ## $1 the nagios vendor/falvour, "nagios" and "icinga" are supported. function install_nagios_monitoring_server() @@ -288,3 +291,36 @@ EOF add_next_step "Munin gatherer admin interface: http://${HOSTNAME}/munin" add_next_step "Make sure all nodes allows its IP to connect to them." } + +## $1 nagios vendor +function create_monitoring_server_overview() +{ + local file=/var/www/index.html + cat > $file < + +

    Welcome to the might monitoring server @ ${HOSTNAME}

    +
      +EOF + if [[ $1 == $MONITORING_VENDOR_NAGIOS ]]; then + echo '
    • Nagios
    • ' \ + else + echo '
    • Icinga (an enhanced Nagios)
    • ' \ + >> $file + fi + cat > $file <Munin +
    + + +EOF + add_next_step "Start page for all monitoring interfaces: http://${HOSTNAME}/" +} + +function install_monitoring_server() +{ + local nagios_flavour=${fai_monitoring_nagios_flavour-$MONITORING_VENDOR_ICINGA} + install_nagios_monitoring_server $nagios_flavour + install_munin_gatherer + create_monitoring_server_overview $nagios_flavour +} From f5dc509453d7f18cb6f59699e43a3e49b040b2d8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 14:03:06 +0800 Subject: [PATCH 0454/2585] - the scope gets too local when using declare -r (even with -x outside a method). --- .../ece-scripts/ece-install.d/constants.sh | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/constants.sh b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh index c743494d..d757dad1 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/constants.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh @@ -1,17 +1,23 @@ # constants used in several/all the ece-install modules. -# constants are just that, constant, this implies they're also -# read-only. -declare -r PROFILE_ALL_IN_ONE=1 -declare -r PROFILE_CACHE_SERVER=5 -declare -r PROFILE_DB_SERVER=4 -declare -r PROFILE_EDITORIAL_SERVER=2 -declare -r PROFILE_PRESENTATION_SERVER=3 -declare -r PROFILE_RMI_HUB=6 -declare -r PROFILE_SEARCH_SERVER=7 -declare -r PROFILE_WIDGET_FRAMEWORK=8 -declare -r PROFILE_CREATE_PUBLICATION=9 -declare -r PROFILE_MONITORING_SERVER=10 -declare -r PROFILE_RESTORE_FROM_BACKUP=11 -declare -r PROFILE_ANALYSIS_SERVER=12 +# installation profiles +PROFILE_ALL_IN_ONE=1 +PROFILE_CACHE_SERVER=5 +PROFILE_DB_SERVER=4 +PROFILE_EDITORIAL_SERVER=2 +PROFILE_PRESENTATION_SERVER=3 +PROFILE_RMI_HUB=6 +PROFILE_SEARCH_SERVER=7 +PROFILE_WIDGET_FRAMEWORK=8 +PROFILE_CREATE_PUBLICATION=9 +PROFILE_MONITORING_SERVER=10 +PROFILE_RESTORE_FROM_BACKUP=11 +PROFILE_ANALYSIS_SERVER=12 + +# database constants +default_db_port=3306 +default_db_host=localhost +default_db_user=ece5user +default_db_password=ece5password +default_db_schema=ece5db From 61732d960e997f2245ee193a18baa1725c401615 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 14:03:41 +0800 Subject: [PATCH 0455/2585] - moved constants to constants.sh --- usr/share/escenic/ece-scripts/ece-install.d/database.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index 95a6d6f3..dfee59bf 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -1,5 +1,6 @@ # ece-install module for installing the database. +# Setting the correct URLs for the percona releas (bootstrap) RPM package. percona_rpm_release_version=0.0-1 percona_rpm_release_package_name=percona-release-${percona_rpm_release_version} percona_rpm_release_url=http://www.percona.com/downloads/percona-release/$percona_rpm_release_package_name.x86_64.rpm @@ -7,12 +8,6 @@ if [[ $(uname -m) != "x86_64" ]]; then percona_rpm_release_url=http://www.percona.com/downloads/percona-release/$percona_rpm_release_package_name.i386.rpm fi -default_db_port=3306 -default_db_host=localhost -default_db_user=ece5user -default_db_password=ece5password -default_db_schema=ece5db - ## $1: optional parameter, binaries_only. If passed, $1=binaries_only, ## the ECE DB schema is not set up. function install_database_server() From 4db69df78b77e5ca32a50f9ca953a74d71ec8597 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 14:36:24 +0800 Subject: [PATCH 0456/2585] - added webservice to the default deployment list for editorial servers. --- usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index a20a5653..600528db 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -12,7 +12,7 @@ function get_deploy_white_list() elif [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then white_list="${white_list} "$(get_publication_short_name_list) elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - white_list="${white_list} escenic studio indexer-webservice" + white_list="${white_list} escenic studio indexer-webservice webservice" white_list="${white_list} "$(get_publication_short_name_list) fi From be0be180759f406b9b8209273bddc93e4431b565 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 2 Mar 2012 14:41:58 +0800 Subject: [PATCH 0457/2585] - moved profile=publication to its own module - moved profile=web server to its own module --- usr/sbin/ece-install | 133 ------------------ .../ece-install.d/create-publication.sh | 79 +++++++++++ .../ece-scripts/ece-install.d/web-server.sh | 85 +++++++++++ 3 files changed, 164 insertions(+), 133 deletions(-) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/create-publication.sh create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/web-server.sh diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index c8833d89..8c6ed3f3 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1355,17 +1355,6 @@ function assemble_deploy_and_restart_type() exit_on_error "su - $ece_user -c \"$ece_command\"" } -function ensure_that_instance_is_running() -{ - ece_command="ece -i $1 -t $type status" - if [ $(su - $ece_user -c "$ece_command" | grep UP | wc -l) -lt 1 ]; then - ece_command="ece -i $1 -t $type start" - su - $ece_user -c "$ece_command" 1>>$log 2>>$log - # TODO improve this by adding a timed while loop - sleep 60 - fi -} - # If the system is installed using the recommended paths, the method # will return a list of the instances configured in # ${escenic_conf_dir}/engine/instance @@ -1380,42 +1369,6 @@ function get_instance_list() echo $instance_list } -function create_publication() -{ - if [ ! -e $escenic_root_dir/engine -o \ - ! -e $escenic_root_dir/assemblytool ]; then - print_and_log "Please install ECE and an assembly environment before" - print_and_log "running this installation profile again." - remove_pid_and_exit_in_error - fi - - print_and_log "Getting ready to create a new publication ..." - create_publication_definition_and_war - - local instance_list=$(get_instance_list) - local default_instance=$(echo ${instance_list} | cut -d' ' -f1) - - if [ $fai_enabled -eq 0 ]; then - print "Which ECE instance do you wish to use to create it?" - print "These instances are available: $instance_list" - echo -n "Your choice [$default_instance]> " - read instance_name - else - instance_name=$(get_conf_value fai_publication_use_instance) - fi - - if [ -z "$instance_name" ]; then - instance_name=$default_instance - fi - - type=engine - ensure_that_instance_is_running $instance_name - create_publication_in_db $publication_war - assemble_deploy_and_restart_type - - add_next_step "A new publication $publication_name has been created." -} - function install_editorial_server() { print_and_log "Installing an editorial server on $HOSTNAME ..." @@ -1491,92 +1444,6 @@ function install_all_in_one_environment() create_publication } -# parameters: -# $1 - what kind of webserver. Available options are: -# * 0 - cache server -# * 1 - monitoring server -# * 2 - both cache & monitoring server -function install_web_server() -{ - print_and_log "Installing a web server on $HOSTNAME ..." - - if [ $on_debian_or_derivative -eq 1 ]; then - packages="nginx" - install_packages_if_missing $packages - else - debug "Web server installation not supported on your system." - return - fi - - file=/etc/nginx/sites-available/default - # in some very unusual cases, this file will not exist (can occur - # when re-running ece-install several times). - if [ -e $file ]; then - run mv $file $file.orig - fi - - if [ $1 -eq 0 ]; then - port=81 - cat > $file < $file < $file < " + read instance_name + else + instance_name=$(get_conf_value fai_publication_use_instance) + fi + + if [ -z "$instance_name" ]; then + instance_name=$default_instance + fi + + type=engine + ensure_that_instance_is_running $instance_name + create_publication_in_db $publication_war + add_publication_to_deployment_lists + assemble_deploy_and_restart_type + + add_next_step "A new publication $publication_name has been created." +} + +## $1 : publication name +function add_publication_to_deployment_lists() { + if [ $fai_enabled -eq 0 ]; then + print "On which ECE instances do you wish to deploy $publication_name" + print "These instances are available: $instance_list" + echo -n "Your choice [$default_instance]> " + read update_instance_list + else + update_instance_list=${fai_publication_instance_list-${default_instance}} + fi + + if [ -z $update_instance_list ]; then + update_instance_list=$default_instance + fi + + for el in $update_instance_list; do + print "Adding $publication_name to instance ${el}'s deployment list ..." + local file=$escenic_conf_dir/ece-${el}.conf + if [ -e $file ]; then + # don't want to source it as it'll pollute the variable name + # space. + local existing_value=$( + grep deploy_webapp_white_list $file | cut -d'=' -f2 | sed 's#"##g' + ) + set_conf_file_value \ + deploy_webapp_white_list \ + $publication_name $existing_value \ + $file + fi + done +} + +function ensure_that_instance_is_running() +{ + local ece_command="ece -i $1 -t $type status" + if [ $(su - $ece_user -c "$ece_command" | grep UP | wc -l) -lt 1 ]; then + ece_command="ece -i $1 -t $type start" + su - $ece_user -c "$ece_command" 1>>$log 2>>$log + # TODO improve this by adding a timed while loop + sleep 60 + fi +} diff --git a/usr/share/escenic/ece-scripts/ece-install.d/web-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/web-server.sh new file mode 100644 index 00000000..c42de569 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/web-server.sh @@ -0,0 +1,85 @@ +# parameters: +# $1 - what kind of webserver. Available options are: +# * 0 - cache server +# * 1 - monitoring server +# * 2 - both cache & monitoring server +function install_web_server() +{ + print_and_log "Installing a web server on $HOSTNAME ..." + + if [ $on_debian_or_derivative -eq 1 ]; then + packages="nginx" + install_packages_if_missing $packages + else + debug "Web server installation not supported on your system." + return + fi + + file=/etc/nginx/sites-available/default + # in some very unusual cases, this file will not exist (can occur + # when re-running ece-install several times). + if [ -e $file ]; then + run mv $file $file.orig + fi + + if [ $1 -eq 0 ]; then + port=81 + cat > $file < $file < $file < Date: Fri, 2 Mar 2012 15:04:18 +0800 Subject: [PATCH 0458/2585] - moved content engine / Nursery specific set up code into its own module - moved more publication and monitoring methods into the designated modules for these tasks. --- usr/sbin/ece-install | 511 ------------------ .../ece-install.d/content-engine.sh | 399 +++++++++++++- .../ece-install.d/create-publication.sh | 88 ++- .../ece-scripts/ece-install.d/monitoring.sh | 67 ++- 4 files changed, 526 insertions(+), 539 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 8c6ed3f3..a62dfb23 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -282,154 +282,6 @@ function set_up_assembly_tool() { # ###################################################################### -function set_up_assembly_tool() -{ - log "Setting up the Assembly Tool ..." - - make_dir $escenic_root_dir/assemblytool/ - cd $escenic_root_dir/assemblytool/ - - if [ -e $download_dir/assemblytool*zip ]; then - run unzip -q -u $download_dir/assemblytool*zip - fi - - # adding an instance layer to the Nursery configuration - cp -r $escenic_root_dir/engine/siteconfig/bootstrap-skeleton \ - $escenic_root_dir/assemblytool/conf - cd $escenic_root_dir/assemblytool/conf/ - cp -r layers/host layers/instance - cat > layers/instance/Files.properties <> Nursery.properties - echo "layer.06 = /layers/instance/Layer" >> Nursery.properties - - # fixing the path to the Nursery configuration according to - # escenic_conf_dir which may have been overridden by the user. - for el in $(find $escenic_root_dir/assemblytool/conf -name Files.properties) - do - sed -i "s#/etc/escenic#${escenic_conf_dir}#g" $el - done - - # set up which plugins to use - cd $escenic_root_dir/assemblytool/ - make_dir plugins - cd plugins - find ../../ -maxdepth 1 -type d | \ - grep -v assemblytool | \ - while read directory; do - if [ $directory = "../../" -o \ - $(echo $directory | grep widget-framework | wc -l) -gt 0 ]; then - continue - fi - - # nuisance to get the community engine and analysis engine, - # but not the engine - if [ $(echo $directory | \ - grep engine | \ - grep -v community | \ - grep -v analysis | \ - wc -l) -gt 0 ]; then - continue - fi - - make_ln $directory - done - - run cd $escenic_root_dir/assemblytool/ - run ant -q initialize - sed -i "s~#\ engine.root\ =\ \.~engine.root=${escenic_root_dir}/engine~g" \ - assemble.properties \ - 1>>$log 2>>$log - exit_on_error "sed on assemble.properties" - - sed -i "s~\#\# plugins\ =\ /path/to/plugins~plugins=${escenic_root_dir}/assemblytool/plugins~g" \ - assemble.properties \ - 1>>$log 2>>$log - exit_on_error "sed on assemble.properties" - - make_dir $escenic_root_dir/assemblytool/publications - - # set up user publication definitions - if [ -n "${fai_publication_war_uri_list}" ]; then - run cd $escenic_root_dir/assemblytool/publications - - for el in ${fai_publication_war_uri_list}; do - if [[ $el == http* ]]; then - run wget $wget_opts $el - elif [[ $el == file://* ]]; then - local file_with_path=$(echo $el | sed "s#file://##g") - run cp $file_with_path . - else - run cp ${el} . - fi - - if [ ! -e $(basename $el) ]; then - print_and_log "Failed to get user publication $el." - print_and_log "I will skip it and continue to the next one." - continue - fi - - local short_name=$(basename $el .war) - cat > ${short_name}.properties < " - read user_host_name - fi - - if [ -n "$user_host_name" ]; then - public_host_name=$user_host_name - fi - - cat > $common_nursery_dir/ServerConfig.properties <= 5.3 -ldapProductName=OpenLdap -ldapProductVersion=2.2.26 -EOF - cat > $common_nursery_dir/neo/io/managers/ContentManager.properties <> $file < $el/RMI.properties - fi - done - - nursery_context=neo/io/managers/HubConnectionManager.properties - file=$escenic_conf_dir/engine/instance/$instance_name/$nursery_context - make_dir $(dirname $file) - - # we don't touch it if the file already exists. - if [ ! -e $file ]; then - run echo "hostname=$HOSTNAME" >> $file - fi -} - -function set_up_proper_logging_configuration() -{ - print_and_log "Setting up proper log4j & Java logging configuration ..." - - cat > $common_nursery_dir/trace.properties < $tomcat_base/conf/logging.properties < " - read publication_name - else - publication_name=$(get_conf_value fai_publication_name) - fi - - if [ -z "$publication_name" ]; then - publication_name=mypub - fi - - print_and_log "Setting up the ${publication_name} publication ..." - make_dir $escenic_root_dir/assemblytool/publications/ - run cd $escenic_root_dir/assemblytool/publications/ - cat > $escenic_root_dir/assemblytool/publications/${publication_name}.properties < definition for core - # widgets in: - # publications/demo-community/src/main/webapp/META-INF/escenic/publication-resources/escenic/content-type -} - function check_for_required_downloads() { if [ $ece_software_setup_completed -eq 1 ]; then @@ -1467,41 +991,6 @@ function add_server_to_runlevels() fi } -## $1 configuration file name -## $2 host group name -## $3 host group alias -## $4..n host group members -function set_up_monitoring_host_group() -{ - local file=$1 - local host_group_name=$2 - local host_group_alias=$3 - # the remainding arguments passed to the methods is the member - # list members - local host_group_member_list=${@:4:$(( $# - 3 ))} - - if [ $(grep "hostgroup_name $host_group_name" $file | wc -l) -gt 0 ]; then - print "Icinga group member" \ - $host_group_name \ - "already defined, skipping it." - return - fi - - cat >> $file <> $file - for el in $host_group_member_list; do - echo -n " ${el}," >> $file - done - cat >> $file < -function install_ece_instance() -{ +function install_ece_instance() { install_ece_third_party_packages ask_for_instance_name $1 @@ -48,8 +45,8 @@ function install_ece_instance() set_archive_files_depending_on_profile - # most likely, the user is _not_ installing from archives (EAR + - # configuration bundle), hence the false test goes first. + # most likely, the user is _not_ installing from archives (EAR + + # configuration bundle), hence the false test goes first. if [ $(is_installing_from_ear) -eq 0 ]; then download_escenic_components check_for_required_downloads @@ -97,3 +94,389 @@ function install_ece_instance() add_next_step "/etc/default/ece lists all instances started at boot time" } +function set_up_engine_and_plugins() { + if [ $ece_software_setup_completed -eq 1 ]; then + return + fi + + log "Setting up the Escenic Content Engine & its plugins ..." + + make_dir $escenic_root_dir + cd $escenic_root_dir/ + + for el in $technet_download_list; do + if [ $(basename $el | \ + grep -E "^engine-[0-9]|^engine-trunk-SNAPSHOT|^engine-dist" | \ + wc -l) -gt 0 ]; then + engine_dir=$(get_base_dir_from_bundle $download_dir/$(basename $el)) + engine_file=$(basename $el) + fi + done + + if [ ! -d "${engine_dir}" ]; then + run unzip -q -u $download_dir/${engine_file} + if [ -h engine ]; then + run rm engine + fi + + run ln -s ${engine_dir} engine + else + debug "${engine_dir} is already there, skipping to next step." + fi + + # we now extract all the plugins. We extract them in $escenic_root_dir + # as we want to re-use them between minor updates of ECE. + cd $escenic_root_dir/ + for el in $download_dir/*.zip; do + if [ $(basename $el | grep ^engine-.*.zip | wc -l) -gt 0 ]; then + continue + elif [ $(basename $el | grep ^assemblytool-.*.zip | wc -l) -gt 0 ]; then + continue + elif [ $(basename $el | grep ^jdk-.*.zip | wc -l) -gt 0 ]; then + continue + fi + + run unzip -q -u $el + done + + ece_software_setup_completed=1 +} + +function set_up_assembly_tool() { + log "Setting up the Assembly Tool ..." + + make_dir $escenic_root_dir/assemblytool/ + cd $escenic_root_dir/assemblytool/ + + if [ -e $download_dir/assemblytool*zip ]; then + run unzip -q -u $download_dir/assemblytool*zip + fi + + # adding an instance layer to the Nursery configuration + cp -r $escenic_root_dir/engine/siteconfig/bootstrap-skeleton \ + $escenic_root_dir/assemblytool/conf + cd $escenic_root_dir/assemblytool/conf/ + cp -r layers/host layers/instance + cat > layers/instance/Files.properties <> Nursery.properties + echo "layer.06 = /layers/instance/Layer" >> Nursery.properties + + # fixing the path to the Nursery configuration according to + # escenic_conf_dir which may have been overridden by the user. + for el in $(find $escenic_root_dir/assemblytool/conf -name Files.properties) + do + sed -i "s#/etc/escenic#${escenic_conf_dir}#g" $el + done + + # set up which plugins to use + cd $escenic_root_dir/assemblytool/ + make_dir plugins + cd plugins + find ../../ -maxdepth 1 -type d | \ + grep -v assemblytool | \ + while read directory; do + if [ $directory = "../../" -o \ + $(echo $directory | grep widget-framework | wc -l) -gt 0 ]; then + continue + fi + + # nuisance to get the community engine and analysis engine, + # but not the engine + if [ $(echo $directory | \ + grep engine | \ + grep -v community | \ + grep -v analysis | \ + wc -l) -gt 0 ]; then + continue + fi + + make_ln $directory + done + + run cd $escenic_root_dir/assemblytool/ + run ant -q initialize + sed -i "s~#\ engine.root\ =\ \.~engine.root=${escenic_root_dir}/engine~g" \ + assemble.properties \ + 1>>$log 2>>$log + exit_on_error "sed on assemble.properties" + + sed -i "s~\#\# plugins\ =\ /path/to/plugins~plugins=${escenic_root_dir}/assemblytool/plugins~g" \ + assemble.properties \ + 1>>$log 2>>$log + exit_on_error "sed on assemble.properties" + + make_dir $escenic_root_dir/assemblytool/publications + + # set up user publication definitions + if [ -n "${fai_publication_war_uri_list}" ]; then + run cd $escenic_root_dir/assemblytool/publications + + for el in ${fai_publication_war_uri_list}; do + if [[ $el == http* ]]; then + run wget $wget_opts $el + elif [[ $el == file://* ]]; then + local file_with_path=$(echo $el | sed "s#file://##g") + run cp $file_with_path . + else + run cp ${el} . + fi + + if [ ! -e $(basename $el) ]; then + print_and_log "Failed to get user publication $el." + print_and_log "I will skip it and continue to the next one." + continue + fi + + local short_name=$(basename $el .war) + cat > ${short_name}.properties < " + read user_host_name + fi + + if [ -n "$user_host_name" ]; then + public_host_name=$user_host_name + fi + + cat > $common_nursery_dir/ServerConfig.properties <= 5.3 +ldapProductName=OpenLdap +ldapProductVersion=2.2.26 +EOF + cat > $common_nursery_dir/neo/io/managers/ContentManager.properties <> $file < $el/RMI.properties + fi + done + + nursery_context=neo/io/managers/HubConnectionManager.properties + file=$escenic_conf_dir/engine/instance/$instance_name/$nursery_context + make_dir $(dirname $file) + + # we don't touch it if the file already exists. + if [ ! -e $file ]; then + run echo "hostname=$HOSTNAME" >> $file + fi +} + +function set_up_proper_logging_configuration() { + print_and_log "Setting up proper log4j & Java logging configuration ..." + + cat > $common_nursery_dir/trace.properties < $tomcat_base/conf/logging.properties < " + read publication_name + else + publication_name=$(get_conf_value fai_publication_name) + fi + + if [ -z "$publication_name" ]; then + publication_name=mypub + fi + + print_and_log "Setting up the ${publication_name} publication ..." + make_dir $escenic_root_dir/assemblytool/publications/ + run cd $escenic_root_dir/assemblytool/publications/ + cat > $escenic_root_dir/assemblytool/publications/${publication_name}.properties < definition for core + # widgets in: + # publications/demo-community/src/main/webapp/META-INF/escenic/publication-resources/escenic/content-type +} diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 6b545ddd..69919fa0 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -268,10 +268,10 @@ function install_munin_gatherer() print_and_log "Adding ${el} to the Munin gatherer on ${HOSTNAME} ..." local file=/etc/munin/munin-conf.d/escenic.conf cat >> $file < $file < $file <

    Welcome to the might monitoring server @ ${HOSTNAME}

      EOF - if [[ $1 == $MONITORING_VENDOR_NAGIOS ]]; then - echo '
    • Nagios
    • ' \ - else - echo '
    • Icinga (an enhanced Nagios)
    • ' \ - >> $file - fi - cat > $file <Nagios' \ + else + echo '
    • Icinga (an enhanced Nagios)
    • ' \ + >> $file + fi + cat > $file <Munin
    EOF - add_next_step "Start page for all monitoring interfaces: http://${HOSTNAME}/" + add_next_step "Start page for all monitoring interfaces: http://${HOSTNAME}/" } function install_monitoring_server() { - local nagios_flavour=${fai_monitoring_nagios_flavour-$MONITORING_VENDOR_ICINGA} - install_nagios_monitoring_server $nagios_flavour - install_munin_gatherer - create_monitoring_server_overview $nagios_flavour + local nagios_flavour=${fai_monitoring_nagios_flavour-$MONITORING_VENDOR_ICINGA} + install_nagios_monitoring_server $nagios_flavour + install_munin_gatherer + create_monitoring_server_overview $nagios_flavour +} + +## $1 configuration file name +## $2 host group name +## $3 host group alias +## $4..n host group members +function set_up_monitoring_host_group() +{ + local file=$1 + local host_group_name=$2 + local host_group_alias=$3 + # the remainding arguments passed to the methods is the member + # list members + local host_group_member_list=${@:4:$(( $# - 3 ))} + + if [ $(grep "hostgroup_name $host_group_name" $file | wc -l) -gt 0 ]; then + print "Icinga group member" \ + $host_group_name \ + "already defined, skipping it." + return + fi + + cat >> $file <> $file + for el in $host_group_member_list; do + echo -n " ${el}," >> $file + done + cat >> $file < Date: Fri, 2 Mar 2012 14:33:15 +0100 Subject: [PATCH 0459/2585] Fixed syntax error --- usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 69919fa0..00a4a6d3 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -271,7 +271,7 @@ function install_munin_gatherer() [${el}] address $(get_ip $el) use_node_name yes - EOF +EOF done # TODO add the priveleged network to the Allowed stanza (i.e. the From 622d2880fe995b7d54a3f89fe34a16fdc31891a7 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Fri, 2 Mar 2012 14:55:47 +0100 Subject: [PATCH 0460/2585] Made wait-for-ssh postinstall hook emit more information and wait for cloud-init to complete, rather than just an apt-get install to complete. --- .../vizrt/vosa/post-install-hooks/wait-for-ssh.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh b/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh index 7f293685..2d530097 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/wait-for-ssh.sh @@ -1,15 +1,23 @@ #!/bin/bash +echo -n "Waiting for ssh access..." for a in $(seq 1 90) ; do + echo -n . ssh -q -F $2/ssh.conf root@guest id > /dev/null if [ $? == 0 ] ; then pid=1 + echo + echo -n "Waiting for cloud-init to complete" # Wait for existing apt-gets to complete. - while [ ! -z "$pid" ] ; + # TODO: drop the boot confi + while [ "$pid" != 0 ] ; do - pid=$(ssh -F $2/ssh.conf root@guest pidof apt-get) + echo -n . + ssh -q -F $2/ssh.conf root@guest ls /var/lib/cloud/instances/nocloud/boot-finished > /dev/null 2>&1 + pid=$? sleep 1; done + echo exit 0 fi sleep 1; From d0bc7ebe5d9c42147506b4f134441c10165608dd Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 5 Mar 2012 13:45:57 +0800 Subject: [PATCH 0461/2585] - added new features to "ece info". The command now lists all the Escenic JNDI resources, the DB handles as well as the log files used by the given ECE instance. $ ece -i dev1 info [ece#engine-dev1] Current instance: dev1 [ece#engine-dev1] Instances available on quanah: dev1 memento web1 [ece#engine-dev1] Conf files parsed: /etc/escenic/ece-dev1.conf /etc/escenic/ece.conf [ece#engine-dev1] ECE location: /opt/escenic/engine [ece#engine-dev1] Assembly Tool location: /opt/escenic/assemblytool [ece#engine-dev1] Java location: /usr/lib/jvm/java-6-sun [ece#engine-dev1] Application server: [ece#engine-dev1] |-> Status: UP 0d 1h 9m 31s [ece#engine-dev1] |-> Port: 8080 [ece#engine-dev1] |-> PID: 16883 [ece#engine-dev1] |-> Type: tomcat [ece#engine-dev1] |-> Tomcat home: /opt/tomcat [ece#engine-dev1] |-> Tomcat base: /opt/tomcat-dev1 [ece#engine-dev1] Application server resources: [ece#engine-dev1] |-> escenic/indexer-webservice: http://localhost:9980/indexer-webservice/index/ [ece#engine-dev1] |-> escenic/index-update-uri: http://quanah:8080/solr/update/ [ece#engine-dev1] |-> escenic/solr-base-uri: http://quanah:8080/solr/ [ece#engine-dev1] |-> escenic/head-tail-storage-file: /var/lib/escenic/engine/head-tail.index [ece#engine-dev1] |-> escenic/failing-documents-storage-file: /var/lib/escenic/engine/failures.index [ece#engine-dev1] Database: [ece#engine-dev1] |-> name: jdbc/ECE_READ_DS [ece#engine-dev1] |-> url: jdbc:mysql://localhost:3306/ece5db?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8 [ece#engine-dev1] |-> username: mementouser [ece#engine-dev1] |-> name: jdbc/ECE_UPDATE_DS [ece#engine-dev1] |-> url: jdbc:mysql://localhost:3306/ece5db?autoReconnect=true&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8 [ece#engine-dev1] |-> username: mementouser [ece#engine-dev1] Log files: [ece#engine-dev1] |-> System out log: /var/log/escenic/engine-dev1.out [ece#engine-dev1] |-> App server log: /opt/tomcat-dev1/logs/localhost.2012-03-05.log [ece#engine-dev1] |-> Log4j log: /var/log/escenic/quanah-dev1-messages Pre-requisite is that the user has xml_grep installed. On Debian, this is available in the xml-twig-tools package. --- usr/bin/ece | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index 5dbd0549..4f8c759c 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -990,6 +990,7 @@ function get_info_for_type() if [ -n "${tomcat_home}" ]; then print "|-> Tomcat home:" $tomcat_home print "|-> Tomcat base:" $tomcat_base + print_tomcat_resources fi ;; resin) @@ -999,6 +1000,51 @@ function get_info_for_type() ;; esac fi + + print "Log files:" + print "|-> System out log:" $log_file + print "|-> App server log:" $(get_app_log) + print "|-> Log4j log: " $(get_log4j_log) +} + +function print_tomcat_resources() { + if [ "$(which xml_grep)x" == "x" ]; then + log "Install xml_grep to get more 'ece info' details" + return + fi + + print "Application server resources:" + + xml_grep --nowrap --cond Context/Environment \ + $tomcat_base/conf/context.xml | \ + sed 's/^[ \t]//g' | \ + sed "s#><#>\n<#g" | \ + cut -d' ' -f2,5 | \ + cut -d'"' -f2,4 | \ + while read line ; do + local key=$(echo $line | cut -d'"' -f1) + local value=$(echo $line | cut -d'"' -f2) + print "|->" ${key}: ${value} + done + + print "Database:" + xml_grep \ + --nowrap \ + --cond 'Server/GlobalNamingResources/Resource[@type="javax.sql.DataSource"]' \ + $tomcat_base/conf/server.xml | \ + sed 's/^[ \t]//g' | \ + sed "s#><#>\n<#g" | \ + while read line ; do + for el in $(echo $line | cut -d' ' -f1- ); do + if [[ $el == "name"* || $el == "url"* || $el == "username"* ]]; then + local key=$(echo $el | cut -d'"' -f1 | cut -d'=' -f1) + local value=$(echo $el | cut -d'"' -f2) + print "|->" ${key}: ${value} + fi + + done + done + } function restart_type() From 5c8806c04c5fceb13baee231b1abe6a7fac0137f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 5 Mar 2012 14:06:05 +0800 Subject: [PATCH 0462/2585] - moved the log files up in the "ece info" output --- usr/bin/ece | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 4f8c759c..6f637cd2 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -976,6 +976,12 @@ function get_info_for_type() if [ -n "${java_home}" ]; then print "Java location: $java_home" fi + + print "Log files:" + print "|-> System out log:" $log_file + print "|-> App server log:" $(get_app_log) + print "|-> Log4j log: " $(get_log4j_log) + if [ -n "${appserver}" ]; then print "Application server:" set_type_port @@ -1000,11 +1006,6 @@ function get_info_for_type() ;; esac fi - - print "Log files:" - print "|-> System out log:" $log_file - print "|-> App server log:" $(get_app_log) - print "|-> Log4j log: " $(get_log4j_log) } function print_tomcat_resources() { From e5b36e28a71a0c00ca6f6fe079c33a0d919ce629 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 6 Mar 2012 13:37:22 +0800 Subject: [PATCH 0463/2585] - fixed bug in pulse method for function set_up_assembly_tool, it was using the actual method name instead of the _p access point. --- usr/sbin/ece-install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a62dfb23..0dc0256b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -273,7 +273,7 @@ function install_ece_third_party_packages_p() { show_pulse $! "Installing 3rd party packages needed by ECE" } -function set_up_assembly_tool() { +function set_up_assembly_tool_p() { $(set_up_assembly_tool) & show_pulse $! "Setting up the Assembly Tool" } From 7ff91c35179926fddd3dd514ad460fa2c537eed7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 6 Mar 2012 13:40:40 +0800 Subject: [PATCH 0464/2585] - speed & size improvement: the analysis profile now doesn't require the technet download list to include ECE and AssemblyTool and will now happily install EAE without access to ECE & AT. This closes issue #42 - ece-install will also no longer set up a deployment white list for the analysis profile - added more run wrappers to set_up_assembly_tool to ensure fail fast (as possible) and consistent error reporting. --- .../ece-install.d/content-engine.sh | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index 55192434..bc71ea22 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -51,22 +51,28 @@ function install_ece_instance() { download_escenic_components check_for_required_downloads set_up_engine_and_plugins - set_up_assembly_tool + + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + set_up_assembly_tool + fi else verify_that_files_exist_and_are_readable \ $ece_instance_ear_file \ $ece_instance_conf_archive fi - - set_up_basic_nursery_configuration - set_up_instance_specific_nursery_configuration + + if [ $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + set_up_basic_nursery_configuration + set_up_instance_specific_nursery_configuration + fi set_up_app_server set_up_proper_logging_configuration # We set a WAR white list for all profiles except all in one - if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE ]; then - file=$escenic_conf_dir/ece-${instance_name}.conf + if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE -a \ + $install_profile_number -ne $PROFILE_ANALYSIS_SERVER ]; then + local file=$escenic_conf_dir/ece-${instance_name}.conf print_and_log "Creating deployment white list in $file ..." set_conf_file_value \ deploy_webapp_white_list \ @@ -113,7 +119,7 @@ function set_up_engine_and_plugins() { fi done - if [ ! -d "${engine_dir}" ]; then + if [ -n "$engine_dir" -a ! -d "${engine_dir}" ]; then run unzip -q -u $download_dir/${engine_file} if [ -h engine ]; then run rm engine @@ -124,8 +130,8 @@ function set_up_engine_and_plugins() { debug "${engine_dir} is already there, skipping to next step." fi - # we now extract all the plugins. We extract them in $escenic_root_dir - # as we want to re-use them between minor updates of ECE. + # we now extract all the plugins. We extract them in $escenic_root_dir + # as we want to re-use them between minor updates of ECE. cd $escenic_root_dir/ for el in $download_dir/*.zip; do if [ $(basename $el | grep ^engine-.*.zip | wc -l) -gt 0 ]; then @@ -153,10 +159,10 @@ function set_up_assembly_tool() { fi # adding an instance layer to the Nursery configuration - cp -r $escenic_root_dir/engine/siteconfig/bootstrap-skeleton \ + run cp -r $escenic_root_dir/engine/siteconfig/bootstrap-skeleton \ $escenic_root_dir/assemblytool/conf - cd $escenic_root_dir/assemblytool/conf/ - cp -r layers/host layers/instance + run cd $escenic_root_dir/assemblytool/conf/ + run cp -r layers/host layers/instance cat > layers/instance/Files.properties < Date: Tue, 6 Mar 2012 14:23:44 +0800 Subject: [PATCH 0465/2585] - in create_publication: don't set/introduce the deployment white list for profile=all. --- .../ece-scripts/ece-install.d/create-publication.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/create-publication.sh b/usr/share/escenic/ece-scripts/ece-install.d/create-publication.sh index 3a7a5729..368e1799 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/create-publication.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/create-publication.sh @@ -28,7 +28,12 @@ function create_publication() { type=engine ensure_that_instance_is_running $instance_name create_publication_in_db $publication_war - add_publication_to_deployment_lists + + # don't set/introduce the deployment white list for profile=all + if [ $install_profile_number -ne $PROFILE_ALL_IN_ONE ]; then + add_publication_to_deployment_lists + fi + assemble_deploy_and_restart_type add_next_step "A new publication $publication_name has been created." From ac9e2f0910968fd062b070c1f3986968f1b539a8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 6 Mar 2012 16:11:29 +0800 Subject: [PATCH 0466/2585] - added local scope to print_status_and_next_steps's local variables. --- usr/sbin/ece-install | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0dc0256b..0321724c 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -426,15 +426,15 @@ function set_correct_permissions() function print_status_and_next_steps() { - now=`date +%s` - started=`stat -c %Y $pid_file` - seconds=$(( now - started )) - days=$(( seconds / ( 60 * 60 * 24 ) )) - seconds_left=$(( seconds - ( $days * 60 * 60 * 24 ) )) - hours=$(( seconds_left / ( 60 * 60 ) )) - seconds_left=$(( seconds_left - ( $hours * 60 * 60 ) )) - minutes=$(( seconds_left / 60 )) - seconds_left=$(( seconds_left - $minutes * 60 )) + local now=`date +%s` + local started=`stat -c %Y $pid_file` + local seconds=$(( now - started )) + local days=$(( seconds / ( 60 * 60 * 24 ) )) + local seconds_left=$(( seconds - ( $days * 60 * 60 * 24 ) )) + local hours=$(( seconds_left / ( 60 * 60 ) )) + local seconds_left=$(( seconds_left - ( $hours * 60 * 60 ) )) + local minutes=$(( seconds_left / 60 )) + local seconds_left=$(( seconds_left - $minutes * 60 )) if [ $install_profile_number -ne $PROFILE_RESTORE_FROM_BACKUP ]; then s="The installation is now $(green complete)!" From d2559624fcb98cac8a3089d58ddd97e5466cf5d8 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Tue, 6 Mar 2012 10:51:09 +0100 Subject: [PATCH 0467/2585] Reverted 7ebf804fb18dd6db90f22d13ceae0b3a62545789 since the problem was with CPU sharing --- usr/share/vizrt/vosa/commands/boot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/vizrt/vosa/commands/boot.sh b/usr/share/vizrt/vosa/commands/boot.sh index 689c724a..5d799057 100755 --- a/usr/share/vizrt/vosa/commands/boot.sh +++ b/usr/share/vizrt/vosa/commands/boot.sh @@ -201,7 +201,7 @@ function boot_kvm() { -balloon "virtio" -drive "file=${img},if=virtio,cache=none" -kernel ${kernel} - -net "nic,model=rtl8139,macaddr=${install_config_macaddr}" + -net "nic,model=virtio,macaddr=${install_config_macaddr}" -net "tap,ifname=$tapinterface,script=no,downscript=no") # http://dwdwwebcache.googleusercontent.com/search?q=cache:mEAjcA2zHosJ:kerneltrap.org/mailarchive/linux-kvm/2010/1/26/6257297/thread+qemu-kvm+acpi+shutdown&cd=1&hl=no&ct=clnk&gl=no From e44bfc3c16763601bca179f216bfdd2e0a0a575d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 6 Mar 2012 19:21:11 +0800 Subject: [PATCH 0468/2585] - new feature: first version of the VIP provider profile For instance, 192.168.1.108 and 192.168.1.109 are to provide the VIP 192.168.1.200 for NFS and .108 is the master. .108 is then installed with: fai_vip_address=192.168.1.200 fai_vip_interface=eth0 fai_vip_sibling_ip=192.168.1.109 fai_vip_service_list="nfs-kernel-server" fai_vip_primary_node_name="nfs1" fai_vip_primary_node_ip="192.168.1.108" fai_vip_secondary_node_name="nfs2" fai_vip_secondary_node_ip="192.168.1.109" # optional fai_vip_log=/var/log/ha.log and .109 is then installed with: fai_vip_address=192.168.1.200 fai_vip_interface=eth0 fai_vip_sibling_ip=192.168.1.108 fai_vip_service_list="nfs-kernel-server" fai_vip_primary_node_name="nfs1" fai_vip_primary_node_ip="192.168.1.108" fai_vip_secondary_node_name="nfs2" fai_vip_secondary_node_ip="192.168.1.109" TODO: test & bug hunt. Find out how to best configure custom services, probably scripts in ha.d for "our" VIP services, such as search-website-vip, search-editor-vip, content-studio-vip --- .../ece-scripts/ece-install.d/vip-provier.sh | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh new file mode 100644 index 00000000..a1663109 --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh @@ -0,0 +1,125 @@ +function get_vip_configuration() { + # TODO add interactive mode if it's needed/requested + + # dependant on the running ece-install process/host + vip_address=${fai_vip_address} + vip_interface=${fai_vip_interface} + vip_log_file=${fai_vip_log-/var/log/ha-debug} + vip_sibling_ip=${fai_vip_sibling_ip} + vip_service_list=${fai_vip_service_list} + + # common to both VIP nodes, node names must be what $(uname -n) + # returns. + vip_primary_node_name=${fai_vip_primary_node_name} + vip_primary_node_ip=${fai_vip_primary_node_ip} + vip_secondary_node_name=${fai_vip_secondary_node_name} + vip_secondary_node_ip=${fai_vip_secondary_node_ip} +} + +function install_vip_provider() { + install_packages_if_missing heartbeat + set_kernel_vip_parameters + create_ha_auth_keys + set_up_ha_conf + set_up_ha_resources + propagate_ha_settings + + run /etc/init.d/heartbeat restart + + add_next_step "Heartbeat installed on $HOSTNAME to provide VIP $vip" + add_next_step "be sure that $node2_name gets similar setup as $HOSTNAME." +} + +function set_kernel_vip_parameters() { + local file=/etc/sysctl.conf + + if [ $(grep "net.ipv4.ip_nonlocal_bind=1" $file | wc -l) -gt 0 ]; then + return + fi + + cat >> $file < $file < +$vip_primary_node_name $vip_address $vip_service_list +EOF +} + +function assure_vip_hosts_are_resolvable() { + local keep_off_etc_hosts=${fai_keep_off_etc_hosts-0} + if [ $keep_off_etc_hosts -eq 1 ]; then + return + fi + + if [ $(grep $vip_primary_node_name /etc/hosts | wc -l) -lt 1 ]; then + cat >> /etc/hosts <> /etc/hosts < $file < $file + run chmod 0600 $file +} From 25cecfe792cb2caaa0a735ab20728bd54a17646a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 11:22:59 +0800 Subject: [PATCH 0469/2585] - implemented installing an NFS server, issue #43 TODO add profile hook. - implemented installing an NFS client, issue #44 TODO add profile hook --- .../ece-install.d/network-file-system.sh | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh new file mode 100644 index 00000000..02648d9d --- /dev/null +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -0,0 +1,47 @@ +# module to install & configure network file systems + +default_nfs_export_list="/exports/multimedia" + +function get_nfs_configuration() { + nfs_export_list=${fai_nfs_export_list-$default_nfs_export_list} + nfs_server=${fai_nfs_server} +} + + +function install_nfs_server() { + print "Installing an NFS server on $HOSTNAME ..." + install_packages_if_missing "portmap nfs-kernel-server nfs-common" + get_nfs_configuration + + for el in $nfs_export_list; do + cat >> /etc/exports <> /etc/fstab < Date: Wed, 7 Mar 2012 11:35:04 +0800 Subject: [PATCH 0470/2585] - added profile hooks for NFS server and client profiles, it should now be possible to install these in FAI mode: ece-install-nfs-server.conf: fai_nfs_server_address=nfs1 fai_nfs_export_list="/exports/multimedia" fai_nfs_allowed_client_network="172.16.230.0/255.255.255.0" ece-install-nfs-client.conf: fai_nfs_server_address=nfs1 fai_nfs_export_list="/exports/multimedia" Related issues #43 & #44 --- usr/sbin/ece-install | 21 +++++++++++++++++++ .../ece-scripts/ece-install.d/constants.sh | 3 +++ .../ece-install.d/network-file-system.sh | 4 ++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 0321724c..88be34fc 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -1124,6 +1124,24 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi + if [ $(get_boolean_conf_value fai_vip_install) -eq 1 ]; then + install_profile_number=$PROFILE_VIP_PROVIDER + install_vip_provider + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_nfs_server_install) -eq 1 ]; then + install_profile_number=$PROFILE_NFS_SERVER + install_nfs_server + no_fai_profile=0 + fi + + if [ $(get_boolean_conf_value fai_nfs_client_install) -eq 1 ]; then + install_profile_number=$PROFILE_NFS_CLIENT + install_nfs_client + no_fai_profile=0 + fi + if [ $no_fai_profile -eq 1 ]; then print_and_log "No install profile selected, be sure to have one of the " print_and_log "fai__install=1 in your $conf_file" @@ -1171,6 +1189,9 @@ else $PROFILE_RESTORE_FROM_BACKUP) restore_from_backup ;; + $PROFILE_VIP_PROVIDER) + install_vip_provider + ;; *) print "Invalid profile number $install_profile_number, must be 1-11" remove_pid_and_exit_in_error diff --git a/usr/share/escenic/ece-scripts/ece-install.d/constants.sh b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh index d757dad1..0000d70a 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/constants.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/constants.sh @@ -13,6 +13,9 @@ PROFILE_CREATE_PUBLICATION=9 PROFILE_MONITORING_SERVER=10 PROFILE_RESTORE_FROM_BACKUP=11 PROFILE_ANALYSIS_SERVER=12 +PROFILE_VIP_PROVIDER=13 +PROFILE_NFS_SERVER=14 +PROFILE_NFS_CLIENT=15 # database constants default_db_port=3306 diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh index 02648d9d..5700ed47 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -4,7 +4,7 @@ default_nfs_export_list="/exports/multimedia" function get_nfs_configuration() { nfs_export_list=${fai_nfs_export_list-$default_nfs_export_list} - nfs_server=${fai_nfs_server} + nfs_server_address=${fai_nfs_server_address} } @@ -38,7 +38,7 @@ function install_nfs_client() { cat >> /etc/fstab < Date: Wed, 7 Mar 2012 11:35:32 +0800 Subject: [PATCH 0471/2585] - added nfs_allowed_network=${fai_nfs_allowed_client_network}. Issue #43 --- .../escenic/ece-scripts/ece-install.d/network-file-system.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh index 5700ed47..7094d9b5 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -5,6 +5,7 @@ default_nfs_export_list="/exports/multimedia" function get_nfs_configuration() { nfs_export_list=${fai_nfs_export_list-$default_nfs_export_list} nfs_server_address=${fai_nfs_server_address} + nfs_allowed_network=${fai_nfs_allowed_client_network} } @@ -15,7 +16,7 @@ function install_nfs_server() { for el in $nfs_export_list; do cat >> /etc/exports < Date: Wed, 7 Mar 2012 12:16:58 +0800 Subject: [PATCH 0472/2585] - hardening: not adding NFS configuration if it's already there. Tested the NFS server profile, it now works in FAI mode with e.g. this configuration: fai_nfs_server_install=1 fai_nfs_export_list="/exports/multimedia" fai_nfs_allowed_client_network="192.168.1.0/255.255.255.0" Issue closes #43 --- .../ece-install.d/network-file-system.sh | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh index 7094d9b5..368f121f 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -5,7 +5,7 @@ default_nfs_export_list="/exports/multimedia" function get_nfs_configuration() { nfs_export_list=${fai_nfs_export_list-$default_nfs_export_list} nfs_server_address=${fai_nfs_server_address} - nfs_allowed_network=${fai_nfs_allowed_client_network} + nfs_allowed_client_network=${fai_nfs_allowed_client_network} } @@ -15,17 +15,23 @@ function install_nfs_server() { get_nfs_configuration for el in $nfs_export_list; do - cat >> /etc/exports <> /etc/exports <> /etc/fstab <> /etc/fstab < Date: Wed, 7 Mar 2012 13:10:17 +0800 Subject: [PATCH 0473/2585] - first success full install of a VIP provider (tested with NFS): fai_vip_install=1 fai_vip_service_list="nfs-kernel-server" fai_vip_primary_node_name=ubiquitous fai_vip_primary_node_ip=192.168.1.112 fai_vip_secondary_node_name=ubiquitous-lts fai_vip_secondary_node_ip=192.168.1.111 fai_vip_address=192.168.1.200 fai_vip_sibling_ip=$fai_vip_secondary_node_ip Issue #34 TODO: hardening & testing. Lots of it. --- .../ece-scripts/ece-install.d/vip-provier.sh | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh index a1663109..c078a970 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh @@ -3,7 +3,7 @@ function get_vip_configuration() { # dependant on the running ece-install process/host vip_address=${fai_vip_address} - vip_interface=${fai_vip_interface} + vip_interface=${fai_vip_interface-eth0} vip_log_file=${fai_vip_log-/var/log/ha-debug} vip_sibling_ip=${fai_vip_sibling_ip} vip_service_list=${fai_vip_service_list} @@ -17,32 +17,36 @@ function get_vip_configuration() { } function install_vip_provider() { + print_and_log "Installing a VIP provider on $HOSTNAME" install_packages_if_missing heartbeat + get_vip_configuration + assure_vip_hosts_are_resolvable set_kernel_vip_parameters create_ha_auth_keys set_up_ha_conf set_up_ha_resources propagate_ha_settings + print_and_log "Restarting hearbeat to activate new configuration ..." run /etc/init.d/heartbeat restart - add_next_step "Heartbeat installed on $HOSTNAME to provide VIP $vip" - add_next_step "be sure that $node2_name gets similar setup as $HOSTNAME." + add_next_step "Heartbeat installed on $HOSTNAME to provide VIP ${vip_address}," + add_next_step "be sure that $vip_sibling_ip gets similar setup as $HOSTNAME." } function set_kernel_vip_parameters() { local file=/etc/sysctl.conf - - if [ $(grep "net.ipv4.ip_nonlocal_bind=1" $file | wc -l) -gt 0 ]; then + local entry="net.ipv4.ip_nonlocal_bind=1" + if [ $(grep "$entry" $file | wc -l) -gt 0 ]; then return fi cat >> $file < $file < $file < -$vip_primary_node_name $vip_address $vip_service_list +$entry EOF + fi } function assure_vip_hosts_are_resolvable() { @@ -67,26 +74,31 @@ function assure_vip_hosts_are_resolvable() { if [ $keep_off_etc_hosts -eq 1 ]; then return fi - - if [ $(grep $vip_primary_node_name /etc/hosts | wc -l) -lt 1 ]; then - cat >> /etc/hosts <> $file <> /etc/hosts <> $file < $file < Date: Wed, 7 Mar 2012 13:39:31 +0800 Subject: [PATCH 0474/2585] - added assumption (RAM) --- usr/share/doc/escenic/ece-install-guide.org | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index cfa461c3..4fa6e1b2 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -773,11 +773,14 @@ set_up_solr.postinst #+END_SRC * Assumptions -** /etc/esecenic is shared -It's assumed that the /etc/escenic directory is either on a shared -file system or rsync-ed among the hosts that are running the ECE -instances (the exception being database and cache servers). - +** The machine on which ece-install has enough RAM +It is assumed that the machine on which ece-install has enough RAM to +install the profiles the user asks for. + +A common error by users, is to run ece-install in a (virtual) machine +that has e.g. only 1GB RAM. For happy running, we recommend having a +machine with at least 3GB when installing a full ECE stack on it. + * Uninstalling Everything That the ece-install Set Up WARNING: this is potentially dangerous as some of these components may be used by other pieces of software you have running on your From 57a12ce33ae7c15d614d87b6dd334b0b909e58f2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 13:40:39 +0800 Subject: [PATCH 0475/2585] - replaced \_ with _, this means the HTML export will produce funny looking subscripts, but at least the .org source is clean and folks using e.g. https://github.com/skybert/ece-scripts/blob/master/usr/share/doc/escenic/ece-install-guide.org can copy and paste code without problems. --- usr/share/doc/escenic/ece-install-guide.org | 136 ++++++++++---------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 4fa6e1b2..f372e9fd 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -27,7 +27,7 @@ will tell you which ones it cannot find and tell you to install these, so you can simply run it to find out which ones you need. You may also look inside the ece-install script for all calls to the -'assert\_pre\_prequesite' method and you'll get a list of the binaries +'assert_pre_prequesite' method and you'll get a list of the binaries ece-install assumes are present. ** Tested Operating Systems @@ -153,7 +153,7 @@ any Escenic server components. *** Sun Java 6 *** Apache Tomcat application server - Compression of textual content (previously, this was typically set - up with Apache and its mod\_deflate module). + up with Apache and its mod_deflate module). - pooling set up tweaked for high read/write performance. - proper logging configuration directing solr messages to its own log. - routing information in the cookies @@ -172,7 +172,7 @@ may be removed after the installation if you want to. *** Compression of content This was was previously accomplished by having a web server in fron t of the application server (or cache server if you used ESI). A typical -system architecture would contain Apache with mod\_deflate. However, +system architecture would contain Apache with mod_deflate. However, this is no longer necessary as Varnish can handles ESI parsing of compressed content (and many other things that we before needed Apache for). Thus, we'll let the application server do the compression for us @@ -350,12 +350,12 @@ restore from: |----------------------------------+---------+------------------------------------------------| | Parameter | Default | Description | |----------------------------------+---------+------------------------------------------------| -| fai\_restore\_all | 0 | Restore everything found in the backup file | -| fai\_restore\_db | 0 | Install the DB server & restore its contents | -| fai\_restore\_data\_files | 0 | Restore the Solr & ECE data files | -| fai\_restore\_configuration | 0 | Restore the Solr & ECE configuration files | -| fai\_restore\_software\_binaries | 0 | Restore the Escenic and Apache Tomcat software | -| fai\_restore\_from\_file | "" | The .tar.gz produced by "ece -i backup" | +| fai_restore_all | 0 | Restore everything found in the backup file | +| fai_restore_db | 0 | Install the DB server & restore its contents | +| fai_restore_data_files | 0 | Restore the Solr & ECE data files | +| fai_restore_configuration | 0 | Restore the Solr & ECE configuration files | +| fai_restore_software_binaries | 0 | Restore the Escenic and Apache Tomcat software | +| fai_restore_from_file | "" | The .tar.gz produced by "ece -i backup" | |----------------------------------+---------+------------------------------------------------| For example, to restore everything possible from a given tarball, you @@ -513,7 +513,7 @@ add entries for these domains in the machine's /etc/hosts: #+END_SRC If you do not want ece-install to touch your /etc/hosts, you can set -fai\_keep\_off\_etc\_hosts=1 in your ece-install.conf +fai_keep_off_etc_hosts=1 in your ece-install.conf ** Overview of All FAI Parameters The ece-install script understands for the following settings in the @@ -522,58 +522,58 @@ $HOME/ece-install.conf file of the root user: |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| | Parameter | Default | Description | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| fai\_all\_install | 0 | Install all components on your server. | -| fai\_analysis\_db\_host | localhost | For the EAE DB (different from ECE's) | -| fai\_analysis\_db\_install | 0 | Install db profile | -| fai\_analysis\_db\_password | read-the-source-luke | For the EAE DB (different from ECE's) | -| fai\_analysis\_db\_port | 3306 | For the EAE DB (different from ECE's) | -| fai\_analysis\_db\_schema | ece5db | For the EAE DB (different from ECE's) | -| fai\_analysis\_db\_user | ece5user | For the EAE DB (different from ECE's) | -| fai\_analysis\_name | analysis1 | EAE instance name | -| fai\_analysis\_port | 8080 | Port of the EAE | -| fai\_analysis\_shutdown | 8005 | Shutdown port for the EAE app server | -| fai\_cache\_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai\_cache\_install | 0 | Install cache server profile | -| fai\_db\_drop\_old\_db\_first | 0 | Warning: this will drop the old database before installing a new one | -| fai\_db\_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai\_db\_install | 0 | Install db profile | -| fai\_db\_password | read-the-source-luke | Useful for DB installation profile | -| fai\_db\_port | 3306 | Useful for editor & presentation profiles | -| fai\_db\_schema | ece5db | Useful for DB installation profile | -| fai\_db\_user | ece5user | Useful for DB installation profile | -| fai\_editor\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_editor\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_editor\_install | 0 | Install the editorial profile | -| fai\_editor\_name | editor1 | Name of the editor instance | -| fai\_editor\_port | 8080 | HTTP port of the editor instance | -| fai\_editor\_shutdown | 8005 | Shutdown port of the editor instance | -| fai\_enabled | 0 | Whether or not to run ece-install in FAI mode | -| fai\_keep\_off\_etc\_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | -| fai\_monitoring\_server\_install | 0 | Install the monitoring server profile. | -| fai\_monitoring\_server\_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai\_presentation\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_presentation\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_presentation\_install | 0 | Install the presentation server profile | -| fai\_presentation\_name | web1 | Name of the presentation server instance | -| fai\_presentation\_port | 8080 | HTTP port of the presentation server instance | -| fai\_presentation\_shutdown | 8005 | Shutdown port of the presentation instance | -| fai\_public\_host\_name | ${HOSTNAME}:8080 | The public address for your website | -| fai\_publication\_create | 0 | Create a new publication | -| fai\_publication\_domain\_mapping\_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | -| fai\_publication\_name | mypub | Name of the publication | -| fai\_publication\_use\_instance | dev1 | Name of local instance to use for creation | -| fai\_publication\_war | "WF or ECE demo WAR" | WAR to base the new publication on | -| fai\_publication\_war\_uri\_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai\_rmi\_install | 0 | Install RMI hub profile | -| fai\_search\_conf\_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai\_search\_ear | "" | EAR to use instead of the Escenic binaries | -| fai\_search\_for\_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | -| fai\_search\_indexer\_ws\_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | -| fai\_search\_install | 0 | Install the search server profile (Solr + indexer) | -| fai\_search\_name | search1 | Name of the search instance | -| fai\_search\_port | 8080 | HTTP port of the search instance | -| fai\_search\_shutdown | 8005 | Shutdown port of the search instance | -| fai\_wf\_install | 0 | Install Widget Framework profile | +| fai_all_install | 0 | Install all components on your server. | +| fai_analysis_db_host | localhost | For the EAE DB (different from ECE's) | +| fai_analysis_db_install | 0 | Install db profile | +| fai_analysis_db_password | read-the-source-luke | For the EAE DB (different from ECE's) | +| fai_analysis_db_port | 3306 | For the EAE DB (different from ECE's) | +| fai_analysis_db_schema | ece5db | For the EAE DB (different from ECE's) | +| fai_analysis_db_user | ece5user | For the EAE DB (different from ECE's) | +| fai_analysis_name | analysis1 | EAE instance name | +| fai_analysis_port | 8080 | Port of the EAE | +| fai_analysis_shutdown | 8005 | Shutdown port for the EAE app server | +| fai_cache_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai_cache_install | 0 | Install cache server profile | +| fai_db_drop_old_db_first | 0 | Warning: this will drop the old database before installing a new one | +| fai_db_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai_db_install | 0 | Install db profile | +| fai_db_password | read-the-source-luke | Useful for DB installation profile | +| fai_db_port | 3306 | Useful for editor & presentation profiles | +| fai_db_schema | ece5db | Useful for DB installation profile | +| fai_db_user | ece5user | Useful for DB installation profile | +| fai_editor_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai_editor_ear | "" | EAR to use instead of the Escenic binaries | +| fai_editor_install | 0 | Install the editorial profile | +| fai_editor_name | editor1 | Name of the editor instance | +| fai_editor_port | 8080 | HTTP port of the editor instance | +| fai_editor_shutdown | 8005 | Shutdown port of the editor instance | +| fai_enabled | 0 | Whether or not to run ece-install in FAI mode | +| fai_keep_off_etc_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | +| fai_monitoring_server_install | 0 | Install the monitoring server profile. | +| fai_monitoring_server_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai_presentation_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai_presentation_ear | "" | EAR to use instead of the Escenic binaries | +| fai_presentation_install | 0 | Install the presentation server profile | +| fai_presentation_name | web1 | Name of the presentation server instance | +| fai_presentation_port | 8080 | HTTP port of the presentation server instance | +| fai_presentation_shutdown | 8005 | Shutdown port of the presentation instance | +| fai_public_host_name | ${HOSTNAME}:8080 | The public address for your website | +| fai_publication_create | 0 | Create a new publication | +| fai_publication_domain_mapping_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | +| fai_publication_name | mypub | Name of the publication | +| fai_publication_use_instance | dev1 | Name of local instance to use for creation | +| fai_publication_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai_publication_war_uri_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | +| fai_rmi_install | 0 | Install RMI hub profile | +| fai_search_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai_search_ear | "" | EAR to use instead of the Escenic binaries | +| fai_search_for_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | +| fai_search_indexer_ws_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | +| fai_search_install | 0 | Install the search server profile (Solr + indexer) | +| fai_search_name | search1 | Name of the search instance | +| fai_search_port | 8080 | HTTP port of the search instance | +| fai_search_shutdown | 8005 | Shutdown port of the search instance | +| fai_wf_install | 0 | Install Widget Framework profile | |-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| #+TBLFM: $2=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ 0 @@ -651,9 +651,9 @@ folder, and only download the missing ones (if any). To get a list of the artifacts it'll pull from http://technet.escenic.com and http://tomcat.apache.org search for the following variables: -- technet\_download\_list -- wf\_download\_list -- tomcat\_download +- technet_download_list +- wf_download_list +- tomcat_download Two other ways of speeding up the installation is (of course) to use the backup/restore feature or install from a EAR and configuration @@ -689,8 +689,8 @@ but these should be the most interesting ones. | /opt/escenic/assemblytool | The assembly tool | | /opt/escenic/assemblytool/plugins | Contains symlinks to all plugins in /opt/escenic | | /opt/escenic/engine | Symlink pointing to the current ECE | -| /opt/tomcat | Symlink pointing to the install Apache Tomcat (catalina\_home) | -| /opt/tomcat- | Instance specific Tomcat files (catalina\_base) | +| /opt/tomcat | Symlink pointing to the install Apache Tomcat (catalina_home) | +| /opt/tomcat- | Instance specific Tomcat files (catalina_base) | | /usr/bin/ece | Script for operating all ECE instances + RMI hub and EAE | | /usr/sbin/drop-and-create-ecedb | DB script used by the ece-install script | | /usr/sbin/ece-install | The installation script described in this guide | @@ -788,7 +788,7 @@ host. However, this may be useful if you're installing a clean environment and want to e.g. undo your previous install to install a different profile. -Open the ece-install script and look for the "un\_install\_ece" +Open the ece-install script and look for the "un_install_ece" function, it has copy and pastable commands for undoing most/all things set up by the script. From 5e8e2b4100f6752fcc3e3f8a86b65b6b65c4892a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 13:42:39 +0800 Subject: [PATCH 0476/2585] - updated test status for CentOS --- usr/share/doc/escenic/ece-install-guide.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index f372e9fd..d494987b 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -36,7 +36,7 @@ ece-install assumes are present. | Debian 6.0 (squeeze) | Everything is 100% automatic | 2012-02-27 13:18 | | Ubuntu LTS 10.04 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | | Ubuntu 11.10 | Everything except Tomcat/APR wrappers are automatic | 2012-02-27 13:19 | -| CentOS 6.2 | APR & monitoring are not automatic | 2012-02-28 12:37 | +| CentOS 6.2 | HA, APR & monitoring are not automatic | 2012-03-07 13:42 | * A Note on Running ece-install On Non-GNU/Linux Systems Please note, it's assumed that the OS uses GNU's tools, i.e. find, cp From c3dbbabcd4e014149e3ea4494000d54b0cf8b6dd Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 13:45:45 +0800 Subject: [PATCH 0477/2585] - removed munin-java-plugins from munin plugins as it pulls down OpenJDK. Unless we really need these plugins, we'd probably be better without them. --- usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 00a4a6d3..9ea2675f 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -160,7 +160,7 @@ function install_munin_node() fi if [ $on_debian_or_derivative -eq 1 ]; then - packages="munin-node munin-plugins-extra munin-java-plugins" + packages="munin-node munin-plugins-extra" install_packages_if_missing $packages else print_and_log "Munin node installation not supported on your system" From caccd1399d42ba552f07ac5662d9e6ba0cee4a1a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 14:32:10 +0800 Subject: [PATCH 0478/2585] - added ensure_variable_is_set --- .../escenic/ece-scripts/common-bashing.sh | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh index 3e6e4fdf..b1afdb20 100644 --- a/usr/share/escenic/ece-scripts/common-bashing.sh +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -179,3 +179,25 @@ function get_base_dir_from_bundle() echo $file_name } +## Will assert that all the passed variable names are set, if not, it +## will exit in error. +## +## $@ : a list of variable names +## +## Requires $conf_file to be set. +function ensure_variable_is_set() { + local requirements_failed=0 + + for el in $@; do + if [ -n "$(eval echo $`echo $el`)" ]; then + continue + fi + + print "You need to specifiy '$el' in your $conf_file" + requirements_failed=1 + done + + if [ $requirements_failed -eq 1 ]; then + remove_pid_and_exit_in_error + fi +} From 77f631d5d57b6c6639d500656716d21b0705ec57 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 14:37:29 +0800 Subject: [PATCH 0479/2585] - NFS client profile install now works! Example configuration: fai_enabled=1 fai_nfs_server_address=192.168.1.200 fai_nfs_client_install=1 fai_nfs_export_list="/exports/multimedia" This closes issue #44. --- .../ece-install.d/network-file-system.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh index 368f121f..e76086a0 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -6,11 +6,17 @@ function get_nfs_configuration() { nfs_export_list=${fai_nfs_export_list-$default_nfs_export_list} nfs_server_address=${fai_nfs_server_address} nfs_allowed_client_network=${fai_nfs_allowed_client_network} + + ensure_variable_is_set fai_nfs_server_address + + if [ $install_profile_number -eq $PROFILE_NFS_SERVER ]; then + fai_nfs_allowed_client_network + fi } function install_nfs_server() { - print "Installing an NFS server on $HOSTNAME ..." + print_and_log "Installing an NFS server on $HOSTNAME ..." install_packages_if_missing "portmap nfs-kernel-server nfs-common" get_nfs_configuration @@ -35,23 +41,24 @@ EOF } function install_nfs_client() { - print "Installing an NFS client on $HOSTNAME ..." + print_and_log "Installing an NFS client on $HOSTNAME ..." install_packages_if_missing "nfs-common" get_nfs_configuration local mount_point_list="" + local file=/etc/fstab for el in $nfs_export_list; do local entry="${nfs_server_address}:$el /mnt/$(basename $0) nfs defaults 0 0" - if [ $(grep "$entry" /etc/exports | wc -l) -lt 1 ]; then - cat >> /etc/fstab <> $file < Date: Wed, 7 Mar 2012 14:39:11 +0800 Subject: [PATCH 0480/2585] - minor change to documentation. This closes #44 --- .../escenic/ece-scripts/ece-install.d/network-file-system.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh index e76086a0..598aa570 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -1,4 +1,5 @@ -# module to install & configure network file systems +# module to install & configure network file systems, both server and +# client side. default_nfs_export_list="/exports/multimedia" @@ -14,7 +15,6 @@ function get_nfs_configuration() { fi } - function install_nfs_server() { print_and_log "Installing an NFS server on $HOSTNAME ..." install_packages_if_missing "portmap nfs-kernel-server nfs-common" From 2def3ea67f09efee694306b90ca45220fb9d7fd9 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 17:35:09 +0800 Subject: [PATCH 0481/2585] - improved NFS server profile, added more export options to make it perform better. - NFS server: now creates exports.d to get rid of warning/error each time nfs-kernel-server starts - for NFS client profile, added possibility to decide mount point parent: fai_nfs_client_mount_point_parent=/data Default is /mnt --- .../ece-install.d/network-file-system.sh | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh index 598aa570..3fde3e09 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -1,17 +1,19 @@ # module to install & configure network file systems, both server and # client side. -default_nfs_export_list="/exports/multimedia" +default_nfs_export_list="/var/exports/multimedia" +default_nfs_mount_point_parent="/mnt" function get_nfs_configuration() { nfs_export_list=${fai_nfs_export_list-$default_nfs_export_list} nfs_server_address=${fai_nfs_server_address} nfs_allowed_client_network=${fai_nfs_allowed_client_network} - - ensure_variable_is_set fai_nfs_server_address + nfs_client_mount_point_parent=${fai_nfs_client_mount_point_parent-${default_nfs_mount_point_parent}} if [ $install_profile_number -eq $PROFILE_NFS_SERVER ]; then - fai_nfs_allowed_client_network + ensure_variable_is_set fai_nfs_allowed_client_network + elif [ $install_profile_number -eq $PROFILE_NFS_CLIENT ]; then + ensure_variable_is_set fai_nfs_server_address fi } @@ -21,11 +23,12 @@ function install_nfs_server() { get_nfs_configuration for el in $nfs_export_list; do - local entry="$el ${nfs_allowed_client_network}(rw,sync)" - if [ $(grep "$entry" /etc/exports | wc -l) -lt 1 ]; then + # using no_subtree_check and async to speed up transfers. + local entry="$el ${nfs_allowed_client_network}(rw,no_subtree_check,async)" + if [ $(grep "$entry" /etc/exports 2>/dev/null | wc -l) -lt 1 ]; then cat >> /etc/exports <> $file < Date: Wed, 7 Mar 2012 17:40:17 +0800 Subject: [PATCH 0482/2585] - added option to pass the primary node auth key. This is needed when installing the secondary node (or indeed for setting up both primary and secondary). fai_vip_primary_node_auth_key This value is the MD5 sum in /etc/ha.d/authkeys Normally, we could use ha_propagate, however, this doesn't work on Ubuntu as we're running as root and ha_propagate uses ssh, so since the target system doesn't allow/know about the root user when logging in as root, the ha_propagate command is useless. Passing this parameter here makes the secondary node installation possible on all distributions, Ubuntu included. Issue #34 - hardening. --- .../ece-scripts/ece-install.d/vip-provier.sh | 50 ++++++++++++------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh index c078a970..03196109 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh @@ -14,6 +14,11 @@ function get_vip_configuration() { vip_primary_node_ip=${fai_vip_primary_node_ip} vip_secondary_node_name=${fai_vip_secondary_node_name} vip_secondary_node_ip=${fai_vip_secondary_node_ip} + + # only useful when installing the secondary node (from its + # /etc/ha.d/authkeys), looks something like: + # e19f39c09e9affafc23fc5bcd0404186 + vip_primary_node_auth_key=${fai_vip_primary_node_auth_key} } function install_vip_provider() { @@ -25,7 +30,6 @@ function install_vip_provider() { create_ha_auth_keys set_up_ha_conf set_up_ha_resources - propagate_ha_settings print_and_log "Restarting hearbeat to activate new configuration ..." run /etc/init.d/heartbeat restart @@ -37,7 +41,7 @@ function install_vip_provider() { function set_kernel_vip_parameters() { local file=/etc/sysctl.conf local entry="net.ipv4.ip_nonlocal_bind=1" - if [ $(grep "$entry" $file | wc -l) -gt 0 ]; then + if [ $(grep "$entry" $file 2>/dev/null | wc -l) -gt 0 ]; then return fi @@ -53,15 +57,11 @@ EOF # print "Making sure the kernel parameters are loaded at boot time" } -function propagate_ha_settings() { - run /usr/share/heartbeat/ha_propagate -} - function set_up_ha_resources() { local file=/etc/ha.d/haresources local entry="$vip_primary_node_name $vip_address $vip_service_list" - if [ $(grep "$entry" $file | wc -l) -lt 1 ]; then + if [ $(grep "$entry" $file 2>/dev/null | wc -l) -lt 1 ]; then cat > $file < $entry @@ -76,17 +76,14 @@ function assure_vip_hosts_are_resolvable() { fi local file=/etc/hosts - local entry="$vip_primary_node_name $vip_primary_node_ip" - if [ $(grep "$entry" $file | wc -l) -lt 1 ]; then - cat >> $file </dev/null | wc -l) -lt 1 ]; then cat >> $file < $file + echo -ne "auth 1\n1 sha1 " > $file + + if [ -n "${vip_primary_node_auth_key}" ]; then + echo "${vip_primary_node_auth_key}" >> $file + else + # the cut -d' ' -f2 is because 1.0.0e produces "(stdin)=" in front + # of the output, something which the old, 0.9.8o didn't do. + dd if=/dev/urandom bs=512 count=1 2>> $log | \ + openssl md5 | \ + cut -d' ' -f2 \ + >> $file + fi + run chmod 0600 $file } From 938ff29e1041c592cab5e543904f16d7f2e8440c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 7 Mar 2012 18:11:24 +0800 Subject: [PATCH 0483/2585] - added fsid to the NFS exports options. This should remedy stale NFS handles after recovery. Issue #43 --- .../ece-scripts/ece-install.d/network-file-system.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh index 3fde3e09..aa621ff4 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/network-file-system.sh @@ -21,10 +21,12 @@ function install_nfs_server() { print_and_log "Installing an NFS server on $HOSTNAME ..." install_packages_if_missing "portmap nfs-kernel-server nfs-common" get_nfs_configuration - + + local i=1 for el in $nfs_export_list; do # using no_subtree_check and async to speed up transfers. - local entry="$el ${nfs_allowed_client_network}(rw,no_subtree_check,async)" + local nfs_opts="(rw,no_subtree_check,sync,fsid=$i)" + local entry="$el ${nfs_allowed_client_network}${nfs_opts}" if [ $(grep "$entry" /etc/exports 2>/dev/null | wc -l) -lt 1 ]; then cat >> /etc/exports < Date: Thu, 8 Mar 2012 13:40:00 +0800 Subject: [PATCH 0484/2585] - hardening: run assert_correct_runtime_environment before init. This ensure that ece-install doesn't try to load all the different modules if the run-time environment isn't right (e.g. if the user is trying to run ece-install as a non-root user). - moved the checking/running of the VIP provider profile so that the user can install any other profile before (possible) the VIP profile, which therefore can depend on a previous installation profile in the same run of ece-install. This means that a user can install both an NFS server as well as an NFS server VIP provider from the same ece-install process. --- usr/sbin/ece-install | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 88be34fc..9b127f68 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -186,7 +186,8 @@ function install_packages_if_missing() { if [ $some_are_missing -eq 0 ]; then return elif [ $one_time_apt_update_done -eq 0 ]; then - log "Running one time APT update to ensure package list is up to date" + print_and_log \ + "Running one time APT update to ensure package list is fresh" run apt-get update one_time_apt_update_done=1 fi @@ -614,15 +615,15 @@ function read_user_input() function assert_correct_runtime_environment() { if [ $(whoami) != "root" ]; then - print "You must be root when running $(basename $0)" + echo "You must be root when running $(basename $0)" exit 1 fi if [ -e $pid_file ]; then - print "There's already one $(basename $0) process running." - print "If you blelieve this is wrong, e.g. if a previous run of" - print "$(basename $0) was aborted before it completed, you" - print "may remove ${pid_file} and run $(basename $0) again." + echo "There's already one $(basename $0) process running." + echo "If you blelieve this is wrong, e.g. if a previous run of" + echo "$(basename $0) was aborted before it completed, you" + echo "may remove ${pid_file} and run $(basename $0) again." exit 1 else echo $BASHPID > $pid_file @@ -630,8 +631,8 @@ function assert_correct_runtime_environment() fi if [ ! -e "$conf_file" ]; then - print $conf_file "doesn't exist." - print "I cannot live without it, so I'm exiting :-(" + echo $conf_file "doesn't exist." + echo "I cannot live without it, so I'm exiting :-(" remove_pid_and_exit_in_error fi } @@ -1038,8 +1039,8 @@ for el in $@; do fi done -init assert_correct_runtime_environment +init fai_enabled=$(get_boolean_conf_value fai_enabled) if [ $fai_enabled -eq 1 ]; then @@ -1124,12 +1125,6 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi - if [ $(get_boolean_conf_value fai_vip_install) -eq 1 ]; then - install_profile_number=$PROFILE_VIP_PROVIDER - install_vip_provider - no_fai_profile=0 - fi - if [ $(get_boolean_conf_value fai_nfs_server_install) -eq 1 ]; then install_profile_number=$PROFILE_NFS_SERVER install_nfs_server @@ -1141,6 +1136,15 @@ if [ $fai_enabled -eq 1 ]; then install_nfs_client no_fai_profile=0 fi + + # checking for VIP profile last so that ece-install can (if so + # configured in the ece-install.con) install all dependent + # services first. + if [ $(get_boolean_conf_value fai_vip_install) -eq 1 ]; then + install_profile_number=$PROFILE_VIP_PROVIDER + install_vip_provider + no_fai_profile=0 + fi if [ $no_fai_profile -eq 1 ]; then print_and_log "No install profile selected, be sure to have one of the " From 1810e375d801667d3c0ab08b5c4aa669dd106485 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Mar 2012 15:21:55 +0800 Subject: [PATCH 0485/2585] - added NFS client documentation, #44 - added NFS server documentation, #43 - added VIP provider documentation, #34 --- usr/share/doc/escenic/ece-install-guide.org | 209 ++++++++++++++------ 1 file changed, 146 insertions(+), 63 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index d494987b..35c0ce2f 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -178,7 +178,7 @@ compressed content (and many other things that we before needed Apache for). Thus, we'll let the application server do the compression for us now. -** Profile - Full Stack on One Host +** Profile :: Full Stack on One Host This profile is suitable for developers and admins wanting to set up a test environment. It will install the full stack including caching server, application server, ECE, assembly host, database, Widget @@ -188,17 +188,17 @@ For further details on each of the different server components, see the different profile descriptions bellow. -** Profile - Editorial Server (Publication Server) +** Profile :: Editorial Server (Publication Server) Will set up a typical editorial server (often referred to as the publication server in Escenic literature). -** Profile - Presentation Server +** Profile :: Presentation Server This will set up a typical presentation server: - Memcached, distributed memory cache - Deployment setup to only deploy escenic-admin and the publication(s). -** Profile - Database Server +** Profile :: Database Server If ece-install is run on a support version of Debian or Ubuntu, this will install the excellent Percona distribution of MySQL with their set of high performance patches. @@ -219,7 +219,7 @@ ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists [ece-install] running tables FAILED, exiting :-( #+END_SRC -** Profile - Cache Server +** Profile :: Cache Server If ece-install is run on a support version of Debian or Ubuntu, it will install the latest Varnish 3.0 caching server from the Varnish APT repository. @@ -255,7 +255,7 @@ TBD: to be in the staff network ACL, defined in the Varnish configuration. -** Profile - Install Widget Framework +** Profile :: Install Widget Framework You'll need a user name and password for accessing the repo.escenic.com Maven repository. You should get these credentials when you bought Widget Framework. If you for some reason do not have @@ -267,7 +267,7 @@ complain: [ece-install] Be sure to set wf_user and wf_password in /root/ece-install.conf [ece-install] If you don't have these, please contact support@escenic.com #+END_SRC -** Profile - Create Publication +** Profile :: Create Publication This profile will create a publication for you, only asking you the name of the publication and which ECE instance to use for its creation. @@ -275,14 +275,14 @@ creation. This installation profile will base the publication on the Widget Framework if its present on the system, if not, ECE's clean demo WAR is used as a basis. -** Profile - Monitoring Server +** Profile :: Monitoring Server This will install a Munin gatherer and web server. The latter for accessing the reports generated by the former. TBD: This profile will also install the Nagios interface for monitoring the different nodes. -** Profile - Restoring from backup +** Profile :: Restoring from backup ece-install can restore from a backup made by the ece script: #+BEGIN_SRC conf $ ece -i backup @@ -296,11 +296,80 @@ what's available on the host where it's run): - ECE, cache and web server configuration - Escenic software -** Profile - Analysis Server +** Profile :: Analysis Server This profile will install the Escenic Analysis Engine and configure it for production use with sane defaults. Be sure to use a different DB than you use for ECE. +** Profile :: NFS server +Will install an NFS server. + +** Profile :: NFS client +Will install an NFS client, create all the mountpoints and mount these +on the host. + +Configuration example for setting up an NFS client mounting the +multimedia archive on the NFS server: +#+BEGIN_SRC sh +fai_enabled=1 +fai_nfs_client_install=1 +fai_nfs_server_address=192.168.1.200 +fai_nfs_export_list="/var/exports/multimedia" +#+END_SRC + +** Profile :: VIP provider + +This profile will make the host capable of serving a certain virtual +IP (VIP) for one or more services. This is a useful thing for all +[[http://en.wikipedia.org/wiki/Single_point_of_failure][SPOFs]] in your architecture, such as the file server or database. + +You will typically install another profile at the same time as the VIP +provider. For instance, you will install both an NFS server and a VIP +provider for that NFS server (and perhaps other services too running +on the same host). + +The following ece-install.conf configuration will install an NFS +server and make it a VIP provider exposing its NFS service. The two +VIP providers have have the IP addresses 192.168.1.111 and +192.168.1.112 and both provide the VIP 192.168.1.200 + +#+BEGIN_SRC sh +# install the NFS server +fai_enabled=1 +fai_nfs_server_install=1 +fai_nfs_export_list="/var/exports/multimedia" +fai_nfs_allowed_client_network="192.168.1.0/255.255.255.0" + +# install the VIP provider, primary node +fai_vip_install=1 +fai_vip_service_list="nfs-kernel-server" +fai_vip_primary_node_name=ubiquitous +fai_vip_primary_node_ip=192.168.1.112 +fai_vip_primary_node_auth_key=d41d8cd98f00b204e9800998ecf8427e +fai_vip_secondary_node_name=ubiquitous-lts +fai_vip_secondary_node_ip=192.168.1.111 +fai_vip_address=192.168.1.200 +fai_vip_sibling_ip=$fai_vip_secondary_node_ip +#+END_SRC + +The secondary node, also an NFS VIP provider can have the exact same +configuration except for having a different sibling: + +#+BEGIN_SRC sh +fai_vip_sibling_ip=$fai_vip_primary_node_ip +#+END_SRC + +Note that the fai_vip_primary_node_auth_key is optional. If not set, +ece-install will generate it for you. However, you will then have to +add this to ece-install.conf when installing the secondary +node. Generate this key, do: + +#+BEGIN_SRC sh +$ dd if=/dev/urandom bs=512 count=1 2>> $log | \ + openssl md5 | \ + cut -d' ' -f2 +#+END_SRC + ** Running interactively *** Start ece-install and choose the Option, "Restore from backup" #+BEGIN_SRC text @@ -519,63 +588,77 @@ fai_keep_off_etc_hosts=1 in your ece-install.conf The ece-install script understands for the following settings in the $HOME/ece-install.conf file of the root user: -|-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| Parameter | Default | Description | -|-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -| fai_all_install | 0 | Install all components on your server. | -| fai_analysis_db_host | localhost | For the EAE DB (different from ECE's) | -| fai_analysis_db_install | 0 | Install db profile | -| fai_analysis_db_password | read-the-source-luke | For the EAE DB (different from ECE's) | -| fai_analysis_db_port | 3306 | For the EAE DB (different from ECE's) | -| fai_analysis_db_schema | ece5db | For the EAE DB (different from ECE's) | -| fai_analysis_db_user | ece5user | For the EAE DB (different from ECE's) | -| fai_analysis_name | analysis1 | EAE instance name | -| fai_analysis_port | 8080 | Port of the EAE | -| fai_analysis_shutdown | 8005 | Shutdown port for the EAE app server | -| fai_cache_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | -| fai_cache_install | 0 | Install cache server profile | -| fai_db_drop_old_db_first | 0 | Warning: this will drop the old database before installing a new one | -| fai_db_host | $HOSTNAME | Useful for editor & presentation profiles | -| fai_db_install | 0 | Install db profile | -| fai_db_password | read-the-source-luke | Useful for DB installation profile | -| fai_db_port | 3306 | Useful for editor & presentation profiles | -| fai_db_schema | ece5db | Useful for DB installation profile | -| fai_db_user | ece5user | Useful for DB installation profile | -| fai_editor_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai_editor_ear | "" | EAR to use instead of the Escenic binaries | -| fai_editor_install | 0 | Install the editorial profile | -| fai_editor_name | editor1 | Name of the editor instance | -| fai_editor_port | 8080 | HTTP port of the editor instance | -| fai_editor_shutdown | 8005 | Shutdown port of the editor instance | -| fai_enabled | 0 | Whether or not to run ece-install in FAI mode | +|-------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| Parameter | Default | Description | +|-------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| +| fai_all_install | 0 | Install all components on your server. | +| fai_analysis_db_host | localhost | For the EAE DB (different from ECE's) | +| fai_analysis_db_install | 0 | Install db profile | +| fai_analysis_db_password | read-the-source-luke | For the EAE DB (different from ECE's) | +| fai_analysis_db_port | 3306 | For the EAE DB (different from ECE's) | +| fai_analysis_db_schema | ece5db | For the EAE DB (different from ECE's) | +| fai_analysis_db_user | ece5user | For the EAE DB (different from ECE's) | +| fai_analysis_name | analysis1 | EAE instance name | +| fai_analysis_port | 8080 | Port of the EAE | +| fai_analysis_shutdown | 8005 | Shutdown port for the EAE app server | +| fai_cache_backends | ${HOSTNAME}:8080 | Space separated, e.g. "app1:8080 app2:8080" | +| fai_cache_install | 0 | Install cache server profile | +| fai_db_drop_old_db_first | 0 | Warning: this will drop the old database before installing a new one | +| fai_db_host | $HOSTNAME | Useful for editor & presentation profiles | +| fai_db_install | 0 | Install db profile | +| fai_db_password | read-the-source-luke | Useful for DB installation profile | +| fai_db_port | 3306 | Useful for editor & presentation profiles | +| fai_db_schema | ece5db | Useful for DB installation profile | +| fai_db_user | ece5user | Useful for DB installation profile | +| fai_editor_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai_editor_ear | "" | EAR to use instead of the Escenic binaries | +| fai_editor_install | 0 | Install the editorial profile | +| fai_editor_name | editor1 | Name of the editor instance | +| fai_editor_port | 8080 | HTTP port of the editor instance | +| fai_editor_shutdown | 8005 | Shutdown port of the editor instance | +| fai_enabled | 0 | Whether or not to run ece-install in FAI mode | | fai_keep_off_etc_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | -| fai_monitoring_server_install | 0 | Install the monitoring server profile. | -| fai_monitoring_server_ip | 127.0.0.1 | The IP of the monitoring server. | -| fai_presentation_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai_presentation_ear | "" | EAR to use instead of the Escenic binaries | -| fai_presentation_install | 0 | Install the presentation server profile | -| fai_presentation_name | web1 | Name of the presentation server instance | -| fai_presentation_port | 8080 | HTTP port of the presentation server instance | -| fai_presentation_shutdown | 8005 | Shutdown port of the presentation instance | -| fai_public_host_name | ${HOSTNAME}:8080 | The public address for your website | -| fai_publication_create | 0 | Create a new publication | +| fai_monitoring_server_install | 0 | Install the monitoring server profile. | +| fai_monitoring_server_ip | 127.0.0.1 | The IP of the monitoring server. | +| fai_nfs_allowed_client_network | "" | IP/netmwask of allowed NFS clients, example: 192.168.1.0/255.255.255.0 | +| fai_nfs_client_install | 0 | Installs an NFS client | +| fai_nfs_client_mount_point_parent | /mnt | Mount point parent directory | +| fai_nfs_export_list | "" | Space separated list of NFS export directories, full paths as seen on the NFS server. E.g.: /var/exports/multimedia | +| fai_nfs_server_address | "" | Address of the NFS server, useful for the NFS client profile | +| fai_nfs_server_install | 0 | Install an NFS server | +| fai_presentation_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai_presentation_ear | "" | EAR to use instead of the Escenic binaries | +| fai_presentation_install | 0 | Install the presentation server profile | +| fai_presentation_name | web1 | Name of the presentation server instance | +| fai_presentation_port | 8080 | HTTP port of the presentation server instance | +| fai_presentation_shutdown | 8005 | Shutdown port of the presentation instance | +| fai_public_host_name | ${HOSTNAME}:8080 | The public address for your website | +| fai_publication_create | 0 | Create a new publication | | fai_publication_domain_mapping_list | "" | Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" | -| fai_publication_name | mypub | Name of the publication | -| fai_publication_use_instance | dev1 | Name of local instance to use for creation | -| fai_publication_war | "WF or ECE demo WAR" | WAR to base the new publication on | +| fai_publication_name | mypub | Name of the publication | +| fai_publication_use_instance | dev1 | Name of local instance to use for creation | +| fai_publication_war | "WF or ECE demo WAR" | WAR to base the new publication on | | fai_publication_war_uri_list | "" | Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, file:///tmp/mypub.war and /tmp/mypub.war. | -| fai_rmi_install | 0 | Install RMI hub profile | -| fai_search_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | -| fai_search_ear | "" | EAR to use instead of the Escenic binaries | -| fai_search_for_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | +| fai_rmi_install | 0 | Install RMI hub profile | +| fai_search_conf_archive | "" | conf.tar.gz to use for Nursery & JAAS configuration | +| fai_search_ear | "" | EAR to use instead of the Escenic binaries | +| fai_search_for_editor | 0 | If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. | | fai_search_indexer_ws_uri | http://${HOSTNAME}:8080/indexer-webservice/index/ | URI of the indexer-webservice that the search instance shall use for knowing what to index. | -| fai_search_install | 0 | Install the search server profile (Solr + indexer) | -| fai_search_name | search1 | Name of the search instance | -| fai_search_port | 8080 | HTTP port of the search instance | -| fai_search_shutdown | 8005 | Shutdown port of the search instance | -| fai_wf_install | 0 | Install Widget Framework profile | -|-----------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| -#+TBLFM: $2=http://$fai_editor_name:$fai_editor_port/indexer-webservice/index/ 0 +| fai_search_install | 0 | Install the search server profile (Solr + indexer) | +| fai_search_name | search1 | Name of the search instance | +| fai_search_port | 8080 | HTTP port of the search instance | +| fai_search_shutdown | 8005 | Shutdown port of the search instance | +| fai_vip_address | "" | The virtual IP the provider will claim | +| fai_vip_install | 0 | Install a VIP provider | +| fai_vip_primary_node_auth_key | "" | Optional, but useful to set to make conf files consistent. Will be generated if not set | +| fai_vip_primary_node_ip | "" | Primary node IP | +| fai_vip_primary_node_name | "" | Primary node name, must be what $(uname -n) returns | +| fai_vip_secondary_node_ip | "" | Secondary node IP | +| fai_vip_secondary_node_name | "" | Secondary node name, must be what $(uname -n) returns | +| fai_vip_service_list | "" | List of init.d scripts to invoke when the VIP is acclaimed/revoked, script must support start & stop | +| fai_vip_sibling_ip | "" | The IP of the other node offering the VIP | +| fai_wf_install | 0 | Install Widget Framework profile | +|-------------------------------------+---------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------| As you've probably have guessed, 0 means "false" and 1 means "true" :-) From 0555c7b7affb8ad92896d72d1e46557887263fed Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Mar 2012 15:22:10 +0800 Subject: [PATCH 0486/2585] - updated HTML --- usr/share/doc/escenic/ece-install-guide.html | 353 ++++++++++++------- 1 file changed, 230 insertions(+), 123 deletions(-) diff --git a/usr/share/doc/escenic/ece-install-guide.html b/usr/share/doc/escenic/ece-install-guide.html index b03ed535..95b62b46 100644 --- a/usr/share/doc/escenic/ece-install-guide.html +++ b/usr/share/doc/escenic/ece-install-guide.html @@ -7,7 +7,7 @@ - + @@ -253,26 +253,29 @@

    Table of Contents

  • 5.2.7 Compression of content
  • -
  • 5.3 Profile - Full Stack on One Host
  • -
  • 5.4 Profile - Editorial Server (Publication Server)
  • -
  • 5.5 Profile - Presentation Server
  • -
  • 5.6 Profile - Database Server
  • -
  • 5.7 Profile - Cache Server
  • -
  • 5.8 Profile - Install Widget Framework
  • -
  • 5.9 Profile - Create Publication
  • -
  • 5.10 Profile - Monitoring Server
  • -
  • 5.11 Profile - Restoring from backup
  • -
  • 5.12 Profile - Analysis Server
  • -
  • 5.13 Running interactively +
  • 5.3 Profile :: Full Stack on One Host
  • +
  • 5.4 Profile :: Editorial Server (Publication Server)
  • +
  • 5.5 Profile :: Presentation Server
  • +
  • 5.6 Profile :: Database Server
  • +
  • 5.7 Profile :: Cache Server
  • +
  • 5.8 Profile :: Install Widget Framework
  • +
  • 5.9 Profile :: Create Publication
  • +
  • 5.10 Profile :: Monitoring Server
  • +
  • 5.11 Profile :: Restoring from backup
  • +
  • 5.12 Profile :: Analysis Server
  • +
  • 5.13 Profile :: NFS server
  • +
  • 5.14 Profile :: NFS client
  • +
  • 5.15 Profile :: VIP provider
  • +
  • 5.16 Running interactively
  • -
  • 5.14 Running in FAI mode
  • -
  • 5.15 Data security
  • +
  • 5.17 Running in FAI mode
  • +
  • 5.18 Data security
  • 6 Full Automatic Install (FAI) @@ -303,7 +306,7 @@

    Table of Contents

  • 13 Assumptions
  • 14 Uninstalling Everything That the ece-install Set Up
  • @@ -363,7 +366,7 @@

    1.3 Other GNU/Linux and U

    You may also look inside the ece-install script for all calls to the -'assert_pre_prequesite' method and you'll get a list of the binaries +'assertpreprequesite' method and you'll get a list of the binaries ece-install assumes are present.

    @@ -385,7 +388,7 @@

    1.4 Tested Operating Syst

    - +
    fai_all_install0Install all components on your server.
    fai_analysis_db_port3306For the EAE DB (different from ECE's)
    fai_analysis_db_hostlocalhostFor the EAE DB (different from ECE's)
    fai_analysis_db_userece5userFor the EAE DB (different from ECE's)
    fai_analysis_db_passwordread-the-source-lukeFor the EAE DB (different from ECE's)
    fai_analysis_db_schemaece5dbFor the EAE DB (different from ECE's)
    fai_analysis_port8080Port of the EAE
    fai_analysis_shutdown8005Shutdown port for the EAE app server
    fai_analysis_nameanalysis1EAE instance name
    fai_cache_backends${HOSTNAME}:8080Space separated, e.g. "app1:8080 app2:8080"
    fai_cache_install0Install cache server profile
    fai_db_host$HOSTNAMEUseful for editor & presentation profiles
    Debian 6.0 (squeeze)Everything is 100% automatic2012-02-27 13:18
    Ubuntu LTS 10.04Everything except Tomcat/APR wrappers are automatic2012-02-27 13:19
    Ubuntu 11.10Everything except Tomcat/APR wrappers are automatic2012-02-27 13:19
    CentOS 6.2APR & monitoring are not automatic2012-02-28 12:37
    CentOS 6.2HA, APR & monitoring are not automatic2012-03-07 13:42
    @@ -419,7 +422,7 @@

    3 Downloading the ece-insta

    -
    $ git clone git@github.com:skybert/ece-scripts.git
    +
    $ git clone https://github.com/skybert/ece-scripts.git
     
    @@ -504,10 +507,14 @@

    4 Running the ece-install S

    -
    # bash ece-install [-v|--verbose]
    +
    # bash ece-install [-v|--verbose|<-f|--file> <conf-file>]
     
    +

    +It will output everything from internal and external commands to its +log file located at /var/log/ece-install.log. +

    ece-install will try to "fail fast", exiting as soon as it detects an error during its execution: @@ -602,7 +609,7 @@

    5.2.2 Apache Tomcat app
    • Compression of textual content (previously, this was typically set - up with Apache and its mod_deflate module). + up with Apache and its moddeflate module).
    • pooling set up tweaked for high read/write performance.
    • @@ -663,7 +670,7 @@

      5.2.7 Compression of co

      This was was previously accomplished by having a web server in fron t of the application server (or cache server if you used ESI). A typical -system architecture would contain Apache with mod_deflate. However, +system architecture would contain Apache with moddeflate. However, this is no longer necessary as Varnish can handles ESI parsing of compressed content (and many other things that we before needed Apache for). Thus, we'll let the application server do the compression for us @@ -675,7 +682,7 @@

      5.2.7 Compression of co

    -

    5.3 Profile - Full Stack on One Host

    +

    5.3 Profile :: Full Stack on One Host

    This profile is suitable for developers and admins wanting to set up a @@ -693,7 +700,7 @@

    5.3 Profile - Full Stack

    -

    5.4 Profile - Editorial Server (Publication Server)

    +

    5.4 Profile :: Editorial Server (Publication Server)

    Will set up a typical editorial server (often referred to as the @@ -704,7 +711,7 @@

    5.4 Profile - Editorial S

    -

    5.5 Profile - Presentation Server

    +

    5.5 Profile :: Presentation Server

    This will set up a typical presentation server: @@ -722,7 +729,7 @@

    5.5 Profile - Presentatio

    -

    5.6 Profile - Database Server

    +

    5.6 Profile :: Database Server

    If ece-install is run on a support version of Debian or Ubuntu, this @@ -756,7 +763,7 @@

    5.6 Profile - Database Se

    -

    5.7 Profile - Cache Server

    +

    5.7 Profile :: Cache Server

    If ece-install is run on a support version of Debian or Ubuntu, it @@ -817,7 +824,7 @@

    5.7 Profile - Cache Serve

    -

    5.8 Profile - Install Widget Framework

    +

    5.8 Profile :: Install Widget Framework

    You'll need a user name and password for accessing the @@ -840,7 +847,7 @@

    5.8 Profile - Install Wid

    -

    5.9 Profile - Create Publication

    +

    5.9 Profile :: Create Publication

    This profile will create a publication for you, only asking you the @@ -856,7 +863,7 @@

    5.9 Profile - Create Publ

    -

    5.10 Profile - Monitoring Server

    +

    5.10 Profile :: Monitoring Server

    This will install a Munin gatherer and web server. The latter for @@ -871,7 +878,7 @@

    5.10 Profile - Monitorin

    -

    5.11 Profile - Restoring from backup

    +

    5.11 Profile :: Restoring from backup

    ece-install can restore from a backup made by the ece script: @@ -903,7 +910,7 @@

    5.11 Profile - Restoring

    -

    5.12 Profile - Analysis Server

    +

    5.12 Profile :: Analysis Server

    This profile will install the Escenic Analysis Engine and configure it @@ -915,15 +922,110 @@

    5.12 Profile - Analysis

    -

    5.13 Running interactively

    +

    5.13 Profile :: NFS server

    +

    Will install an NFS server. +

    +
    -
    -

    5.13.1 Start ece-install and choose the Option, "Restore from backup"

    -
    +
    +

    5.14 Profile :: NFS client

    +
    + +

    Will install an NFS client, create all the mountpoints and mount these +on the host. +

    +
    + +
    + +
    +

    5.15 Profile :: VIP provider

    +
    + + +

    +This profile will make the host capable of serving a certain virtual +IP (VIP) for one or more services. This is a useful thing for all +SPOFs in your architecture, such as the file server or database. +

    +

    +You will typically install another profile at the same time as the VIP +provider. For instance, you will install both an NFS server and a VIP +provider for that NFS server (and perhaps other services too running +on the same host). +

    +

    +The following ece-install.conf configuration will install an NFS +server and make it a VIP provider exposing its NFS service. The two +VIP providers have have the IP addresses 192.168.1.111 and +192.168.1.112 and both provide the VIP 192.168.1.200 +

    + + + +
    # install the NFS server
    +fai_enabled=1
    +fai_nfs_server_install=1
    +fai_nfs_export_list="/var/exports/multimedia"
    +fai_nfs_allowed_client_network="192.168.1.0/255.255.255.0"
    +
    +# install the VIP provider, primary node
    +fai_vip_install=1
    +fai_vip_service_list="nfs-kernel-server"
    +fai_vip_primary_node_name=ubiquitous
    +fai_vip_primary_node_ip=192.168.1.112
    +fai_vip_primary_node_auth_key=d41d8cd98f00b204e9800998ecf8427e
    +fai_vip_secondary_node_name=ubiquitous-lts
    +fai_vip_secondary_node_ip=192.168.1.111
    +fai_vip_address=192.168.1.200
    +fai_vip_sibling_ip=$fai_vip_secondary_node_ip
    +
    + + +

    +The secondary node, also an NFS VIP provider can have the exact same +configuration except for having a different sibling: +

    + + + +
    fai_vip_sibling_ip=$fai_vip_primary_node_ip
    +
    + + +

    +Note that the faivipprimarynodeauthkey is optional. If not set, +ece-install will generate it for you. However, you will then have to +add this to ece-install.conf when installing the secondary +node. Generate this key, do: +

    + + + +
    $ dd if=/dev/urandom bs=512 count=1 2>> $log | \
    +    openssl md5 | \
    +    cut -d' ' -f2   
    +
    + + +
    + +
    + +
    +

    5.16 Running interactively

    +
    + + +
    + +
    +

    5.16.1 Start ece-install and choose the Option, "Restore from backup"

    +
    @@ -936,9 +1038,9 @@

    5.13.1 Start ece-insta

    -
    -

    5.13.2 Select Which Backup to Restore

    -
    +
    +

    5.16.2 Select Which Backup to Restore

    +
    @@ -967,9 +1069,9 @@

    5.13.2 Select Which Ba

    -
    -

    5.13.3 Choose What to Restore

    -
    +
    +

    5.16.3 Choose What to Restore

    +
    @@ -987,9 +1089,9 @@

    5.13.3 Choose What to

    -
    -

    5.13.4 Sit Back and Watch ece-install Restore the Data for You

    -
    +
    +

    5.16.4 Sit Back and Watch ece-install Restore the Data for You

    +
    @@ -1007,9 +1109,9 @@

    5.13.4 Sit Back and Wa

    -
    -

    5.14 Running in FAI mode

    -
    +
    +

    5.17 Running in FAI mode

    +

    If you're running in FAI mode, you can choose between these settings to decide what to restore and where to find the backup file to @@ -1023,12 +1125,12 @@

    5.14 Running in FAI mode ParameterDefaultDescription -fai_restore_all0Restore everything found in the backup file -fai_restore_db0Install the DB server & restore its contents -fai_restore_data_files0Restore the Solr & ECE data files -fai_restore_configuration0Restore the Solr & ECE configuration files -fai_restore_software_binaries0Restore the Escenic and Apache Tomcat software -fai_restore_from_file""The .tar.gz produced by "ece -i <instance> backup" +fairestoreall0Restore everything found in the backup file +fairestoredb0Install the DB server & restore its contents +fairestoredatafiles0Restore the Solr & ECE data files +fairestoreconfiguration0Restore the Solr & ECE configuration files +fairestoresoftwarebinaries0Restore the Escenic and Apache Tomcat software +fairestorefromfile""The .tar.gz produced by "ece -i <instance> backup" @@ -1051,9 +1153,9 @@

    5.14 Running in FAI mode

    -
    -

    5.15 Data security

    -
    +
    +

    5.18 Data security

    +

    You should take heed when running restore, so that you're not restoring a system which you didn't want to change (yes, this mishap @@ -1278,7 +1380,7 @@

    6.2 Setting up virtual ho

    If you do not want ece-install to touch your /etc/hosts, you can set -fai_keep_off_etc_hosts=1 in your ece-install.conf +faikeepoffetchosts=1 in your ece-install.conf

    @@ -1299,57 +1401,58 @@

    6.3 Overview of All FAI P ParameterDefaultDescription -fai_all_install0Install all components on your server. -fai_analysis_db_port3306For the EAE DB (different from ECE's) -fai_analysis_db_hostlocalhostFor the EAE DB (different from ECE's) -fai_analysis_db_userece5userFor the EAE DB (different from ECE's) -fai_analysis_db_passwordread-the-source-lukeFor the EAE DB (different from ECE's) -fai_analysis_db_schemaece5dbFor the EAE DB (different from ECE's) -fai_analysis_port8080Port of the EAE -fai_analysis_shutdown8005Shutdown port for the EAE app server -fai_analysis_nameanalysis1EAE instance name -fai_cache_backends${HOSTNAME}:8080Space separated, e.g. "app1:8080 app2:8080" -fai_cache_install0Install cache server profile -fai_db_host$HOSTNAMEUseful for editor & presentation profiles -fai_db_drop_old_db_first0Warning: this will drop the old database before installing a new one -fai_db_install0Install db profile -fai_db_passwordread-the-source-lukeUseful for DB installation profile -fai_db_port3306Useful for editor & presentation profiles -fai_db_schemaece5dbUseful for DB installation profile -fai_db_userece5userUseful for DB installation profile -fai_editor_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration -fai_editor_ear""EAR to use instead of the Escenic binaries -fai_editor_install0Install the editorial profile -fai_editor_nameeditor1Name of the editor instance -fai_editor_port8080HTTP port of the editor instance -fai_editor_shutdown8005Shutdown port of the editor instance -fai_enabled0Whether or not to run ece-install in FAI mode -fai_monitoring_server_install0Install the monitoring server profile. -fai_monitoring_server_ip127.0.0.1The IP of the monitoring server. -fai_presentation_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration -fai_presentation_ear""EAR to use instead of the Escenic binaries -fai_presentation_install0Install the presentation server profile -fai_presentation_nameweb1Name of the presentation server instance -fai_presentation_port8080HTTP port of the presentation server instance -fai_presentation_shutdown8005Shutdown port of the presentation instance -fai_public_host_name${HOSTNAME}:8080The public address for your website -fai_publication_create0Create a new publication -fai_publication_namemypubName of the publication -fai_publication_use_instancedev1Name of local instance to use for creation -fai_publication_war"WF or ECE demo WAR"WAR to base the new publication on -fai_publication_war_uri_list""Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, ///tmp/mypub.war and /tmp/mypub.war. -fai_publication_domain_mapping_list""Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" -fai_rmi_install0Install RMI hub profile -fai_search_conf_archive""conf.tar.gz to use for Nursery & JAAS configuration -fai_search_ear""EAR to use instead of the Escenic binaries -fai_search_for_editor0If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. -fai_search_indexer_ws_urihttp://${HOSTNAME}:8080/indexer-webservice/index/URI of the indexer-webservice that the search instance shall use for knowing what to index. -fai_search_install0Install the search server profile (Solr + indexer) -fai_search_namesearch1Name of the search instance -fai_search_port8080HTTP port of the search instance -fai_search_shutdown8005Shutdown port of the search instance -fai_keep_off_etc_hosts0Set this to 1 if you don't want ece-install adding entries to /etc/hosts -fai_wf_install0Install Widget Framework profile +faiallinstall0Install all components on your server. +faianalysisdbhostlocalhostFor the EAE DB (different from ECE's) +faianalysisdbinstall0Install db profile +faianalysisdbpasswordread-the-source-lukeFor the EAE DB (different from ECE's) +faianalysisdbport3306For the EAE DB (different from ECE's) +faianalysisdbschemaece5dbFor the EAE DB (different from ECE's) +faianalysisdbuserece5userFor the EAE DB (different from ECE's) +faianalysisnameanalysis1EAE instance name +faianalysisport8080Port of the EAE +faianalysisshutdown8005Shutdown port for the EAE app server +faicachebackends${HOSTNAME}:8080Space separated, e.g. "app1:8080 app2:8080" +faicacheinstall0Install cache server profile +faidbdropolddbfirst0Warning: this will drop the old database before installing a new one +faidbhost$HOSTNAMEUseful for editor & presentation profiles +faidbinstall0Install db profile +faidbpasswordread-the-source-lukeUseful for DB installation profile +faidbport3306Useful for editor & presentation profiles +faidbschemaece5dbUseful for DB installation profile +faidbuserece5userUseful for DB installation profile +faieditorconfarchive""conf.tar.gz to use for Nursery & JAAS configuration +faieditorear""EAR to use instead of the Escenic binaries +faieditorinstall0Install the editorial profile +faieditornameeditor1Name of the editor instance +faieditorport8080HTTP port of the editor instance +faieditorshutdown8005Shutdown port of the editor instance +faienabled0Whether or not to run ece-install in FAI mode +faikeepoffetchosts0Set this to 1 if you don't want ece-install adding entries to /etc/hosts +faimonitoringserverinstall0Install the monitoring server profile. +faimonitoringserverip127.0.0.1The IP of the monitoring server. +faipresentationconfarchive""conf.tar.gz to use for Nursery & JAAS configuration +faipresentationear""EAR to use instead of the Escenic binaries +faipresentationinstall0Install the presentation server profile +faipresentationnameweb1Name of the presentation server instance +faipresentationport8080HTTP port of the presentation server instance +faipresentationshutdown8005Shutdown port of the presentation instance +faipublichostname${HOSTNAME}:8080The public address for your website +faipublicationcreate0Create a new publication +faipublicationdomainmappinglist""Mapping between publication names and their corresponding domains on the form: "one#one.com other#other.com" +faipublicationnamemypubName of the publication +faipublicationuseinstancedev1Name of local instance to use for creation +faipublicationwar"WF or ECE demo WAR"WAR to base the new publication on +faipublicationwarurilist""Publication WARs used for setting up Assembly tool. Supported: http[s]://tmp.com/mypub.war, ///tmp/mypub.war and /tmp/mypub.war. +fairmiinstall0Install RMI hub profile +faisearchconfarchive""conf.tar.gz to use for Nursery & JAAS configuration +faisearchear""EAR to use instead of the Escenic binaries +faisearchforeditor0If 1 (true), will configure Solr for use with an editorial server, if not conf for presentation servers will be chosen. +faisearchindexerwsurihttp://${HOSTNAME}:8080/indexer-webservice/index/URI of the indexer-webservice that the search instance shall use for knowing what to index. +faisearchinstall0Install the search server profile (Solr + indexer) +faisearchnamesearch1Name of the search instance +faisearchport8080HTTP port of the search instance +faisearchshutdown8005Shutdown port of the search instance +faiwfinstall0Install Widget Framework profile @@ -1493,11 +1596,11 @@

    8 Re-running ece-install (a http://technet.escenic.com and http://tomcat.apache.org search for the following variables:

      -
    • technet_download_list +
    • technetdownloadlist
    • -
    • wf_download_list +
    • wfdownloadlist
    • -
    • tomcat_download +
    • tomcatdownload
    @@ -1559,8 +1662,8 @@

    10 Overview of File Paths /opt/escenic/assemblytoolThe assembly tool /opt/escenic/assemblytool/pluginsContains symlinks to all plugins in /opt/escenic /opt/escenic/engineSymlink pointing to the current ECE -/opt/tomcatSymlink pointing to the install Apache Tomcat (catalina_home) -/opt/tomcat-<instance>Instance specific Tomcat files (catalina_base) +/opt/tomcatSymlink pointing to the install Apache Tomcat (catalinahome) +/opt/tomcat-<instance>Instance specific Tomcat files (catalinabase) /usr/bin/eceScript for operating all ECE instances + RMI hub and EAE /usr/sbin/drop-and-create-ecedbDB script used by the ece-install script /usr/sbin/ece-installThe installation script described in this guide @@ -1710,12 +1813,16 @@

    13 Assumptions

    -

    13.1 /etc/esecenic is shared

    +

    13.1 The machine on which ece-install has enough RAM

    -

    It's assumed that the /etc/escenic directory is either on a shared -file system or rsync-ed among the hosts that are running the ECE -instances (the exception being database and cache servers). +

    It is assumed that the machine on which ece-install has enough RAM to +install the profiles the user asks for. +

    +

    +A common error by users, is to run ece-install in a (virtual) machine +that has e.g. only 1GB RAM. For happy running, we recommend having a +machine with at least 3GB when installing a full ECE stack on it.

    @@ -1733,7 +1840,7 @@

    14 Uninstalling Everything different profile.

    -Open the ece-install script and look for the "un_install_ece" +Open the ece-install script and look for the "uninstallece" function, it has copy and pastable commands for undoing most/all things set up by the script.

    @@ -1814,7 +1921,7 @@

    15 Example Output FAI


    Author: Torstein Krause Johansen (tkj@vizrt.com) - Date: 2012-02-28 18:55:47 CST + Date: 2012-03-08 15:10:00 CST

    From d42ed571b297a8623c24e5fedc7d41b039ba2036 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Mar 2012 15:26:51 +0800 Subject: [PATCH 0487/2585] - removed >>log in auth key generation example --- usr/share/doc/escenic/ece-install-guide.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 35c0ce2f..7cf18563 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -365,7 +365,7 @@ add this to ece-install.conf when installing the secondary node. Generate this key, do: #+BEGIN_SRC sh -$ dd if=/dev/urandom bs=512 count=1 2>> $log | \ +$ dd if=/dev/urandom bs=512 count=1 | \ openssl md5 | \ cut -d' ' -f2 #+END_SRC From d69ac823a7b609c549c784760b80bf3714d7362e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 8 Mar 2012 18:02:07 +0800 Subject: [PATCH 0488/2585] - first implementation of client/server replication master/slave. #45 --- .../ece-scripts/ece-install.d/database.sh | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index dfee59bf..419bf886 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -100,6 +100,17 @@ function install_database_server() set_up_engine_and_plugins set_up_ecedb fi + + if [ $db_replication -eq 1 ]; then + if [ $db_master -eq 1 ]; then + create_replication_user + fi + configure_mysql_for_replication + + if [ $db_master -eq 0 ]; then + set_slave_to_replicate_master + fi + fi } function set_up_ecedb() @@ -179,6 +190,18 @@ function set_db_settings_from_fai_conf() print_and_log "$(yellow WARNING): fai_db_drop_old_db_first is 1 (true)" fi fi + + # replication is only available in FAI mode + db_replication=${fai_db_replication-0} + db_replication_user=${fai_db_replication_user-replicationuser} + db_replication_password=${fai_db_replication_password-replicationpassword} + + db_master=${fai_db_master-0} + db_master_host=${fai_db_master_host} + db_master_log_file=${fai_db_master_log_file} + db_master_log_position=${fai_db_master_log_position} + + # TODO assert set } # Method used both from interactive mode to set any missing values @@ -204,5 +227,91 @@ function set_db_defaults_if_not_set() if [ -z "$db_schema" ]; then db_schema=${default_db_schema} fi + +} + +function create_replication_user() { + print_and_log "Creating replication user $db_replication_user ..." + mysql -u $db_user -p${db_password} < $file <> $file + fi + + run /etc/init.d/mysql restart + + # replication log configuration of the master + if [ $db_master -eq 1 ]; then + old="bind-address.*= 127.0.0.1" + new="# bind-address = 127.0.0.1" + if [ $(grep "$old" $file | wc -l) -gt 0 ]; then + sed -i "s#$old#$new#g" $file + else + echo "$new" >> $file + fi + + + old="#log_bin.*= /var/log/mysql/mysql-bin.log" + new="log_bin = /var/log/mysql/mysql-bin.log" + if [ $(grep "$old" $file | wc -l) -gt 0 ]; then + sed -i "s#$old#$new#g" $file + else + echo "$new" >> $file + fi + + old="#binlog_do_db.* =.*" + new="binlog_do_db = ${db_schema}" + + if [ $(grep "$old" $file | wc -l) -gt 0 ]; then + sed -i "s#$old#$new#g" $file + else + echo "$new" >> $file + fi + fi } +function set_slave_to_replicate_master() { + print_and_log "Setting slave to replicate master DB @ $db_master_host ..." + mysql ${db_schema} < Date: Thu, 8 Mar 2012 18:22:00 +0800 Subject: [PATCH 0489/2585] - minor bug fixes to configure_mysql_for_replication --- .../escenic/ece-scripts/ece-install.d/database.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index 419bf886..8d8bf6b6 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -232,7 +232,7 @@ function set_db_defaults_if_not_set() function create_replication_user() { print_and_log "Creating replication user $db_replication_user ..." - mysql -u $db_user -p${db_password} <> $file fi @@ -274,7 +274,7 @@ EOF old="bind-address.*= 127.0.0.1" new="# bind-address = 127.0.0.1" if [ $(grep "$old" $file | wc -l) -gt 0 ]; then - sed -i "s#$old#$new#g" $file + sed -i "s~$old~$new~g" $file else echo "$new" >> $file fi @@ -283,7 +283,7 @@ EOF old="#log_bin.*= /var/log/mysql/mysql-bin.log" new="log_bin = /var/log/mysql/mysql-bin.log" if [ $(grep "$old" $file | wc -l) -gt 0 ]; then - sed -i "s#$old#$new#g" $file + sed -i "s~$old~$new~g" $file else echo "$new" >> $file fi @@ -292,7 +292,7 @@ EOF new="binlog_do_db = ${db_schema}" if [ $(grep "$old" $file | wc -l) -gt 0 ]; then - sed -i "s#$old#$new#g" $file + sed -i "s~$old~$new~g" $file else echo "$new" >> $file fi From 44a8967d7ad9adbe85eb9da794bc9ef1892cf2cb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Mar 2012 19:00:17 +0800 Subject: [PATCH 0490/2585] - improved add_next_step to accept multiple arguments --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 9b127f68..75d4edad 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -722,13 +722,13 @@ function add_apt_source() { # $1 : your added line function add_next_step() { - next_steps[${#next_steps[@]}]="${1}" + next_steps[${#next_steps[@]}]="$@" return if [ -n "$next_steps" ]; then next_steps=${next_steps}$'\n'"[$(basename $0)] "${1} else - next_steps="[$(basename $0)] "${1} + next_steps="[$(basename $0)] $@" fi } From e3679849ef5d451bc92528ff7cba9ceccc0c050e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 13 Mar 2012 19:32:40 +0800 Subject: [PATCH 0491/2585] - fix to create_replication_user - hardening when re-running the DB profile several times with replication settings. Issue #45 --- .../ece-scripts/ece-install.d/database.sh | 40 +++++++++++-------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index 8d8bf6b6..cb2206f5 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -12,7 +12,7 @@ fi ## the ECE DB schema is not set up. function install_database_server() { - print_and_log "Installing the database server on $HOSTNAME ..." + print_and_log "Installing database server on $HOSTNAME ..." source $(dirname $0)/drop-and-create-ecedb @@ -232,10 +232,9 @@ function set_db_defaults_if_not_set() function create_replication_user() { print_and_log "Creating replication user $db_replication_user ..." - mysql <> $file fi @@ -273,32 +272,39 @@ EOF if [ $db_master -eq 1 ]; then old="bind-address.*= 127.0.0.1" new="# bind-address = 127.0.0.1" - if [ $(grep "$old" $file | wc -l) -gt 0 ]; then - sed -i "s~$old~$new~g" $file - else - echo "$new" >> $file + if [ $(grep ^"${old}" $file | wc -l) -gt 0 ]; then + sed -i "s~^${old}~$new~g" $file fi - old="#log_bin.*= /var/log/mysql/mysql-bin.log" new="log_bin = /var/log/mysql/mysql-bin.log" - if [ $(grep "$old" $file | wc -l) -gt 0 ]; then - sed -i "s~$old~$new~g" $file - else + if [ $(grep ^"$old" $file | wc -l) -gt 0 ]; then + sed -i "s~^${old}~${new}~g" $file + elif [ $(grep ^$new $file | wc -l) -lt 1 ]; then echo "$new" >> $file fi - old="#binlog_do_db.* =.*" + old="#binlog_do_db.*=.*" new="binlog_do_db = ${db_schema}" - if [ $(grep "$old" $file | wc -l) -gt 0 ]; then - sed -i "s~$old~$new~g" $file - else + if [ $(grep ^"$old" $file | wc -l) -gt 0 ]; then + sed -i "s~^${old}~${new}~g" $file + elif [ $(grep ^$new $file | wc -l) -lt 1 ]; then echo "$new" >> $file fi + + # report the needed settings for a slave + local master_status=$(mysql $db_schema -e "show master status" | tail -1) + local file=$(echo $master_status | cut -d' ' -f1) + local position=$(echo $master_status | cut -d' ' -f2) + add_next_step "ece-install.conf for slave: fai_db_master_log_file=$file" + add_next_step "ece-install.conf for slave:" \ + fai_db_master_log_position=$position fi } + + function set_slave_to_replicate_master() { print_and_log "Setting slave to replicate master DB @ $db_master_host ..." mysql ${db_schema} < Date: Tue, 13 Mar 2012 19:33:00 +0800 Subject: [PATCH 0492/2585] - added section on adding DB replication support, issue #45 --- usr/share/doc/escenic/ece-install-guide.org | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 7cf18563..ade45951 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -218,6 +218,47 @@ ERROR 1007 (HY000) at line 1: Can't create database 'ece5db'; database exists ERROR 1050 (42S01) at line 2: Table 'DBChangeLog' already exists [ece-install] running tables FAILED, exiting :-( #+END_SRC +If you for some reason wish to make ece-install drop the DB fro you +before creating a new one, you can set +#+BEGIN_SRC sh + fai_db_drop_old_db_first=1 +#+END_SRC + +*** Master & slave setup +It's easy to set up master and slave DBs, here's how you'd configure +ece-install to install these: + +On the master, create an ece-install.conf with: +#+BEGIN_SRC sh +fai_enabled=1 +fai_db_install=1 +fai_db_master=1 +fai_db_replication=1 +#+END_SRC + +The script will itself output some replication log settings you need +for the slave: +#+BEGIN_SRC sh +[ece-install-35] - DB is now set up on localhost:3306 +[ece-install-35] - ece-install.conf for slave: fai_db_master_log_file=mysql-bin.000013 +[ece-install-35] - ece-install.conf for slave: fai_db_master_log_position=106 +#+END_SRC + +Once this is done, create and use an ece-install.conf on the slave +with this: +#+BEGIN_SRC sh +fai_enabled=1 +fai_db_install=1 +fai_db_replication=1 +fai_db_master=0 +fai_db_master_host=my-db-master +fai_db_master_log_file=mysql-bin.000013 +fai_db_master_log_position=106 +#+END_SRC + +The script will set up and use the replication user & credentials with +default values, if you wish to override these, have a look in the FAI +parameter overview. ** Profile :: Cache Server If ece-install is run on a support version of Debian or Ubuntu, it From 3c571ba29fe2f0264bdf0c19beb7a9d18d4e83c5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:05:14 +0800 Subject: [PATCH 0493/2585] - moved control files to package specific location --- .../debian/{DEBIAN => escenic-content-engine-installer}/control | 0 .../debian/{DEBIAN => escenic-content-engine-installer}/postinst | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename usr/local/src/debian/{DEBIAN => escenic-content-engine-installer}/control (100%) rename usr/local/src/debian/{DEBIAN => escenic-content-engine-installer}/postinst (100%) diff --git a/usr/local/src/debian/DEBIAN/control b/usr/local/src/debian/escenic-content-engine-installer/control similarity index 100% rename from usr/local/src/debian/DEBIAN/control rename to usr/local/src/debian/escenic-content-engine-installer/control diff --git a/usr/local/src/debian/DEBIAN/postinst b/usr/local/src/debian/escenic-content-engine-installer/postinst similarity index 100% rename from usr/local/src/debian/DEBIAN/postinst rename to usr/local/src/debian/escenic-content-engine-installer/postinst From 8d3bc769a15e8108677207b5c6f242915f0d48e3 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:05:25 +0800 Subject: [PATCH 0494/2585] - can now build different packages --- usr/local/src/create-packages | 110 +++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 41 deletions(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 89f2056f..412f5704 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -5,60 +5,88 @@ # by tkj@vizrt.com -rm -rf debian/{etc,usr} - -target_dir=debian +target_dir=target package_name=escenic-content-engine-installer +package_version=$(date +%s) -function get_version() -{ - engine_file=$(grep http ../../sbin/ece-install | grep "/engine") - version=$(basename $engine_file .zip | cut -d'-' -f2) - echo $version +function clean_up() { + rm -rf $target_dir } -function get_name() -{ - name=$(grep $USER /etc/passwd | cut -d':' -f5 | cut -d',' -f1) - echo "${name}" +function get_user_options() { + while getopts ":p:r:" opt; do + case $opt in + r) + package_version=${OPTARG} + ;; + p) + echo "-p was triggered, Parameter: $OPTARG" >&2 + package_name=${OPTARG} + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac + done + } -function get_email() -{ - echo ${USER}@${HOSTNAME} +function set_up_target_directory() { + mkdir -p $target_dir/{etc,usr} + mkdir -p $target_dir/usr/{bin,share/doc} + + if [[ $package_name == "escenic-content-engine-installer" ]]; then + cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ + cp -r ../../share/escenic/ $target_dir/usr/share/ + cp -r ../../share/doc/escenic/*.{org,html} $target_dir/usr/share/doc/ + cp -r ../../../etc/escenic/ $target_dir/etc/ + fi + + debian_dir=$target_dir/DEBIAN + mkdir -p $debian_dir + cp debian/${package_name}/* $debian_dir } -version=$(get_version) - -cp -r ../../../etc $target_dir - -mkdir -p $target_dir/usr -cp -r ../../../usr/bin $target_dir/usr -cp -r ../../../usr/sbin $target_dir/usr -cp -r ../../../usr/share $target_dir/usr +function get_name() { + name=$(grep $USER /etc/passwd | cut -d':' -f5 | cut -d',' -f1) + echo "${name}" +} -cp $target_dir/DEBIAN/control $target_dir/DEBIAN/control.orig +function get_email() { + echo ${USER}@${HOSTNAME} +} -sed -i "s#VERSION#${version}#g" $target_dir/DEBIAN/control -sed -i "s#MAINTAINER_NAME#$(get_name)#g" $target_dir/DEBIAN/control -sed -i "s#MAINTAINER_EMAIL#$(get_email)#g" $target_dir/DEBIAN/control +function create_packages() { + sed -i "s#VERSION#${package_version}#g" $debian_dir/control + sed -i "s#MAINTAINER_NAME#$(get_name)#g" $debian_dir/control + sed -i "s#MAINTAINER_EMAIL#$(get_email)#g" $debian_dir/control -# fix permissions -chmod 755 $target_dir/DEBIAN/postinst + # fix permissions + chmod 755 $debian_dir/postinst -if [ ! -x /usr/bin/dpkg-deb ]; then + if [ ! -x /usr/bin/dpkg-deb ]; then echo "You must have dpkg-deb installed :-(" exit 1 -fi - -dpkg-deb --build debian -mv debian.deb ${package_name}-${version}.deb - -if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then - fakeroot alien --to-rpm --scripts ${package_name}-${version}.deb -else + fi + + dpkg-deb --build $target_dir + mv ${target_dir}.deb $target_dir/${package_name}-${package_version}.deb + + if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then + cd $target_dir + fakeroot alien --to-rpm --scripts \ + ${package_name}-${package_version}.deb + else echo "You must have 'alien' and 'fakeroot' installed to create RPMs" -fi + fi +} -# cleanup -cp $target_dir/DEBIAN/control.orig $target_dir/DEBIAN/control +clean_up +get_user_options $@ +set_up_target_directory +create_packages From fcf5572d194c82081b920194d235ac93488d10bb Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:05:41 +0800 Subject: [PATCH 0495/2585] - too complicated, deleting it. --- .../usr/share/doc/escenic/create_hook_overview | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 usr/local/src/debian/usr/share/doc/escenic/create_hook_overview diff --git a/usr/local/src/debian/usr/share/doc/escenic/create_hook_overview b/usr/local/src/debian/usr/share/doc/escenic/create_hook_overview deleted file mode 100644 index b68ce6f3..00000000 --- a/usr/local/src/debian/usr/share/doc/escenic/create_hook_overview +++ /dev/null @@ -1,14 +0,0 @@ -#! /usr/bin/env bash - -dir=~/src/ece-scripts - -echo "" > $dir/usr/share/doc/escenic/ece-install-hooks.org - -grep run_hook $dir/usr/sbin/ece-install | \ - grep -v function | \ - awk '{print $2}' | \ - while read l; do - echo "- $l" >> $dir/usr/share/doc/escenic/ece-install-hooks.org -done - - From 346c9256896c37dff0c3f7c43f211005d0a8c660 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:16:50 +0800 Subject: [PATCH 0496/2585] - can now generate both escenic-content-engine-installer and vosa packages: $ create-packages -p vosa Sir, the packages are now ready: target/vosa-1331709369-2.noarch.rpm target/vosa-1331709369.deb --- usr/local/src/create-packages | 37 ++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 412f5704..87ea9eea 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -20,7 +20,6 @@ function get_user_options() { package_version=${OPTARG} ;; p) - echo "-p was triggered, Parameter: $OPTARG" >&2 package_name=${OPTARG} ;; \?) @@ -37,14 +36,22 @@ function get_user_options() { } function set_up_target_directory() { - mkdir -p $target_dir/{etc,usr} + mkdir -p $target_dir/{etc,usr,/etc/default} mkdir -p $target_dir/usr/{bin,share/doc} if [[ $package_name == "escenic-content-engine-installer" ]]; then + mkdir -p $target_dir/usr/share/doc/escenic + cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ cp -r ../../share/escenic/ $target_dir/usr/share/ - cp -r ../../share/doc/escenic/*.{org,html} $target_dir/usr/share/doc/ + cp -r ../../share/doc/escenic/*.{org,html} $target_dir/usr/share/doc/escenic cp -r ../../../etc/escenic/ $target_dir/etc/ + cp -r ../../../etc/default/ece $target_dir/etc/default + elif [[ $package_name == "vosa" ]]; then + mkdir -p $target_dir/usr/share/doc/vosa + cp -r ../../bin/vosa $target_dir/usr/bin/ + cp -r ../../share/vizrt/ $target_dir/usr/share/ + cp -r ../../share/doc/vizrt/*.org $target_dir/usr/share/doc/vosa/ fi debian_dir=$target_dir/DEBIAN @@ -67,26 +74,38 @@ function create_packages() { sed -i "s#MAINTAINER_EMAIL#$(get_email)#g" $debian_dir/control # fix permissions - chmod 755 $debian_dir/postinst - + if [ -e $debian_dir/postinst ]; then + chmod 755 $debian_dir/postinst + fi + if [ ! -x /usr/bin/dpkg-deb ]; then echo "You must have dpkg-deb installed :-(" exit 1 fi - dpkg-deb --build $target_dir + dpkg-deb --build $target_dir \ + 2>&1 > /dev/null mv ${target_dir}.deb $target_dir/${package_name}-${package_version}.deb if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then - cd $target_dir - fakeroot alien --to-rpm --scripts \ - ${package_name}-${package_version}.deb + ( + cd $target_dir + fakeroot alien --to-rpm --scripts \ + ${package_name}-${package_version}.deb \ + 2>&1 > /dev/null + ) else echo "You must have 'alien' and 'fakeroot' installed to create RPMs" fi } +function list_packages() { + echo "Sir, the packages are now ready:" + ls $target_dir/*.{deb,rpm} +} + clean_up get_user_options $@ set_up_target_directory create_packages +list_packages From a00c8b2c1727acc7c4b1592f559e1a772d9e1828 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:38:46 +0800 Subject: [PATCH 0497/2585] - forgot the echo --- .../debian/escenic-content-engine-installer/postinst | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/usr/local/src/debian/escenic-content-engine-installer/postinst b/usr/local/src/debian/escenic-content-engine-installer/postinst index d8b24474..4147fc4a 100755 --- a/usr/local/src/debian/escenic-content-engine-installer/postinst +++ b/usr/local/src/debian/escenic-content-engine-installer/postinst @@ -3,17 +3,7 @@ chmod +x /usr/sbin/ece-install /usr/bin/{ece,system-info} -echo "Do you wish to run the Escenic Content Engine installer now?" - -answer=no -echo -n "Your choice: yes/[no] " -read answer - -if [ "$answer" = "yes" ]; then - ece-install -else - echo "Run the Escenic Content Engine installer by typing 'ece-install'" -fi +echo "Run the Escenic Content Engine installer by typing 'ece-install'" exit 0 From 6a6a11dd99c227e34861f3477548e25923a3adea Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:39:34 +0800 Subject: [PATCH 0498/2585] - "as root", mind you --- usr/local/src/debian/escenic-content-engine-installer/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/local/src/debian/escenic-content-engine-installer/postinst b/usr/local/src/debian/escenic-content-engine-installer/postinst index 4147fc4a..946b0b50 100755 --- a/usr/local/src/debian/escenic-content-engine-installer/postinst +++ b/usr/local/src/debian/escenic-content-engine-installer/postinst @@ -3,7 +3,7 @@ chmod +x /usr/sbin/ece-install /usr/bin/{ece,system-info} -echo "Run the Escenic Content Engine installer by typing 'ece-install'" +echo "Run the Escenic Content Engine installer by typing 'ece-install' as root" exit 0 From 21d5019c574f39bf85364bc43d6ea3a5f4d0685c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:40:18 +0800 Subject: [PATCH 0499/2585] - some more user feedback --- usr/local/src/debian/escenic-content-engine-installer/postinst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/local/src/debian/escenic-content-engine-installer/postinst b/usr/local/src/debian/escenic-content-engine-installer/postinst index 946b0b50..ca61ce55 100755 --- a/usr/local/src/debian/escenic-content-engine-installer/postinst +++ b/usr/local/src/debian/escenic-content-engine-installer/postinst @@ -3,7 +3,8 @@ chmod +x /usr/sbin/ece-install /usr/bin/{ece,system-info} -echo "Run the Escenic Content Engine installer by typing 'ece-install' as root" +echo "You can now run the Escenic Content Engine installer " +echo "by typing 'ece-install' as root" exit 0 From 6bef64b2654d28c116549c5ecacabab26edbe241 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 15:42:56 +0800 Subject: [PATCH 0500/2585] - some small bug fixes - better version string, e.g.: 2012-03-14.154031 --- usr/local/src/create-packages | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 87ea9eea..8c431f61 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -7,7 +7,7 @@ target_dir=target package_name=escenic-content-engine-installer -package_version=$(date +%s) +package_version=$(date +%Y-%m-%d.%H%M%S) function clean_up() { rm -rf $target_dir @@ -37,16 +37,19 @@ function get_user_options() { function set_up_target_directory() { mkdir -p $target_dir/{etc,usr,/etc/default} - mkdir -p $target_dir/usr/{bin,share/doc} + mkdir -p $target_dir/usr/{bin,sbin,share/doc} if [[ $package_name == "escenic-content-engine-installer" ]]; then mkdir -p $target_dir/usr/share/doc/escenic + cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ + cp -r ../../sbin/{ece-install,drop-and-create-ecedb} $target_dir/usr/sbin/ cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ cp -r ../../share/escenic/ $target_dir/usr/share/ cp -r ../../share/doc/escenic/*.{org,html} $target_dir/usr/share/doc/escenic cp -r ../../../etc/escenic/ $target_dir/etc/ cp -r ../../../etc/default/ece $target_dir/etc/default + elif [[ $package_name == "vosa" ]]; then mkdir -p $target_dir/usr/share/doc/vosa cp -r ../../bin/vosa $target_dir/usr/bin/ From 73092133f89ae26a8cd408b4c941d53bd3f87db9 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 16:49:10 +0800 Subject: [PATCH 0501/2585] - added, thanks Mogsie ;-) --- usr/local/src/debian/vosa/control | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 usr/local/src/debian/vosa/control diff --git a/usr/local/src/debian/vosa/control b/usr/local/src/debian/vosa/control new file mode 100644 index 00000000..5287045b --- /dev/null +++ b/usr/local/src/debian/vosa/control @@ -0,0 +1,10 @@ +Package: vosa +Version: VERSION +Section: base +Priority: optional +Architecture: all +Maintainer: MAINTAINER_NAME +Description: The Vizrt Online System Administration command line tools. + See the documentation in /usr/share/doc/vizrt/ for further + details. + \ No newline at end of file From a8c084fd3be71782eec6bf5c304c9d3006cc3ac1 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 17:06:20 +0800 Subject: [PATCH 0502/2585] - now exists on failed fakeroot/alien --- usr/local/src/create-packages | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 8c431f61..26db3bc2 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -99,6 +99,7 @@ function create_packages() { ) else echo "You must have 'alien' and 'fakeroot' installed to create RPMs" + exit 1 fi } From 39140c478c223d9bbebb51782337eaac6fb05006 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 18:10:40 +0800 Subject: [PATCH 0503/2585] - added --- usr/share/doc/escenic/images/nfs-vip.nwdiag | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 usr/share/doc/escenic/images/nfs-vip.nwdiag diff --git a/usr/share/doc/escenic/images/nfs-vip.nwdiag b/usr/share/doc/escenic/images/nfs-vip.nwdiag new file mode 100644 index 00000000..68a4cd88 --- /dev/null +++ b/usr/share/doc/escenic/images/nfs-vip.nwdiag @@ -0,0 +1,13 @@ +nwdiag { + group { + color = "orange"; + nfs1; + nfs2; + } + + network nfs-vip { + address = "192.168.1.200"; + nfs1 [ address = "192.168.1.122", description = "NFS master" ]; + nfs2 [ address = "192.168.1.123", description = "NFS slave" ]; + } +} From 71d9e31e759d584121d249a89b9b58013f33dae5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 18:10:58 +0800 Subject: [PATCH 0504/2585] - added --- usr/share/doc/escenic/images/nfs-vip.png | Bin 0 -> 9721 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 usr/share/doc/escenic/images/nfs-vip.png diff --git a/usr/share/doc/escenic/images/nfs-vip.png b/usr/share/doc/escenic/images/nfs-vip.png new file mode 100644 index 0000000000000000000000000000000000000000..283e0f2c0e7134091115b772452cd294eb5ab580 GIT binary patch literal 9721 zcmdsdXH-*Nw{Advv7;y*0R=&+(yM?d2uKkygdQdI-XWBLBJfJ@y$C1@1VV42i1Z)= zA_NE!DWQcTC4>+XZhXIQe0Pj{?)iVl`LX9-WA8oZn(LWm%{|9@_gw$!m5Uq~0RX@i zZLKFp007+p?Gicv7wySYUp^QBaQBP$lSeNC^S0)O6Q#9n(7WI#9WQ`HsgZA+-?ytb zk8y9RuW4mmK2Hj_JuArsZ}v3rxo~^B!{V^;*TXN@5zTiH@}7@|dg<8*8F(@@8sCB> z_s#_em?9jY_63>SRaM{OekHM+8BQC{SKFAP9Hkb{FNGjnHLW}NhVFdqO= z-NhZu006M@(K7+U&;HLRU%J>Jvo%DzzDO_)!4B4Tt2BkUa#!F~2 zi4GVR-JuKL<>jRVX#GQ8UiMyI^Y8s>7$w2{YE$C+B7on%QGQ}{7XWJX{}4#h zjmJ&GrBRZx_t_Fub?SKqBV+yoT;=M7 zL{eCm-x}@$zAU*_JiDL;05F}`0(_~mk3u;4~cIvr+gh`nX52T*g;5_qnz%16v zPvD;^t>Eu*?;?sr7Ru%({DQz|m)~I;yo)8fI}A&I-&}m&R`G34PVK_L8Hv7TPi$}} zv1(I!B%dD$Z1BSnE3={yCF7qkF((Tr(5C}3h^tJpSG^$Z*c1abxQc&5yhKp`O^#+(@olb!NsabmZ%)l`z@<(B~Q-IDGL zkx7>;`%9g;XFqaygn8D@2jlbVzPEf+aA+qAgiN$*+`pa{_#WroB2QEC!`qi>tqB~z z>wzY*onMt>N&@JQtrmt+KbAUqL*hni;X*^+W*IoYf>q&F%XGW!1!+cC=2lY$FU~Jz zQ1amxE4-oVUJRZT0&6j^6x(R+@cmA(f+O8iB?cULt#1kX)^P_U0$z4jRaJe@9KuaI zH{^hTG8OpNzejeqt?i8-8)9^;iZ2U>l@=g(_8}OmEX^pMv4tc1vpDE>)F=P9%)Vcr z7-VCArI!T>B6(&$Oh9o z5uaUi_yWxErdY$zSyx#fG#X85N!tBsuYH`VN=ogd`lu3pPIHsaJ^&U`j|t#Y!l}F^ z7q5Ac{*PhB-zow(k}xbM9e4tHmI^UxpY}UN^1Y9ywqAL49lH(dsFV_BFic)v@Vg1zfej9#4v&q7Wyj9qwRN&zyT;xw|Yp z#FP-ZXALxbTTDgfMOk8FX&};|uBE~0m<(V&hl?kec}Qi4&kDGAeeCE&JTth`Bz4jr zZ7%6v=u)FOBuO~uBAaVfke}aPS?)Ypd1{_%*8(qhW@#1vN_y-VdbAL&szM7XU&9V} zbi>*9hQ`5K__hGfGOw_(Q$bZVEY#os(4QtDN`s!}3LwyqHRh$EwF-Ud{$tv_(rQ(iJuu+#_WR5dP$2RJx5?6jIJ0}EC|)vosQ?LZ{ypceSwjaA=Ypj|c^ zknX3ASaQv812aK*@eP5Ex9sp;8{C*;#5~(lt%yrZv1ufwZc>&O z_bdA(yDV-!;)Xw%x|bQ|xf%Lpy*OmfZE84Y6$-rBOYjt(tU5OxP_wOQ=Z?s!F{}rCc8U>(#8%VU+_~BpJUBlSg z;Pv5`A7Ys;BEKfLEJlk_oV}bs-p@UQY zj;7O5%sD#J>=pB*L9w45)oz!1j;52sewQxZx$D6hWvG$znZx~( z?s``d>DpO-**V2|wnWoDF!J2F8RpmD6#3;?Ii^lpF6{exZCZV~kQ+j3z5dc#(oH^S z>F&;i7-Ai{=;BMvX^Vc9w6UeM(l(gEs*8uOIt#qe_SrdpA@6ALYa_? z%dF9t)US=#ogA1l7pS;vF_fh_FFS(^iZ}RK%+FiHa&fNm5qs+kD}fy!QfX4fGg6QM zIDOCWHumP8p1_W})>V4%#d0~YNXJ1_hoJVvr&GK~^rvngVklL%e~KzWWJop^0;`g! z;sd7W`C3hZ3fn!MQq#d@bS+5-Rld0CdRKd54;7UpYF|gy(F8VMi;xhCv7v+n$drU? z$%rl9?e(<9RLy%~`-5kb470M*tB zPw$lno>rVXED@~KSoWd83Y`-umbtqpvfBZf#7o2`N;{Ixa`X|cg4M>5vmihEY`Od( zu_p64Z@>FZ=AX?vp8F7thH;lBGFP25F11YPR-tfs-T6pQ>0C@+>4rhrDmtd~P@J02 zzrOKQwPAPoXm=tJR(?E=B{lrqZ*sFpamP^eF(Ka$cSy%>wO`L25~pn9+&FXY z@Cv$%uaY+v57*k=`nm9CoJk;YU_0eGA(!z*dnlOD+mch!GF$HWxYk8u`xvQ-vOM|a z5JkkR9#?v-sGjC_j)9aT(6~RN>(-yvNLH}S`l_joKd(z+JCLSMJiduy#YhSEqQ?na zm*MxKRp%lhA6;u+z+ktD^Da0&C1dnd>8;pJ>z(#@{de|jx9lrRT;^5`SzVfCw>vK5 zjp>7)=k+jg<*L(Ppw9N)IwZBqos%R^J(SxkJRQcKCmN5a`jU}tFbtOPxsz4! zc}H7Sf=@Z4{SHflO)?1tQ@;XLy4R`j7J)s52 z&}iGZ@Sdx!#&XMXKW%AFPcw6eGKg>(TB$;QK2#X;VZwpuGt8&z*jucL!?&*B%Z3uG z>JjQUjG9Ot64y4OfKS)M5<6K{u^`!97k`qK1&mlp449)n1b1%j1RY>woiY&%PVAYg zCo!&O!m1WRVbI%6LKrkIeRI=j`a~V(UqgM}m@w!NtC_Mf(J+kk>JMtDEh3ome6Aq$ z6?XII?>3L{=S9RRxaX#gV?9Q;+>juv+u-3Dul`uZhU1fG=G0|jbLvq=>v1f?k+dZ4 zLP%w72%;2h1o*Vz)vIDGl2XMyNa)j5tnNc_*eWYko@!0W2OllDA$VZo%CLdMXw}pp z7CMJDc z@K~9G593%t-u|Vvx);WYa%XC91^Z4|x|0evn!uxkio}MK1Da-?;BlNFcw+AMVDLHwthg6UDOv#^KdBQxF(m~fY*#((Vn6Dd46&gT>~e;ExM@F0+d(Hu z?TPd$VkVos-=1ta)S8n7QY*Y$fJOFZ+(>hQ)c%5_d#t*===nO8>QOXZV1kpLuu8K=yy@^>(_?u8~W{w-2MU{-FW z-=3MSR0(l&4Kr>o$xW3eWw4!Z4x{#e?P2vdo9Dj^ogK|!sF{HeAH$m}s4E~I``Puh{qa}Wq#y{Ct>x;9YOWStXI?K`#Yj0}iSjEMmd`rXe zimyVyaEFQ??_R&n>(w0*v#z<2*s3S%o}&Lt9R({UhwAQzO&yhYd1X72VjL!q4pyTe_2d=X(WI^8Wgxs3%#H|ySUO)@(tn$_j{FN!$?8dqCLO;e2aj3j$74rM`hFq$484! zddggM2ktO;gUy%G;XM+%FacqJwjk9^fY-~En6-KF*W0{NhXM($%}r}NY8C5TSSfx@ z*D-Ewa1*lU_j|UVpO%ySdG0}%gM;9w*XWrH@Yx{Pm%)pLZc@#vuY7A3f0u@_YXOK2 zp|6t$i_X34-i<&C?sX{)im9cxDY3)w5$oBmq*jNnY$MVg-d#tf&;m+dCYfz+)v4IU z&fV@qE`wA6<=YJ^qcZ=4(j18&_lD3Es#3HS+@9izj*rKwhKKj0vRSV?*FGK0IqbkX zKZ}$-2lI45d5meNA+Dvo>$KCs#fJBE9n6)(>gaQW&OIPdcw)r6tGnWmlCj7wITYH< z-@!w_JjAas;6}9F0rA%E%m7hc&1T0v@2`d}Z_gW*=kt^fxi`n&PdFB@>fMwznEn`) zDX(_CC)3&X)8;nUV+D(oim}N`Mt?CM7zO&+;D%E9UG7*QIVqDT?;7RaypjWXj+%@97?tqlD8$QRpLfNPT%cT&DIr zN|6uWTAg;xg3CgIXvu%x)weQ8YJ$7U!NH8(&IA<)vi~rvHQD&+e&u6tyeaMmdjcdi zcDx!cWtR<{G`0IK6refDIlZkl^Q4^~;_{TXl!~LZd6E~i|A%%9oDkgjReQ|0Aq%X@ zWh<@2r6+t^104bCDSW(w3v)uCSCY&vKtl_X7(Xr!t{bKYmzZRsY5zSSV&{8&%>ZPL z0DNWNGA`nigNwJ^zA1)tSj<>roB#fqAiq8+GBSX-tIEDOTx(e;nOae~JPAowan?w< zLig_JJHZcqki_qt6K9wCL;Qh`hSKIY<8!vJH1rL!VH23Mhqw2;puZT>MZnG$i3BZS zpzHBw?N9Yc7HCx|>O&t&)(HH#7?}`{%V>R+D%i@97otOlOI~Tz~l^5);Eq zRznB5A$~L5 z%fci(gHXN4k>?_xL97LHh40BQukSUu%OsS#-o2JIg*^N_EEIO5&kZ+Y0>Q;OV@<5Z z%D?oPgk1qSe{H78MKv_dXy7W8Zn=K*;G=sNQAD2`CE#18$hWF1Eg-#pCa-3P$6W8H z-Hv+Gh9-I={^Q=$i140A*Am7Til^UpXH1)7`)jsY-JORr-4C^nyqdW0(lZ4fPIku` zHf>CiJ3j30@bEX(yzK&#I0&?+T1>k*lYUOdP&gw@buV4TO^aOm}38TKV6oMkkt@oV=54hFI>k!vTc=zI=#Zd<|S|87Pm#I_~m z%2><3u;%Cp=%1;;&{QrF1>;{@HVHf}Or%+-pKYD-beI*}7WC<4JfMX#)D%fNu(78_ z*UGRY+Aba6T2tz;IsdGR(#%GRe?_SYi_#o9IMr%&xw|sylA(~#+#qQoTxXq4sBKjG zAveb3f0>3N6rJ#aJTcc;rzO>zjhBd$$i zuSEuy!5JW1=XcM-O#WJ|dFobah;v$LyU9WzIHs%13=G^Vd&F zW9Jj(+X*FKXUDGE9v4>`3nONO(#^;!X_IBu){z91o@Mftgrlc-+}CH^!+e&=ov-)& z&U%p~jS95w?13vr*$T{!?KrRJsKnZk-t$^qPmZc>`-JO^#(?Jgvrg91<(A}{flA<3 zWu8akUK#SL;MMfm`knq_M`>}6;r-Ie0ZH_&tpF<{28pX$CW+q zfuGi1Pcjkm`J_?cWIgqSWX=;F|4)1d)<5qEC3<9SjzvK*-XcCF(D^m^XQ0fM!Ock* z9Gt^p=Av%Or@xf4p5LC{4!!RdT)kHe_cLki`oXr}kEO-e znq$KKR>Dx*!yk;yS!v2&Kj~mnpYQhU3@dv)#_KU+-}!O%82^5VA&|CBb6dp--a1PCEtgx@OP5t4AOX}?tY<-xX; zD>R?A!TMWk_+@5OY4U^o6ax~&lYqx0$uT<9u>%G$bkcvzu%kfhraNUcMn*`oqV}6i zA!q(btRD%5G_jSYRXnr;*+4oyxP*QvKj+6YZ&ds^+_ApAvb?YSQ1km3h4A8-_$fE( zM&CAzMu`RuK}YE!PG&QutLz3NJQ0C;!cs=b(BVBzJ&^xiRa>PF3N2ATht-_qz-0rI zmC~h3dW(PBO#0)R9^NcE#~x6Zz)0H#FFbtdMYkjP##Fpep=DNk{=Tsi#CzTps26OD z4dfd+8zXpwzR$N#mF)g|)+s|_c+c-#Ms)bk;lGxRiPbq>K9gE4$!!ETk`ZT?-HObJ z71sMh$ivFU5O#K#v}s+lg%@d>bSXEN)GZi5llqUG_`o!v#hB4jAnxCeIFD*EQ z9*DclTx{~y++UpBi;R%4GLx_pmNd?Q+Sygvn3QEr2*qe>a7Qjpq0j)@rh4%Vp@=?% zAeo(+jBoAv(ATm(U_F;iIUav>*Iz!j&Jdk$EYB=>X%(I$8xmF7Ujnkx+%hUKZ@Odm zxZKX=?iUP=3`#`de>2?>n&~|=^W{R*&m4GvRnPGrGvx}UBk2Bq%D%Oj zG4h_;&q^JVViw`F+=&(B##q{VrurLdIo@?;#)36LU@VAhNf|=VL}mtk(cGIGHXS_P zt^tEgi-^zq5n+Y-9Sw+KwYW~9W5tU_%ubW9*C!%FKm|;=OVPg2Hm>+LN3avfQb-BSd9CgdLvJ-sYpk{>C+~DT zT12r?Yc~)1m;?DyPKVEIVgBcNa0qV1ZBf>7x~vd~V{4 zp&Fb*a^u9P$db_V+UsoAqCzCi*~9Hka(*mBopEQshYA-JCUsex3pIR-OM*pkfABM% z7x92-s@XiX|Lqs38YJYN4u5Q9&ZF{!U6!7)3eiaURmG`$Zjtx$&J5MjymbVHA>d*W!x48yI)uZS2XBg+RMx!=WAH&)u^74@L6Kz>et73BKX);w7dl8M=j+Lq=a6NXaR6@SkUz1f(bKyRgkBi*W$5#| z^-`+IP<5c{Q$rtNaDY}EY4~3FP5u1fW{_a2%$8Y~2A{EXt%j(Dg+iH?BrkiCJTv1J z4b5T$-J3~`6$~@S?<;w9Cl3TQgg6q+&onT}$v#uaP6#yjLdGSeiaOiHHmX$D=`?O`yeoF^mi&2} z0_80U>c611$r~rfAtC!lr9|%Y zj57yis;lJ`8_FWmI-8BCHf1NSWv(45q7}0I$jJ|J7PMUSDXbv8N37WE%#E)#DWZsU z(F_#WlGgeu@&@XpxS9=Gv3dgqc4oGLJN?GTUV%;Rn6nE`8=0t@jxC#fAPOu3>^LSbfchfG@Gc{duARvY+Gv)OAWF^h+T2!SCBhm`NpF>ER zRgC@V8~ZJ@LYW9m_(2I*G($D0=$uoi(uvWe+sKchOX)nYsy!ShZj}|a0;FU` ztSPSwy7dJOxi8bbQ>q+tWaF#0MsZ}f(R5EMv>Iq9ns>M(ff!F^<2n&sg=L3X6YmAB z%cCDXozM6Xeo7~Mi$=I1IU&^(-QHkCI*ls>BniE1rzcnE_hKpOEWba_R+kUr zl5KNW32Eeis1F^FS6E4}t-B-x@@WCy2I2O;>r~|y%I4HE{QGFKR_Plqcn#+$p1gdKGI&^$l^@Y~UTAJ&r2L`0c(x0#r33ylTGywvuFrs(w4z^`^8&GRCnqF z%Kca7h4`$78)RMUhPiT*A7_Hu#8pgIW3b5pEh^4RnGLKpWTLyUWiGI!apqX_Re4Hk=AbAj~Xd`PdNWiRw~S{=eWQArPY zJYLRqUTb!%&k#HMQa70}K=)yG(@U|>dX|%pba|^&>&Kh3o|I7NnZ*&q?-8N1!B{|8{j&uREHT0iUslR;le*k8a5~2VA literal 0 HcmV?d00001 From 052764d141f0e3178427f99df4956833f55c3de7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 18:11:12 +0800 Subject: [PATCH 0505/2585] - added diagram for NFS + VIP --- usr/share/doc/escenic/ece-install-guide.org | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index ade45951..dfe0e641 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -245,7 +245,7 @@ for the slave: #+END_SRC Once this is done, create and use an ece-install.conf on the slave -with this: +with this configuration: #+BEGIN_SRC sh fai_enabled=1 fai_db_install=1 @@ -359,6 +359,7 @@ fai_nfs_export_list="/var/exports/multimedia" #+END_SRC ** Profile :: VIP provider +[[file:images/nfs-vip.png]] This profile will make the host capable of serving a certain virtual IP (VIP) for one or more services. This is a useful thing for all From 303be1c0fc6fa3a095ea6a4a86bb2c6cb4bdb9c2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 18:22:51 +0800 Subject: [PATCH 0506/2585] - minor change to log message --- usr/sbin/ece-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 75d4edad..52b2dab4 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -186,8 +186,8 @@ function install_packages_if_missing() { if [ $some_are_missing -eq 0 ]; then return elif [ $one_time_apt_update_done -eq 0 ]; then - print_and_log \ - "Running one time APT update to ensure package list is fresh" + log "First running APT update to ensure fresh package list, " \ + "then continuing the above" run apt-get update one_time_apt_update_done=1 fi From ac63c70dff06266660c2bf5df551babad133d420 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 14 Mar 2012 18:26:06 +0800 Subject: [PATCH 0507/2585] - added get_percona_supported_list. This means that ece-install will dynamically determine if the OS you're installing it on supports package based installation of the Percona database. - (another) fix to installing the Percona DB if the(ir) GPG key server doesn't respond at the time of installation. Since this key server is so unstable, ece-install turns on force package installation of the DB related packages if the key server couldn't be reached. - simplification + some hardening of the DB replication configuration All of this relates to issue #45 --- .../ece-scripts/ece-install.d/database.sh | 77 ++++++++++++------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index cb2206f5..4a83a7c7 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -8,6 +8,15 @@ if [[ $(uname -m) != "x86_64" ]]; then percona_rpm_release_url=http://www.percona.com/downloads/percona-release/$percona_rpm_release_package_name.i386.rpm fi +function get_percona_supported_list() { + echo $( + curl -s http://repo.percona.com/apt/dists/ | \ + grep ^"
  • >$log 2>>$log - # There has been twice now, during six months, that - # the key cannot be retrieved from - # keys.gnupg.net. Therefore, we're checking if it - # failed and if yes, force the package installation. + # There has been three times now, during six months, that the + # key cannot be retrieved from keys.gnupg.net. Therefore, + # we're checking if it failed and if yes, force the package + # installation. if [ $? -gt 0 ]; then s="Failed retrieving the Percona key from keys.gnupg.net" print_and_log $s @@ -62,8 +72,7 @@ function install_database_server() fi add_apt_source "deb http://repo.percona.com/apt ${code_name} main" - packages="percona-server-server percona-server-client" - force_packages=0 + packages="percona-server-server percona-server-client libmysqlclient16" else print_and_log "The Percona APT repsository doesn't have packages" print_and_log "for your Debian (or derivative) version with code" @@ -86,6 +95,7 @@ function install_database_server() fi install_packages_if_missing $packages + force_packages=0 if [ $on_redhat_or_derivative -eq 1 ]; then run chkconfig --level 35 mysql on @@ -251,25 +261,17 @@ function configure_mysql_for_replication() { EOF fi - # server ID - local old="#server-id.*= 1" - - if [ $db_master -eq 1 ]; then - local new="server-id = 1" - else - local new="server-id = 2" - fi - - if [ $(grep ^"$old" $file | wc -l) -gt 0 ]; then - sed -i "s~^$old~$new~g" $file - else - echo "$new" >> $file - fi - - run /etc/init.d/mysql restart - # replication log configuration of the master if [ $db_master -eq 1 ]; then + local old="#server-id.*= 1" + local new="server-id = 1" + + if [ $(grep ^"$old" $file | wc -l) -gt 0 ]; then + sed -i "s~^$old~$new~g" $file + elif [ $(grep ^"$new" $file | wc -l) -lt 1 ]; then + echo "$new" >> $file + fi + old="bind-address.*= 127.0.0.1" new="# bind-address = 127.0.0.1" if [ $(grep ^"${old}" $file | wc -l) -gt 0 ]; then @@ -280,7 +282,7 @@ EOF new="log_bin = /var/log/mysql/mysql-bin.log" if [ $(grep ^"$old" $file | wc -l) -gt 0 ]; then sed -i "s~^${old}~${new}~g" $file - elif [ $(grep ^$new $file | wc -l) -lt 1 ]; then + elif [ $(grep ^"$new" $file | wc -l) -lt 1 ]; then echo "$new" >> $file fi @@ -289,10 +291,12 @@ EOF if [ $(grep ^"$old" $file | wc -l) -gt 0 ]; then sed -i "s~^${old}~${new}~g" $file - elif [ $(grep ^$new $file | wc -l) -lt 1 ]; then + elif [ $(grep ^"$new" $file | wc -l) -lt 1 ]; then echo "$new" >> $file fi + run /etc/init.d/mysql restart + # report the needed settings for a slave local master_status=$(mysql $db_schema -e "show master status" | tail -1) local file=$(echo $master_status | cut -d' ' -f1) @@ -300,6 +304,17 @@ EOF add_next_step "ece-install.conf for slave: fai_db_master_log_file=$file" add_next_step "ece-install.conf for slave:" \ fai_db_master_log_position=$position + else + local old="#server-id.*= 1" + local new="server-id = 2" + + if [ $(grep ^"$old" $file | wc -l) -gt 0 ]; then + sed -i "s~^$old~$new~g" $file + elif [ $(grep ^"$new" $file | wc -l) -lt 1 ]; then + echo "$new" >> $file + fi + + run /etc/init.d/mysql restart fi } @@ -307,7 +322,9 @@ EOF function set_slave_to_replicate_master() { print_and_log "Setting slave to replicate master DB @ $db_master_host ..." - mysql ${db_schema} < Date: Thu, 15 Mar 2012 13:05:13 +0800 Subject: [PATCH 0508/2585] - added new ece.conf parameter, jvm_gc_settings with sane defaults for production use. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default GC settings are based on the concurrent mark sweep GC profile for the Hotspot VM. Credits go to Ola Schön, Axel Springer & UnitB for providing these settings. The default settings should be fine for most users, but they're override-able on a per ECE/EAE/Search instance basis if that's desirable. The GC logging related settings are still maintained by the default ece_args variable inside ece as this should be applicable to all ECE users regardless of GC profile. Also, the existing JVM settings related ece.conf variables are still kept in place. This closes issue #46 - simplified the declaration of the default JVM parameters in ece_args. Sorted them so that they're easier to read. --- etc/escenic/ece.conf | 22 ++++++++++++++++++++++ usr/bin/ece | 23 +++++++++++++++-------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/etc/escenic/ece.conf b/etc/escenic/ece.conf index 166b5e01..08d9edef 100644 --- a/etc/escenic/ece.conf +++ b/etc/escenic/ece.conf @@ -122,6 +122,28 @@ remote_debugging_port=5005 enable_remote_monitoring=0 remote_monitoring_port=5792 +######################################################################## +# Settings related to the JVM garbage collection, don't touch these if +# you're not sure of what you're doing and have tested your changes +# properly. Note that ece takes care of the GC logging, so you don't +# have to cater for that here. +######################################################################## +jvm_gc_settings=" +-XX:+CMSClassUnloadingEnabled +-XX:+CMSIncrementalPacing +-XX:+CMSPermGenSweepingEnabled +-XX:+ExplicitGCInvokesConcurrent +-XX:+UseCMSInitiatingOccupancyOnly +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=65 +-XX:InitialCodeCacheSize=50m +-XX:MaxNewSize=250m +-XX:NewSize=250m +-XX:ParallelGCThreads=1 +-XX:ReservedCodeCacheSize=50m +-XX:SurvivorRatio=1 +" + ##################################################################### # Java will use IPv6 if it's available on the OS. This however, can # cause problems with applications that insists on using IPv4 diff --git a/usr/bin/ece b/usr/bin/ece index 6f637cd2..97a38e6c 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -340,14 +340,21 @@ function set_instance_settings() # * java.security for configuring the Java security framework with ECE. # * garbage collection log: this is paramount to keep an eye on # when running in production. - ece_args="-Descenic.server=$ece_server\ - -Dsolr.solr.home=$solr_home\ - -Djava.awt.headless=true\ - -Djava.security.auth.login.config=$ece_security_configuration_dir/jaas.config\ - -Djava.security.policy=$ece_security_configuration_dir/java.policy\ - -XX:+PrintGCTimeStamps\ - -XX:+PrintGCDetails\ - -Xloggc:${log_dir}/${type}-${instance}-gc.log" + ece_args=" + -Descenic.server=$ece_server + -Djava.awt.headless=true + -Djava.security.auth.login.config=$ece_security_configuration_dir/jaas.config + -Djava.security.policy=$ece_security_configuration_dir/java.policy + -Dsolr.solr.home=$solr_home + -XX:+PrintGCDetails + -XX:+PrintGCTimeStamps + -Xloggc:${log_dir}/${type}-${instance}-gc.log + " + + if [ "$jvm_gc_settings" ]; then + ece_args=$ece_args" "$jvm_gc_settings + fi + if [ "$instance" != "default" ]; then ece_args=$ece_args" -Dcom.escenic.instance=$instance" From d03e4aa436b166dbbbf8494e7c687f45c1d6e1a6 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 14:00:53 +0800 Subject: [PATCH 0509/2585] - ece-install will now output the required slave DB FAI settings in a designated .conf.add file. The file location is listed in the installation report summary as well as in the log: [ece-install-37] - See /root/ece-install-db-slave.conf.add for settings needed for the slave DB(s) Typical contents of this file looks like this: fai_db_master_log_file=mysql-bin.000006 fai_db_master_log_position=107 This is related to issue #45 --- .../escenic/ece-scripts/ece-install.d/database.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index 4a83a7c7..85a5a492 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -301,9 +301,14 @@ EOF local master_status=$(mysql $db_schema -e "show master status" | tail -1) local file=$(echo $master_status | cut -d' ' -f1) local position=$(echo $master_status | cut -d' ' -f2) - add_next_step "ece-install.conf for slave: fai_db_master_log_file=$file" - add_next_step "ece-install.conf for slave:" \ - fai_db_master_log_position=$position + local slave_conf_file=$HOME/ece-install-db-slave.conf.add + cat > $slave_conf_file < Date: Thu, 15 Mar 2012 16:38:19 +0800 Subject: [PATCH 0510/2585] - added support for escenic-content-engine-scripts - removed /usr/bin/ece, init.d & BASH completion from escenic-content-engine-installer --- usr/local/src/create-packages | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 26db3bc2..f546d5c8 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -42,16 +42,26 @@ function set_up_target_directory() { if [[ $package_name == "escenic-content-engine-installer" ]]; then mkdir -p $target_dir/usr/share/doc/escenic - cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ cp -r ../../sbin/{ece-install,drop-and-create-ecedb} $target_dir/usr/sbin/ - cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ cp -r ../../share/escenic/ $target_dir/usr/share/ - cp -r ../../share/doc/escenic/*.{org,html} $target_dir/usr/share/doc/escenic + cp -r ../../share/doc/escenic/ece-install-guide.{org,html} \ + $target_dir/usr/share/doc/escenic + + elif [[ $package_name == "escenic-content-engine-scripts" ]]; then + mkdir -p $target_dir/usr/share/doc/escenic + mkdir -p $target_dir/usr/share/escenic/ece-scripts + + cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ + cp -r ../../share/escenic/ece-scripts/common* \ + $target_dir/usr/share/escenic/ece-scripts/ + cp -r ../../share/doc/escenic/{ece,system-info}-guide.{org,html} \ + $target_dir/usr/share/doc/escenic cp -r ../../../etc/escenic/ $target_dir/etc/ cp -r ../../../etc/default/ece $target_dir/etc/default elif [[ $package_name == "vosa" ]]; then mkdir -p $target_dir/usr/share/doc/vosa + cp -r ../../bin/vosa $target_dir/usr/bin/ cp -r ../../share/vizrt/ $target_dir/usr/share/ cp -r ../../share/doc/vizrt/*.org $target_dir/usr/share/doc/vosa/ From fb31538a21a0789bd9433c80495bab8f19584fbe Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 16:38:36 +0800 Subject: [PATCH 0511/2585] - added escenic-content-engine-scripts package --- .../debian/escenic-content-engine-scripts/control | 15 +++++++++++++++ .../escenic-content-engine-scripts/postinst | 7 +++++++ 2 files changed, 22 insertions(+) create mode 100644 usr/local/src/debian/escenic-content-engine-scripts/control create mode 100755 usr/local/src/debian/escenic-content-engine-scripts/postinst diff --git a/usr/local/src/debian/escenic-content-engine-scripts/control b/usr/local/src/debian/escenic-content-engine-scripts/control new file mode 100644 index 00000000..b15f692c --- /dev/null +++ b/usr/local/src/debian/escenic-content-engine-scripts/control @@ -0,0 +1,15 @@ +Package: escenic-content-engine-scripts +Version: VERSION +Section: base +Priority: optional +Architecture: all +Maintainer: MAINTAINER_NAME +Description: Scripts for operating the Escenic Content Engine. + Included in this packages are scripts for operating the Escenic + Content Engine, Escenic Analysis Engine, Search servers (solr + + indexer-webapp), BASH completion of all commands as well as init.d + scripts. + . + See the documentation in /usr/share/doc/escenic/ for further + details. + \ No newline at end of file diff --git a/usr/local/src/debian/escenic-content-engine-scripts/postinst b/usr/local/src/debian/escenic-content-engine-scripts/postinst new file mode 100755 index 00000000..cc6f2bb3 --- /dev/null +++ b/usr/local/src/debian/escenic-content-engine-scripts/postinst @@ -0,0 +1,7 @@ +#! /usr/bin/env bash +# set -e + +chmod +x /usr/bin/{ece,system-info} + +exit 0 + From 7f0e7da0a18a9b4312a65caab5027147f84cc95a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 16:46:03 +0800 Subject: [PATCH 0512/2585] - added --keep-version to alien option list --- usr/local/src/create-packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index f546d5c8..5c38862d 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -103,7 +103,7 @@ function create_packages() { if [[ -x /usr/bin/alien && -x /usr/bin/fakeroot ]]; then ( cd $target_dir - fakeroot alien --to-rpm --scripts \ + fakeroot alien --keep-version --to-rpm --scripts \ ${package_name}-${package_version}.deb \ 2>&1 > /dev/null ) From 47df743d13cf47358e464604b7e9ae879b43132c Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 15 Mar 2012 10:36:30 +0100 Subject: [PATCH 0513/2585] Made it possible to run several ece-installs in succession --- .../vosa/post-install-hooks/install-engine.sh | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh index 1f6d2851..a49cf881 100755 --- a/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh +++ b/usr/share/vizrt/vosa/post-install-hooks/install-engine.sh @@ -14,7 +14,14 @@ # The VM is assumed to be booted and ready for SSH using the passwordless # key referenced by the $2/ssh.conf file, as the ubuntu or root users. -# ensure we have an ece-install image. +# ensure we have some ece-install*.conf file(s) +ece_install_conf_files=$(ls $1 | grep "^ece-install.*\\.conf$") +if [ -z "$ece_install_conf_files" ] ; then + echo "No ece-install.conf files present in $1" + exit 1; +fi + +# ensure we have an ece-install image, download one. if [ ! -r $2/ece-install ] ; then if [ -r $1/ece-install ] ; then cp $1/ece-install $2 @@ -22,12 +29,13 @@ if [ ! -r $2/ece-install ] ; then wget "https://github.com/skybert/ece-scripts/tarball/master" -O $2/ece-install.tar.gz fi fi -if [ ! -r $1/ece-install.conf ] ; then - echo "No ece-install.conf file present in $1" - exit 1; -fi -scp -F $2/ssh.conf $2/ece-install.tar.gz $1/ece-install.conf root@guest: +scp -F $2/ssh.conf $2/ece-install.tar.gz $1/ece-install*.conf root@guest: ssh -F $2/ssh.conf root@guest tar xfz ece-install.tar.gz -ssh -F $2/ssh.conf root@guest bash *-ece-scripts-*/usr/sbin/ece-install +for conf in $ece_install_conf_files ; do + echo "Performing ece-install with -f $conf" + ssh -F $2/ssh.conf root@guest bash *-ece-scripts-*/usr/sbin/ece-install -f "$conf" || exit $? + echo "---------8<-------------------------" +done + From 2d62105759a966b61543dd2706b9c6a235f36fc8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 18:06:00 +0800 Subject: [PATCH 0514/2585] - re-implemented get_instance_list to read the /etc/escenic/ece[type]-/dev/null); do + + local instance=$(basename $el .conf) + instance=${instance##ece-} + + for ele in $allowed_types; do + if [[ $instance == $ele ]]; then + instance="" + continue + fi + + local away="${ele}-" + instance=${instance##${away}} + done + instance_list="$instance_list $instance" + done echo $instance_list } @@ -456,6 +471,10 @@ function sanity_check() exit 1 fi + if [[ "$command"=="list-instances" ]]; then + return + fi + # checks if the user is using the default instance when he/she # really wants to start one of the instances installed on the # system. @@ -1804,6 +1823,9 @@ for el in $command; do versions) list_versions ;; + list-instances) + print $(get_instance_list) + ;; update) update_publication_resources ;; From e67e786068e602adf55264ebf10fabcd40716ea2 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 18:07:18 +0800 Subject: [PATCH 0515/2585] - made use of ece's get_instance_list instead of re-implementing the method ourselves - simpler declaration of commands, options and resources --- etc/bash_completion.d/ece | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/etc/bash_completion.d/ece b/etc/bash_completion.d/ece index ad53165e..b781f4d5 100644 --- a/etc/bash_completion.d/ece +++ b/etc/bash_completion.d/ece @@ -5,25 +5,19 @@ _ece_commands() local cur=${COMP_WORDS[COMP_CWORD]} local prev=${COMP_WORDS[COMP_CWORD-1]} - commands="applog assemble backup clean deploy edit help info kill log loglist \ - flush outlog package restart start status stop threaddump update versions" - options="-i --instance -p --publication -r --publication-resource \ + commands="applog assemble backup clean deploy edit help info kill log loglist + flush outlog package restart start status stop threaddump update versions + list-instances + " + options="-i --instance -p --publication -r --publication-resource -t --type -u --user -w --password" - resources="content-type feature layout layout-group image-version menu \ + resources="content-type feature layout layout-group image-version menu security root-section-parameters" types="engine search analysis rmi-hub" # default completions is the list of commands completions=$commands" "$options - instances="" - dir=/etc/escenic/engine/instance - if [ -r $dir ] ; then - for el in ${dir}/*; do - instances=$(basename ${el})" "$instances - done - fi - publications="" dir=/opt/escenic/assemblytool/publications if [ -r $dir ] ; then @@ -40,7 +34,7 @@ _ece_commands() completions=$types ;; -i|--instance) - completions=$instances + completions=$(ece -q list-instances) ;; -p|--publication) completions=$publications From d9b598a1791d9a2971bf36a5496ffcd3d3f56439 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 15 Mar 2012 11:32:55 +0100 Subject: [PATCH 0516/2585] Added documentation for fai_monitoring_munin_node_list --- usr/share/doc/escenic/ece-install-guide.org | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index dfe0e641..8ca35305 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -660,6 +660,7 @@ $HOME/ece-install.conf file of the root user: | fai_editor_shutdown | 8005 | Shutdown port of the editor instance | | fai_enabled | 0 | Whether or not to run ece-install in FAI mode | | fai_keep_off_etc_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | +| fai_monitoring_munin_node_list | "" | Set this to a whitespace separated list of nodes that munin should monitor | | fai_monitoring_server_install | 0 | Install the monitoring server profile. | | fai_monitoring_server_ip | 127.0.0.1 | The IP of the monitoring server. | | fai_nfs_allowed_client_network | "" | IP/netmwask of allowed NFS clients, example: 192.168.1.0/255.255.255.0 | From ae6b147e0316685466235c5525b759f1e62c1004 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 18:34:09 +0800 Subject: [PATCH 0517/2585] - sanity_check now considers the get_instance_list for all commands, except when the commands are help or list-instances (undocumented command for now, only used by the BASH completion). This means that any command which passes a non-existent instance (non existent being defined as an instance which doesn't have a corresponding ece-.conf in /etc/escenic), will fail to execute with a message like: $ ece -i peace-on-earth start [ece#engine-peace-on-earth] Instance 'peace-on-earth' doesn't exist on quanah Another benefit of this feature, is that the TAB completion will return all instances on the system, also the ones that are not ECE (like EAE and search). This closes #48 --- usr/bin/ece | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 5d5a0ea9..f5f27f17 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -471,17 +471,17 @@ function sanity_check() exit 1 fi - if [[ "$command"=="list-instances" ]]; then + if [ $command == "list-instances" ]; then return fi + local instance_list=$(get_instance_list) # checks if the user is using the default instance when he/she # really wants to start one of the instances installed on the # system. if [ "$instance" == "default" -a \ "$type" == "engine" -a \ $(echo $command | egrep 'help' | wc -l) -eq 0 ]; then - instance_list=$(get_instance_list) if [[ -n "$instance_list" && \ $(echo $instance_list | grep ' ' | wc -l) -gt 0 ]]; then print "You must specify the instance with -i " @@ -491,6 +491,19 @@ function sanity_check() fi fi + local instance_exists=0 + for el in $instance_list; do + if [ $el == $instance ]; then + instance_exists=1 + break + fi + done + + if [ $instance_exists -eq 0 ]; then + print "Instance '$instance' doesn't exist on $HOSTNAME" + exit 1 + fi + # verifies that java_home exists and has the java executable if [ ! -d $java_home ]; then print "java_home $java_home doesn't exist" From 6bfc3d18fcf318967304d16166ebdb188490795f Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 15 Mar 2012 11:46:24 +0100 Subject: [PATCH 0518/2585] Fixed documentation to match code. the monitoring server profile is fai_monitoring_install=1 --- usr/share/doc/escenic/ece-install-guide.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/doc/escenic/ece-install-guide.org b/usr/share/doc/escenic/ece-install-guide.org index 8ca35305..82e6af81 100644 --- a/usr/share/doc/escenic/ece-install-guide.org +++ b/usr/share/doc/escenic/ece-install-guide.org @@ -660,8 +660,8 @@ $HOME/ece-install.conf file of the root user: | fai_editor_shutdown | 8005 | Shutdown port of the editor instance | | fai_enabled | 0 | Whether or not to run ece-install in FAI mode | | fai_keep_off_etc_hosts | 0 | Set this to 1 if you don't want ece-install adding entries to /etc/hosts | +| fai_monitoring_install | 0 | Install the monitoring server profile. | | fai_monitoring_munin_node_list | "" | Set this to a whitespace separated list of nodes that munin should monitor | -| fai_monitoring_server_install | 0 | Install the monitoring server profile. | | fai_monitoring_server_ip | 127.0.0.1 | The IP of the monitoring server. | | fai_nfs_allowed_client_network | "" | IP/netmwask of allowed NFS clients, example: 192.168.1.0/255.255.255.0 | | fai_nfs_client_install | 0 | Installs an NFS client | From 76861506e6f3fc933b217aee79d764c9e9da925f Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 19:22:53 +0800 Subject: [PATCH 0519/2585] - what a wonderful chicken and egg problem :-) assert_correct_runtime_environment cannot use any methods pulled in from the modules through the (later processed) init method. --- usr/sbin/ece-install | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 52b2dab4..cf6a4f7b 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -633,7 +633,12 @@ function assert_correct_runtime_environment() if [ ! -e "$conf_file" ]; then echo $conf_file "doesn't exist." echo "I cannot live without it, so I'm exiting :-(" - remove_pid_and_exit_in_error + + if [ -e $pid_file ]; then + rm $pid_file + fi + + exit 1 fi } From fd2e3c1ffb6ec04b98f3ed25348a637c61672787 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 15 Mar 2012 20:04:15 +0800 Subject: [PATCH 0520/2585] - fix for Ubuntu LTS which doesn't have Icinga - fix for conf which doesn't have all the node list variables defined (no host group will be created) --- .../ece-scripts/ece-install.d/monitoring.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 9ea2675f..a027bbc9 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -320,6 +320,14 @@ EOF function install_monitoring_server() { local nagios_flavour=${fai_monitoring_nagios_flavour-$MONITORING_VENDOR_ICINGA} + + if [ "$(lsb_release -s -c 2>/dev/null)" = "lucid" ]; then + log "Version $(lsb_release -s -c 2>/dev/null) of" \ + $(lsb_release -s -c 2>/dev/null) \ + "doesn't support Icinga, will use vanilla Nagios instead." + nagios_flavour=$MONITORING_VENDOR_NAGIOS + fi + install_nagios_monitoring_server $nagios_flavour install_munin_gatherer create_monitoring_server_overview $nagios_flavour @@ -334,10 +342,16 @@ function set_up_monitoring_host_group() local file=$1 local host_group_name=$2 local host_group_alias=$3 - # the remainding arguments passed to the methods is the member - # list members + # the remainding arguments passed to the methods is the member + # list members local host_group_member_list=${@:4:$(( $# - 3 ))} + # don't set up host groups for empty node lists, so we exit here if + # the member list is empty. + if [ -z "${host_group_member_list}" ]; then + return + fi + if [ $(grep "hostgroup_name $host_group_name" $file | wc -l) -gt 0 ]; then print "Icinga group member" \ $host_group_name \ From 13d4514724960b22f038509d48efbf5d85102540 Mon Sep 17 00:00:00 2001 From: Erik Mogensen Date: Thu, 15 Mar 2012 13:54:02 +0100 Subject: [PATCH 0521/2585] Restart munin-node even when engine is not installed. --- usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 9ea2675f..18e86fa4 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -189,6 +189,11 @@ EOF if [ -z "${instance_list}" ]; then print_and_log "No ECE instances found on $HOSTNAME, so I'm not adding" print_and_log "additional Munin configuration" + + if [ $on_debian_or_derivative -eq 1 ]; then + run service munin-node restart + fi + add_next_step "A Munin node has been installed on $HOSTNAME" return fi From 42d202d5507250e49727d8885d0054ca25dc74b7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 10:24:51 +0800 Subject: [PATCH 0522/2585] - ece & system-info are no longer a part of the escenic-content-engine-installer package --- usr/local/src/debian/escenic-content-engine-installer/postinst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/local/src/debian/escenic-content-engine-installer/postinst b/usr/local/src/debian/escenic-content-engine-installer/postinst index ca61ce55..d1b78243 100755 --- a/usr/local/src/debian/escenic-content-engine-installer/postinst +++ b/usr/local/src/debian/escenic-content-engine-installer/postinst @@ -1,7 +1,7 @@ #! /usr/bin/env bash # set -e -chmod +x /usr/sbin/ece-install /usr/bin/{ece,system-info} +chmod +x /usr/sbin/ece-install echo "You can now run the Escenic Content Engine installer " echo "by typing 'ece-install' as root" From a0e8ade5d793fd15c9d4872e92af84f5604aaaec Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 10:49:23 +0800 Subject: [PATCH 0523/2585] - removed user output: "You can now run the Escenic Content Engine installer " "by typing 'ece-install' as root" As this is rather unusual for a DEB package. --- usr/local/src/debian/escenic-content-engine-installer/postinst | 3 --- 1 file changed, 3 deletions(-) diff --git a/usr/local/src/debian/escenic-content-engine-installer/postinst b/usr/local/src/debian/escenic-content-engine-installer/postinst index d1b78243..9f4c9f1c 100755 --- a/usr/local/src/debian/escenic-content-engine-installer/postinst +++ b/usr/local/src/debian/escenic-content-engine-installer/postinst @@ -3,8 +3,5 @@ chmod +x /usr/sbin/ece-install -echo "You can now run the Escenic Content Engine installer " -echo "by typing 'ece-install' as root" - exit 0 From 9086a5fe5bff96c8bffedc5d25ffa586c1503962 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 10:52:22 +0800 Subject: [PATCH 0524/2585] - removed the pulse methods --- usr/sbin/ece-install | 60 -------------------------------------------- 1 file changed, 60 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index cf6a4f7b..7886f246 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -223,66 +223,6 @@ function install_common_os_packages() done } -###################################################################### -# -###################################################################### -function download_escenic_components_p() { - $(download_escenic_components) & - show_pulse $! "Downloading Escenic components from Technet" -} - -function set_up_instance_specific_nursery_configuration_p() { - $(set_up_instance_specific_nursery_configuration) & - show_pulse $! "Setting up instance specific Nursery configuration" -} - -function assemble_deploy_and_restart_type_p() { - echo $(assemble_deploy_and_restart_type) & - show_pulse $! "Assembling, deploying & starting $instance_name" -} - -function set_up_engine_and_plugins_p() { - $(set_up_engine_and_plugins) & - show_pulse $! "Setting up the Escenic Content Engine & its plugins" -} - -function check_for_required_downloads_p() { - $(check_for_required_downloads) & - show_pulse $! "Asserting that required downloads succeeded" -} - -function set_up_user_environment_p() { - $(set_up_user_environment) & - show_pulse $! "Setting up the ${ece_user} user's UNIX environment" -} - -function set_up_ece_scripts_p() { - $(set_up_ece_scripts) & - show_pulse $! 'Setting up the ece UNIX scripts' - if [ -n "${messages}" ]; then - echo $message - fi -} - -function install_common_os_packages_p() { - $(install_common_os_packages) & - show_pulse $! "Installing 3rd party packages needed by $(basename $0)" -} - -function install_ece_third_party_packages_p() { - $(install_ece_third_party_packages) & - show_pulse $! "Installing 3rd party packages needed by ECE" -} - -function set_up_assembly_tool_p() { - $(set_up_assembly_tool) & - show_pulse $! "Setting up the Assembly Tool" -} - -###################################################################### -# -###################################################################### - function set_up_ece_scripts() { print_and_log 'Setting up the ece UNIX scripts ...' From 45bcd40034a1c865c435f251d319298dec525981 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 10:53:21 +0800 Subject: [PATCH 0525/2585] - fix for $command being more than one command (bug introduced with yesterday's "list-instances" command test in sanity_check --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index f5f27f17..d066ae82 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -471,7 +471,7 @@ function sanity_check() exit 1 fi - if [ $command == "list-instances" ]; then + if [ "$command" == "list-instances" ]; then return fi From 40439fa03fbd651c403e27fe7dc4efa10e6f0888 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 10:54:04 +0800 Subject: [PATCH 0526/2585] - more user feedback in set_up_engine_and_plugins, before the message was only logged, not displayed to standard out. --- usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index bc71ea22..422b2d8b 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -105,7 +105,7 @@ function set_up_engine_and_plugins() { return fi - log "Setting up the Escenic Content Engine & its plugins ..." + print_and_log "Setting up the Escenic Content Engine & its plugins ..." make_dir $escenic_root_dir cd $escenic_root_dir/ From dfa632e6329336edc2064dde07d6d8a58c8e0dee Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 11:21:28 +0800 Subject: [PATCH 0527/2585] - bug fix in set_up_basic_nursery_configuration: provided JAAS configuration in the conf archive bundle wasn't heeded. --- usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index 422b2d8b..ed0de9da 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -264,6 +264,8 @@ function set_up_basic_nursery_configuration() { print_and_log "Using the supplied Nursery & JAAS configuration from" print_and_log "bundle: $ece_instance_conf_archive" local a_tmp_dir=$(mktemp -d) + run cd $a_tmp_dir + run tar xzf $ece_instance_conf_archive if [ ! -d ${a_tmp_dir}/engine/security ]; then print "Archive $ece_instance_conf_archive doesn't have JAAS config," @@ -271,8 +273,6 @@ function set_up_basic_nursery_configuration() { run cp -r $escenic_root_dir/engine/security/ $common_nursery_dir/ fi - run cd $a_tmp_dir - run tar xzf $ece_instance_conf_archive run cp -r engine/siteconfig/config-skeleton/* $common_nursery_dir/ else run cp -r $escenic_root_dir/engine/siteconfig/config-skeleton/* \ From 6ed77b19d67611e4241c2e795ed990c80f7821dd Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 11:44:35 +0800 Subject: [PATCH 0528/2585] - moved content engine specific code to the content-engine.sh module - improved user feedback when using EAR based deployment. --- usr/sbin/ece-install | 66 ------------------ .../ece-install.d/content-engine.sh | 68 +++++++++++++++++++ 2 files changed, 68 insertions(+), 66 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 7886f246..92cef523 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -735,53 +735,6 @@ function update_type_instances_to_start_up() fi } -ece_instance_ear_file="" -ece_instance_conf_archive="" - -function set_archive_files_depending_on_profile() -{ - if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then - ece_instance_ear_file=$fai_presentation_ear - ece_instance_conf_archive=$fai_presentation_conf_archive - elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then - ece_instance_ear_file=$fai_editor_ear - ece_instance_conf_archive=$fai_editor_conf_archive - elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then - ece_instance_ear_file=$fai_search_ear - ece_instance_conf_archive=$fai_search_conf_archive - fi -} - -# Returns 1 if we're installing the ECE instances from a provided EAR -# file -function is_installing_from_ear() -{ - log install_profile_number=$install_profile_number - log ece_instance_ear_file=$ece_instance_ear_file - - if [[ -z "$ece_instance_ear_file" || \ - $fai_enabled -eq 0 ]]; then - echo 0 - return - fi - - echo 1 -} - -# Returns 1 if we're using a tarball with the Nursery & JAAS configuration. -function is_using_conf_archive(){ - log install_profile_number=$install_profile_number - log ece_instance_conf_archive=$ece_instance_conf_archive - - if [[ -z "$ece_instance_conf_archive" || \ - $fai_enabled -eq 0 ]]; then - echo 0 - return - fi - - echo 1 -} - # verifies that the passed file(s) exist and are readable, depends on # set_archive_files_depending_on_profile function verify_that_files_exist_and_are_readable() @@ -806,25 +759,6 @@ function install_presentation_server() install_ece_instance "web1" $PROFILE_PRESENTATION_SERVER } -function assemble_deploy_and_restart_type() -{ - print_and_log "Assembling, deploying & starting $instance_name ..." - - set_correct_permissions - - # need to run clean here since we might be running multiple - # profiles in the same ece-install process. - ece_command="ece -i $instance_name -t $type clean assemble deploy restart" - if [ $(is_installing_from_ear) -eq 1 ]; then - run cp $ece_instance_ear_file $escenic_cache_dir/engine.ear - ece_command="ece -i $instance_name -t $type deploy restart" - log "Using the supplied EAR instead of running an assembly." - fi - - su - $ece_user -c "$ece_command" 1>>$log 2>>$log - exit_on_error "su - $ece_user -c \"$ece_command\"" -} - # If the system is installed using the recommended paths, the method # will return a list of the instances configured in # ${escenic_conf_dir}/engine/instance diff --git a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh index ed0de9da..51dd34e9 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/content-engine.sh @@ -486,3 +486,71 @@ function install_ece_third_party_packages assert_pre_requisite $el done } + +ece_instance_ear_file="" +ece_instance_conf_archive="" + +function set_archive_files_depending_on_profile() +{ + if [ $install_profile_number -eq $PROFILE_PRESENTATION_SERVER ]; then + ece_instance_ear_file=$fai_presentation_ear + ece_instance_conf_archive=$fai_presentation_conf_archive + elif [ $install_profile_number -eq $PROFILE_EDITORIAL_SERVER ]; then + ece_instance_ear_file=$fai_editor_ear + ece_instance_conf_archive=$fai_editor_conf_archive + elif [ $install_profile_number -eq $PROFILE_SEARCH_SERVER ]; then + ece_instance_ear_file=$fai_search_ear + ece_instance_conf_archive=$fai_search_conf_archive + fi +} + +# Returns 1 if we're installing the ECE instances from a provided EAR +# file +function is_installing_from_ear() +{ + log install_profile_number=$install_profile_number + log ece_instance_ear_file=$ece_instance_ear_file + + if [[ -z "$ece_instance_ear_file" || \ + $fai_enabled -eq 0 ]]; then + echo 0 + return + fi + + echo 1 +} + +# Returns 1 if we're using a tarball with the Nursery & JAAS configuration. +function is_using_conf_archive(){ + log install_profile_number=$install_profile_number + log ece_instance_conf_archive=$ece_instance_conf_archive + + if [[ -z "$ece_instance_conf_archive" || \ + $fai_enabled -eq 0 ]]; then + echo 0 + return + fi + + echo 1 +} + +function assemble_deploy_and_restart_type() +{ + set_correct_permissions + + # need to run clean here since we might be running multiple + # profiles in the same ece-install process. + ece_command="ece -i $instance_name -t $type clean assemble deploy restart" + if [ $(is_installing_from_ear) -eq 1 ]; then + run cp $ece_instance_ear_file $escenic_cache_dir/engine.ear + ece_command="ece -i $instance_name -t $type deploy restart" + print_and_log "Deploying the supplied EAR on instance $instance_name" + print_and_log "and restarting it ..." + else + print_and_log "Assembling, deploying & starting $instance_name ..." + fi + + su - $ece_user -c "$ece_command" 1>>$log 2>>$log + exit_on_error "su - $ece_user -c \"$ece_command\"" +} + From 001053176a01bdbe3e694ce0f0d1bc985020af2d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 12:07:00 +0800 Subject: [PATCH 0529/2585] - fix to "ece list-instances" in sanity_check. This should also fix the issue with TAB completion where the error message (you have to specify instance with -i ...) would be included in the completion list. --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index d066ae82..bf3f1419 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -471,7 +471,7 @@ function sanity_check() exit 1 fi - if [ "$command" == "list-instances" ]; then + if [ "$(echo $command)" == "list-instances" ]; then return fi From 5a47f0aaba7683f754f9d8c17c55ceb3b52beb4c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 12:14:53 +0800 Subject: [PATCH 0530/2585] - new method: get_gc_log - the GC log is now included in the output both from "ece log" and "ece loglist" --- usr/bin/ece | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index bf3f1419..8f51bf89 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -233,6 +233,8 @@ function set_common_settings() log_file=$log_dir/$type-$instance.out pid_file=$pid_dir/$type-$instance.pid fi + + gc_log=${log_dir}/${type}-${instance}-gc.log log_file_list=" $log_dir/${HOSTNAME}-${instance}-messages @@ -363,7 +365,7 @@ function set_instance_settings() -Dsolr.solr.home=$solr_home -XX:+PrintGCDetails -XX:+PrintGCTimeStamps - -Xloggc:${log_dir}/${type}-${instance}-gc.log + -Xloggc:${gc_log} " if [ "$jvm_gc_settings" ]; then @@ -1020,6 +1022,7 @@ function get_info_for_type() print "|-> System out log:" $log_file print "|-> App server log:" $(get_app_log) print "|-> Log4j log: " $(get_log4j_log) + print "|-> GC log: " $(get_gc_log) if [ -n "${appserver}" ]; then print "Application server:" @@ -1290,6 +1293,10 @@ EOF run rm -rf ${dir} } +function get_gc_log() { + echo ${gc_log} +} + function get_log4j_log() { for el in $log_file_list; do @@ -1684,10 +1691,12 @@ function show_all_log_paths() print "System out log: "$log_file print "App server log: "$(get_app_log) print "log4j log: "$(get_log4j_log) + print "GC log: "$(get_gc_log) else echo $log_file echo $(get_app_log) echo $(get_log4j_log) + echo $(get_gc_log) fi } From 11d1d76abc675223b3fb245fcee5cac8b995e1da Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 12:24:51 +0800 Subject: [PATCH 0531/2585] - The common files are also provided in the installer. Later, when we've got a repo up and running, we'll let the -installer depend on the scripts package, and then remove the common scripts from the installer. For now though, the -scripts package doesn't include the common files. - included the init.d script itself in the -scripts package --- usr/local/src/create-packages | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index 5c38862d..f14d99d7 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -49,15 +49,23 @@ function set_up_target_directory() { elif [[ $package_name == "escenic-content-engine-scripts" ]]; then mkdir -p $target_dir/usr/share/doc/escenic + mkdir -p $target_dir/etc/init.d mkdir -p $target_dir/usr/share/escenic/ece-scripts - + cp -r ../../bin/{ece,system-info} $target_dir/usr/bin/ - cp -r ../../share/escenic/ece-scripts/common* \ - $target_dir/usr/share/escenic/ece-scripts/ + + # The common files are also provided in the installer. Later, when + # we've got a repo up and running, we'll let the -installer depend + # on the scripts package, and then remove the common scripts from + # the installer. + # cp -r ../../share/escenic/ece-scripts/common* \ + # $target_dir/usr/share/escenic/ece-scripts/ + cp -r ../../share/doc/escenic/{ece,system-info}-guide.{org,html} \ $target_dir/usr/share/doc/escenic cp -r ../../../etc/escenic/ $target_dir/etc/ - cp -r ../../../etc/default/ece $target_dir/etc/default + cp -r ../../../etc/default/ece $target_dir/etc/default/ + cp -r ../../../etc/init.d/ece $target_dir/etc/init.d/ elif [[ $package_name == "vosa" ]]; then mkdir -p $target_dir/usr/share/doc/vosa From 7ab99202357e751ab2edccb4f4f930873b0beee7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 12:25:49 +0800 Subject: [PATCH 0532/2585] - updated comment header to include the vosa scripts :) --- usr/local/src/create-packages | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/local/src/create-packages b/usr/local/src/create-packages index f14d99d7..f8a70f11 100644 --- a/usr/local/src/create-packages +++ b/usr/local/src/create-packages @@ -1,7 +1,7 @@ #! /usr/bin/env bash -# script fro creating DEB and RPM packages of the ece-install and ece -# scripts. +# script fro creating DEB and RPM packages of the vosa, ece-install +# and ece scripts. # by tkj@vizrt.com From e3dd4789a09a18fc6e2bfbdb2e640b2a18df1d6c Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 12:59:48 +0800 Subject: [PATCH 0533/2585] - fixed bug in nagios-nrpe-server, restart fails (sometimes) if the PID isn't there (!) --- usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 89956481..4e0b1c1a 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -130,7 +130,11 @@ function install_nagios_node() $file dont_quote_conf_values=0 + # bug in nagios-nrpe-server, restart fails (sometimes) if the PID + # isn't there (!) + run touch /var/run/nagios/nrpe.pid run /etc/init.d/nagios-nrpe-server restart + add_next_step "A Nagios NRPE node has been installed on ${HOSTNAME}" } From c93d5c3331867ed77ca105e157547370667e981d Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 13:45:24 +0800 Subject: [PATCH 0534/2585] - improved "taking care of the user": provider=1 will now test to see if all the needed variables are set in the .conf file. --- .../escenic/ece-scripts/ece-install.d/vip-provier.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh index 03196109..1f890e5c 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/vip-provier.sh @@ -19,6 +19,14 @@ function get_vip_configuration() { # /etc/ha.d/authkeys), looks something like: # e19f39c09e9affafc23fc5bcd0404186 vip_primary_node_auth_key=${fai_vip_primary_node_auth_key} + + ensure_variable_is_set \ + fai_vip_address \ + fai_vip_sibling_ip \ + fai_vip_primary_node_name \ + fai_vip_primary_node_ip \ + fai_vip_secondary_node_name \ + fai_vip_secondary_node_ip } function install_vip_provider() { From db175593c99f874fa431d42b0c1935e4eb61e25b Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 14:05:45 +0800 Subject: [PATCH 0535/2585] - taking care of the user, making sure that required variables: fai_db_master_host fai_db_master_log_file fai_db_master_log_position are provided in the ece-install.conf before attempting setting up a DB slave. --- usr/share/escenic/ece-scripts/ece-install.d/database.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index 85a5a492..dcd81f06 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -210,8 +210,13 @@ function set_db_settings_from_fai_conf() db_master_host=${fai_db_master_host} db_master_log_file=${fai_db_master_log_file} db_master_log_position=${fai_db_master_log_position} - - # TODO assert set + + if [ $fai_db_master -eq 0 ]; then + ensure_variable_is_set \ + fai_db_master_host \ + fai_db_master_log_file \ + fai_db_master_log_position + fi } # Method used both from interactive mode to set any missing values From f8b68528cfde11b51a30f2ba6b78962e4fd8a44a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 15:03:25 +0800 Subject: [PATCH 0536/2585] - improved cache server profile: you can now have two or more backends on the same host. This is useful for cases were you want to run the cache server on the same host as the ECE presentation servers and you have more than one ECE you want to serve the site with. --- .../escenic/ece-scripts/ece-install.d/cache-server.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh index 3fff7e19..b98d0e45 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/cache-server.sh @@ -108,18 +108,20 @@ backend static { } EOF + local i=0 for el in $backend_servers; do appserver_id=$(echo $el | cut -d':' -f1 | sed 's/-/_/g') appserver_host=$(echo $el | cut -d':' -f1) appserver_port=$(echo $el | cut -d':' -f2) cat >> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl <> /etc/varnish/default.vcl < Date: Fri, 16 Mar 2012 15:18:50 +0800 Subject: [PATCH 0537/2585] - small bug fix in DB master test --- usr/share/escenic/ece-scripts/ece-install.d/database.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index dcd81f06..1060667a 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -211,7 +211,7 @@ function set_db_settings_from_fai_conf() db_master_log_file=${fai_db_master_log_file} db_master_log_position=${fai_db_master_log_position} - if [ $fai_db_master -eq 0 ]; then + if [ $db_master -eq 0 ]; then ensure_variable_is_set \ fai_db_master_host \ fai_db_master_log_file \ From 21e3f80d2c234220e427e46871b8f1709d5237b8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Fri, 16 Mar 2012 15:21:59 +0800 Subject: [PATCH 0538/2585] - another wee fix for DB installation which are not a part of a master/slave setup but plain old DB setup. --- usr/share/escenic/ece-scripts/ece-install.d/database.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/database.sh b/usr/share/escenic/ece-scripts/ece-install.d/database.sh index 1060667a..c30c1286 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/database.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/database.sh @@ -211,7 +211,7 @@ function set_db_settings_from_fai_conf() db_master_log_file=${fai_db_master_log_file} db_master_log_position=${fai_db_master_log_position} - if [ $db_master -eq 0 ]; then + if [ $db_master -eq 0 -a $db_replication -eq 1 ]; then ensure_variable_is_set \ fai_db_master_host \ fai_db_master_log_file \ From c2552e7ef578899df0fc98280c245887c5a8c1f7 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 19 Mar 2012 12:14:10 +0800 Subject: [PATCH 0539/2585] - moved the installation of cache server after the presentation server profile so that one ece-install process can install a presentation server first and then a cache server from the same ece-install.conf file. --- usr/sbin/ece-install | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index 92cef523..5294e5b0 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -955,13 +955,6 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi - if [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then - install_profile_number=$PROFILE_CACHE_SERVER - install_cache_server - install_web_server 0 - no_fai_profile=0 - fi - if [ $(get_boolean_conf_value fai_wf_install) -eq 1 ]; then install_profile_number=$PROFILE_WIDGET_FRAMEWORK install_widget_framework @@ -974,6 +967,13 @@ if [ $fai_enabled -eq 1 ]; then no_fai_profile=0 fi + if [ $(get_boolean_conf_value fai_cache_install) -eq 1 ]; then + install_profile_number=$PROFILE_CACHE_SERVER + install_cache_server + install_web_server 0 + no_fai_profile=0 + fi + if [ $(get_boolean_conf_value fai_publication_create) -eq 1 ]; then install_profile_number=$PROFILE_CREATE_PUBLICATION create_publication From 9248e09f5d31e0e257ba9ff1ded77511a81c8a6e Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Mon, 19 Mar 2012 12:43:05 +0800 Subject: [PATCH 0540/2585] - any method coming after install_memory_cache that used a shell command (not a BASH built-in), would get a PWD error because install_memory_cache removed the tmp directory in which it was CD-ed into, e.g.: [ece-install-850] Installing a caching server on staging ... sh: getcwd() failed: No such file or directory This had no practical implications, but was confusing nevertheless. This has now been fixed. --- usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh index f3a71e89..5c73fe3c 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/memory-cache.sh @@ -23,6 +23,7 @@ function install_memory_cache() make_dir ${escenic_root_dir}/assemblytool/lib run cp $name/$name.jar ${escenic_root_dir}/assemblytool/lib run rm -rf $tmp_dir + run cd ~/ memcached_set_up_common_nursery From c1096b533bfae100c2fd1da21eaf3f399361d917 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Mar 2012 11:36:32 +0800 Subject: [PATCH 0541/2585] - fix in get_privileged_hosts: if the host you connect from is on the server identified with a : Date: Tue, 20 Mar 2012 13:36:55 +0800 Subject: [PATCH 0542/2585] - fix in "ece info" in case the user doesn't have read access to the app server configuration files, "ece info" would produce a number of big errors due to xml_grep's verbosity. --- usr/bin/ece | 64 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 8f51bf89..9c19cb7a 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1058,36 +1058,46 @@ function print_tomcat_resources() { print "Application server resources:" - xml_grep --nowrap --cond Context/Environment \ - $tomcat_base/conf/context.xml | \ - sed 's/^[ \t]//g' | \ - sed "s#><#>\n<#g" | \ - cut -d' ' -f2,5 | \ - cut -d'"' -f2,4 | \ - while read line ; do - local key=$(echo $line | cut -d'"' -f1) - local value=$(echo $line | cut -d'"' -f2) - print "|->" ${key}: ${value} - done - + local file=$tomcat_base/conf/context.xml + if [ -r $file ]; then + xml_grep --nowrap --cond Context/Environment \ + $file | \ + sed 's/^[ \t]//g' | \ + sed "s#><#>\n<#g" | \ + cut -d' ' -f2,5 | \ + cut -d'"' -f2,4 | \ + while read line ; do + local key=$(echo $line | cut -d'"' -f1) + local value=$(echo $line | cut -d'"' -f2) + print "|->" ${key}: ${value} + done + else + print "|-> $USER cannot read $file" + fi + print "Database:" - xml_grep \ - --nowrap \ - --cond 'Server/GlobalNamingResources/Resource[@type="javax.sql.DataSource"]' \ - $tomcat_base/conf/server.xml | \ - sed 's/^[ \t]//g' | \ - sed "s#><#>\n<#g" | \ - while read line ; do - for el in $(echo $line | cut -d' ' -f1- ); do - if [[ $el == "name"* || $el == "url"* || $el == "username"* ]]; then - local key=$(echo $el | cut -d'"' -f1 | cut -d'=' -f1) - local value=$(echo $el | cut -d'"' -f2) - print "|->" ${key}: ${value} - fi + file=$tomcat_base/conf/server.xml + if [ -r $file ]; then + xml_grep \ + --nowrap \ + --cond 'Server/GlobalNamingResources/Resource[@type="javax.sql.DataSource"]' \ + $file | \ + sed 's/^[ \t]//g' | \ + sed "s#><#>\n<#g" | \ + while read line ; do + for el in $(echo $line | cut -d' ' -f1- ); do + if [[ $el == "name"* || $el == "url"* || $el == "username"* ]]; then + local key=$(echo $el | cut -d'"' -f1 | cut -d'=' -f1) + local value=$(echo $el | cut -d'"' -f2) + print "|->" ${key}: ${value} + fi + done done - done - + else + print "|-> $USER cannot read $file" + fi + } function restart_type() From 608288e1205d32064156262aab92840795171db0 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Mar 2012 13:38:09 +0800 Subject: [PATCH 0543/2585] - improved error string if user cannot read the configuration file --- usr/bin/ece | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/bin/ece b/usr/bin/ece index 9c19cb7a..cd51df77 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1072,7 +1072,7 @@ function print_tomcat_resources() { print "|->" ${key}: ${value} done else - print "|-> $USER cannot read $file" + print "|-> user $USER cannot read $file" fi print "Database:" @@ -1095,7 +1095,7 @@ function print_tomcat_resources() { done done else - print "|-> $USER cannot read $file" + print "|-> user $USER cannot read $file" fi } From 49314f84ceecadd8802570b0147f445b3ca82a71 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Mar 2012 15:14:23 +0800 Subject: [PATCH 0544/2585] - changed the header of the "ece versions" command (partly to make it better parseable by ece-overview. --- usr/bin/ece | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index cd51df77..fb847796 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1588,7 +1588,7 @@ function list_versions() version_manager=browser/Global/neo/io/managers/VersionManager url=$(get_escenic_admin_url)/$version_manager - print "Installed on the ${type} running on ${host}:${port} is:" + print "Installed on the ${instance} instance running on port ${port}:" wget $wget_auth -O - $url 2>/dev/null | \ grep "\[\[" | \ sed 's/\[//g' | \ From be3e79e0b326a05679c1d16afa9d4df201bcf223 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Mar 2012 15:21:40 +0800 Subject: [PATCH 0545/2585] - improved output from "ece info", creating better URIs for MySQL DBs. --- usr/bin/ece | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usr/bin/ece b/usr/bin/ece index fb847796..d819811e 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1089,6 +1089,10 @@ function print_tomcat_resources() { if [[ $el == "name"* || $el == "url"* || $el == "username"* ]]; then local key=$(echo $el | cut -d'"' -f1 | cut -d'=' -f1) local value=$(echo $el | cut -d'"' -f2) + if [[ $value == "jdbc:mysql"* ]]; then + value=${value:5} + fi + print "|->" ${key}: ${value} fi From bbca038fe43742be59723cc1acdb702bd250003a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Mar 2012 18:42:19 +0800 Subject: [PATCH 0546/2585] - new feature: ece info also lists deployed web applications, e.g.: [ece#engine-web1] Deployed web applications: [ece#engine-web1] |-> http://quanah:8080/solr [ece#engine-web1] |-> http://quanah:8080/webservice [ece#engine-web1] |-> http://quanah:8080/escenic [ece#engine-web1] |-> http://quanah:8080/inpage-ws [ece#engine-web1] |-> http://quanah:8080/ip [ece#engine-web1] |-> http://quanah:8080/escenic-admin --- usr/bin/ece | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/usr/bin/ece b/usr/bin/ece index d819811e..e619adfc 100755 --- a/usr/bin/ece +++ b/usr/bin/ece @@ -1039,17 +1039,31 @@ function get_info_for_type() print "|-> Tomcat home:" $tomcat_home print "|-> Tomcat base:" $tomcat_base print_tomcat_resources + print_deployed_webapps $tomcat_base fi ;; resin) if [ -n "${resin_home}" ]; then print " |-> Resin home:" $resin_home + print_deployed_webapps $resin_home fi ;; esac fi } +## $1: dir +function print_deployed_webapps() { + local webapps="" + + print "Deployed web applications:" + for el in $(find $1/webapps -maxdepth 1 -type d | \ + grep -v webapps$); do + print "|-> http://${HOSTNAME}:${port}/"$(basename $el) + done + +} + function print_tomcat_resources() { if [ "$(which xml_grep)x" == "x" ]; then log "Install xml_grep to get more 'ece info' details" @@ -1101,7 +1115,6 @@ function print_tomcat_resources() { else print "|-> user $USER cannot read $file" fi - } function restart_type() From b6da6b9d9f4caf703f8028c54002cee81400cc60 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Mar 2012 19:34:14 +0800 Subject: [PATCH 0547/2585] - added support for org, html and potentially other output formats, definable by the -f parameter. --- usr/bin/ece-overview | 218 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 usr/bin/ece-overview diff --git a/usr/bin/ece-overview b/usr/bin/ece-overview new file mode 100644 index 00000000..859967c4 --- /dev/null +++ b/usr/bin/ece-overview @@ -0,0 +1,218 @@ +#! /usr/bin/env bash + +## Script which creates an overview of system information related to +## the specified ECE/EAE/Search instance, example invocation: +## +## $ ece-overview -i web1 | xmllint --format - > /var/www/web1.html + +instance=dev1 + +## 1 : html +## 2 : org +format=html + +function create_header() { + local title="Overview of the $instance instance on $HOSTNAME" + + if [ $format == "org" ]; then + echo "* $title" + echo "Generated @ $(date)" + return + fi + + local header=" + + + $title @ $(date) + + + +$(cat ../share/escenic/ece-scripts/html-vizrt-logo-svg.html) +

    $title

    +

    Generated @ $(date)

    +
  • +EOF + fi +} + +function print_h2_header() { + if [ $format == "org" ]; then + echo "" + echo "** $@" + elif [ $format == "html" ]; then + echo "

    $@

    " + fi +} + +function print_h1_header() { + if [ $format == "org" ]; then + echo "* $@" + elif [ $format == "html" ]; then + echo "

    $@

    " + fi +} + +function print_p_text() { + if [ $format == "org" ]; then + echo "$@" + elif [ $format == "html" ]; then + cat < + $@ +

    +EOF + fi +} + +function list_hw() { + print_h2_header "Disks used on $HOSTNAME" + print_pre_text "$(df -hT)" + + print_h2_header "Memory on $HOSTNAME

    " + print_pre_text "$(free -m)" + + print_h2_header "CPU(s) on $HOSTNAME" + print_p_text "Number of CPUs:" $(grep "model name" /proc/cpuinfo | wc -l) + print_pre_text "$(grep "model name" /proc/cpuinfo | sort | uniq)" +} + + + +function list_services_running() { + cat < +

    Overview of all server services on $HOSTNAME

    +
    +$(netstat -nlp | egrep -v "tcp6|ACC" | grep LISTEN)
    +    
    +EOF + + if [ $(whoami) != "root" ]; then + echo "

    Run $(basename $0) as root to get more details in this listing

    " + fi +} + +function get_user_options() { + while getopts ":i:f:" opt; do + case $opt in + i) + instance=${OPTARG} + ;; + f) + format=${OPTARG} + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + exit 1 + ;; + esac + done + +} + +get_user_options $@ +create_header +do_create-overview +create_footer + + From 95b707abd1888a36a556eba19c5a6f2c4b57cd5a Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Tue, 20 Mar 2012 19:34:36 +0800 Subject: [PATCH 0548/2585] - added Vizrt SVG and CSS static files --- .../ece-scripts/html-vizrt-logo-svg.html | 44 ++++++++++ usr/share/escenic/ece-scripts/vizrt.css | 85 +++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 usr/share/escenic/ece-scripts/html-vizrt-logo-svg.html create mode 100644 usr/share/escenic/ece-scripts/vizrt.css diff --git a/usr/share/escenic/ece-scripts/html-vizrt-logo-svg.html b/usr/share/escenic/ece-scripts/html-vizrt-logo-svg.html new file mode 100644 index 00000000..c51f5b9a --- /dev/null +++ b/usr/share/escenic/ece-scripts/html-vizrt-logo-svg.html @@ -0,0 +1,44 @@ + diff --git a/usr/share/escenic/ece-scripts/vizrt.css b/usr/share/escenic/ece-scripts/vizrt.css new file mode 100644 index 00000000..ce7826bc --- /dev/null +++ b/usr/share/escenic/ece-scripts/vizrt.css @@ -0,0 +1,85 @@ +body { + font-family: Arial, Helvetica, sans-serif; + font-size: small; + #width : 600; + margin: 70px 70px 0px 50px; +} + +pre { + background-color: grey; +} + +h1, h2 { + color: #e88424; +} + +h1 { + font-size: 150%; +} + +h2 { + font-size: 120%; + margin-top: 3em; + margin-bottom: 0em; +} + +h3, h4, h5, h6 { + font-weight: bold; + margin-top: 2em; + margin-bottom: 0.2em; +} + +h3, h4, h5 { + font-size: 110%; +} + +h6 { + font-size: 80%; +} + +a { + color: #333; + font-weight: bold; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +p, dl { + font-size: 95%; +} + +p { + margin-top: 0em; + margin-bottom: 0.6em; +} + +pre.programlisting { + margin-top: 0em; + background-color: #ffddaa; + padding-top: 0.5em; + padding-bottom: 0.5em; + padding-left: 0.5em; + padding-right: 0.5em; +} + +em { + font-weight: bold; + font-style: normal; +} + +em.replaceable { + font-weight: normal; + font-style: italic; +} + +div.note { + background-color: #ffddaa; + padding-top: 0.5em; + padding-bottom: 0.5em; + padding-left: 0.5em; + padding-right: 0.5em; +} + From cbf7874b8d7d4f1ea0e267062971ca3f60d00511 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Wed, 21 Mar 2012 11:27:59 +0800 Subject: [PATCH 0549/2585] - moved (and will later merge) the new ece-overview to the system-info command as I realise the goals of the two are SO similar they ought to be the same. TODO make system-info pluggable/extendable. --- usr/bin/ece-overview | 218 --------------------------- usr/bin/system-info | 343 +++++++++++++++++++++++-------------------- 2 files changed, 184 insertions(+), 377 deletions(-) delete mode 100644 usr/bin/ece-overview diff --git a/usr/bin/ece-overview b/usr/bin/ece-overview deleted file mode 100644 index 859967c4..00000000 --- a/usr/bin/ece-overview +++ /dev/null @@ -1,218 +0,0 @@ -#! /usr/bin/env bash - -## Script which creates an overview of system information related to -## the specified ECE/EAE/Search instance, example invocation: -## -## $ ece-overview -i web1 | xmllint --format - > /var/www/web1.html - -instance=dev1 - -## 1 : html -## 2 : org -format=html - -function create_header() { - local title="Overview of the $instance instance on $HOSTNAME" - - if [ $format == "org" ]; then - echo "* $title" - echo "Generated @ $(date)" - return - fi - - local header=" - - - $title @ $(date) - - - -$(cat ../share/escenic/ece-scripts/html-vizrt-logo-svg.html) -

    $title

    -

    Generated @ $(date)

    -
      -" - echo "$header" -} - -function create_footer() { - if [ $format != "html" ]; then - return - fi - - local footer=" - - -" - echo "$footer" -} - -function wrap_in_anchor_if_applicable() { - if [ $format == "org" ]; then - echo "$@" - return - fi - - local result="" - for el in "$@"; do - if [[ $el == "/"* || $el == "http://"* || $el == "mysql://"* ]]; then - result="$result $el" - else - result="$result $el" - fi - - done - echo "$result" -} - -function print_un_ordered_list_start() { - if [ $format == "org" ]; then - echo "" - elif [ $format == "html" ]; then - echo "
        " - fi -} - -function print_un_ordered_list_end() { - if [ $format == "org" ]; then - echo "" - elif [ $format == "html" ]; then - echo "
      " - fi -} - -function print_list_item() { - if [ $format == "org" ]; then - echo "- $@" - elif [ $format == "html" ]; then - echo "
    • " \ - "$@" \ - "
    • " - fi -} - - -function do_create-overview() { - local data="$(ece -q -i $instance info)"$'\n' - data="$data $(ece -q -i $instance versions | cut -d'*' -f2-)" - - echo "$data" | while read line; do - if [[ $line == "|->"* ]]; then - print_list_item $(wrap_in_anchor_if_applicable ${line:3}) - elif [ $(echo $line | cut -d':' -f2- | wc -c) -gt 1 ]; then - print_list_item $(wrap_in_anchor_if_applicable $line) - else - print_un_ordered_list_end - print_h2_header $(echo $line | cut -d: -f1) - print_un_ordered_list_start - fi - - done - - list_services_running - list_hw -} - -function print_pre_text() { - if [ $format == "org" ]; then - cat < -$@ - -EOF - fi -} - -function print_h2_header() { - if [ $format == "org" ]; then - echo "" - echo "** $@" - elif [ $format == "html" ]; then - echo "

      $@

      " - fi -} - -function print_h1_header() { - if [ $format == "org" ]; then - echo "* $@" - elif [ $format == "html" ]; then - echo "

      $@

      " - fi -} - -function print_p_text() { - if [ $format == "org" ]; then - echo "$@" - elif [ $format == "html" ]; then - cat < - $@ -

      -EOF - fi -} - -function list_hw() { - print_h2_header "Disks used on $HOSTNAME" - print_pre_text "$(df -hT)" - - print_h2_header "Memory on $HOSTNAME

    " - print_pre_text "$(free -m)" - - print_h2_header "CPU(s) on $HOSTNAME" - print_p_text "Number of CPUs:" $(grep "model name" /proc/cpuinfo | wc -l) - print_pre_text "$(grep "model name" /proc/cpuinfo | sort | uniq)" -} - - - -function list_services_running() { - cat < -

    Overview of all server services on $HOSTNAME

    -
    -$(netstat -nlp | egrep -v "tcp6|ACC" | grep LISTEN)
    -    
    -EOF - - if [ $(whoami) != "root" ]; then - echo "

    Run $(basename $0) as root to get more details in this listing

    " - fi -} - -function get_user_options() { - while getopts ":i:f:" opt; do - case $opt in - i) - instance=${OPTARG} - ;; - f) - format=${OPTARG} - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 - ;; - esac - done - -} - -get_user_options $@ -create_header -do_create-overview -create_footer - - diff --git a/usr/bin/system-info b/usr/bin/system-info index 3df03f28..859967c4 100755 --- a/usr/bin/system-info +++ b/usr/bin/system-info @@ -1,193 +1,218 @@ #! /usr/bin/env bash -# Getting system info, especially useful before reporting to Escenic -# select Support. - -# can be: ascii, confluence -output_format=ascii -on_debian_or_derivate=0 -on_gentoo_or_derivate=0 -on_redhat_or_derivate=0 -on_linux=0 - -important_packages_on_debian=" -ant -apache2 -libapr1 -libmysql-java -libtcnative-1 -maven2 -mysql-server -nginx -percona-server-server -slapd -sun-java6-jdk -sun-java6-jre -tomcat6 -tomcat6-user -varnish +## Script which creates an overview of system information related to +## the specified ECE/EAE/Search instance, example invocation: +## +## $ ece-overview -i web1 | xmllint --format - > /var/www/web1.html + +instance=dev1 + +## 1 : html +## 2 : org +format=html + +function create_header() { + local title="Overview of the $instance instance on $HOSTNAME" + + if [ $format == "org" ]; then + echo "* $title" + echo "Generated @ $(date)" + return + fi + + local header=" + + + $title @ $(date) + + + +$(cat ../share/escenic/ece-scripts/html-vizrt-logo-svg.html) +

    $title

    +

    Generated @ $(date)

    + From c9931cf5678f7537c30653abb5e6c8987daee2aa Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 3 May 2012 16:00:29 +0200 Subject: [PATCH 0748/2585] - moved add_next_step to common-bashing - added new method to list the next steps (array), available in common-bashing - moved the add_apt_source to common-os --- usr/sbin/ece-install | 37 +------------------ .../escenic/ece-scripts/common-bashing.sh | 22 +++++++++++ usr/share/escenic/ece-scripts/common-os.sh | 12 ++++++ 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index a99024cb..bd10c6cc 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -75,11 +75,6 @@ force_packages=0 # ECE software components. ece_software_setup_completed=0 -# the next steps printed when the user has installed his/her -# components. -next_steps=() - - technet_download_list=" http://technet.escenic.com/downloads/assemblytool-2.0.2.zip http://technet.escenic.com/downloads/release/53/analysis-engine-2.3.7.0.zip @@ -329,10 +324,8 @@ function print_status_and_next_steps() add_next_step "Install info: "\ "/usr/share/doc/escenic/ece-install-guide.txt" add_next_step "Guide books: http://documentation.vizrt.com/ece-5.3.html" fi - - for (( i = 0; i < ${#next_steps[@]}; i++ )); do - print " - " ${next_steps[$i]} - done + + print_next_step_list print $'\n'"Enjoy your time with Escenic Content Engine!"$'\n' print "-$(red Vizrt) Online" @@ -605,32 +598,6 @@ function common_pre_install() set_up_user_environment } -function add_apt_source() { - local escenic_sources=/etc/apt/sources.list.d/escenic.list - if [[ ! -e $escenic_sources && - $(grep "$@" /etc/apt/sources.list | wc -l) -lt 1 ]]; then - echo "$@" >> $escenic_sources - run apt-get update - elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then - echo "$@" >> $escenic_sources - run apt-get update - fi -} - -# Parameters: -# $1 : your added line -function add_next_step() -{ - next_steps[${#next_steps[@]}]="$@" - return - - if [ -n "$next_steps" ]; then - next_steps=${next_steps}$'\n'"[$(basename $0)] "${1} - else - next_steps="[$(basename $0)] $@" - fi -} - # $1 is the default instance name, the calee is responsible for # setting this. function ask_for_instance_name() diff --git a/usr/share/escenic/ece-scripts/common-bashing.sh b/usr/share/escenic/ece-scripts/common-bashing.sh index 902f0524..2e7aca77 100644 --- a/usr/share/escenic/ece-scripts/common-bashing.sh +++ b/usr/share/escenic/ece-scripts/common-bashing.sh @@ -243,3 +243,25 @@ function extract_archive() { fi } +# the next steps printed when the user has installed his/her +# components. +next_steps=() + +## Parameters: +## $1 : your added line +function add_next_step() { + next_steps[${#next_steps[@]}]="$@" + return + + if [ -n "$next_steps" ]; then + next_steps=${next_steps}$'\n'"[$(basename $0)] "${1} + else + next_steps="[$(basename $0)] $@" + fi +} + +function print_next_step_list() { + for (( i = 0; i < ${#next_steps[@]}; i++ )); do + print " - " ${next_steps[$i]} + done +} diff --git a/usr/share/escenic/ece-scripts/common-os.sh b/usr/share/escenic/ece-scripts/common-os.sh index a0c0685a..6b3d8822 100644 --- a/usr/share/escenic/ece-scripts/common-os.sh +++ b/usr/share/escenic/ece-scripts/common-os.sh @@ -164,3 +164,15 @@ function get_total_memory_in_mega_bytes() { echo $(( $total_in_kb / 1024 )) fi } + +function add_apt_source() { + local escenic_sources=/etc/apt/sources.list.d/escenic.list + if [[ ! -e $escenic_sources && + $(grep "$@" /etc/apt/sources.list | wc -l) -lt 1 ]]; then + echo "$@" >> $escenic_sources + run apt-get update + elif [ $(grep "$@" $escenic_sources | wc -l) -lt 1 ]; then + echo "$@" >> $escenic_sources + run apt-get update + fi +} From 3cbe422d1b3d7fcd6bb835034773033220f3dca8 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 3 May 2012 16:07:17 +0200 Subject: [PATCH 0749/2585] - common_pre_install now sets up the Vizrt APT repository and installs escenic-common-scripts. - simplified the set up of the ece-scripts package in ece-scripts.sh --- usr/sbin/ece-install | 14 ++++++++++++++ .../ece-scripts/ece-install.d/ece-scripts.sh | 11 +---------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/usr/sbin/ece-install b/usr/sbin/ece-install index bd10c6cc..a9d7d025 100644 --- a/usr/sbin/ece-install +++ b/usr/sbin/ece-install @@ -211,6 +211,20 @@ function install_common_os_packages() for el in lsb_release curl wget git unzip; do assert_pre_requisite $el done + + # all hosts need the system-info package, hence we'll install the + # escenic-content-engine package here if on a Debian based + # system. If not, we'll just leave it, it'll be made available on + # ther systems through git. + if [ $on_debian_or_derivative -eq 1 ]; then + curl -s http://apt.vizrt.com/archive.key 2>> $log | \ + apt-key add - 1>> $log 2>> $log + local package_pool=${fai_apt_vizrt_pool-stable} + add_apt_source "deb http://apt.vizrt.com ${package_pool} main" + + install_packages_if_missing escenic-common-scripts + fi + } # don't quote values when setting conf file values with diff --git a/usr/share/escenic/ece-scripts/ece-install.d/ece-scripts.sh b/usr/share/escenic/ece-scripts/ece-install.d/ece-scripts.sh index 8a8d948c..7809dc33 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/ece-scripts.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/ece-scripts.sh @@ -2,15 +2,6 @@ # code for the ece-scripts themselves -function install_ece_scripts_with_apt() { - curl -s http://apt.vizrt.com/archive.key 2>> $log | \ - apt-key add - 1>> $log 2>> $log - - local package_pool=${fai_apt_vizrt_pool-stable} - add_apt_source "deb http://apt.vizrt.com ${package_pool} main" - install_packages_if_missing escenic-content-engine-scripts -} - function install_ece_scripts_with_git() { run cd $download_dir if [ -d ece-scripts ]; then @@ -50,7 +41,7 @@ function set_up_ece_scripts() print_and_log 'Setting up the ece UNIX scripts ...' if [ $on_debian_or_derivative -eq 1 ]; then - install_ece_scripts_with_apt + install_packages_if_missing escenic-content-engine-scripts else install_ece_scripts_with_git fi From f2d47e0e744f5929dbb726956cc71462f1261773 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 3 May 2012 16:22:01 +0200 Subject: [PATCH 0750/2585] - added remove_dir --- usr/share/escenic/ece-scripts/common-io.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/usr/share/escenic/ece-scripts/common-io.sh b/usr/share/escenic/ece-scripts/common-io.sh index c4c42b17..15370c28 100644 --- a/usr/share/escenic/ece-scripts/common-io.sh +++ b/usr/share/escenic/ece-scripts/common-io.sh @@ -67,6 +67,12 @@ function make_dir() { fi } +function remove_dir() { + if [ -d $1 ]; then + run rmdir $1 + fi +} + function make_ln() { if [ $2 ]; then if [ -e $1 -a ! -h $2 ]; then From df0cc9d09c5992f383dc615af1e2212206c570b5 Mon Sep 17 00:00:00 2001 From: Torstein Krause Johansen Date: Thu, 3 May 2012 16:37:13 +0200 Subject: [PATCH 0751/2585] - the nagios & icinga have been excluded from the summary for a while :( Fixed now --- usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh index 9a755cfc..cc36a2b5 100644 --- a/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh +++ b/usr/share/escenic/ece-scripts/ece-install.d/monitoring.sh @@ -312,10 +312,9 @@ function create_monitoring_server_overview()