#!/bin/sh
#
# Copyright (C) 1994-2018 Altair Engineering, Inc.
# For more information, contact Altair at www.altair.com.
#
# This file is part of the PBS Professional ("PBS Pro") software.
#
# Open Source License Information:
#
# PBS Pro is free software. You can redistribute it and/or modify it under the
# terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# PBS Pro is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Commercial License Information:
#
# For a copy of the commercial license terms and conditions,
# go to: (http://www.pbspro.com/UserArea/agreement.html)
# or contact the Altair Legal Department.
#
# Altair’s dual-license business model allows companies, individuals, and
# organizations to create proprietary derivative works of PBS Pro and
# distribute them - whether embedded or bundled with other software -
# under a commercial license agreement.
#
# Use of Altair’s trademarks, including but not limited to "PBS™",
# "PBS Professional®", and "PBS Pro™" and Altair’s logos is subject to Altair's
# trademark licensing policies.

if [ $# -eq 1 -a "$1" = "--version" ]; then
   echo pbs_version = 18.1.1
   exit 0
fi

PBS_VERSION=18.1.1
tmpdir=${PBS_TMPDIR:-${TMPDIR:-"/var/tmp"}}
PBS_CURRENT_SCHEMA_VER='1.3.0'
PBS_AES_SWITCH_VER='14.0'

get_db_user() {
	[ -f "${dbuser_fl}" ] && dbuser_name=`cat "${dbuser_fl}" | tr -d '[:space:]'`
	[ -z "${dbuser_name}" ] && dbuser_name="${PBS_DATA_SERVICE_USER:-postgres}"
	if [ ! -f "${dbuser_fl}" ]; then
		printf "%s" "$dbuser_name" >"${dbuser_fl}"
		chmod 0600 "${dbuser_fl}"
	fi
	cat "${dbuser_fl}"
	return $?
}

chk_dataservice_user() {

	chk_usr="$1"

	id=`id ${chk_usr} 2>&1`
	if [ $? -ne 0 ]; then
		echo "PBS Dataservice user ${chk_usr} does not exist"
		return 1;
	fi

	id=`echo ${id} | cut -c5- | cut -d "(" -f1`
	if [ "$id" = "0" ]; then
		echo "PBS Data Service user should not have root priviledges"
		return 1;
	fi

	# login as ${chk_usr} and try to cd to user home dir
	su - ${chk_usr} -c "cd" > /dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable to login as user ${chk_usr}. Is the user enabled/home directory accessible?"
		return 1
	fi
	return 0
}

check_switch() {
	adapters=`ntblstatus 2> /dev/null | awk '/^Adapter/ {print $2}' | uniq`
	if [ -n "$adapters" ]; then
		switch="HPS"
		return 0
	fi
	adapters=`ibstat 2> /dev/null | \
			awk '/INFINIBAND DEVICE INFORMATION/ {print $NF}'`
	if [ -n "$adapters" ]; then
		switch="InfiniBand"
		return 0
	fi
	echo "*** INFO: No High Performance Network detected"
	switch=""
	return 1
}

is_cray_xt() {
	if [ -f /proc/cray_xt/cname ] ; then
		return 0
	fi
	return 1
}

set_db_trust_login() {
	datastore_dir=$1
	err=`cp -p ${datastore_dir}/pg_hba.conf ${datastore_dir}/pg_hba.conf.orig 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`chown ${PBS_DATA_SERVICE_USER} ${datastore_dir}/pg_hba.conf.orig`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`sed 's/md5/trust/g' ${datastore_dir}/pg_hba.conf > ${datastore_dir}/pg_hba.conf.new 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`chown ${PBS_DATA_SERVICE_USER} ${datastore_dir}/pg_hba.conf.new`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`mv ${datastore_dir}/pg_hba.conf.new ${datastore_dir}/pg_hba.conf 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
}

revoke_db_trust_login() {
	datastore_dir=$1
	err=`cp -p ${datastore_dir}/pg_hba.conf.orig ${datastore_dir}/pg_hba.conf 2>&1`
	if [ $? -eq 0 ]; then
		rm -f ${datastore_dir}/pg_hba.conf.orig
	else
		echo "${err}"
		return 1
	fi
}

utf8_to_sql_ascii() {
	#
	# If the existing database has UTF-8 character encoding,
	# change the encoding to SQL_ASCII for database upgrade.
	#

	${PBS_EXEC}/sbin/pbs_dataservice status > /dev/null
	if [ $? -eq 0 ]; then
		# stop the dataservice
		${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null
		if [ $? -ne 0 ]; then
			echo "Error stopping PBS Data Service"
			kill -s SIGTERM `ps -ef | grep "${inst_dir}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
			return 1
		fi
	fi

	set_db_trust_login "${PBS_HOME}/datastore"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	# restart with new credentials
	# redirect the output,
	# Use old pgsql binaries since we
	# are working with old database.
	#

	PGSQL_INST_DIR="${PBS_HOME}/pgsql.old"
	export PGSQL_INST_DIR

	${PBS_EXEC}/sbin/pbs_dataservice start > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error starting PBS Data Service"
		revoke_db_trust_login "${PBS_HOME}/datastore"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		return 1
	fi

	enc_type=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "select pg_encoding_to_char(encoding) from pg_database where datname = 'pbs_datastore'"`
	if [ ! -z "${enc_type}" -a "${enc_type}" = "UTF8" ]; then
		db_list=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "select datname from pg_database"`
		if [ $? -eq 0 ]; then
			for db_entry in ${db_list}
			do
				update=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "update pg_database set encoding = pg_char_to_encoding('SQL_ASCII') where datname = '$db_entry'"`
				ret=$?
				if [ $ret -ne 0 ]; then
					return $ret
				fi
			done
		fi
	fi

	locale=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "SHOW LC_COLLATE"`
	if [ ! -z ${locale} -a ${locale} != "C" ]; then
		change_locale=1
	fi

	# stop the dataservice
	${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error stopping PBS Data Service"
		kill -s SIGTERM `ps -ef | grep "${PGSQL_INST_DIR}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
		return 1
	fi

	unset PGSQL_INST_DIR

	revoke_db_trust_login "${PBS_HOME}/datastore"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	return 0
}

upgrade_pbs_database() {

	user="${PBS_DATA_SERVICE_USER}"
	inst_dir="${PGSQL_DIR}"
	data_dir="${PBS_HOME}/datastore"
	old_inst_dir="${PBS_HOME}/pgsql.old"
	old_data_dir="${PBS_HOME}/datastore.old"

	if [ ! -f "${data_dir}/PG_VERSION" ]; then
		echo "Database version file: ${data_dir}/PG_VERSION not found, cannot continue"
		return 1
	fi
	
	sys_pgsql_ver=$(echo `${PGSQL_DIR}/bin/psql -V` | awk 'NR==1 {print $NF}' | cut -d '.' -f 1,2)
	old_pgsql_ver=`cat ${data_dir}/PG_VERSION`

	[ ${sys_pgsql_ver%.*} -eq ${old_pgsql_ver%.*} ] && [ ${sys_pgsql_ver#*.} \> ${old_pgsql_ver#*.} ] || [ ${sys_pgsql_ver%.*} -gt ${old_pgsql_ver%.*} ];
	result=$?
	if [ ${result} -eq 0 ]; then
		#Start upgrade process of datastore
		upgrade_db
		return $?
	elif [ "${old_pgsql_ver}" = "${sys_pgsql_ver}" ]; then
		return 0
	fi

	echo "Upgrade from version ${old_pgsql_ver} unsupported"
	return 1
}

upgrade_db() {
	#
	# This routine will insatll a 9.3 database cluster,
	# will perform the pre-upgrade checks for datastore
	# with appropriate authentication management.
	#

	change_locale=0
	server_ctl="${PBS_EXEC}/sbin/pbs_dataservice"
	if [ ! -x "${server_ctl}" ]; then
		echo "${server_ctl} not found"
		return 1
	fi

	if [ ! -x "${PBS_EXEC}/sbin/pbs_ds_password" ]; then
		echo "${PBS_EXEC}/sbin/pbs_ds_password not found"
		return 1
	fi

	if [ ! -x "${inst_dir}/bin/pg_upgrade" ]; then
		echo "${inst_dir}/bin/pg_upgrade not found"
		return 1
	fi

	# Check for existence of old data service directory.
	if [ ! -d "${old_inst_dir}" ]; then
		echo "Data service directory from previous PBS installation not found,"
		echo "Datastore upgrade cannot continue"
		return 1
	fi

	# Check and convert UTF-8 character encoded database to SQL_ASCII
	utf8_to_sql_ascii
	if [ $? -ne 0 ]; then
		return 1
	fi

	# Backup datastore directory, if backup directory already
	# present then exit.
	if [ -d "${old_data_dir}" ]; then
		echo "Files from previous datastore upgrade found,"
		echo "Datastore upgrade cannot continue"
		return 1
	else
		err=`mv ${data_dir} ${old_data_dir} 2>&1`
		if [ $? -ne 0 ]; then
			echo "${err}"
			return 1
		fi
	fi

	# Invoke the dataservice creation script for pbs
	if [ ! -x "${PBS_EXEC}/libexec/install_db" ]; then
		echo "${PBS_EXEC}/libexec/install_db not found"
		return 1
	fi

	resp=`${PBS_EXEC}/libexec/install_db "upgrade" ${change_locale} 2>&1`
	ret=$?

	if [ ${ret} -ne 0 ]; then
		echo "*** Error initializing the PBS dataservice"
		echo "Error details:"
		echo "$resp"
		return ${ret}
	fi

	# Copy the pg_hba.conf from old cluster.
	err=`cp -p ${old_data_dir}/pg_hba.conf ${data_dir}/pg_hba.conf`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi

	set_db_trust_login "${data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi
	set_db_trust_login "${old_data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	CWD=`pwd`
	cd "${data_dir}"
	#Perform pg_upgrade -c to check if we can upgrade the cluster or not
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${inst_dir}/bin/pg_upgrade -b ${old_inst_dir}/bin -B ${inst_dir}/bin -d ${old_data_dir} -D ${data_dir} -c'" 2>&1`
	if [ $? -ne 0 ]; then
		echo "Refer pg_upgrade log files at $PBS_HOME/datastore/pg_upgrade_internal.log,"
		echo "$PBS_HOME/datastore/pg_upgrade_server.log and"
		echo "$PBS_HOME/datastore/pg_upgrade_utility.log for more information"
		revoke_db_trust_login "${data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		revoke_db_trust_login "${old_data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		return 1
	fi

	#Perform pg_upgrade for database upgrade
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${inst_dir}/bin/pg_upgrade -b ${old_inst_dir}/bin -B ${inst_dir}/bin -d ${old_data_dir} -D ${data_dir}'" 2>&1`
	if [ $? -ne 0 ]; then
		echo "Refer pg_upgrade log files at $PBS_HOME/datastore/pg_upgrade_internal.log,"
		echo "$PBS_HOME/datastore/pg_upgrade_server.log and"
		echo "$PBS_HOME/datastore/pg_upgrade_utility.log for more information"
		revoke_db_trust_login "${data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		revoke_db_trust_login "${old_data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		return 1
	fi

	# start the dataservice
	${server_ctl} start > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error starting PBS Data Service"
		return 1
	fi

	# Optimizer statistics are not transferred by pg_upgrade, so do it manually.
	ENVSTR="PGPORT=${PBS_DATA_SERVICE_PORT}; export PGPORT; PGHOST=${PBS_SERVER}; export PGHOST; PGUSER=${user}; export PGUSER; "
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${ENVSTR} ${data_dir}/analyze_new_cluster.sh'"`

	# Update locale of pbs database to C
	if [ ${change_locale} -eq 1 ]; then
		${inst_dir}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${user} -c "update pg_database set datcollate='C', datctype='C'" > /dev/null
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
	fi

	# stop the dataservice
	${server_ctl} stop > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error stopping PBS Data Service"
		kill -s SIGTERM `ps -ef | grep "${inst_dir}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
		return 1
	fi
	revoke_db_trust_login "${data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	cd "${CWD}"
	# Delete old cluster
	err=`${data_dir}/delete_old_cluster.sh`
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi
}

upgrade_pbs_schema() {
	#
	# pbs_dataservice command now has more diagnostic output.
	# It can tell why it could not start, for example, that
	# dataservice might be running on another host.
	# So capture pbs_dataservice output and display in case of
	# errors during starting.
	#
	outfile="${tmpdir}/pbs_dataservice_output_$$"

	${PBS_EXEC}/sbin/pbs_dataservice status > /dev/null
	if [ $? -eq 0 ]; then
		# running stop now
		${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null
		if [ $? -ne 0 ]; then
			echo "Failed to stop PBS Dataservice"
			return 1
		fi
	fi

	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	# restart with new credentials
	# redirect the output, dont execute inside `` since
	# postgres processes would otherwise cause the script
	# to hang forever
	#
	${PBS_EXEC}/sbin/pbs_dataservice start > ${outfile}
	if [ $? -ne 0 ]; then
		cat ${outfile}
		rm -f ${outfile}
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		return 1
	fi
	rm -f ${outfile}

	ver=`${PGSQL_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "select pbs_schema_version from pbs.info"`
	if [ "$ver" = "${PBS_CURRENT_SCHEMA_VER}" ]; then
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		return 0
	fi

	if [ "$ver" = "1.0.0" ]; then
		upgrade_pbs_schema_from_v1_0_0
		ret=$?
		if [ $ret -ne 0 ]; then
			ret=$?
			if [ $ret -ne 0 ]; then
				return $ret
			fi
			return 1
		fi
		ver="1.1.0"
	fi

	${PBS_EXEC}/sbin/pbs_dataservice status > /dev/null
	if [ $? -eq 1 ]; then
		# not running, start now
		${PBS_EXEC}/sbin/pbs_dataservice start > ${outfile}
		if [ $? -ne 0 ]; then
			cat ${outfile}
			rm -f ${outfile}
			ret=$?
			if [ $ret -ne 0 ]; then
				return $ret
			fi
			return 1
		fi
		rm -f ${outfile}
	fi

	if [ "$ver" = "1.1.0" ]; then
		upgrade_pbs_schema_from_v1_1_0
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		ver="1.2.0"
	fi
	
	if [ "$ver" = "1.2.0" ]; then
		upgrade_pbs_schema_from_v1_2_0
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		ver="1.3.0"
	else
		echo "Cannot upgrade PBS datastore version $ver"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		return 1
	fi
	
	${PBS_EXEC}/sbin/pbs_dataservice status > /dev/null
	if [ $? -eq 1 ]; then
		# not running, start now
		${PBS_EXEC}/sbin/pbs_dataservice start > ${outfile}
		if [ $? -ne 0 ]; then
			cat ${outfile}
			rm -f ${outfile}
			ret=$?
			if [ $ret -ne 0 ]; then
				return $ret
			fi
			return 1
		fi
	fi
	rm -f ${outfile}

	${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null

	ret=$?
	return $ret
}


upgrade_pbs_schema_from_v1_0_0() {

	${PGSQL_DIR}/bin/psql -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} <<-EOF > /dev/null
		alter table pbs.node alter column nd_index drop default;
		alter table pbs.node drop constraint node_nd_index_key;
		create index job_src_idx on pbs.job_scr (ji_jobid);
		drop sequence pbs.node_sequence;

		\set PBS_MAXHOSTNAME	'64'
		create sequence pbs.svr_id_seq;
		alter table pbs.server add column sv_hostname varchar(:PBS_MAXHOSTNAME);
		update pbs.server set sv_hostname = sv_name;
		alter table pbs.server alter column sv_hostname set not null;
	EOF
	ret=$?
	if [ $ret -ne 0 ]; then
		echo "Some datastore transformations failed to complete"
		echo "Please check dataservice logs"
		return $ret
	fi

	${PBS_EXEC}/sbin/pbs_server -t updatedb > /dev/null
	ret=$?
	if [ $ret -ne 0 -o -f ${PBS_HOME}/server_priv/serverdb ] ; then
		echo "*** Error during overlay upgrade"
		return $ret
	fi
}

upgrade_pbs_schema_from_v1_1_0() {

	${PGSQL_DIR}/bin/psql -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} <<-EOF > /dev/null
		ALTER TABLE pbs.info ALTER COLUMN pbs_schema_version TYPE text;
		ALTER TABLE pbs.job ALTER COLUMN ji_jobid TYPE text;
		ALTER TABLE pbs.job ALTER COLUMN ji_sv_name TYPE text;
		ALTER TABLE pbs.job ALTER COLUMN ji_queue TYPE text;
		ALTER TABLE pbs.job ALTER COLUMN ji_destin TYPE text;
		ALTER TABLE pbs.job ALTER COLUMN ji_4jid TYPE text;
		ALTER TABLE pbs.job ALTER COLUMN ji_4ash TYPE text;
		ALTER TABLE pbs.job_attr ALTER COLUMN ji_jobid TYPE text;
		ALTER TABLE pbs.job_attr ALTER COLUMN attr_name TYPE text;
		ALTER TABLE pbs.job_attr ALTER COLUMN attr_resource TYPE text;
		ALTER TABLE pbs.job_scr ALTER COLUMN ji_jobid TYPE text;
		ALTER TABLE pbs.node ALTER COLUMN nd_name TYPE text;
		ALTER TABLE pbs.node ALTER COLUMN nd_hostname TYPE text;
		ALTER TABLE pbs.node ALTER COLUMN nd_pque TYPE text;
		ALTER TABLE pbs.node_attr ALTER COLUMN nd_name TYPE text;
		ALTER TABLE pbs.node_attr ALTER COLUMN attr_name TYPE text;
		ALTER TABLE pbs.node_attr ALTER COLUMN attr_resource TYPE text;
		ALTER TABLE pbs.queue ALTER COLUMN qu_name TYPE text;
		ALTER TABLE pbs.queue ALTER COLUMN qu_sv_name TYPE text;
		ALTER TABLE pbs.queue_attr ALTER COLUMN qu_name TYPE text;
		ALTER TABLE pbs.queue_attr ALTER COLUMN attr_name TYPE text;
		ALTER TABLE pbs.queue_attr ALTER COLUMN attr_resource TYPE text;
		ALTER TABLE pbs.resv ALTER COLUMN ri_resvid TYPE text;
		ALTER TABLE pbs.resv ALTER COLUMN ri_sv_name TYPE text;
		ALTER TABLE pbs.resv ALTER COLUMN ri_queue TYPE text;
		ALTER TABLE pbs.resv_attr ALTER COLUMN ri_resvid TYPE text;
		ALTER TABLE pbs.resv_attr ALTER COLUMN attr_name TYPE text;
		ALTER TABLE pbs.resv_attr ALTER COLUMN attr_resource TYPE text;
		ALTER TABLE pbs.scheduler ALTER COLUMN sched_name TYPE text;
		ALTER TABLE pbs.scheduler ALTER COLUMN sched_sv_name TYPE text;
		ALTER TABLE pbs.scheduler_attr ALTER COLUMN sched_name TYPE text;
		ALTER TABLE pbs.scheduler_attr ALTER COLUMN attr_name TYPE text;
		ALTER TABLE pbs.scheduler_attr ALTER COLUMN attr_resource TYPE text;
		ALTER TABLE pbs.server ALTER COLUMN sv_name TYPE text;
		ALTER TABLE pbs.server ALTER COLUMN sv_hostname TYPE text;
		ALTER TABLE pbs.server_attr ALTER COLUMN sv_name TYPE text;
		ALTER TABLE pbs.server_attr ALTER COLUMN attr_name TYPE text;
		ALTER TABLE pbs.server_attr ALTER COLUMN attr_resource TYPE text;
		ALTER TABLE pbs.subjob_track ALTER COLUMN ji_jobid TYPE text;
		alter table pbs.job drop constraint job_pkey cascade;
		create index ji_jobid_idx on pbs.job (ji_jobid);
		drop index pbs.job_attr_idx;
		create index job_attr_idx on pbs.job_attr (ji_jobid, attr_name, attr_resource);
		alter table pbs.subjob_track drop constraint subjob_track_pkey cascade;
		create index subjob_jid_idx on pbs.subjob_track (ji_jobid, trk_index);
		update pbs.info set pbs_schema_version = '1.2.0';
	EOF
	ret=$?
	if [ $ret -ne 0 ]; then
		echo "Some datastore transformations failed to complete"
		echo "Please check dataservice logs"
		return $ret
	fi
}

upgrade_pbs_schema_from_v1_2_0() {

	${PGSQL_DIR}/bin/psql -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} <<-EOF > /dev/null
		DROP TABLE pbs.subjob_track;
		INSERT INTO pbs.scheduler (sched_name, sched_sv_name, sched_savetm, sched_creattm) VALUES ('default', '1', localtimestamp, localtimestamp);
		ALTER TABLE pbs.scheduler_attr DROP CONSTRAINT scheduler_attr_fk;
		UPDATE pbs.scheduler_attr SET sched_name='default' WHERE sched_name='1';
		ALTER TABLE pbs.scheduler_attr ADD CONSTRAINT scheduler_attr_fk
			FOREIGN KEY (sched_name)
			REFERENCES pbs.scheduler (sched_name)
			ON DELETE CASCADE
			ON UPDATE NO ACTION
			NOT DEFERRABLE;
		DELETE FROM pbs.scheduler WHERE sched_name='1';
		UPDATE pbs.info SET pbs_schema_version = '1.3.0';
	EOF
	ret=$?
	if [ $ret -ne 0 ]; then
		echo "Some datastore transformations failed to complete"
		echo "Please check dataservice logs"
		return $ret
	fi
}

chkenv() {
	line=`grep -s "^${1}=" $envfile`
	if [ -z "$line" ]; then
		echo "*** setting ${1}=$2"
		echo "${1}=$2" >> $envfile
	else
		echo "*** leave existing $line"
	fi
}

createdir() {
	if [ -n "$1" -a ! -d "$1" ]; then
		if ! mkdir -p "$1"; then
			echo "*** Could not create $1"
			exit 1
		fi
	fi
	if [ -n "$1" -a -n "$2" ]; then
		chmod "$2" "$1"
	fi
}

createpath() {
	while read mode dir ;do
		createdir "${PBS_HOME}/${dir}" $mode
	done
}

# Return the name of the PBS server host
get_server_hostname() {
	shn=""
	if [ -z "${PBS_PRIMARY}" -o -z "${PBS_SECONDARY}" ] ; then
		if [ -z "${PBS_SERVER_HOST_NAME}" ]; then
			shn="${PBS_SERVER}"
		else
			shn="${PBS_SERVER_HOST_NAME}"
		fi
	else
		shn="${PBS_PRIMARY}"
	fi
	echo ${shn} | awk '{print tolower($0)}'
}

# Ensure the supplied hostname is valid
check_hostname() {
	# Check the hosts file
	getent hosts "${1}" >/dev/null 2>&1 && return 0
	# Check DNS
	host "${1}" >/dev/null 2>&1 && return 0
	return 1
}

#
# Start of the pbs_habitat script
#
conf=${PBS_CONF_FILE:-/etc/pbs.conf}
ostype=`uname 2>/dev/null`
umask 022

echo "***"

# Source pbs.conf to get paths: PBS_EXEC and PBS_HOME and PBS_START_*
. $conf

# Ensure certain variables are set
if [ -z "$PBS_EXEC" ]; then
	echo "*** PBS_EXEC is not set"
	exit 1
fi
if [ ! -d "$PBS_EXEC" ]; then
	echo "*** $PBS_EXEC directory does not exist"
	exit 1
fi
if [ -z "$PBS_HOME" ]; then
	echo "*** PBS_HOME is not set"
	exit 1
fi
if [ ! -d "$PBS_HOME" ]; then
	echo "*** WARNING: PBS_HOME not found in $PBS_HOME"
	if [ -x "$PBS_EXEC/sbin/pbs_server" ]; then
		component="server"
	elif [ -x "$PBS_EXEC/sbin/pbs_mom" ]; then
		component="execution"
	else
		echo "*** No need to execute pbs_habitat in client installation."
		exit 0
	fi
	${PBS_EXEC}/libexec/pbs_postinstall $component $PBS_VERSION $PBS_EXEC $PBS_HOME "" "sameconf"
fi

# Store the old PBS VERSION for later use
if [ -f "$PBS_HOME/pbs_version" ]; then
	old_pbs_version=`cat $PBS_HOME/pbs_version`
fi

# Get the current PBS Pro version from qstat
if [ -x "$PBS_EXEC/bin/qstat" ]; then
	pbs_version=`"${PBS_EXEC}/bin/qstat" --version | sed -e 's/^.* = //'`
	if [ -z "$pbs_version" ]; then
		echo "*** Could not obtain PBS Pro version from qstat"
		exit 1
	fi
	if [ "$pbs_version" != "$PBS_VERSION" ]; then
		echo "*** Version mismatch."
		echo "*** Build version is $PBS_VERSION"
		echo "*** qstat version is $pbs_version"
		exit 1
	fi
else
	echo "*** File not found: $PBS_EXEC/bin/qstat"
	exit 1
fi

# Perform sanity check on server name in pbs.conf
server_hostname=`get_server_hostname`
[ "$server_hostname" = 'change_this_to_pbs_pro_server_hostname' ] && server_hostname=''
if [ -z "${server_hostname}" ] ; then
	echo "***" >&2
	echo "*** The hostname of the PBS Pro server in ${conf} is invalid." >&2
	echo "*** Update the configuration file before starting PBS Pro." >&2
	echo "***" >&2
	exit 1
fi
check_hostname "${server_hostname}"
if [ $? -ne 0 ]; then
	echo "***" >&2
	echo "*** The PBS Pro server could not be found: $server_hostname" >&2
	echo "*** This value must resolve to a valid IP address." >&2
	echo "***" >&2
	exit 1
fi
server=`echo ${server_hostname} | awk -F. '{print $1}'`

# Check the clienthost entries if present
if [ -f "$PBS_HOME/mom_priv/config" ]; then
	cmd='cat "$PBS_HOME/mom_priv/config"'
	cmd="$cmd | sed -e 's/\t/ /g' -e 's/ \\+/ /g' -e 's/^ //' -e 's/ $//'"
	cmd="$cmd | grep '\\\$clienthost' | cut -d' ' -f2"
	for host in `eval $cmd`; do
		if [ -z "$host" -o "$host" = 'CHANGE_THIS_TO_PBS_PRO_SERVER_HOSTNAME' ]; then
			echo "***" >&2
			echo "*** Invalid entry in $PBS_HOME/mom_priv/config" >&2
			echo "*** for clienthost: $host" >&2
			echo "***" >&2
			exit 1
		fi
		if ! check_hostname "${host}"; then
			echo "***" >&2
			echo "*** Invalid entry in $PBS_HOME/mom_priv/config" >&2
			echo "*** for clienthost: $host" >&2
			echo "*** This value must resolve to a valid IP address." >&2
			echo "***" >&2
			exit 1
		fi
	done
fi

if [ "${PBS_START_SERVER:-0}" != 0 ] ; then
	# Source the file that sets PGSQL_LIBSTR
	. "$PBS_EXEC"/libexec/pbs_pgsql_env.sh

	PBS_licensing_loc_file=PBS_licensing_loc

	# On some platforms LD_LIBRARY_PATH etc is not passed on after su
	# so we set it again after the su. PGSQL_LIBSTR contains the line to
	# set the variable again
	export PGSQL_LIBSTR

	dbuser_fl="${PBS_HOME}/server_priv/db_user"
	PBS_DATA_SERVICE_USER=`get_db_user`
	if [ $? -ne 0 ]; then
		echo "Could not retrieve PBS Data Service User"
		exit 1
	fi

	# Do not export the PBS_DATA_SERVICE_USER as a env var
	# since that would cause a false warning message
	# that "deprecated" variable PBS_DATA_SERVICE_USER is
	# being ignored.

	chk_dataservice_user ${PBS_DATA_SERVICE_USER}
	if [ $? -ne 0 ]; then
		exit 1
	fi

	# create the database user file if it does not exist
	if [ ! -f "${dbuser_fl}" ]; then
		printf "${PBS_DATA_SERVICE_USER}" >"${dbuser_fl}"
		chmod 0600 "${dbuser_fl}"
	fi

	server_started=0
	PBS_DATA_SERVICE_PORT=${PBS_DATA_SERVICE_PORT:-"15007"}
	export PBS_DATA_SERVICE_PORT

	create_new_svr_data=1

	# invoke the dataservice creation script for pbs
	if [ ! -x "${PBS_EXEC}/libexec/install_db" ]; then
		echo "${PBS_EXEC}/libexec/install_db not found"
		exit 1
	fi
	resp=`${PBS_EXEC}/libexec/install_db 2>&1`
	ret=$?
	if [ $ret -eq 2 ]; then
		create_new_svr_data=0
	elif [ $ret -ne 0 ]; then
		echo "*** Error initializing the PBS dataservice"
		echo "Error details:"
		echo "$resp"
		exit $ret
	fi

	export PBS_HOME
	export PBS_EXEC
	export PBS_SERVER
	export PBS_ENVIRONMENT

	if [ $create_new_svr_data -eq 0 ]; then
		# datastore directory already exists
		# do database upgrade activities
		upgrade_pbs_database
		ret=$?
		if [ $ret -ne 0 ]; then
			echo "Failed to upgrade PBS Datastore"
			exit $ret
		else
			#Remove old dataservice directory
			if [ `echo "${old_inst_dir}"| grep "pgsql.old"` ]; then
				err=`rm -rf "${old_inst_dir}"`
				if [ $? -ne 0 ]; then
					echo "$err"
				fi
			fi
		fi
		# do schema upgrade activities
		set_db_trust_login "${PBS_HOME}/datastore"
		upgrade_pbs_schema
		ret=$?
		if [ $ret -ne 0 ]; then
			revoke_db_trust_login "${PBS_HOME}/datastore"
			echo "Failed to upgrade PBS Datastore"
			exit $ret
		fi
		# We need to regenerate the db_password file since we have changed encryption/decryption
		# library from DES to AES in PBS Version PBS_AES_SWITCH_VER
		if [ "$old_pbs_version" \< "${PBS_AES_SWITCH_VER}" ] ;then
			rm -f  "${PBS_HOME}/server_priv/db_password"
			err=`${PBS_EXEC}/sbin/pbs_ds_password -r`
			if [ $? -ne 0 ]; then
				echo $err
				echo "Error setting password for PBS Data Service"
				${server_ctl} stop > /dev/null 2>&1
				revoke_db_trust_login "${PBS_HOME}/datastore"
				exit 1
			fi
		fi
		revoke_db_trust_login "${PBS_HOME}/datastore"
	else
		# Upgrade from FS version to DB
		if [ -f ${PBS_HOME}/server_priv/serverdb ] ; then
			# migrate filesystem data into db version
			${PBS_EXEC}/sbin/pbs_server -t updatedb > /dev/null
			ret=$?
			if [ $ret -ne 0 -o -f ${PBS_HOME}/server_priv/serverdb ] ; then
				echo "*** Error during overlay upgrade"
				exit $ret
			fi
			create_new_svr_data=0
		fi
	fi

	if [ $create_new_svr_data -eq 1 ] ; then
		echo "*** Setting default queue and resource limits."
		echo "***"

		${PBS_EXEC}/sbin/pbs_server -t create -a 0 > /dev/null
		ret=$?
		if [ $ret -ne 0 ]; then
			echo "*** Error starting pbs server"
			exit $ret
		fi
		server_started=1
		if is_cray_xt; then
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				set server restrict_res_to_release_on_suspend = ncpus
			EOF
		fi
		${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
			create queue workq
			set queue workq queue_type = Execution
			set queue workq enabled = True
			set queue workq started = True
			set server default_queue = workq
			set server log_events = 511
			set server mail_from = adm
			set server query_other_jobs = True
			set server scheduler_iteration = 600
			set server resources_default.ncpus = 1
			set server scheduling = True
		EOF
		if [ -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file ]; then
			read ans < ${PBS_HOME}/server_priv/$PBS_licensing_loc_file
			echo "*** Setting license file location(s)."
			echo "***"
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				set server pbs_license_info = $ans
			EOF
			if ! is_cray_xt; then
				rm -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file	# clean up after INSTALL
			fi
		fi
	else
		# the upgrade case:  serverdb already exists, but license file
		# information is new
		if [ -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file ]; then
			read ans < ${PBS_HOME}/server_priv/$PBS_licensing_loc_file
			echo "*** Setting license file location(s)."
			echo "***"
			${PBS_EXEC}/sbin/pbs_server > /dev/null
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				unset server pbs_license_file_location
				set server pbs_license_info = $ans
			EOF
			if ! is_cray_xt; then
				rm -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file	# clean up after INSTALL
			else
				${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
					set server restrict_res_to_release_on_suspend += ncpus
				EOF
			fi
			server_started=1
		fi
	fi

	if [ $PBS_START_MOM != 0 ]; then
		if [ $server_started -eq 0 ]; then
			${PBS_EXEC}/sbin/pbs_server > /dev/null
			server_started=1
		fi

		if ${PBS_EXEC}/bin/pbsnodes localhost > /dev/null 2>&1 ||
		   ${PBS_EXEC}/bin/pbsnodes $server > /dev/null 2>&1; then
			:
		else
			# node $server is not already available, create
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				create node $server
			EOF
		fi
	fi

	if [ $server_started -eq 1 ]; then
		${PBS_EXEC}/bin/qterm
	fi
fi

#
# For overlay upgrades PBS_START_MOM will be disabled per the install
# instructions. There may still be job and task files present that
# require updating.
#
if [ -d ${PBS_HOME}/mom_priv/jobs ]; then
	upgrade_cmd="${PBS_EXEC}/sbin/pbs_upgrade_job"
	if [ -x ${upgrade_cmd} ]; then
		total=0
		upgraded=0
		for file in ${PBS_HOME}/mom_priv/jobs/*.JB; do
			if [ -f ${file} ]; then
				${upgrade_cmd} -f ${file}
				if [ $? -ne 0 ]; then
					echo "Failed to upgrade ${file}"
				else
					upgraded=`expr ${upgraded} + 1`
				fi
				total=`expr ${total} + 1`
			fi
		done
		if [ ${total} -gt 0 ]; then
			echo "Upgraded ${upgraded} of ${total} job files."
		fi
	else
		echo "WARNING: $upgrade_cmd not found!"
	fi
fi

# Update the version file at the very end, after everything else succeeds.
# This allows it to be re-run, in case a previous update attempt failed.
# Clobber the existing pbs_version file and populate it with current version.
# This will prevent updating PBS_HOME if this same version is re-installed
#
echo "${pbs_version}" >"$PBS_HOME/pbs_version"

echo "*** End of ${0}"
exit 0
