#!/bin/bash
#############################################################
# Name:        Supportconfig Plugin for SUSE CaaSP
# Description: Gathers important troubleshooting information
#              about SUSE CaaSP
# License:     GPLv2
# Author:      containers-bugowner@suse.de
# Modified:    2017 May 25
#############################################################

SVER=1.0.0
RCFILE="/usr/lib/supportconfig/resources/scplugin.rc"
LOG_LINES=10000  # 0 means include the entire file

if [ -s $RCFILE ]; then
    if ! source $RCFILE; then
        echo "ERROR: Initializing resource file: $RCFILE" >&2
        exit 1
    fi
fi

pconf_files() {
    _log_files 0 "Configuration File" "$@"
}

plog_files() {
    loglines="$1"
    shift
    _log_files "$loglines" "Log File" "$@"
}

_log_files() {
    loglines="$1" title="$2"
    shift 2

    for file in "$@"; do
        _log_file "$loglines" "$title" "$file"
    done
}

_log_file() {
    loglines="$1" title="$2" file="$3"

    case "$file" in
        *.tbz|*.bz2|*.gz|*.zip|*.xz)
            continue
            ;;
    esac

    if ! [ -f "$file" ]; then
        plugin_tag "$title" "$file - File not found"
        continue
    fi

    if [ "$loglines" -eq 0 ]; then
        plugin_tag "$title" "$file"
    else
        plugin_tag "$title" "$file - Last $loglines Lines"
    fi

    _capture "$loglines" "$file" | _capture_filter
    echo
}

_capture() {
    loglines="$1" file="$2"
    if [ "$loglines" -eq 0 ]; then
        cat "$file"
    else
        tail -"$loglines" "$file"
    fi
}

HIDDEN='<<HIDDEN BY SUPPORTCONFIG PLUGIN>>'

_capture_filter() {
    sed_args=(
        # "password": "s3kr1t",
        # "PASWORD": "s3kr1t",
        -e 's/\("\([a-z_]\+\)\?\(passw\(or\)\?d\|PASWORD\)":\? \+"\+\)[^"]\+"/\1'"$CENSORED"'"/gi'
    )
    if (( ADD_OPTION_LOGS )); then
        sed_args+=( -e 's/^M//g' )
    else
        sed_args+=(
            -e '/^[[:space:]]*#/d'
            -e '/^[[:space:]]*;/d'
            -e '/^[[:space:]]*\/\//d'
            -e 's/^M//g'
            -e '/^$/d'
        )
    fi
    sed "${sed_args[@]}"
}

check_rpm() {
        #echo
        RPM_NAME=$1
        #echo "#==[ Checking RPM ]=================================#"
        #echo "rpm -q $RPM_NAME"
        if rpm -q $RPM_NAME &>/dev/null; then
                #echo "Status: Package is installed"
                return 0
        else
                #echo "Status: Package is not installed"
                return 1
        fi
}

wait_trace_on() {
        if (( $VAR_OPTION_WAIT_TRACE )); then
                OPT=$1
                case $OPT in
                -t) TEE=0; shift ;;
                *) TEE=1 ;;
                esac
                LOGGING="$@"
                WT_START=$(date +%T:%N)
                log2sys "<$WT_START> $LOGGING"
                if (( $TEE )); then
                        printf "%s" "    <$WT_START> $LOGGING  " | tee -a ${LOG}/${CSFILE}
                else
                        printf "%s" "    <$WT_START> $LOGGING  "
                fi
        fi
}

wait_trace_off() {
        if (( $VAR_OPTION_WAIT_TRACE )); then
                OPT=$1
                case $OPT in
                -t) TEE=0 ;;
                *) TEE=1 ;;
                esac
                WT_END=$(date +%T:%N)
                if (( $TEE )); then
                        echo "<$WT_END>" | tee -a ${LOG}/${CSFILE}
                else
                        echo "<$WT_END>"
                fi
        fi
}

# Input: logfilename logfiles...
ADD_OPTION_LOGS=0
conf_files() {
        LOGFILE=$LOG/$1
        shift
        for CONF in $@
        do
                echo "#==[ Configuration File ]===========================#" >> $LOGFILE
                if [ -f $CONF ]; then
                        echo "# $CONF" >> $LOGFILE
                        wait_trace_on "$CONF"
                        if (( ADD_OPTION_LOGS )); then
                                cat $CONF 2>> $LOGFILE | sed -e 's/\r//g' >> $LOGFILE
                        else
                                cat $CONF 2>> $LOGFILE | sed -e '/^[[:space:]]*#/d;/^[[:space:]]*;/d;s/\r//g;/^[[:space:]]*$/d' >> $LOGFILE
                        fi
                        echo >> $LOGFILE
                        wait_trace_off
                else
                        echo "# $CONF - File not found" >> $LOGFILE
                fi
                echo >> $LOGFILE
        done
}

# Input: logfilename lines logfiles...
# If lines = 0, includes the entire log file
log_files() {
        LOGFILE=$LOG/$1
        shift
        LOGLINES=$1
        shift
        for CONF in $@
        do
                BAD_FILE=$(echo "$CONF" | egrep "\.tbz$|\.bz2$|\.gz$|\.zip$|\.xz$")
                if [ -n "$BAD_FILE" ]; then
                        continue
                fi
                echo "#==[ Log File ]=====================================#" >> $LOGFILE
                CONF=$(echo $CONF | sed -e "s/%7B%20%7D%7B%20%7D/ /g")
                if [ -f "$CONF" ]; then
                        wait_trace_on "$CONF"
                        if [ $LOGLINES -eq 0 ]; then
                                echo "# $CONF" >> $LOGFILE
                                sed -e 's/\r//g' "$CONF" >> $LOGFILE
                        else
                                echo "# $CONF - Last $LOGLINES Lines" >> $LOGFILE
                                tail -$LOGLINES "$CONF" | sed -e 's/\r//g' >> $LOGFILE
                        fi
                        echo >> $LOGFILE
                        wait_trace_off
                else
                        echo "# $CONF - File not found" >> $LOGFILE
                fi
                echo >> $LOGFILE
        done
}

# Input: logfilename command
log_cmd() {
        EXIT_STATUS=0
        LOGFILE=$LOG/$1
        shift
        CMDLINE_ORIG="$@"
        CMDBIN=$(echo $CMDLINE_ORIG | awk '{print $1}')
        CMD=$(\which $CMDBIN 2>/dev/null | awk '{print $1}')
        echo "#==[ Command ]======================================#" >> $LOGFILE
        if [ -x "$CMD" ]; then
                CMDLINE=$(echo $CMDLINE_ORIG | sed -e "s!${CMDBIN}!${CMD}!")
                echo "# $CMDLINE" >> $LOGFILE
                wait_trace_on "$CMDLINE"
                echo "$CMDLINE" | bash  >> $LOGFILE 2>&1
                EXIT_STATUS=$?
                wait_trace_off
        else
                echo "# $CMDLINE_ORIG" >> $LOGFILE
                echo "ERROR: Command not found or not executible" >> $LOGFILE
                EXIT_STATUS=1
        fi
        echo >> $LOGFILE
        return $EXIT_STATUS
}

# Input: logfilename rpm
# Assumes the rpm is installed and $LOG/$RPMFILE has been created
rpm_verify() {
        RPMPATH=$LOG/$RPMFILE
        LOGFILE=$LOG/$1
        INPUT_RPM=$2
        echo "#==[ Verification ]=================================#" >> $LOGFILE
        if rpm -q $INPUT_RPM &>/dev/null
        then
                for RPM in $(rpm -q $INPUT_RPM)
                do
                        echo "# rpm -V $RPM" >> $LOGFILE
                        wait_trace_on "rpm -V $RPM"
                        rpm -V $RPM >> $LOGFILE 2>&1
                        ERR=$?
                        wait_trace_off
                        if [ $ERR -gt 0 ]; then
                                echo "# Verification Status: Differences Found" >> $LOGFILE
                        else
                                echo "# Verification Status: Passed" >> $LOGFILE
                        fi
                        echo >> $LOGFILE
                done
                #cat $RPMPATH | grep "^$INPUT_RPM " >> $LOGFILE
                #echo >> $LOGFILE
                return 0
        else
                echo "# RPM Not Installed: $INPUT_RPM" >> $LOGFILE
                echo >> $LOGFILE
                return 1
        fi
}

# Input logfilename containername command
docker_exec() {
        LOGFILE=$LOG/$1
        CONTAINERNAME=$2
        shift; shift;
        COMMAND="$@"
        CONTAINER=$(docker ps | grep $CONTAINERNAME | awk '{print $1}')

        echo "#==[ $COMMAND ]==" >> $LOGFILE
        timeout 300 docker exec $CONTAINER $COMMAND >> $LOGFILE 2>&1
        EXIT_STATUS=$?

        echo >> $LOGFILE
        return $EXIT_STATUS
}

# Check if supportconfig gets run on the CaaSP Admin
on_admin() {
        if $(docker ps -a | grep -q velum-dashboard); then
                return 0
        else
                return 1
        fi
}

#############################################################
section_header "Supportconfig Plugin for SUSE CaaSP, v${SVER}"
# The plugin already hardcodes a reference to this directory above, so we're
# not introducing a new coupling with .spec files by using this absolute path here.

_find_uncompressed () {
    find "$@" ! -name \*.gz ! -name \*.bz2 ! -name \*.xz ! -name \*.key ! -name \*.crt ! -name \*.pub ! -name \*.pem
}

find_and_pconf_files () {
    [ -d "$1" ] || return 0
    files=$( _find_uncompressed "$@" )
    if [ -n "$files" ]; then
        pconf_files $files
    fi
}

find_and_plog_files () {
    [ -d "$1" ] || return 0
    files=$( _find_uncompressed "$@" )
    if [ -n "$files" ]; then
        plog_files "$LOG_LINES" $files
    fi
}

find_and_plog_files_0 () {
    [ -d "$1" ] || return 0
    files=$( _find_uncompressed "$@" )
    if [ -n "$files" ]; then
        plog_files 0 $files
    fi
}

find_and_conf_files () {
    [ -d "$2" ] || return 0
    files=$( _find_uncompressed "$2" -type f)
    if [ -n "$files" ]; then
        conf_files $1 $files
    fi
}

find_and_log_files () {
    [ -d "$2" ] || return 0
    files=$( _find_uncompressed "$2" -type f)
    if [ -n "$files" ]; then
        log_files $1 0 $files
    fi
}

#############################################################
section_header "Checking transactional-update service ..."
OF=transactional-update.txt
plugin_message "All data stored in $OF"

if check_rpm transactional-update; then
    rpm_verify $OF transactional-update
    log_cmd $OF 'systemctl status transactional-update.service'
    log_cmd $OF 'systemctl status transactional-update.timer'
    log_cmd $OF 'journalctl -u transactional-update'
    log_cmd $OF 'snapper list'
    log_files $OF 0 /var/log/transactional-update.log
    log_files $OF 0 /tmp/transactional-update.\*
fi
plugin_message Done

#############################################################
section_header "Checking flanneld service ..."
OF=flannel.txt
plugin_message "All data stored in $OF"

if check_rpm flannel && [ -e /usr/sbin/flanneld ]; then
    rpm_verify $OF flannel
    log_cmd $OF 'flanneld -version'
    log_cmd $OF 'systemctl status flanneld.service'
    log_cmd $OF 'journalctl -u flanneld'
    conf_files $OF /etc/sysconfig/flanneld
fi
plugin_message Done

#############################################################
section_header "Checking etcd service ..."
OF=etcd.txt
plugin_message "All data stored in $OF"

set -a
[ -f /etc/sysconfig/etcdctl ] && source /etc/sysconfig/etcdctl
set +a
if check_rpm etcd && [ -e /usr/bin/etcdctl ]; then
    rpm_verify $OF etcd
    log_cmd $OF 'etcdctl --version'
    log_cmd $OF 'etcdctl member list'
    log_cmd $OF 'etcdctl cluster-health'
    log_cmd $OF 'etcdctl get /flannel/network/config'
    log_cmd $OF 'etcdctl ls /flannel/network/subnets'
    log_cmd $OF 'systemctl status etcd.service'
    log_cmd $OF 'journalctl -u etcd'
    conf_files $OF /etc/sysconfig/etcd
fi
plugin_message Done

#############################################################
section_header "Checking kubernetes services ..."
OF=kubernetes.txt
plugin_message "All data stored in $OF"

KUBECTL_LOG=/var/log/kubernetes
if check_rpm kubernetes-master && ! on_admin; then
    rpm_verify $OF kubernetes-client
    log_cmd $OF 'kubectl version'
    log_cmd $OF 'kubectl api-versions'
    log_cmd $OF 'kubectl config view -a'
    log_cmd $OF 'kubectl get nodes'
    if kubectl cluster-info &>/dev/null; then
        timeout 3 kubectl cluster-info dump --output-directory=$KUBECTL_LOG; echo
        find_and_log_files $OF $KUBECTL_LOG
        rm ${KUBECTL_LOG} -fr
    fi
    find_and_conf_files $OF /etc/kubernetes

    rpm_verify $OF kubernetes-master
    for K8S_SERVICE in kube-apiserver kube-scheduler kube-controller-manager kube-proxy kubelet; do
        log_cmd $OF "systemctl status $K8S_SERVICE"
        log_cmd $OF "journalctl -u $K8S_SERVICE"
    done
fi
plugin_message Done

#############################################################
section_header "Checking salt services ..."
OF=salt-minion.txt
plugin_message "All data stored in $OF"

if check_rpm salt-minion && [ -e /usr/bin/salt-minion ]; then
    rpm_verify $OF salt-minion
    log_cmd $OF 'salt-minion --versions-report'
    log_cmd $OF 'systemctl status salt-minion'
    log_cmd $OF 'journalctl -u salt-minion'
    conf_files $OF /etc/salt/minion
    conf_files $OF /etc/salt/grains
    find_and_conf_files $OF /etc/salt/minion.d
    find_and_log_files $OF /var/log/salt
fi
plugin_message Done

# checks that only apply to the admin node
[ -e /usr/bin/docker ] && DOCKER_PKG=`rpm -q --whatprovides /usr/bin/docker`
if check_rpm $DOCKER_PKG && on_admin; then
    #############################################################
    section_header "Getting minion objects ..."
    OF=velum-minions.yml
    plugin_message "All data stored in $OF"

    docker_exec $OF velum-dashboard entrypoint.sh bundle exec rails runner 'puts(Minion.all.map(&:to_yaml))'
    plugin_message Done

    #############################################################
    section_header "Getting salt events ..."
    OF=salt-events.json
    OFS=salt-events-summary.txt
    plugin_message "JSON stored in $OF - Summary stored in $OFS"

    /var/lib/supportutils-plugin-suse-caasp/debug-salt --json_output=$OF --summary_output=$OFS --text-status --no-color
    plugin_message Done

    #############################################################
    section_header "Getting salt pillars ..."
    OF=velum-salt-pillars.yml
    plugin_message "All data stored in $OF"

    docker_exec $OF velum-dashboard entrypoint.sh bundle exec rails runner 'puts(Pillar.all.pluck(:pillar,:value).to_yaml)'
    plugin_message Done

    #############################################################
    section_header "Checking velum migrations ..."
    OF=velum-migrations.txt
    plugin_message "All data stored in $OF"

    docker_exec $OF velum-dashboard entrypoint.sh bundle exec rake db:migrate:status
    plugin_message Done

    #############################################################
    section_header "Checking velum routes ..."
    OF=velum-routes.txt
    plugin_message "All data stored in $OF"

    docker_exec $OF velum-dashboard entrypoint.sh bundle exec rake routes
    plugin_message Done

    #############################################################
    section_header "Listing velum files ..."
    OF=velum-files.txt
    plugin_message "All data stored in $OF"

    docker_exec $OF velum-dashboard entrypoint.sh ls -lR
    plugin_message Done

    #############################################################
    section_header "Checking salt master ..."
    OF=salt-master.txt
    plugin_message "All data stored in $OF"

    docker_exec $OF salt-master salt-key
    plugin_message Done
fi

#############################################################
section_header "Checking crio service ..."
OF=crio.txt
plugin_message "All data stored in $OF"

if check_rpm cri-o && [ -e /usr/bin/crictl ] && [ -S /var/run/crio/crio.sock ]; then
    rpm_verify $OF cri-o
    log_cmd $OF 'crictl version'
    log_cmd $OF 'systemctl status crio.service'
    log_cmd $OF 'crictl info'
    log_cmd $OF 'crictl images'
    log_cmd $OF 'crictl ps --all'
    conf_files $OF /etc/crictl.yaml
    find_and_conf_files $OF /etc/crio/
    for i in $(crictl  ps -a 2>/dev/null | grep -v "COMMAND" | awk '{print $1}');
    do
        log_cmd $OF "crictl top $i"
        log_cmd $OF "crictl logs $i"
        log_cmd $OF "crictl inspect $i"
    done
    log_cmd $OF 'journalctl -u crio'
fi
plugin_message Done
