-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathinsanity-service-check
186 lines (173 loc) · 7.92 KB
/
insanity-service-check
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
#!/usr/bin/env bash
# VARS PART 1
HOSTSDIR='hosts'
MODE=$3
CONFDIR='conf'
SERVER_LIST=$1
ESC="\e["
LOGDIR='./log'
LOGFILE="${MODE}-${SERVER_LIST}-${DATE}.log"
GREEN="${ESC}32m"
RED="${ESC}31m"
YELLOW="${ESC}33m"
BOLD="${ESC}1m"
NORMAL="${ESC}0m"
# Test if SERVER_LIST exists
if [[ ! -e $HOSTSDIR/$SERVER_LIST ]]; then
printf "${RED}${BOLD}FAIL${NORMAL}: Can't find server list, please check it.\n" | tee -a $LOGDIR/$LOGFILE; exit 1
fi
# Create log direrctory
if [[ ! -d $LOGDIR ]]; then
mkdir $LOGDIR
fi
# Usage function
usage() {
echo -e "${BOLD}USAGE${NORMAL}:
${YELLOW}$0 server-list user mode (service|marker) [sudo]${NORMAL}\n${BOLD}Examples${NORMAL}:
${YELLOW}$0 risdevinteractive root marker
$0 risqainteractive1 ricie service sudo
$0 risbdcinteractive2 pkowalczyk marker sudo${NORMAL}"; exit 1
}
# Test nummber of parameters
if [[ ( $# != 3 && $# != 4 ) ]]; then
usage
fi
# VARS PART 2
PASS=$(cat ${CONFDIR}/.passwd)
USER=$2
SUDO=$4
DATE=`date +%m%d%y_%H%M`
TMPDIR='./tmp'
DISABLED_LIST="${CONFDIR}/disabled-list"
FAIL=0
WARN=0
RANDOM_SERVER=`head -$((${RANDOM} % \`wc -l < ${HOSTSDIR}/${SERVER_LIST}\` + 1)) ${HOSTSDIR}/${SERVER_LIST} | awk {'print $1'} | tail -1`
SERVERS=()
for SERVER in PNWEB PNAPP PNBOWEB PNBOAPP PNBROKER PNCRM PNGAME PN2DB PN2PAM PN2CONNECT PN2BROKER PN2BOWEB PN2BOAPP; do
eval ${SERVER}=$(cat ${HOSTSDIR}/${SERVER_LIST} | grep -i $SERVER | awk {'print $1'})
if [ ! -z ${!SERVER} ]; then
SERVERS+=(${SERVER})
fi
done
# SSH test
sshtest() {
printf "${ESC}4m${BOLD}Testing SSH connection with user ${USER} on ${RANDOM_SERVER} server...${NORMAL}\n${BOLD}Log:${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
SSHTEST=`sshpass -p $PASS ssh -t $USER@${RANDOM_SERVER} true`; RC=$?
if [[ $RC -eq 0 ]]; then
printf "\n${GREEN}SSH Test ${BOLD}passed!${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
elif [[ $RC -eq 255 ]]; then
printf "\n${RED}SSH Test ${BOLD}failed${NORMAL}!\nExample server (${RANDOM_SERVER}) did not respond!${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; exit 1
elif [[ $RC -eq 5 ]]; then
printf "\n${RED}SSH Test ${BOLD}failed${NORMAL}!\nCheck username (current: ${USER}) or password (in conf/.passwd)!${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; exit 1
fi
}
sshtest
# Check sudo parameter
if [[ $SUDO == 'sudo' ]]; then
SUDO_PREFIX="echo ${PASS} | sudo -S --"
fi
# Cleanup TMPDIR
rm -rf $TMPDIR
mkdir $TMPDIR
# Service Function
service() {
for SERVER in ${SERVERS[@]}; do
if [[ -e ${CONFDIR}/${SERVER}-service-list ]]; then
printf "\n${ESC}4m${BOLD}Checking services on ${!SERVER}${NORMAL}${ESC}24m\n" | tee -a $LOGDIR/$LOGFILE
for SERVICE in `cat ${CONFDIR}/${SERVER}-service-list`; do
COMMAND=`sshpass -p ${PASS} ssh -q -t -o StrictHostKeyChecking=no $USER@${!SERVER} "${SUDO_PREFIX} /bin/sh -c 'systemctl status ${SERVICE} > /dev/null 2>&1'"`; RC=$?
if [[ $RC -ne 0 ]]; then
if egrep -q "^${SERVICE}$" ${DISABLED_LIST}; then
printf "${YELLOW}${BOLD}WARN${NORMAL}: ${BOLD}Disabled${NORMAL} service check ${SERVICE} return code: ${TAB}${RC}${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; WARN=$((WARN+1))
else
printf "${RED}${BOLD}FAIL${NORMAL}: Service check ${SERVICE} return code: ${TAB}${RC}${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; FAIL=$((FAIL+1))
fi
else
if egrep -q "^${SERVICE}$" ${DISABLED_LIST}; then
printf "${RED}${BOLD}FAIL${NORMAL}: ${BOLD}disabled${NORMAL} Service check ${SERVICE} return code: ${TAB}${RC}${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; FAIL=$((FAIL+1))
else
printf "${GREEN}${BOLD}OKAY${NORMAL}: Service check ${SERVICE} return code: ${TAB}${RC}${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
fi
fi
done
fi
done
}
# Marker function
marker() {
for SERVER in ${SERVERS[@]}; do
if [[ -e ${CONFDIR}/${SERVER}-container-list ]]; then
for CONTAINER in `cat ${CONFDIR}/${SERVER}-container-list`; do
printf "\n${ESC}4m${BOLD}Searching for markers on ${!SERVER} in /usr/share/jbossas/${CONTAINER}${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
EXIST_CHECK=`sshpass -p ${PASS} ssh -q -t -o StrictHostKeyChecking=no $USER@${!SERVER} "${SUDO_PREFIX} /bin/sh -c \"[[ -e /usr/share/jbossas/${CONTAINER}/deployments ]]\""`; RC=$?
if [[ $RC -eq 0 ]]; then
sshpass -p ${PASS} ssh -q -t -o StrictHostKeyChecking=no $USER@${!SERVER} "${SUDO_PREFIX} /bin/sh -c \"find /usr/share/jbossas/${CONTAINER}/deployments -name '*.deployed'\"" > ${TMPDIR}/${SERVER}-${CONTAINER}.deployed
sshpass -p ${PASS} ssh -q -t -o StrictHostKeyChecking=no $USER@${!SERVER} "${SUDO_PREFIX} /bin/sh -c \"find /usr/share/jbossas/${CONTAINER}/deployments -name '*.failed'\"" > ${TMPDIR}/${SERVER}-${CONTAINER}.failed
DEPLOYED=`grep -v sudo $TMPDIR/${SERVER}-${CONTAINER}.deployed | wc -l`
FAILED=`grep -v sudo $TMPDIR/${SERVER}-${CONTAINER}.failed | wc -l`
if [[ $DEPLOYED -gt 0 ]]; then
if egrep -q "^${CONTAINER}$" ${DISABLED_LIST}; then
echo -e "${RED}${BOLD}FAIL${NORMAL}: Found *.deployed files, even though container ${CONTAINER} is ${BOLD}disabled${NORMAL}!\n\n
${BOLD}Deployed:${NORMAL}\n`cat $TMPDIR/${SERVER}-${CONTAINER}.deployed | grep -v sudo`${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
FAIL=$((FAIL+1))
else
printf "${GREEN}${BOLD}OKAY${NORMAL}: Found *.deployed files in ${CONTAINER} container.\n\n${BOLD}Deployed:${NORMAL}\n`cat $TMPDIR/${SERVER}-${CONTAINER}.deployed | grep -v sudo`${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
fi
else
if egrep -q "^${CONTAINER}$" ${DISABLED_LIST}; then
printf "${YELLOW}${BOLD}WARN${NORMAL}: No *.deployed files found in ${BOLD}disabled${NORMAL} container ${CONTAINER}.${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; WARN=$((WARN+1))
else
printf "${RED}${BOLD}FAIL${NORMAL}: No *.deployed files found in container ${CONTAINER}!${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; FAIL=$((FAIL+1))
fi
fi
if [[ $FAILED -gt 0 ]]; then
if egrep -q "^${CONTAINER}$" ${DISABLED_LIST}; then
echo -e "${RED}${BOLD}FAIL${NORMAL}: Found *.failed files, even though container ${CONTAINER} is ${BOLD}disabled${NORMAL}!\n\n${BOLD}Failed:${NORMAL}\n`cat $TMPDIR/${SERVER}-${CONTAINER}.failed | grep -v sudo`${NORMAL}" | tee -a $LOGDIR/$LOGFILE; FAIL=$((FAIL+1))
else
printf "${RED}\n${BOLD}Failed:${NORMAL}\n`cat $TMPDIR/${SERVER}-${CONTAINER}.failed | grep -v sudo`${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; FAIL=$((FAIL+1))
fi
fi
else
if egrep -q "^${CONTAINER}$" ${DISABLED_LIST}; then
printf "${YELLOW}${BOLD}WARN${NORMAL}: Directory /usr/share/jbossas/${CONTAINER} (${BOLD}disabled${NORMAL}) doesn't exists.${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; WARN=$((WARN+1))
else
printf "${RED}FAIL: Directory /usr/share/jbossas/${CONTAINER} doesn't exists.${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE; FAIL=$((FAIL+1))
fi
fi
done
fi
done
rm -rf $TMPDIR
}
# Runmode
case "$MODE" in
service)
service
;;
marker)
marker
;;
*)
usage
;;
esac
# Status
status() {
printf "\n\n${ESC}4m${BOLD}Status:${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
if [ $FAIL -gt 0 ]; then
printf "${RED}${BOLD}FAIL${NORMAL}: number of fails: ${FAIL}.${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE;
if [ $WARN -gt 0 ]; then
printf "${YELLOW}${BOLD}WARN${NORMAL}: number of warnings: ${WARN}.${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
fi
printf 'Check the log!'
exit 1
else
printf "${GREEN}${BOLD}OKAY${NORMAL}: number of fails: ${FAIL}.${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
if [ $WARN -gt 0 ]; then
printf "${YELLOW}${BOLD}WARN${NORMAL}: number of warnings: ${WARN}. ${BOLD}Check the log just in case...${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
fi
printf "${GREEN}${BOLD}You\'re all set!${NORMAL}\n" | tee -a $LOGDIR/$LOGFILE
exit 0
fi
}
status