-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmemUsage.py
182 lines (160 loc) · 9.58 KB
/
memUsage.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
#!/usr/bin/python
# Authors: Olga I. Zolotareva, Sergey I. Mitrofanov
from __future__ import print_function
import argparse
import sys, os, socket
import psutil
import time, datetime
import subprocess
LastUpdate = "02.08.2019 01:45"
Version = "0.2.0"
### argparse ###
parser = argparse.ArgumentParser(description="""Monitors the process by PID and directory size by path, writes .mem.log file with the following TAB-separated fields:
- time, s
- PID
- available RAM, GB
- residen set size, GB - the portion of RAM occupied by a process
- % of total RAM occupied by a process, %
- % of CPU used by a process
- size of given folder, GB
- free disk space on partition with given folder, GB
OR plots the provided memlog file.
""" , formatter_class=argparse.RawTextHelpFormatter)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-p', '--pid', dest='pid', type=int, help='PID to be watched. All children of this PID are watched too.', metavar='12345')
parser.add_argument('-d', '--wdir', dest='wdir', type=str, help='Diretory to be watched (size). Default is current directory (.).', default='.', required=False)
parser.add_argument('-t', '--time-step', dest='time_step', type=int, help='Write metrics to memlog file every N sec. Default is 5 sec.', default=5, required=False, metavar='N')
parser.add_argument('-o', '--out-memlog', dest='out_mem_log', type=str, help='Output memlog filename (common extention is .mem.log). If not provided output will be written into memlog file PID.mem.log in the current directory.', default='', required=False, metavar='a.mem.log')
group.add_argument('-i', '--input-memlog', dest='in_mem_log', help='Input memlog file, generated by this script. This flag switches off log writing.', default=False, required=False, metavar='12345.mem.log')
#group.add_argument('--pname', dest='pname', type=str, nargs=1, help='Process name, if no PID provided.')
parser.add_argument('--avail-mem', dest='avail_pmem', action='store_true', help='Plot free RAM (GB) vs time (sec).', default=False, required=False)
parser.add_argument('--mem-rss', dest='mem_rss', action='store_true', help='Plot used by this PID RAM (GB) [= RSS - the portion of memory occupied by a process] vs time (sec).', default=False, required=False)
parser.add_argument('--pmem', dest='pmem', action='store_true', help='Plot used by this PID RAM (percent) vs time (sec).', default=False, required=False)
parser.add_argument('--pcpu', dest='pcpu', action='store_true', help='Plot used by this PID CPU (percent) vs time (sec).', default=False, required=False)
parser.add_argument('--avail-space', dest='avail_space', action='store_true', help='Plot free disk space on partition with watched dir (GB) vs time (sec).', default=False, required=False)
parser.add_argument('--dir-size', dest='dir_size', action='store_true', help='Plot watched dir size (GB) vs time (sec).', default=False, required=False)
parser.add_argument('--plot-all', dest='plot_all', action='store_true', help='Switches on all plotting.', default=False, required=False)
def get_pcpu_pmem(pid):
try:
process = psutil.Process(pid)
pmem = process.memory_percent() # % of memory used
mem_rss = process.memory_info().rss # rss = residen set size is the portion of memory occupied by a process
pcpu = process.cpu_percent(interval=0.1) # % of CPU used
for child in process.children(recursive=True):
try:
pmem += child.memory_percent()
pcpu += child.cpu_percent(interval=0.1)
mem_rss += child.memory_info().rss
except:
pass
mem = psutil.virtual_memory() # total memory available
mem_rss = (mem_rss*1.0)/(1024*1024*1024)
mem_avail = (mem.available*1.0)/(1024*1024*1024)
exited = 0
except:
mem_avail, mem_rss, pmem, pcpu = (0, 0, 0, 0)
exited = 1
return mem_avail, mem_rss, pmem, pcpu, exited
def get_avail_space(wdir):
'''Monitors available disk space.'''
command = "df "+wdir
process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
avail_space = str(out.split(b'\n')[1]).split()[3] # e.g. Filesystem Size Used Avail Use% Mounted on'\n'/dev/xvdb1 1008G 721G 236G 76% /mnt/data1
return (int(avail_space)*1.0)/(1024*1024) # disk space in Gb
def get_dir_size(wdir):
'''Iterates over all files and subfolders in 'wdir' and returns total size of the folder in Gb.'''
dir_size = 0
for dirpath, dirnames, filenames in os.walk(wdir):
for f in filenames:
fp = os.path.join(dirpath, f)
if os.path.isfile(fp):
dir_size += os.path.getsize(fp)
else:
pass # e.g. Permission Denied
return (dir_size*1.0)/(1024*1024*1024)
# read arguments
args = parser.parse_args()
if not args.in_mem_log: # then write logfile
pid = args.pid # this is the only required argument
if not psutil.pid_exists(pid):
print ("[memusage] No process with PID", pid, "found.", file=sys.stderr)
exit(1)
machineName = socket.gethostname() # hostname
cpuAmount = psutil.cpu_count() # virtual machine CPU count
if psutil.cpu_freq().max == 0.0:
cpuFreqMax = psutil.cpu_freq().current # Max CPU frequency inside Docker is 0.0, that's why using current
else:
cpuFreqMax = psutil.cpu_freq().max # CPU frequency [psutil.cpu_freq() should return 'scpufreq(current=1200.0, min=1200.0, max=3601.0)'], also 'psutil.cpu_freq(percpu=True)' to get info about every vCore
memTotalGb = (psutil.virtual_memory().total*1.0)/(1024*1024*1024) # total physical memory size in GB
t_start = time.time()
if not args.out_mem_log:
out_file_name = str(pid)
else:
out_file_name = args.out_mem_log
if not out_file_name.endswith(".mem.log"):
out_file_name = out_file_name + ".mem.log"
if os.path.exists(out_file_name):
append_write = 'a' # append if already exists
else:
append_write = 'w' # make a new file if not
outfile = open(out_file_name,append_write)
outfile.write('\t'.join(["#Hostname", machineName]) + '\n')
outfile.write('\t'.join(["#Cores", str(cpuAmount)]) + '\n')
outfile.write('\t'.join(["#CoreMaxFreq", str(int(cpuFreqMax))+" MHz"]) + '\n')
outfile.write('\t'.join(["#TotalRAM", str(int(memTotalGb))+" GB"]) + '\n')
outfile.write('\t'.join(["#Date", datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')]) + '\n')
outfile.write('\t'.join(["#memUsageVersion", Version+" ("+LastUpdate+")"]) + '\n')
outfile.write('\n')
colNames = ["time", "pid", "avail_mem", "mem_rss", "%mem", "%cpu", "wdir_size", "free_space"]
outfile.write('\t'.join(colNames) + '\n')
warnCounter1 = 0
while True:
if psutil.pid_exists(pid):
currTime = time.time()-t_start
mem_avail,mem_rss,pmem,pcpu,exited = get_pcpu_pmem(pid)
if exited:
print ("[memusage] Process with PID", pid, "was terminated [1].", file=sys.stderr)
break
if (mem_rss > memTotalGb):
warnCounter1 += 1
if (warnCounter1 > 10):
print ("[memusage] Process with PID", pid, "gained a warning (n=", warnCounter1, ", time=", currTime, "): mem_rss (", mem_rss, ") is greater than total RAM (", memTotalGb, ").", file=sys.stderr)
continue
dir_size = get_dir_size(args.wdir)
space_avail = get_avail_space(args.wdir)
outfile.write('%d\t%d\t%.1f\t%.1f\t%.2f\t%.2f\t%.1f\t%.1f\n' % (currTime,pid,mem_avail,mem_rss,pmem,pcpu,dir_size,space_avail))
outfile.flush()
else:
print ("[memusage] Process with PID", pid, "was terminated [2].", file=sys.stderr)
break
time.sleep(args.time_step)
outfile.close()
time.sleep(1)
else:
in_file_name = args.in_mem_log # otherwise log file is provided by user
#read the resulting file to pandas
if args.plot_all or args.avail_pmem or args.mem_rss or args.pmem or args.pcpu or args.dir_size or args.avail_space:
import pandas as pd
import matplotlib
matplotlib.use('Agg')
df = pd.read_csv(in_file_name, sep='\t', comment='#', skip_blank_lines=True) # , names=colNames
#plot with pylab
if args.avail_pmem or args.plot_all: # avail_mem
fig1 = df.plot(x='time', y=["avail_mem"],title="available memory, GB",kind='line').get_figure()
fig1.savefig(in_file_name.split('.')[0]+'.avail_mem.png')
if args.mem_rss or args.plot_all: # RSS
fig2 = df.plot(x='time', y=["mem_rss"],title ="RSS, GB",kind='line').get_figure()
fig2.savefig(in_file_name.split('.')[0]+'.mem_rss.png')
if args.pmem or args.plot_all: #%mem
fig3 = df.plot(x='time', y=["%mem"],title = "% of memory used",kind='line').get_figure()
fig3.savefig(in_file_name.split('.')[0]+'.pmem.png')
if args.pcpu or args.plot_all: #%cpu
fig4 = df.plot(x='time', y=["%cpu"],title = "% of CPU used",kind='line').get_figure()
fig4.savefig(in_file_name.split('.')[0]+'.pcpu.png')
if args.dir_size or args.plot_all:
fig5 = df.plot(x='time', y=["wdir_size"],title = "Directory "+str(args.wdir)+" size, GB",kind='line').get_figure()
fig5.savefig(in_file_name.split('.')[0]+'.dir_size.png')
if args.avail_space or args.plot_all:
fig6 = df.plot(x='time', y=["free_space"],title = "Remaining disk space, GB",kind='line').get_figure()
fig6.savefig(in_file_name.split('.')[0]+'.avail_space.png')