cleaned commented lines from scripts
grantwra
parent
437a6c68d4
commit
01ee7d46bf
|
|
@ -1,10 +1,3 @@
|
|||
# find latency numbers
|
||||
# Usage: python findlatency.py (BDB|SQL) (Workload.log)
|
||||
|
||||
# Detailed json = [starttime, endtime, runtime, insert op, complete op, addr]
|
||||
# Summary json = [IOcount, min runtime, max runtime, average runtime, IOcount of events that were 50% more or less then the average]
|
||||
|
||||
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
|
|
@ -16,19 +9,12 @@ parser.add_argument('input', type=str, help="Input data file.")
|
|||
parser.add_argument('--summary', action="store_true", help="Don't print detailed information.")
|
||||
args = parser.parse_args()
|
||||
|
||||
#print args
|
||||
|
||||
dbType = args.dbType
|
||||
workload = args.input
|
||||
#dbType = sys.argv[1]
|
||||
#workload = sys.argv[2]
|
||||
log = []
|
||||
block_insert = []
|
||||
block_complete1 = []
|
||||
|
||||
#print dbType
|
||||
#print workload
|
||||
|
||||
start_time = 0.0
|
||||
end_time = 0.0
|
||||
|
||||
|
|
@ -47,27 +33,18 @@ with gzip.open(workload, 'r') as log:
|
|||
if dbType + '_START' in line:
|
||||
found = 1
|
||||
if dbType + '_END' in line:
|
||||
#count += 1
|
||||
found = 2
|
||||
if found == 0:
|
||||
#count += 1
|
||||
continue
|
||||
if found == 2:
|
||||
break
|
||||
if 'block_rq_insert' in line:
|
||||
timestamp = float(line.split()[3].split(':')[0])
|
||||
insertaddr = line.split()[9] + ' + ' + line.split()[11]
|
||||
|
||||
#addr = ad[9]
|
||||
#addr = addr + ' + ' + ad[11]
|
||||
#print timestamp
|
||||
if 'withjson' not in line:
|
||||
continue
|
||||
#print insertaddr
|
||||
#print line
|
||||
|
||||
with gzip.open(workload, 'r') as log2:
|
||||
#count += 1
|
||||
sameline = 0
|
||||
for line2 in log2:
|
||||
if line == line2:
|
||||
|
|
@ -80,24 +57,17 @@ with gzip.open(workload, 'r') as log:
|
|||
count += 1
|
||||
break
|
||||
|
||||
#print count
|
||||
|
||||
data = dict()
|
||||
for d in insert_complete:
|
||||
#print insert_complete[d]
|
||||
insert_string = insert_complete[d][0]
|
||||
insertaddr = insert_string.split()[9] + ' + ' + insert_string.split()[11]
|
||||
inserttimestamp = float(insert_string.split()[3].split(':')[0])
|
||||
insertop = insert_string.split()[6]
|
||||
#print insertaddr
|
||||
|
||||
complete_string = insert_complete[d][1]
|
||||
completeaddr = complete_string.split()[8] + ' + ' + complete_string.split()[10]
|
||||
completetimestamp = float(complete_string.split()[3].split(':')[0])
|
||||
completeop = complete_string.split()[6]
|
||||
#print completeaddr
|
||||
#print insert_string
|
||||
#data[d] = []
|
||||
assert insertaddr == completeaddr
|
||||
assert completetimestamp > inserttimestamp
|
||||
start = (inserttimestamp - start_time) * 1000
|
||||
|
|
@ -107,10 +77,6 @@ for d in insert_complete:
|
|||
assert (end - start) > 0
|
||||
data[d] = [start, end, end - start, insertop, completeop, insertaddr]
|
||||
|
||||
'''
|
||||
for i in data:
|
||||
print data[i]
|
||||
'''
|
||||
if args.summary:
|
||||
summary = dict()
|
||||
r_min = 10000.0
|
||||
|
|
|
|||
|
|
@ -1,11 +1,3 @@
|
|||
# find latency numbers
|
||||
# Usage: python findlatency.py (BDB|SQL) (Workload.log) {--summary}
|
||||
# ^ Optional
|
||||
|
||||
# Detailed json = [starttime, endtime, runtime, insert op, complete op, addr]
|
||||
# Summary json = [IOcount, min runtime, max runtime, average runtime, IOcount of events that were 50% more or less then the average]
|
||||
|
||||
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
|
|
@ -17,8 +9,6 @@ parser.add_argument('input', type=str, help="Input data file.")
|
|||
parser.add_argument('--summary', action="store_true", help="Don't print detailed information.")
|
||||
args = parser.parse_args()
|
||||
|
||||
#print args
|
||||
|
||||
dbType = args.dbType
|
||||
workload = args.input
|
||||
|
||||
|
|
@ -36,21 +26,14 @@ with gzip.open(workload,'r') as log:
|
|||
break
|
||||
f.close()
|
||||
|
||||
#print 'done'
|
||||
|
||||
workload = 'info.info'
|
||||
|
||||
#dbType = sys.argv[1]
|
||||
#workload = sys.argv[2]
|
||||
pids = []
|
||||
my_pid = 0
|
||||
block_insert = []
|
||||
block_complete1 = []
|
||||
data = dict()
|
||||
|
||||
#print dbType
|
||||
#print workload
|
||||
|
||||
start_time = 0.0
|
||||
end_time = 0.0
|
||||
|
||||
|
|
@ -59,18 +42,13 @@ with open(workload, 'r') as log:
|
|||
count = 0
|
||||
between = 0
|
||||
for line in log:
|
||||
#count += 1
|
||||
#print count
|
||||
if dbType + '_START' in line:
|
||||
start_time = float(line.split()[3].split(':')[0])
|
||||
pid = line.split('[0')[0].split('-')[1].split()[0]
|
||||
my_pid = pid
|
||||
#print pid
|
||||
name = 'benchmark'
|
||||
#print name
|
||||
data[pid] = [name,0,0,0]
|
||||
pids.append(pid)
|
||||
#count += 1
|
||||
between = 1
|
||||
if between == 0:
|
||||
continue
|
||||
|
|
@ -88,11 +66,6 @@ with open(workload, 'r') as log:
|
|||
pids.append(pid)
|
||||
name = line.split('[0')[0].split('-')[0].split()[0]
|
||||
data[pid] = [name,0.0,0.0]
|
||||
#print name
|
||||
#count += 1
|
||||
#print count
|
||||
#print data
|
||||
#print (end_time - start_time)* 1000
|
||||
count = 0
|
||||
|
||||
with open(workload, 'r') as log:
|
||||
|
|
@ -107,13 +80,11 @@ with open(workload, 'r') as log:
|
|||
between = 2
|
||||
if between == 2:
|
||||
break
|
||||
#count += 1
|
||||
if 'sched_switch' in line:
|
||||
piddd = line.split()[11].split('=')
|
||||
if 'next_pid' in piddd[0]:
|
||||
switch_in_pid = piddd[1]
|
||||
switch_in = float(line.split()[3].split(':')[0])
|
||||
#print switch_in_pid
|
||||
with open(workload, 'r') as log2:
|
||||
sameline = 0
|
||||
for line2 in log2:
|
||||
|
|
@ -130,8 +101,6 @@ with open(workload, 'r') as log:
|
|||
data[switch_in_pid][1] += 1
|
||||
data[switch_in_pid][2] += runtime
|
||||
count += 1
|
||||
#print count
|
||||
#print switch_in_pid
|
||||
break
|
||||
|
||||
if args.summary:
|
||||
|
|
@ -141,7 +110,6 @@ if args.summary:
|
|||
added_up_cpu_time = 0.0
|
||||
benchmarkcputime = 0.0
|
||||
benchmarkswitchcount = 0
|
||||
# name, pid, num switch, cpu time
|
||||
maxstuff = ['',0,0.0,0.0]
|
||||
for i in data:
|
||||
if 'benchmark' in data[i][0]:
|
||||
|
|
@ -156,13 +124,9 @@ if args.summary:
|
|||
maxstuff[1] = i
|
||||
maxstuff[2] = data[i][1]
|
||||
maxstuff[3] = data[i][2]
|
||||
#summary[dbType] = [['Wall Clock Time',(end_time-start_time)*1000],0,0,0,0,0,0]
|
||||
#array = [['Wall Clock Time',total_wallclock_time],['Total Number of Sched Switches', num_of_sched_switch],['BENCHMARK INFO',[['PID',my_pid],['CPUTime', benchmarkcputime],['Number of Sched Switches',benchmarkswitchcount]]],['Most CPU Cycles',[['Name',maxstuff[0]],['PID',maxstuff[1]],['CPUTime', maxstuff[3]],['Number of Sched Switches',maxstuff[2]]]]
|
||||
|
||||
summary[dbType] = [['Wall Clock Time',total_wallclock_time],['Added Up CPUTime', added_up_cpu_time/2],['Total Number of Sched Switches', num_of_sched_switch],['BENCHMARK INFO',[['PID',my_pid],['CPUTime', benchmarkcputime],['Number of Sched Switches',benchmarkswitchcount]]],['Most CPU Cycles',[['Name',maxstuff[0]],['PID',maxstuff[1]],['CPUTime', maxstuff[3]],['Number of Sched Switches',maxstuff[2]]]]]
|
||||
|
||||
#summary[dbType] = array
|
||||
|
||||
print json.dumps(summary, indent=2)
|
||||
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,11 +1,3 @@
|
|||
# find latency numbers
|
||||
# Usage: python findtimeblockedonio.py (BDB|SQL) (Workload.log) {--summary}
|
||||
# ^ Optional
|
||||
|
||||
# Detailed json = [starttime, endtime, runtime, insert op, complete op, addr]
|
||||
# Summary json = [IOcount, min runtime, max runtime, average runtime, IOcount of events that were 50% more or less then the average]
|
||||
|
||||
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
|
|
@ -17,14 +9,11 @@ parser.add_argument('input', type=str, help="Input data file.")
|
|||
parser.add_argument('--summary', action="store_true", help="Don't print detailed information.")
|
||||
args = parser.parse_args()
|
||||
|
||||
#print args
|
||||
|
||||
dbType = args.dbType
|
||||
workload = args.input
|
||||
|
||||
mypid = 0
|
||||
sced_on_core = 999
|
||||
#deltas = []
|
||||
data = dict()
|
||||
dict_count = 0
|
||||
|
||||
|
|
@ -43,11 +32,9 @@ with gzip.open(workload,'r') as log:
|
|||
mypid = line.split('[0')[0].split('-')[1].split()[0]
|
||||
core = int(line.split(']')[0].split('[')[1])
|
||||
sced_on_core = core
|
||||
#print core
|
||||
assert(mypid != 0)
|
||||
if between == 0:
|
||||
continue
|
||||
#if 'sched_switch' in line or 'START' in line or 'END' in line or 'block':
|
||||
if resetline == True:
|
||||
if line == holdline:
|
||||
resetline = False
|
||||
|
|
@ -58,36 +45,23 @@ with gzip.open(workload,'r') as log:
|
|||
if 'prev_pid='+mypid in line:
|
||||
sced_on_core = 999
|
||||
core = int(line.split(']')[0].split('[')[1])
|
||||
#print core
|
||||
#break
|
||||
if 'next_pid='+mypid in line:
|
||||
core = int(line.split(']')[0].split('[')[1])
|
||||
sced_on_core = core
|
||||
#print core
|
||||
#break
|
||||
if dbType + '_END' in line:
|
||||
end_time = float(line.split()[3].split(':')[0])
|
||||
break
|
||||
if 'block_rq_insert' in line:
|
||||
#timestampstart = float(line.split()[3].split(':')[0])
|
||||
insertaddr = line.split()[9] + ' + ' + line.split()[11]
|
||||
|
||||
#addr = ad[9]
|
||||
#addr = addr + ' + ' + ad[11]
|
||||
#print timestamp
|
||||
#if sced_on_core != 999:
|
||||
# continue
|
||||
if 'withjson' not in line:
|
||||
continue
|
||||
#print insertaddr
|
||||
#print line
|
||||
|
||||
with gzip.open(workload, 'r') as log2:
|
||||
linecount = 0
|
||||
desced = False
|
||||
iocomplete = False
|
||||
|
||||
#count += 1
|
||||
sameline = 0
|
||||
for line2 in log2:
|
||||
|
||||
|
|
@ -96,8 +70,6 @@ with gzip.open(workload,'r') as log:
|
|||
if sameline == 0:
|
||||
continue
|
||||
linecount += 1
|
||||
#if sced_on_core != 999:
|
||||
# break
|
||||
if 'prev_pid='+mypid in line2:
|
||||
timestampstart = float(line2.split()[3].split(':')[0])
|
||||
sced_on_core = 999
|
||||
|
|
@ -105,19 +77,13 @@ with gzip.open(workload,'r') as log:
|
|||
if linecount <= 10:
|
||||
desced = True
|
||||
|
||||
#print core
|
||||
#break
|
||||
if 'next_pid='+mypid in line2:
|
||||
core = int(line2.split(']')[0].split('[')[1])
|
||||
sced_on_core = core
|
||||
#print core
|
||||
#break
|
||||
if iocomplete != True:
|
||||
break
|
||||
else:
|
||||
timestampend = float(line2.split()[3].split(':')[0])
|
||||
#print 'hello'
|
||||
#deltas.append((timestampend - timestampstart)*1000)
|
||||
total += (timestampend - timestampstart)*1000
|
||||
holdline = line2
|
||||
resetline = True
|
||||
|
|
@ -127,22 +93,11 @@ with gzip.open(workload,'r') as log:
|
|||
|
||||
if 'block_rq_complete' in line2:
|
||||
if insertaddr in line2:
|
||||
#timestampend = float(line.split()[3].split(':')[0])
|
||||
#insert_complete[count] = [line,line2]
|
||||
#count += 1
|
||||
#deltas.append(timestampend - timestampstart)
|
||||
#break
|
||||
if desced == True:
|
||||
iocomplete = True
|
||||
|
||||
assert(mypid != 0)
|
||||
|
||||
#print mypid
|
||||
#print deltas
|
||||
#total = 0.0
|
||||
#for i in deltas:
|
||||
# total += i
|
||||
|
||||
if args.summary:
|
||||
summary = dict()
|
||||
|
||||
|
|
@ -155,7 +110,6 @@ if args.summary:
|
|||
|
||||
for key in data:
|
||||
op = data[key][0][1].split('179,0 ')[1].split()[0]
|
||||
#print op
|
||||
if 'R' in op:
|
||||
readnum += 1
|
||||
waitreadtime += float(data[key][1][1])
|
||||
|
|
@ -165,21 +119,11 @@ if args.summary:
|
|||
if 'F' in op:
|
||||
flushnum += 1
|
||||
waitflushtime += float(data[key][1][1])
|
||||
#break
|
||||
wallclocktime = (end_time - start_time)*1000
|
||||
summary['Summary'] = [['Time Waiting on IO',['Total',total],['Percentage',(total / wallclocktime)*100]],['Reads',['Number of Ops',readnum],['Time Waiting for Reads',waitreadtime]],['Writes',['Number of Ops',writenum],['Time Waiting for Writes',waitwritetime]],['Flushes',['Number of Ops',flushnum],['Time Waiting for Flushes',waitflushtime]]]
|
||||
print json.dumps(summary, indent=2)
|
||||
|
||||
else:
|
||||
|
||||
#print total
|
||||
|
||||
print json.dumps(data, indent=2)
|
||||
|
||||
#data = dict()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue