PocketData
/
data-TPCTC-Pocketdata
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Some preliminary graphs

main
Oliver Kennedy 2015-06-06 19:15:22 -04:00
parent 12436761f2
commit 977531bdd8
15 changed files with 416 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
*.swp
/build/*
currentStats*

198
Rakefile
View File

@ -1,41 +1,20 @@
require 'rubygems'
require 'gnuplot'
$:.push("lib")
require 'json'
require 'util.rb'
require 'plot.rb'
$files = Dir["/Volumes/Data/sqlite-logs/*.out.gz"]
$files = Dir["/Volumes/Storage/sqlite-logs/*.out.gz"]
task :default => :graphs
plot_output :pdf
auto_open_plots
$query_actions = [
:SELECT, :DELETE, :INSERT, :UPDATE
]
$parser_jar = "build/jar/SQLProcessor.jar"
$parser = nil
def parser
if $parser.nil?
$parser =
IO.popen([
"java",
"-cp",
([$parser_jar] + Dir["lib/*.jar"]).join(":"),
"sqlanalysis.Parser"
].join(" "), "w+")
else $parser end
end
def parse(q)
return ["###SUCCESS: #{q}"] if /^PRAGMA/ =~ q
parser.puts(
q.
gsub(/^SQLiteProgram: */, "").
gsub(/;?$/, ";")
);
parser.flush()
data = parser.lines_until { |l| /^###/ =~ l }
raise "Parse Failed" if /^###ERROR/ =~ data[-1]
data.join.gsub(/^###SUCCESS: /, "")
end
class String
def escape
@ -106,10 +85,6 @@ def log_lines(file)
end
end
def all_log_lines
$files.flatmap { |f| log_lines(f) { |meta, data| yield(meta, data) } }
end
task :actions do
actions =
all_log_lines do |meta, data|
@ -134,7 +109,7 @@ $files.each do |source|
(/^SQLiteProgram: / =~ data["Results"]) ? 1 : 0,
data["Time"],
data["Rows returned"],
data["Results"].gsub(/^SQLiteProgram: /, "")
data["Results"].gsub(/^SQLiteProgram: /, "").gsub(/\n/, " ")
].join("\t"))
end
end
@ -150,33 +125,6 @@ $files.each do |source|
task :flatten => target
end
task :qstats => :compile do
i = 0;
errs = 0
all_log_lines do |meta, data|
data if $query_actions.include? data["Action"].to_sym
end.compact.each do |data|
i += 1
q = data["Results"]
stats = nil
unless /^PRAGMA/ =~ q then
begin
p JSON.parse(parse(q));
rescue => e
if e.to_s == "Parse Failed" then
errs += 1
else
raise e
end
# puts "#{i}: #{e}: #{data["Results"]}"
# puts "======================"
end
end
end
puts "#{errs} out of #{i} errored out"
end
task :test => :compile do
# p JSON.parse(parse("SELECT GROUP_CONCAT(imagesGroupConcatSourceTable._id, ' / ') FROM R"));
@ -186,6 +134,132 @@ end
file $parser_jar => Dir["src/**/*.java"] { system("ant jar") or raise "Build Failed"; }
task :compile => $parser_jar
task :stats => [:actions]
$stats = Hash.new { |h,k| h[k] = Hash.new }
$stat_classes = ["SELECT", "INSERT", "UPSERT", "UPDATE", "DELETE"]
task :default => :qstats
task :loadStats do
File.open("graphs/stats") do |f|
segment = nil
field_name = nil;
field_value = nil;
attr_name = nil;
attr_value = nil;
f.each do |l|
case l.chomp
when /----- ([a-zA-Z]+) Stats -----/ then segment = $1
when / ([^:]+): +([0-9.]+)/ then
field_name, field_value = $1, $2.to_f
$stats[segment][field_name] = field_value
when / ([^:]+): *$/ then
field_name, field_value = $1, Hash.new
$stats[segment][field_name] = field_value
when / +\| *([^ ]+) +-> *([^ ]+)/
attr_name, attr_value = $1, $2.to_f
field_value[attr_name] = attr_value
else raise "Huh? : #{l.chomp}"
end
end
end
end
["SELECT", "DELETE", "UPDATE"].each do |segment|
plot "graphs/#{segment.downcase}_breakdown_by_width" => :loadStats do |graph|
pretty_plot(graph, :logy => true)
graph.yrange '[1:100000000]'
graph.xlabel 'Number of Tables Accessed'
graph.ylabel "Number of #{segment} Queries"
counts, qcounts =
$stats[segment]["Query Counts by Number of Tables Referenced"].to_a.
sort { |a,b| a[0] <=> b[0] }.
unzip
draw_bar_plot(graph, data: qcounts, labels: counts, box_style: lambda { |i| "boxes fill solid lc rgb \"#A00000\""})
end
task :graphs => "graphs/#{segment.downcase}_breakdown_by_width"
end
plot "graphs/spj_breakdown_by_width" => :loadStats do |graph|
pretty_plot(graph, :logy => true)
graph.yrange '[1:100000000]'
graph.xlabel 'Number of Tables Accessed'
graph.ylabel 'Number of Select/Project/Join Queries'
counts, qcounts =
$stats["SELECT"]["SPJ Query Counts by Join Width"].to_a.
sort { |a,b| a[0] <=> b[0] }.
unzip
draw_bar_plot(graph, data: qcounts, labels: counts, box_style: lambda { |i| "boxes fill solid lc rgb \"#A00000\""})
end
task :graphs => "graphs/spj_breakdown_by_width"
["SELECT", "UPDATE"].each do |segment|
plot "graphs/#{segment.downcase}_breakdown_by_nesting" => :loadStats do |graph|
pretty_plot(graph, :logy => true)
graph.yrange '[1:100000000]'
graph.xlabel 'Number of Tables Accessed'
graph.ylabel "Number of #{segment} Queries"
counts, qcounts =
$stats[segment]["Query Counts by Query Nesting Depth"].to_a.
sort { |a,b| a[0] <=> b[0] }.
unzip
draw_bar_plot(graph, data: qcounts, labels: counts, box_style: lambda { |i| "boxes fill solid lc rgb \"#A00000\""})
end
task :graphs => "graphs/#{segment.downcase}_breakdown_by_nesting"
end
# def plot(type, args)
# fname =
# case args
# when Hash then args.keys[0]
# when String solid
# else raise "Invalid plot arguments: #{args}"
# end
# file(args) do
# g = type.new
# g.theme_greyscale
# yield g
# g.write(fname)
# system("open #{fname}")
# end
# task :graphs => fname
# end
# plot Gruff::Pie, "graphs/breakdown_by_category.pdf" => :loadStats do |g|
# g.title = "Breakdown by SQL/SDL Operation"
# $stat_classes.each do |qtype|
# g.data qtype, $stats[qtype]["Total Queries"]
# end
# end
# ["SELECT"].each do |segment|
# plot Gruff::Bar, "graphs/#{segment.downcase}_breakdown_by_width.pdf" => :loadStats do |g|
# g.title = "Join Width in #{segment} Queries"
# counts, qcounts =
# $stats[segment]["Query Counts by Number of Tables Referenced"].to_a.
# sort { |a,b| a[0] <=> b[0] }.
# unzip
# g.minimum_value = 0;
# g.maximum_value = Math.log10(qcounts.max).ceil
# # counts.shift
# g.labels = counts.map.with_index { |k,v| [v, k] }.to_h
# g.data(:SELECT, qcounts.map {|i| Math.log10(i) })
# end
# end

2
build.xml
View File

@ -19,7 +19,7 @@
<target name="compile">
<mkdir dir="${classes.dir}"/>
<javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="classpath" includeantruntime="false"/>
<javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="classpath" includeantruntime="false" debug="true"/>
</target>
<target name="jar" depends="compile">

BIN
graphs/breakdown_by_category.pdf Normal file
View File

Binary file not shown.

BIN
graphs/delete_breakdown_by_nesting.pdf Normal file
View File

Binary file not shown.

BIN
graphs/delete_breakdown_by_width.pdf Normal file
View File

Binary file not shown.

BIN
graphs/select_breakdown_by_nesting.pdf Normal file
View File

Binary file not shown.

BIN
graphs/select_breakdown_by_width.pdf Normal file
View File

Binary file not shown.

BIN
graphs/spj_breakdown_by_width.pdf Normal file
View File

Binary file not shown.

186
graphs/stats Normal file
View File

@ -0,0 +1,186 @@
----- Global Stats -----
Total Queries: 45090798
OUTER JOIN Queries: 391288
DISTINCT Queries: 1893624
LIMIT Queries: 1165518
ORDER BY Queries: 3169109
Aggregate Queries: 641352
GROUP BY Aggregate Queries: 438944
UNION Queries: 13866
WITH Queries: 0
Average Runtime: 1.3473690060371963 ms
Call Sites per Function:
| CAST -> 38208
| MAX -> 318153
| LENGTH -> 102747
| JULIANDAY -> 587
| SUM -> 321387
| UPPER -> 20487
| DATE -> 44
| COALESCE -> 3494
| AVG -> 15
| MIN -> 19566
| IFNULL -> 657
| PHONE_NUMBERS_EQUAL -> 2017
| LOWER -> 3110
| COUNT -> 173038
| GROUP_CONCAT -> 583474
| SUBSTR -> 88462
| STRFTIME -> 1147
Query Counts by Number of Tables Referenced:
| 0 -> 11574173
| 1 -> 31846350
| 2 -> 1347385
| 3 -> 200258
| 4 -> 32896
| 5 -> 83744
| 6 -> 1765
| 7 -> 4216
| 8 -> 11
Query Counts by Query Nesting Depth:
| 0 -> 11574173
| 1 -> 33021712
| 2 -> 389294
| 3 -> 88951
| 4 -> 16668
Query Runtimes by Query Nesting Depth:
Total Parse Errors: 308752
Percent Success Rate: 99.31992277456494
----- DELETE Stats -----
Total Queries: 1248594
OUTER JOIN Queries: 236
DISTINCT Queries: 5586
LIMIT Queries: 422
ORDER BY Queries: 194
Aggregate Queries: 3190
GROUP BY Aggregate Queries: 0
UNION Queries: 65
WITH Queries: 0
Average Runtime: 3.7761548616708076 ms
Call Sites per Function:
| MAX -> 3183
| COUNT -> 7
Query Counts by Number of Tables Referenced:
| 0 -> 1202472
| 1 -> 42528
| 2 -> 1736
| 3 -> 782
| 4 -> 460
| 7 -> 616
Query Counts by Query Nesting Depth:
| 0 -> 1202472
| 1 -> 46122
Query Runtimes by Query Nesting Depth:
----- SELECT Stats -----
Total Queries: 33470310
OUTER JOIN Queries: 391052
DISTINCT Queries: 1888013
LIMIT Queries: 1165096
ORDER BY Queries: 3168915
Aggregate Queries: 638137
GROUP BY Aggregate Queries: 438919
UNION Queries: 13801
WITH Queries: 0
Average Runtime: 1.1290731669464669 ms
Call Sites per Function:
| CAST -> 38208
| MAX -> 314970
| LENGTH -> 102747
| JULIANDAY -> 587
| SUM -> 321387
| UPPER -> 20487
| DATE -> 44
| COALESCE -> 3494
| AVG -> 15
| MIN -> 19566
| IFNULL -> 657
| PHONE_NUMBERS_EQUAL -> 2017
| LOWER -> 3110
| COUNT -> 173031
| GROUP_CONCAT -> 583474
| SUBSTR -> 88462
| STRFTIME -> 1147
Query Counts by Number of Tables Referenced:
| 1 -> 31803710
| 2 -> 1345568
| 3 -> 199476
| 4 -> 32436
| 5 -> 83744
| 6 -> 1765
| 7 -> 3600
| 8 -> 11
Query Counts by Query Nesting Depth:
| 1 -> 32975453
| 2 -> 389238
| 3 -> 88951
| 4 -> 16668
Query Runtimes by Query Nesting Depth:
SP Queries: 28723834
SP Runtime: 0.6330146528792779 ms
SPJ Queries: 29147210
SPJ Runtime: 0.6557674440088433 ms
SPA Queries: 28947579
SPA Runtime: 0.6698433789751812 ms
SPJA Queries: 29405360
SPJA Runtime: 0.6921697210216438 ms
SPJ Query Counts by Join Width:
| 1 -> 28723834
| 2 -> 375061
| 3 -> 47187
| 4 -> 1125
| 6 -> 3
----- INSERT Stats -----
Total Queries: 1953279
OUTER JOIN Queries: 0
DISTINCT Queries: 0
LIMIT Queries: 0
ORDER BY Queries: 0
Aggregate Queries: 0
GROUP BY Aggregate Queries: 0
UNION Queries: 0
WITH Queries: 0
Average Runtime: 2.3116640830337087 ms
Call Sites per Function:
Query Counts by Number of Tables Referenced:
| 0 -> 1953279
Query Counts by Query Nesting Depth:
| 0 -> 1953279
Query Runtimes by Query Nesting Depth:
----- UPDATE Stats -----
Total Queries: 1041967
OUTER JOIN Queries: 0
DISTINCT Queries: 25
LIMIT Queries: 0
ORDER BY Queries: 0
Aggregate Queries: 25
GROUP BY Aggregate Queries: 25
UNION Queries: 0
WITH Queries: 0
Average Runtime: 6.588152973392632 ms
Call Sites per Function:
Query Counts by Number of Tables Referenced:
| 0 -> 1041774
| 1 -> 112
| 2 -> 81
Query Counts by Query Nesting Depth:
| 0 -> 1041774
| 1 -> 137
| 2 -> 56
Query Runtimes by Query Nesting Depth:
----- UPSERT Stats -----
Total Queries: 7376648
OUTER JOIN Queries: 0
DISTINCT Queries: 0
LIMIT Queries: 0
ORDER BY Queries: 0
Aggregate Queries: 0
GROUP BY Aggregate Queries: 0
UNION Queries: 0
WITH Queries: 0
Average Runtime: 0.9311366858334572 ms
Call Sites per Function:
Query Counts by Number of Tables Referenced:
| 0 -> 7376648
Query Counts by Query Nesting Depth:
| 0 -> 7376648
Query Runtimes by Query Nesting Depth:

BIN
graphs/update_breakdown_by_nesting.pdf Normal file
View File

Binary file not shown.

BIN
graphs/update_breakdown_by_width.pdf Normal file
View File

Binary file not shown.

BIN
lib/jsqlparser.jar
View File

Binary file not shown.

1
lib/plot.rb
View File

@ -173,6 +173,7 @@ def draw_clustered_bar_plot plot, args = {}
box_style = args.fetch(:box_style,
lambda { |i| "boxes fill pattern #{i}" });
plot.grid "noxtics"
group_offset = base_offset + margins
group_size = interbar_offset * data.length + intergroup_offset;
plot.boxwidth bar_width.to_s;

98
src/sqlanalysis/AggStats.java
View File

@ -20,9 +20,11 @@ public class AggStats {
long groupBys = 0;
long withs = 0;
long totalCount = 0;
double runtime = 0;
CountableHash<String> functionCounts = new CountableHash<String>();
CountableHash<Integer> tableCounts = new CountableHash<Integer>();
CountableHash<Integer> nestingDepths = new CountableHash<Integer>();
AverageableHash<Integer> runtimeByNestingInMS = new AverageableHash<Integer>();
public void incorporate(QueryStats q)
{
@ -35,11 +37,11 @@ public class AggStats {
if(q.union) unions++;
if(q.groupBy) groupBys++;
if(q.with) withs++;
for(Map.Entry<String, Integer> entry : q.functions.entrySet()){
functionCounts.incr(entry.getKey(), entry.getValue());
}
runtime += q.duration;
functionCounts.mergeWith(q.functions);
tableCounts.incr(q.tables.size());
nestingDepths.incr(q.maxNesting);
runtimeByNestingInMS.incr(q.maxNesting, q.duration / 1000000l);
}
public void mergeWith(AggStats other)
@ -53,9 +55,11 @@ public class AggStats {
unions += other.unions;
groupBys += other.groupBys;
withs += other.withs;
runtime += other.runtime;
functionCounts.mergeWith(other.functionCounts);
tableCounts.mergeWith(other.tableCounts);
nestingDepths.mergeWith(other.nestingDepths);
runtimeByNestingInMS.mergeWith(runtimeByNestingInMS);
}
public String toString()
@ -70,39 +74,79 @@ public class AggStats {
sb.append(" GROUP BY Aggregate Queries: " + groupBys + "\n");
sb.append(" UNION Queries: " + unions + "\n");
sb.append(" WITH Queries: " + withs + "\n");
sb.append(" Average Runtime: " + (runtime / totalCount /1000000.0) + " ms\n");
sb.append(" Call Sites per Function: \n"+functionCounts.toString());
sb.append(" Query Counts by Number of Tables Referenced: \n"+tableCounts.toString());
sb.append(" Query Counts by Query Nesting Depth: \n"+nestingDepths.toString());
sb.append(" Query Runtimes by Query Nesting Depth: \n"+runtimeByNestingInMS.toString(" ms"));
return sb.toString();
}
public static class CountableHash<K> extends HashMap<K,Integer> {
public static class CountableHash<K> extends HashMap<K,Long> {
public void incr(K key) { incr(key, 1); }
public void incr(K key, int v){
public void incr(K key, long v){
if(!containsKey(key)){ put(key, v); }
else { put(key, get(key) + v); }
}
public String toString()
{
StringBuilder sb = new StringBuilder();
for(Map.Entry<K, Integer> entry : entrySet()) {
for(Map.Entry<K, Long> entry : entrySet()) {
sb.append(" | "+entry.getKey()+" -> " + entry.getValue() + "\n");
}
return sb.toString();
}
public void mergeWith(CountableHash<K> other)
{
for(Map.Entry<K, Integer> entry : other.entrySet()) {
for(Map.Entry<K, Long> entry : other.entrySet()) {
incr(entry.getKey(), entry.getValue());
}
}
}
public static class AverageableHash<K> extends HashMap<K,Double> {
int count = 0;
public void incr(K key, double v){
count++;
if(!containsKey(key)){ put(key, v); }
else { put(key, get(key) + v); }
}
public String toString() { return toString(""); }
public String toString(String units)
{
StringBuilder sb = new StringBuilder();
for(Map.Entry<K, Double> entry : entrySet()) {
sb.append(" | "+entry.getKey()+" -> " + (entry.getValue() / Math.max(count,1)) + units + "\n");
}
return sb.toString();
}
public void mergeWith(CountableHash<K> other)
{
count += other.count;
for(Map.Entry<K, Double> entry : other.entrySet()) {
if(!containsKey(entry.getKey())){
put(entry.getKey(), entry.getValue());
} else {
put(entry.getKey(), get(entry.getKey()) + entry.getValue());
}
}
}
}
public static class RecursiveStats extends AggStats
{
HashMap<QueryType,AggStats> statsByType = new HashMap<QueryType,AggStats>();
long errors = 0;
long spQueries = 0;
long spjQueries = 0;
long spaQueries = 0;
long spjaQueries = 0;
double spRuntime = 0;
double spjRuntime = 0;
double spaRuntime = 0;
double spjaRuntime = 0;
CountableHash<Integer> spJoinWidths = new CountableHash<Integer>();
public RecursiveStats()
{
@ -118,6 +162,26 @@ public class AggStats {
{
super.incorporate(q);
statsByType.get(q.type).incorporate(q);
if(q.type == QueryType.SELECT){
if( (!q.distinct) &&
(!q.orderBy) &&
(q.maxNesting == 1))
{
spjaQueries++;
spjaRuntime += q.duration;
if(!q.aggregate){
spjQueries++;
spjRuntime += q.duration;
spJoinWidths.incr(q.tables.size());
}
if(q.tables.size() == 1){
spaQueries++;
spaRuntime += q.duration;
if(!q.aggregate){ spQueries++; spRuntime += q.duration;}
}
}
}
}
public void mergeWithRecursive(RecursiveStats other)
{
@ -126,6 +190,15 @@ public class AggStats {
statsByType.get(t).mergeWith(other.statsByType.get(t));
}
errors += other.errors;
spjaQueries += other.spjaQueries;
spaQueries += other.spaQueries;
spjQueries += other.spjQueries;
spQueries += other.spQueries;
spjaRuntime += other.spjaRuntime;
spaRuntime += other.spaRuntime;
spjRuntime += other.spjRuntime;
spRuntime += other.spRuntime;
spJoinWidths.mergeWith(other.spJoinWidths);
}
public void error()
@ -142,6 +215,17 @@ public class AggStats {
for(Map.Entry<QueryType,AggStats> entry : statsByType.entrySet()){
sb.append("----- "+entry.getKey().name()+" Stats -----\n");
sb.append(entry.getValue());
if(entry.getKey() == QueryType.SELECT){
sb.append(" SP Queries: " + spQueries + "\n");
sb.append(" SP Runtime: " + (spRuntime/spQueries/1000000.0) + " ms\n");
sb.append(" SPJ Queries: " + spjQueries + "\n");
sb.append(" SPJ Runtime: " + (spjRuntime/spjQueries/1000000.0) + " ms\n");
sb.append(" SPA Queries: " + spaQueries + "\n");
sb.append(" SPA Runtime: " + (spaRuntime/spaQueries/1000000.0) + " ms\n");
sb.append(" SPJA Queries: " + spjaQueries + "\n");
sb.append(" SPJA Runtime: " + (spjaRuntime/spjaQueries/1000000.0) + " ms\n");
sb.append(" SPJ Query Counts by Join Width: \n" + spJoinWidths.toString());
}
}
return sb.toString();
}