more aggstats
parent
76915e0c52
commit
036f9bbfbf
Binary file not shown.
|
|
@ -3,8 +3,10 @@ package sqlanalysis;
|
|||
import java.io.*;
|
||||
import java.util.*;
|
||||
|
||||
import net.sf.jsqlparser.schema.*;
|
||||
import net.sf.jsqlparser.statement.*;
|
||||
import net.sf.jsqlparser.statement.select.*;
|
||||
import net.sf.jsqlparser.statement.update.*;
|
||||
import net.sf.jsqlparser.expression.*;
|
||||
import net.sf.jsqlparser.expression.operators.conditional.*;
|
||||
import net.sf.jsqlparser.expression.operators.relational.*;
|
||||
|
|
@ -27,6 +29,7 @@ public class AggStats {
|
|||
CountableHash<Integer> nestingDepths = new CountableHash<Integer>();
|
||||
CountableHash<Integer> runtimeByNestingInMS = new CountableHash<Integer>();
|
||||
CountableHash<String> whereClausesByQuery = new CountableHash<String>();
|
||||
CountableHash<String> queriesByApp = new CountableHash<String>();
|
||||
List<String> notes = new ArrayList<String>();
|
||||
|
||||
public void incorporate(QueryStats q)
|
||||
|
|
@ -42,9 +45,10 @@ public class AggStats {
|
|||
if(q.with) withs++;
|
||||
runtime += q.duration;
|
||||
functionCounts.mergeWith(q.functions);
|
||||
tableCounts.incr(q.tables.size());
|
||||
tableCounts.incr(q.tableSet.size());
|
||||
nestingDepths.incr(q.maxNesting);
|
||||
runtimeByNestingInMS.incr(q.maxNesting, q.duration / 1000000l);
|
||||
queriesByApp.incr(q.app);
|
||||
Set<String> tempWhereClausesByQuery = new HashSet<String>();
|
||||
for(Expression e : q.whereClauses){ tempWhereClausesByQuery.add(getWhereClauseType(e)); }
|
||||
for(String t : tempWhereClausesByQuery) { whereClausesByQuery.incr(t); }
|
||||
|
|
@ -68,6 +72,7 @@ public class AggStats {
|
|||
runtimeByNestingInMS.mergeWith(other.runtimeByNestingInMS);
|
||||
whereClausesByQuery.mergeWith(other.whereClausesByQuery);
|
||||
notes.addAll(other.notes);
|
||||
queriesByApp.mergeWith(other.queriesByApp);
|
||||
}
|
||||
|
||||
public void note(String note){ System.out.println("!!!"+note); notes.add(note); }
|
||||
|
|
@ -90,6 +95,7 @@ public class AggStats {
|
|||
sb.append(" Query Counts by Query Nesting Depth: \n"+nestingDepths.toString());
|
||||
sb.append(" Query Runtimes by Query Nesting Depth: \n"+runtimeByNestingInMS.toRatioString(nestingDepths));
|
||||
sb.append(" Query Counts by Use of Each Where Clause Type: \n"+whereClausesByQuery.toString());
|
||||
sb.append(" Query Counts by App: \n"+queriesByApp.toString());
|
||||
if(notes.size() > 0){
|
||||
sb.append(" Other Notes: \n");
|
||||
int i = 0;
|
||||
|
|
@ -100,6 +106,24 @@ public class AggStats {
|
|||
return sb.toString();
|
||||
}
|
||||
|
||||
public static class Box<A> {
|
||||
public A a;
|
||||
public Box(){}
|
||||
public Box(A a){ this.a = a; }
|
||||
public int hashCode(){ return a.hashCode(); }
|
||||
public boolean equals(Box <A> other){
|
||||
return a.equals(other.a);
|
||||
}
|
||||
public boolean equals(Object other){
|
||||
if(other instanceof Box){
|
||||
return equals((Box<A>)other);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
public String toString() { return "<"+a.toString()+">"; }
|
||||
}
|
||||
|
||||
public static class Pair<A,B> {
|
||||
public A a;
|
||||
public B b;
|
||||
|
|
@ -119,6 +143,26 @@ public class AggStats {
|
|||
public String toString() { return "<"+a.toString()+", "+b.toString()+">"; }
|
||||
}
|
||||
|
||||
public static class Triple<A,B,C> {
|
||||
public A a;
|
||||
public B b;
|
||||
public C c;
|
||||
public Triple(){}
|
||||
public Triple(A a, B b, C c){ this.a = a; this.b = b; this.c = c; }
|
||||
public int hashCode(){ return a.hashCode() ^ b.hashCode() ^ c.hashCode(); }
|
||||
public boolean equals(Triple <A,B,C> other){
|
||||
return a.equals(other.a) && b.equals(other.b) && c.equals(other.c);
|
||||
}
|
||||
public boolean equals(Object other){
|
||||
if(other instanceof Triple){
|
||||
return equals((Triple<A,B,C>)other);
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
public String toString() { return "<"+a.toString()+", "+b.toString()+", "+c.toString()+">"; }
|
||||
}
|
||||
|
||||
public static class CountableHash<K> extends HashMap<K,Long> {
|
||||
public void incr(K key) { incr(key, 1); }
|
||||
public void incr(K key, long v){
|
||||
|
|
@ -250,7 +294,10 @@ public class AggStats {
|
|||
CountableHash<SelectQueryType> selectRuntimesByType = new CountableHash<SelectQueryType>();
|
||||
CountableHash<Integer> spjJoinWidths = new CountableHash<Integer>();
|
||||
CountableHash<Pair<Integer,Integer>> spjsJoinWidthsAndWhereClauses = new CountableHash<Pair<Integer,Integer>>();
|
||||
CountableHash<Integer> updateJoinWidths = new CountableHash<Integer>();
|
||||
CountableHash<Triple<Integer,Integer,Integer>> updateTargetCountsByTotalsAndConstsAndDeltas = new CountableHash<Triple<Integer,Integer,Integer>>();
|
||||
CountableHash<String> spsSingleClausesByType = new CountableHash<String>();
|
||||
CountableHash<String> spsQueriesByApp = new CountableHash<String>();
|
||||
|
||||
public RecursiveStats()
|
||||
{
|
||||
|
|
@ -275,6 +322,7 @@ public class AggStats {
|
|||
if(q.type == QueryType.SELECT){
|
||||
int joinWidth = q.tableSet.size();
|
||||
int whereClauseCount = q.whereClauses.size();
|
||||
|
||||
if( (!q.distinct) &&
|
||||
(!q.union) &&
|
||||
(q.maxNesting == 1))
|
||||
|
|
@ -312,13 +360,48 @@ public class AggStats {
|
|||
spjsJoinWidthsAndWhereClauses.incr(new Pair<Integer,Integer>(joinWidth, whereClauseCount));
|
||||
if((joinWidth == 1)&&(whereClauseCount == 1)){
|
||||
// Learn more about simple lookup queries.
|
||||
Expression clause = q.whereClauses.get(0);
|
||||
spsSingleClausesByType.incr(getWhereClauseType(clause));
|
||||
spsSingleClausesByType.incr(getWhereClauseType(q.whereClauses.get(0)));
|
||||
}
|
||||
if((joinWidth == 1)&&(
|
||||
((whereClauseCount == 1)&&(getWhereClauseType(q.whereClauses.get(0)).equals("CONST_CMP_EQUALITY")))
|
||||
|| (whereClauseCount == 0)
|
||||
)) {
|
||||
spsQueriesByApp.incr(q.app);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
markQueryOfType(SelectQueryType.COMPLEX, q.duration);
|
||||
}
|
||||
} else if(q.type == QueryType.UPDATE) {
|
||||
int joinWidth = q.tableSet.size();
|
||||
if(joinWidth == 0){
|
||||
updateJoinWidths.incr(q.whereClauses.size());
|
||||
}
|
||||
try {
|
||||
Update upd = (Update)q.stmt;
|
||||
List<Column> targets = upd.getColumns();
|
||||
List<Expression> values = upd.getExpressions();
|
||||
int constants = 0;
|
||||
int deltas = 0;
|
||||
for(int i = 0; i < targets.size(); i++){
|
||||
final Column target = targets.get(i);
|
||||
final Expression expr = values.get(i);
|
||||
final Box<Boolean> hasColRef = new Box(false);
|
||||
if(expressionIsAConstant(expr)){ constants++; }
|
||||
target.accept(new ExpressionVisitorBase() {
|
||||
public void visitColumn(Column cmp){
|
||||
if(cmp.getColumnName().equals(target.getColumnName())){ hasColRef.a = true; }
|
||||
}
|
||||
});
|
||||
if(hasColRef.a){ deltas++; }
|
||||
}
|
||||
// System.out.println(""+targets.size()+","+constants+","+deltas);
|
||||
updateTargetCountsByTotalsAndConstsAndDeltas.incr(new Triple<Integer,Integer,Integer>(targets.size(), constants, deltas));
|
||||
} catch(ClassCastException e) { e.printStackTrace();
|
||||
} catch(Exception e){
|
||||
System.out.println("BOOM : " + e);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
public void mergeWithRecursive(RecursiveStats other)
|
||||
|
|
@ -333,6 +416,9 @@ public class AggStats {
|
|||
spjJoinWidths.mergeWith(other.spjJoinWidths);
|
||||
spjsJoinWidthsAndWhereClauses.mergeWith(other.spjsJoinWidthsAndWhereClauses);
|
||||
spsSingleClausesByType.mergeWith(other.spsSingleClausesByType);
|
||||
spsQueriesByApp.mergeWith(other.spsQueriesByApp);
|
||||
updateJoinWidths.mergeWith(other.updateJoinWidths);
|
||||
updateTargetCountsByTotalsAndConstsAndDeltas.mergeWith(other.updateTargetCountsByTotalsAndConstsAndDeltas);
|
||||
}
|
||||
|
||||
public void error()
|
||||
|
|
@ -356,6 +442,10 @@ public class AggStats {
|
|||
sb.append(" SP[+Sort] Query Counts by Join Width and Number of Conjunctive Where Clauses: \n"+
|
||||
spjsJoinWidthsAndWhereClauses.toString());
|
||||
sb.append(" SP[+Sort] With Lone Where Clause by Clause Type: \n" + spsSingleClausesByType.toString());
|
||||
sb.append(" Simple Select queries by app: \n" + spsQueriesByApp.toString());
|
||||
} else if(entry.getKey() == QueryType.UPDATE) {
|
||||
sb.append(" Simple Update Join Widths: \n" + updateJoinWidths.toString());
|
||||
sb.append(" Update Target Columns By Constants and Deltas: \n" + updateTargetCountsByTotalsAndConstsAndDeltas.toString());
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ public class Parser {
|
|||
|
||||
}
|
||||
|
||||
public static void processOneFile(File path, AggStats.RecursiveStats aggStats) throws IOException
|
||||
public static void processOneFile(File path, AggStats.RecursiveStats aggStats, HashMap<Integer,String> appNamesByPID) throws IOException
|
||||
{
|
||||
long lineCount = 0;
|
||||
long errorCount = 0;
|
||||
|
|
@ -89,6 +89,14 @@ public class Parser {
|
|||
stats.duration = duration;
|
||||
stats.rows = rows;
|
||||
stats.isAProgram = isAProgram;
|
||||
stats.pid = pid;
|
||||
stats.did = path.getName().split("-",2)[0];
|
||||
if(appNamesByPID != null){
|
||||
stats.app = appNamesByPID.get(pid);
|
||||
} else {
|
||||
stats.app = null;
|
||||
}
|
||||
if(stats.app == null){ stats.app = "???"; }
|
||||
aggStats.incorporate(stats);
|
||||
} else {
|
||||
errorCount++;
|
||||
|
|
@ -138,14 +146,31 @@ public class Parser {
|
|||
{
|
||||
File path;
|
||||
public AggStats.RecursiveStats aggStats = new AggStats.RecursiveStats();
|
||||
HashMap<Integer,String> appNamesByPID = new HashMap<Integer,String>();
|
||||
|
||||
public FileTask(File path){ this.path = path; }
|
||||
|
||||
public void loadAppNames()
|
||||
throws IOException
|
||||
{
|
||||
BufferedReader in = new BufferedReader(new FileReader(
|
||||
"/staging/sqlite-apps/"+path.getName().replaceAll("flat","appnames")));
|
||||
String line;
|
||||
String[] fields;
|
||||
while((line = in.readLine()) != null) {
|
||||
fields = line.split(" +", 3);
|
||||
appNamesByPID.put(Integer.parseInt(fields[0]), fields[2]);
|
||||
}
|
||||
}
|
||||
|
||||
public void run()
|
||||
{
|
||||
try {
|
||||
processOneFile(path, aggStats);
|
||||
loadAppNames();
|
||||
processOneFile(path, aggStats, appNamesByPID);
|
||||
mergeStats(path, aggStats);
|
||||
} catch(IOException e){
|
||||
System.out.println("ERROR: " + e.getMessage());
|
||||
} catch(Exception e){
|
||||
System.out.println(e.getMessage());
|
||||
e.printStackTrace();
|
||||
|
|
|
|||
|
|
@ -26,15 +26,51 @@ public class QueryStats
|
|||
public long duration;
|
||||
public long rows;
|
||||
public boolean isAProgram;
|
||||
public int pid;
|
||||
public String did;
|
||||
public int maxNesting = 0; // The maximum level of select nesting
|
||||
public QueryType type = QueryType.SELECT; // The type of query
|
||||
public String target = ""; // The target of an update/insert/delete/select into
|
||||
public String app;
|
||||
public Map<String, Long> functions = new HashMap<String, Long>();
|
||||
public List<Expression> whereClauses = new ArrayList<Expression>();
|
||||
public List<Expression> havingClauses = new ArrayList<Expression>();
|
||||
public Set<String> tableSet = new HashSet<String>();
|
||||
public List<String> tables = new ArrayList<String>();
|
||||
|
||||
// public static PreparedStatement[] prepare(Connection conn)
|
||||
// {
|
||||
// return new PreparedStatement[] {
|
||||
// conn.prepareStatement("
|
||||
// INSERT INTO query_stats(
|
||||
// id, has_outer_join, has_distinct, has_limit,
|
||||
// has_order_by, has_aggregate, has_union, has_group_by,
|
||||
// query_ts, query_duration, row_count, is_a_program,
|
||||
// max_nesting, query_type, target, table_count
|
||||
// ) VALUES (
|
||||
// ?, ?, ?, ?,
|
||||
// ?, ?, ?, ?,
|
||||
// ?, ?, ?, ?,
|
||||
// ?, ?, ?, ?
|
||||
// )
|
||||
// "),
|
||||
// conn.prepareStatement("
|
||||
// INSERT INTO query_stats(
|
||||
// id, has_outer_join, has_distinct, has_limit,
|
||||
// has_order_by, has_aggregate, has_union, has_group_by,
|
||||
// query_ts, query_duration, row_count, is_a_program,
|
||||
// max_nesting, query_type, target, table_count
|
||||
// ) VALUES (
|
||||
// ?, ?, ?, ?,
|
||||
// ?, ?, ?, ?,
|
||||
// ?, ?, ?, ?,
|
||||
// ?, ?, ?, ?
|
||||
// )
|
||||
// "),
|
||||
|
||||
// }
|
||||
// }
|
||||
|
||||
public String toString()
|
||||
{
|
||||
String sep;
|
||||
|
|
|
|||
Loading…
Reference in New Issue