Stats processing now also supports aggregate analysis
parent
54e4f99cbd
commit
12436761f2
|
|
@ -0,0 +1,8 @@
|
|||
#!/bin/bash
|
||||
|
||||
ME=$(dirname $0)/..
|
||||
LIBS=$(exec ls $ME/lib/*.jar $ME/build/jar/*.jar | xargs echo | tr ' ' ':')
|
||||
|
||||
CMD="java -cp $LIBS sqlanalysis.Parser $*"
|
||||
#echo $CMD
|
||||
$CMD
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
package sqlanalysis;
|
||||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
|
||||
import net.sf.jsqlparser.statement.*;
|
||||
import net.sf.jsqlparser.statement.select.*;
|
||||
import net.sf.jsqlparser.expression.*;
|
||||
import net.sf.jsqlparser.expression.operators.conditional.*;
|
||||
|
||||
import sqlanalysis.QueryStats.QueryType;
|
||||
|
||||
public class AggStats {
|
||||
long outerJoins = 0;
|
||||
long distincts = 0;
|
||||
long limits = 0;
|
||||
long orderBys = 0;
|
||||
long aggregates = 0;
|
||||
long unions = 0;
|
||||
long groupBys = 0;
|
||||
long withs = 0;
|
||||
long totalCount = 0;
|
||||
CountableHash<String> functionCounts = new CountableHash<String>();
|
||||
CountableHash<Integer> tableCounts = new CountableHash<Integer>();
|
||||
CountableHash<Integer> nestingDepths = new CountableHash<Integer>();
|
||||
|
||||
public void incorporate(QueryStats q)
|
||||
{
|
||||
totalCount++;
|
||||
if(q.outerJoin) outerJoins++;
|
||||
if(q.distinct) distincts++;
|
||||
if(q.limit) limits++;
|
||||
if(q.orderBy) orderBys++;
|
||||
if(q.aggregate) aggregates++;
|
||||
if(q.union) unions++;
|
||||
if(q.groupBy) groupBys++;
|
||||
if(q.with) withs++;
|
||||
for(Map.Entry<String, Integer> entry : q.functions.entrySet()){
|
||||
functionCounts.incr(entry.getKey(), entry.getValue());
|
||||
}
|
||||
tableCounts.incr(q.tables.size());
|
||||
nestingDepths.incr(q.maxNesting);
|
||||
}
|
||||
|
||||
public void mergeWith(AggStats other)
|
||||
{
|
||||
totalCount += other.totalCount;
|
||||
outerJoins += other.outerJoins;
|
||||
distincts += other.distincts;
|
||||
limits += other.limits;
|
||||
orderBys += other.orderBys;
|
||||
aggregates += other.aggregates;
|
||||
unions += other.unions;
|
||||
groupBys += other.groupBys;
|
||||
withs += other.withs;
|
||||
functionCounts.mergeWith(other.functionCounts);
|
||||
tableCounts.mergeWith(other.tableCounts);
|
||||
nestingDepths.mergeWith(other.nestingDepths);
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(" Total Queries: " + totalCount + "\n");
|
||||
sb.append(" OUTER JOIN Queries: " + outerJoins + "\n");
|
||||
sb.append(" DISTINCT Queries: " + distincts + "\n");
|
||||
sb.append(" LIMIT Queries: " + limits + "\n");
|
||||
sb.append(" ORDER BY Queries: " + orderBys + "\n");
|
||||
sb.append(" Aggregate Queries: " + aggregates + "\n");
|
||||
sb.append(" GROUP BY Aggregate Queries: " + groupBys + "\n");
|
||||
sb.append(" UNION Queries: " + unions + "\n");
|
||||
sb.append(" WITH Queries: " + withs + "\n");
|
||||
sb.append(" Call Sites per Function: \n"+functionCounts.toString());
|
||||
sb.append(" Query Counts by Number of Tables Referenced: \n"+tableCounts.toString());
|
||||
sb.append(" Query Counts by Query Nesting Depth: \n"+nestingDepths.toString());
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static class CountableHash<K> extends HashMap<K,Integer> {
|
||||
public void incr(K key) { incr(key, 1); }
|
||||
public void incr(K key, int v){
|
||||
if(!containsKey(key)){ put(key, v); }
|
||||
else { put(key, get(key) + v); }
|
||||
}
|
||||
public String toString()
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for(Map.Entry<K, Integer> entry : entrySet()) {
|
||||
sb.append(" | "+entry.getKey()+" -> " + entry.getValue() + "\n");
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
public void mergeWith(CountableHash<K> other)
|
||||
{
|
||||
for(Map.Entry<K, Integer> entry : other.entrySet()) {
|
||||
incr(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public static class RecursiveStats extends AggStats
|
||||
{
|
||||
HashMap<QueryType,AggStats> statsByType = new HashMap<QueryType,AggStats>();
|
||||
long errors = 0;
|
||||
|
||||
public RecursiveStats()
|
||||
{
|
||||
super();
|
||||
statsByType.put(QueryType.SELECT, new AggStats());
|
||||
statsByType.put(QueryType.UPDATE, new AggStats());
|
||||
statsByType.put(QueryType.INSERT, new AggStats());
|
||||
statsByType.put(QueryType.DELETE, new AggStats());
|
||||
statsByType.put(QueryType.UPSERT, new AggStats());
|
||||
}
|
||||
|
||||
public void incorporate(QueryStats q)
|
||||
{
|
||||
super.incorporate(q);
|
||||
statsByType.get(q.type).incorporate(q);
|
||||
}
|
||||
public void mergeWithRecursive(RecursiveStats other)
|
||||
{
|
||||
mergeWith(other);
|
||||
for(QueryType t : QueryType.values()){
|
||||
statsByType.get(t).mergeWith(other.statsByType.get(t));
|
||||
}
|
||||
errors += other.errors;
|
||||
}
|
||||
|
||||
public void error()
|
||||
{
|
||||
errors ++;
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
StringBuilder sb = new StringBuilder("----- Global Stats -----\n");
|
||||
sb.append(super.toString());
|
||||
sb.append(" Total Parse Errors: " + errors + "\n");
|
||||
sb.append(" Percent Success Rate: " + (100.0 * (double)totalCount / ((double)(totalCount + errors))) + "\n");
|
||||
for(Map.Entry<QueryType,AggStats> entry : statsByType.entrySet()){
|
||||
sb.append("----- "+entry.getKey().name()+" Stats -----\n");
|
||||
sb.append(entry.getValue());
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ package sqlanalysis;
|
|||
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.*;
|
||||
import java.sql.SQLException;
|
||||
|
||||
import net.sf.jsqlparser.*;
|
||||
|
|
@ -24,6 +25,7 @@ public class Parser {
|
|||
public QueryStats stats;
|
||||
|
||||
public Parser(Reader r){ source = new CCJSqlParser(r); }
|
||||
public Parser(String s){ source = new CCJSqlParser(new StringReader(s)); }
|
||||
|
||||
public QueryStats nextStmt()
|
||||
{
|
||||
|
|
@ -34,16 +36,19 @@ public class Parser {
|
|||
stmt.accept(statementHandler);
|
||||
return stats;
|
||||
} catch(ParseException e) {
|
||||
// System.err.println(e.toString());
|
||||
System.err.println(e.toString().split("\n")[0]);
|
||||
// e.printStackTrace();
|
||||
} catch(TokenMgrError e) {
|
||||
// System.err.println(e.toString());
|
||||
// e.printStackTrace();
|
||||
System.err.println(e.toString());
|
||||
//a e.printStackTrace();
|
||||
} catch(NumberFormatException e){
|
||||
System.err.println(e.toString());
|
||||
//a e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static void main(String[] args)
|
||||
public static void processQueries() throws IOException
|
||||
{
|
||||
Parser parser = new Parser(new InputStreamReader(System.in));
|
||||
QueryStats stmt;
|
||||
|
|
@ -61,24 +66,101 @@ public class Parser {
|
|||
|
||||
}
|
||||
|
||||
public void feature(String feature)
|
||||
public static void processOneFile(File path, AggStats.RecursiveStats aggStats) throws IOException
|
||||
{
|
||||
System.out.println("Feature: " + feature);
|
||||
long lineCount = 0;
|
||||
long errorCount = 0;
|
||||
BufferedReader r = new BufferedReader(new FileReader(path));
|
||||
String line;
|
||||
while((line = r.readLine()) != null){
|
||||
if(!line.equals("")){
|
||||
lineCount++;
|
||||
try {
|
||||
String[] cols = line.split("\t");
|
||||
int pid = Integer.parseInt(cols[0]);
|
||||
long timestamp = Long.parseLong(cols[1]);
|
||||
boolean isAProgram = Integer.parseInt(cols[2]) != 0;
|
||||
long duration = Long.parseLong(cols[3]);
|
||||
long rows = (cols[4].equals("") ? 0 : Long.parseLong(cols[4]));
|
||||
Parser p = new Parser(cols[5]);
|
||||
QueryStats stats = p.nextStmt();
|
||||
if(stats != null){
|
||||
stats.timestamp = timestamp;
|
||||
stats.duration = duration;
|
||||
stats.rows = rows;
|
||||
stats.isAProgram = isAProgram;
|
||||
aggStats.incorporate(stats);
|
||||
} else {
|
||||
errorCount++;
|
||||
aggStats.error();
|
||||
}
|
||||
} catch(Exception e) {
|
||||
System.err.println("On Line: "+line);
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
System.out.println(path.getName()+"\n -> "+errorCount+" errors on "+lineCount+" lines ("+
|
||||
(int)(100.0*((double)(lineCount-errorCount)/((double)lineCount)))+
|
||||
"% success rate)");
|
||||
}
|
||||
|
||||
public List<String> getFunctions(Expression e)
|
||||
public static void processFlatlines(File path, ForkJoinPool exec) throws IOException
|
||||
{
|
||||
final List<String> functions = new ArrayList<String>();
|
||||
e.accept(new ExpressionVisitorBase(true){
|
||||
public void visit(Function f){ functions.add(f.getName()); }
|
||||
});
|
||||
return functions;
|
||||
}
|
||||
public void dumpFunctions(Expression e)
|
||||
{
|
||||
for(String f : getFunctions(e)){
|
||||
feature("FUNCTION: " + f);
|
||||
if(path.isDirectory()){
|
||||
for(String subpath : path.list()){
|
||||
processFlatlines(new File(path,subpath), exec);
|
||||
}
|
||||
} else {
|
||||
System.out.println("Queuing: "+path.getName());
|
||||
exec.execute(new FileTask(path));
|
||||
}
|
||||
}
|
||||
|
||||
public static AggStats.RecursiveStats globalStats = new AggStats.RecursiveStats();
|
||||
|
||||
public static synchronized void mergeStats(File src, AggStats.RecursiveStats other)
|
||||
{
|
||||
System.out.println("Merging: "+src.getName());
|
||||
globalStats.mergeWithRecursive(other);
|
||||
try {
|
||||
FileWriter out = new FileWriter(new File("currentStats"), false);
|
||||
out.write(globalStats.toString());
|
||||
out.close();
|
||||
} catch(IOException e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public static class FileTask implements Runnable
|
||||
{
|
||||
File path;
|
||||
public AggStats.RecursiveStats aggStats = new AggStats.RecursiveStats();
|
||||
|
||||
public FileTask(File path){ this.path = path; }
|
||||
|
||||
public void run()
|
||||
{
|
||||
try {
|
||||
processOneFile(path, aggStats);
|
||||
mergeStats(path, aggStats);
|
||||
} catch(Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String args[]) throws Exception
|
||||
{
|
||||
if(args.length == 0){ processQueries(); }
|
||||
else {
|
||||
ForkJoinPool threadPool = new ForkJoinPool(6);
|
||||
for(String path : args){ processFlatlines(new File(path), threadPool); }
|
||||
threadPool.shutdown();
|
||||
while(!threadPool.isTerminated()){
|
||||
threadPool.awaitTermination(1, TimeUnit.DAYS);
|
||||
}
|
||||
System.out.println(globalStats.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,21 +14,25 @@ public class QueryStats
|
|||
Statement stmt;
|
||||
int currentNesting = 0;
|
||||
|
||||
boolean outerJoin = false; // The query contains an outer join
|
||||
boolean distinct = false; // The query contains a distinct
|
||||
boolean limit = false; // The query contains a limit / top
|
||||
boolean orderBy = false; // The query contains an order by / top
|
||||
boolean aggregate = false; // The query contains an aggregate
|
||||
boolean union = false; // The query contains a union
|
||||
boolean groupBy = false; // The query contains a Group By
|
||||
boolean with = false; // The query is a WITH/USING ... SELECT query
|
||||
int maxNesting = 0; // The maximum level of select nesting
|
||||
QueryType type = QueryType.SELECT; // The type of query
|
||||
String target = ""; // The target of an update/insert/delete/select into
|
||||
Map<String, Integer> functions = new HashMap<String, Integer>();
|
||||
List<Expression> whereClauses = new ArrayList<Expression>();
|
||||
List<Expression> havingClauses = new ArrayList<Expression>();
|
||||
Set<String> tables = new HashSet<String>();
|
||||
public boolean outerJoin = false; // The query contains an outer join
|
||||
public boolean distinct = false; // The query contains a distinct
|
||||
public boolean limit = false; // The query contains a limit / top
|
||||
public boolean orderBy = false; // The query contains an order by / top
|
||||
public boolean aggregate = false; // The query contains an aggregate
|
||||
public boolean union = false; // The query contains a union
|
||||
public boolean groupBy = false; // The query contains a Group By
|
||||
public boolean with = false; // The query is a WITH/USING ... SELECT query
|
||||
public long timestamp;
|
||||
public long duration;
|
||||
public long rows;
|
||||
public boolean isAProgram;
|
||||
public int maxNesting = 0; // The maximum level of select nesting
|
||||
public QueryType type = QueryType.SELECT; // The type of query
|
||||
public String target = ""; // The target of an update/insert/delete/select into
|
||||
public Map<String, Integer> functions = new HashMap<String, Integer>();
|
||||
public List<Expression> whereClauses = new ArrayList<Expression>();
|
||||
public List<Expression> havingClauses = new ArrayList<Expression>();
|
||||
public Set<String> tables = new HashSet<String>();
|
||||
|
||||
public String toString()
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue