Prototype

.gitignore

@@ -0,0 +1,2 @@
+*.swp
+/build/*

README

@@ -0,0 +1,14 @@
+GENERAL:
+Clone the project and just type 'ant'.
+Change build.xml as required.
+
+FOLDERS:
+/data - Contains data generated by JoinsFileGenerator.
+/lib  - Contains all libs used in this project.
+/logs - Necessary logs for further investigation.
+/scripts - python scripts to generate plots.
+/src  - Java code to parse the SQLLite queries.
+
+NOTE:
+Original data for generating the files in the data directory is obtained from
+PhoneLab SQLLite logs.

build.xml

@@ -0,0 +1,56 @@
+<project name="SQLProcessor" basedir="." default="main">
+
+    <property name="src.dir"  value="src"/>
+    <property name="build.dir"  value="build"/>
+    <property name="lib.dir" value="lib"/>
+    <property name="classes.dir"  value="${build.dir}/classes"/>
+    <property name="jar.dir"  value="${build.dir}/jar"/>
+
+    <property name="main-class"  value="SQLAnalyser.JoinsFileProcessor"/>
+    <property name="test-class"  value="SQLAnalyser.TestRunner"/>
+
+    <path id="classpath">
+      <fileset dir="${lib.dir}" includes="**/*.jar"/>
+    </path>
+
+    <target name="clean">
+      <delete dir="${build.dir}"/>
+    </target>
+
+    <target name="compile">
+        <mkdir dir="${classes.dir}"/>
+        <javac srcdir="${src.dir}" destdir="${classes.dir}" classpathref="classpath"/>
+    </target>
+
+    <target name="jar" depends="compile">
+        <mkdir dir="${jar.dir}"/>
+        <jar destfile="${jar.dir}/${ant.project.name}.jar" basedir="${classes.dir}">
+            <manifest>
+              <attribute name="Main-Class" value="${main-class}"/>
+            </manifest>
+        </jar>
+    </target>
+
+    <target name="run" depends="jar">
+      <java fork="true" classname="${main-class}">
+        <classpath>
+          <path refid="classpath"/>
+          <path location="${jar.dir}/${ant.project.name}.jar"/>
+        </classpath>
+				<arg value="${arg0}"/>
+      </java>
+    </target>
+
+		<target name="test" depends="clean,jar">
+      <java fork="true" classname="${test-class}">
+        <classpath>
+          <path refid="classpath"/>
+          <path location="${jar.dir}/${ant.project.name}.jar"/>
+        </classpath>
+      </java>
+    </target>
+
+    <target name="clean-build" depends="clean,jar"/>
+    <target name="main" depends="clean,run"/>
+
+</project>

data/table-ref-count.txt

@@ -0,0 +1,53 @@
+edition                         16
+reminders                        4
+conversation_labels            367
+stickers                         2
+game_instances                  71
+contacts                         2
+threads                       1986
+interest                         1
+LISTS                            4
+tree_entity                     56
+DataSources                     76
+ac_people                     2169
+games                           19
+alert                          171
+SHOULDKEEPON                     2
+collaborators                    8
+items                           37
+notifications                    1
+note_label                       1
+owner_sync_requests             55
+images                         371
+event_pending_ops                1
+blob_node                       69
+invitations                    106
+reminder                         1
+request_pending_ops              1
+data                           116
+Account                         17
+messages                      1075
+view_data                      194
+sharing                         18
+LISTITEMS                        6
+notification                     5
+AccountMetadata121               2
+accounts                      3053
+PrimePlaylistTracks              4
+Message                        236
+application_sessions             1
+pdu                           1477
+stream_item_photos             134
+sms                           1477
+all_photos                      14
+MUSIC                            7
+list_item_conflict               9
+section                         46
+leaderboard_pending_scores       1
+pack_types                      79
+search_sequence               3987
+raw_contacts                    39
+view_contacts                  171
+DocumentContent121               8
+notes                            5
+list_item                      238

scripts/plot.py

@@ -0,0 +1,38 @@
+import re
+import numpy as np
+import matplotlib.pyplot as plt
+
+entries = []
+
+def process_entries():
+    tables = []
+    hits = []
+    p = re.compile(r'^(?P<table>\w+)\s*(?P<hits>.*?)$')
+    for entry in entries:
+        m = p.match(entry)
+        tables.append(m.group('table'))
+        hits.append(int(m.group('hits')))
+    print tables
+    print hits
+
+    #ax.plot(hits)
+    #ax.bar(np.arange(len(hits)), hits, align='center', width=1.0)
+    #ax.set(xticks=range(len(tables)), xticklabels=tables, rotation='60')
+    plt.bar(range(len(hits)), hits, align='center', width=0.7)
+    plt.xticks(range(len(tables)), tables, size='small')
+    plt.setp(plt.xticks()[1], rotation=90)
+    plt.grid()
+    plt.xlabel("Table Name")
+    plt.ylabel("Hits");
+    plt.title("Table-Name vs Hits")
+    plt.show()
+
+def readfile(filename):
+    f = open(filename, 'r')
+    for line in f:
+        entries.append(line.rstrip())
+
+if __name__ == "__main__":
+    readfile("/home/jerry/Desktop/SQLLite-experiment/java2/data/"\
+            "table-ref-count.txt")
+    process_entries()

src/SQLAnalyser/JoinsFileGenerator.java

@@ -0,0 +1,65 @@
+package SQLAnalyser;
+
+import java.io.*;
+import java.util.*;
+import java.util.regex.*;
+
+
+public class JoinsFileGenerator {
+
+  ArrayList<String> joins = new ArrayList<String>();
+
+  private void writeJOINsToFile() {
+    try {
+      File file = new File("data/joins.dat");
+      BufferedWriter output = new BufferedWriter(new FileWriter(file));
+      for(String join:joins) {
+        output.write(join);
+        output.write("\n");
+      }
+      output.close();
+    } catch(IOException e) {
+      System.out.println("Unable to write to file data/joins.dat");
+    }
+  }
+
+  private void extractJOINs(String line) {
+    Pattern p = Pattern.compile("\"(.*?)\"");
+    Matcher m;
+    if(line.contains("JOIN")) {
+      m = p.matcher(line);
+      if(m.find()){
+        String temp = m.group(1).replaceAll("\\\\n"," ");
+        //System.out.println(temp);
+        joins.add(temp);
+      }
+    }
+  }
+
+  private void parseFile(File file) {
+    try {
+      BufferedReader br = new BufferedReader(new FileReader(file));
+      String line;
+      while((line=br.readLine()) != null) {
+        extractJOINs(line);
+      }
+    } catch(IOException e) {
+      System.out.println("Unable to open file"+file.getAbsolutePath());
+    }
+  }
+
+  private void parseDirectory(String dirName) {
+    File dir = new File(dirName);
+    File[] dirListing = dir.listFiles();
+    for (File file:dirListing) {
+      this.parseFile(file);
+    }
+    this.writeJOINsToFile();
+  }
+
+  public static void main(String[] args) {
+    JoinsFileGenerator main = new JoinsFileGenerator();
+    main.parseDirectory(args[0]);       //Assuming arg0 is the dirName
+    //main.parseFile(new File("data/joins-experiment.dat")); 
+  }
+}

src/SQLAnalyser/JoinsFileProcessor.java

@@ -0,0 +1,86 @@
+package SQLAnalyser;
+
+import java.io.*;
+import java.util.*;
+
+import net.sf.jsqlparser.*;
+import net.sf.jsqlparser.parser.CCJSqlParserManager;
+import net.sf.jsqlparser.statement.*;
+import net.sf.jsqlparser.statement.select.*;
+
+import SQLAnalyser.visitor.*;
+
+public class JoinsFileProcessor {
+
+
+  private String sql;
+  private CCJSqlParserManager pm = new CCJSqlParserManager();
+
+  private void parseSQL() {
+    try {
+      Statement stmt = pm.parse(new StringReader(sql));
+      FileWriter fw;
+      try {
+        File unparsable = new File("data/parsable.dat");
+        unparsable.createNewFile();
+        fw = new FileWriter("data/parsable.dat",true);
+        fw.write(sql);
+        fw.write("\n");
+        fw.close();
+      } catch(IOException ioe) {
+        System.err.println(ioe.getMessage());
+      }
+    } catch(JSQLParserException e) {
+      FileWriter fw;
+      try {
+        File unparsable = new File("data/unparsable.dat");
+        unparsable.createNewFile();
+        fw = new FileWriter("data/unparsable.dat",true);
+        fw.write(sql);
+        fw.write("\n");
+        fw.close();
+      } catch(IOException ioe) {
+        System.err.println(ioe.getMessage());
+      }
+    }
+  }
+
+  public void parseJoinFile(String joinsFileName) {
+    try {
+      BufferedReader br = new BufferedReader(new FileReader(joinsFileName));
+      while((sql=br.readLine()) != null)
+        this.parseSQL();
+    } catch(IOException e) {
+      e.printStackTrace();
+    }
+
+  }
+
+  public void parseParsableJOINs(String filename) {
+    MySelectVisitor mySelectVisitor = new MySelectVisitor();
+    try {
+      BufferedReader br = new BufferedReader(new FileReader(filename));
+      while((sql=br.readLine()) != null) {
+        Statement stmt = pm.parse(new StringReader(sql));
+        if(stmt instanceof Select) {
+          SelectBody selectBody = ((Select)stmt).getSelectBody();
+          System.out.println("");
+          System.out.println("=================================");
+          System.out.println(((Select)stmt).toString());
+          selectBody.accept(new MySelectVisitor());
+        }
+      }
+    } catch(IOException | JSQLParserException e) {
+      e.printStackTrace();
+    }
+  }
+
+  public static void main(String[] args){
+    JoinsFileProcessor joinsFileProcessor = new JoinsFileProcessor();
+    //joinsFileProcessor.parseJoinFile(args[0]);
+    joinsFileProcessor.parseParsableJOINs("data/parsable.dat");
+    MyFromItemVisitor myFromItemVisitor = new MyFromItemVisitor();
+    myFromItemVisitor.printTableRefCount();
+  }
+
+}

src/SQLAnalyser/TestRunner.java

@@ -0,0 +1,14 @@
+package SQLAnalyser;
+
+import org.junit.runner.*;
+import org.junit.runner.notification.*;
+
+public class TestRunner {
+  public static void main(String[] args) {
+    Result result = JUnitCore.runClasses(JoinsFileGenerator.class);
+    for(Failure failure: result.getFailures()) {
+      System.out.println(failure.toString());
+    }
+    System.out.println(result.wasSuccessful());
+  }
+}

src/SQLAnalyser/visitor/MyExpressionVisitor.java

@@ -0,0 +1,218 @@
+package SQLAnalyser.visitor;
+
+import java.util.*;
+
+import net.sf.jsqlparser.*;
+import net.sf.jsqlparser.schema.*;
+import net.sf.jsqlparser.expression.*;
+import net.sf.jsqlparser.statement.select.*;
+import net.sf.jsqlparser.expression.operators.arithmetic.*;
+import net.sf.jsqlparser.expression.operators.conditional.*;
+import net.sf.jsqlparser.expression.operators.relational.*;
+
+public class MyExpressionVisitor implements ExpressionVisitor {
+
+  @Override
+  public void visit(Addition addition) {
+    System.out.println("In ADDITION:"); 
+  }
+
+  @Override
+  public void visit(AllComparisonExpression allComparisonExpression) {
+    System.out.println("IN ALLCOMPARISON EXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(AndExpression andExpression) {
+    System.out.println("IN ANDEXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(AnyComparisonExpression anyComparisonExpression) {
+    System.out.println("IN ANYCOMPARISON EXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(BitwiseXor bitwiseXor) {
+    System.out.println("IN BITWISE XOR:"); 
+  }
+
+  @Override
+  public void visit(BitwiseOr bitwiseOr) {
+    System.out.println("IN BITWISE OR:"); 
+  }
+
+  @Override
+  public void visit(BitwiseAnd bitwiseAnd) {
+    System.out.println("IN BITWISE AND:"); 
+  }
+
+  @Override
+  public void visit(Between between) {
+    System.out.println("IN BETWEEN:"); 
+  }
+
+  @Override
+  public void visit(CaseExpression caseExpression) {
+    System.out.println("IN CASEEXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(Column column) {
+    System.out.println("IN COLUMN:"); 
+    System.out.println("COLUMN NAME:"+column.getColumnName());
+    System.out.println("WHOLE COLUMN NAME:"+column.getWholeColumnName());
+    //Table table = column.getTable();
+    //if(!table.toString().equals("null")) table.accept(new MyFromItemVisitor());
+  }
+
+  @Override
+  public void visit(Concat concat) {
+    System.out.println("IN CONCAT:"); 
+    System.out.println(concat.toString());
+  }
+
+  @Override
+  public void visit(DateValue dateValue) {
+    System.out.println("IN DATEVALUE:"); 
+  }
+
+  @Override
+  public void visit(Division division) {
+    System.out.println("IN DIVISION:"); 
+  }
+
+  @Override
+  public void visit(DoubleValue doubleValue) {
+    System.out.println("IN DOUBLEVALUE:"); 
+  }
+
+  @Override
+  public void visit(EqualsTo equalsTo) {
+    System.out.println("IN EQUALSTO:");  
+  }
+
+  @Override
+  public void visit(ExistsExpression existsExpression) {
+    System.out.println("IN EXISTSEXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(Function function) {
+    System.out.println("IN FUNCTION:");
+  }
+
+  @Override
+  public void visit(GreaterThan greaterThan) {
+    System.out.println("IN GREATER THAN:");
+  }
+
+  @Override
+  public void visit(GreaterThanEquals greaterThanEquals) {
+    System.out.println("IN GREATER THAN EQUALS:");
+  }
+
+  @Override
+  public void visit(InExpression inExpression) {
+    System.out.println("IN INEXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(InverseExpression inverseExpression) {
+    System.out.println("IN INVERSE EXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(IsNullExpression isNullExpression) {
+    System.out.println("IN ISNULL EXPRESSSION:"); 
+  }
+
+  @Override
+  public void visit(JdbcParameter jdbcParameter) {
+    System.out.println("IN JDBC PARAMETER:"); 
+  }
+
+  @Override
+  public void visit(LikeExpression likeExpression) {
+    System.out.println("IN LIKE EXPRESSION:");  
+  }
+
+  @Override
+  public void visit(LongValue longValue) {
+    System.out.println("IN LONG VALUE");  
+  }
+
+  @Override
+  public void visit(Matches matches) {
+    System.out.println("IN MATCHES:");  
+  }
+
+  @Override
+  public void visit(MinorThan minorThan) {
+    System.out.println("IN MINOR THAN"); 
+  }
+
+  @Override
+  public void visit(MinorThanEquals minorThanEquals) {
+    System.out.println("IN MINOR THAN EQUALS:"); 
+  }
+
+  @Override
+  public void visit(Multiplication multiplication) {
+    System.out.println("IN MULTIPLICATION:"); 
+  }
+
+  @Override
+  public void visit(NotEqualsTo notEqualsTo) {
+    System.out.println("IN NOT EQUALS TO:"); 
+  }
+
+  @Override
+  public void visit(NullValue nullValue) {
+    System.out.println("IN NULL VALUE:"); 
+  }
+
+  @Override
+  public void visit(OrExpression orExpression) {
+    System.out.println("IN OR EXPRESSION:"); 
+  }
+
+  @Override
+  public void visit(Parenthesis parenthesis) {
+    System.out.println("IN PARENTHESIS:"); 
+  }
+
+  @Override
+  public void visit(StringValue stringValue) {
+    System.out.println("IN STRING VALUE:");  
+  }
+
+  @Override
+  public void visit(SubSelect subSelect) {
+    System.out.println("IN SUBSELECT:"); 
+    System.out.println(subSelect.toString());
+    SelectBody selectBody = subSelect.getSelectBody();
+    selectBody.accept(new MySelectVisitor());
+  }
+
+  @Override
+  public void visit(Subtraction substraction) {
+    System.out.println("IN SUBTRACTION:"); 
+  }
+
+  @Override
+  public void visit(TimestampValue timestampValue) {
+    System.out.println("IN TIMESTAMP VALUE:"); 
+  }
+
+  @Override
+  public void visit(TimeValue timeValue) {
+    System.out.println("IN TIMEVALUE:"); 
+  }
+
+  @Override
+  public void visit(WhenClause whenClause) {
+    System.out.println("IN WHEN CLAUSE:"); 
+  }
+
+}

src/SQLAnalyser/visitor/MyFromItemVisitor.java

@@ -0,0 +1,77 @@
+package SQLAnalyser.visitor;
+
+import java.util.*;
+
+import net.sf.jsqlparser.statement.select.*;
+import net.sf.jsqlparser.schema.*;
+
+public class MyFromItemVisitor implements FromItemVisitor {
+
+  /*
+   * HashMap to keep count of the number of accesses to a table.
+   */
+  static HashMap<String,Integer> tableRefCount = 
+                      new HashMap<String,Integer>(0);
+  
+  @Override
+  public void visit(SubJoin subjoin) {
+    System.out.println("IN SUBJOIN:");
+  }
+  
+  @Override
+  public void visit(SubSelect subSelect) {
+    System.out.println("IN SUBSELECT:");
+    System.out.println(subSelect.toString());
+    SelectBody selectBody = subSelect.getSelectBody();
+    selectBody.accept(new MySelectVisitor());
+  }
+  
+  @Override
+  public void visit(Table tableName) {
+    System.out.println("IN TABLE:");
+    System.out.println(tableName.toString());
+    System.out.println("WholeTableName: "+tableName.getWholeTableName());
+    String alias = tableName.getAlias();
+    if(alias != null) System.out.println("Alias: "+alias);
+    String name = tableName.getName();
+    System.out.println("Name: "+name);
+    String schemaName = tableName.getSchemaName();
+    if(schemaName != null) System.out.println("SchemaName: "+schemaName);
+
+    /*
+     * Logic to update tableRefCount
+     */
+    boolean found = false;
+    Integer count = 0;
+    for(String key:tableRefCount.keySet()){
+      if(key.equals(name)){
+        found=true;
+        count = tableRefCount.get(name);
+        count += 1;
+      }
+    }
+    if(!found) tableRefCount.put(name,1);
+    else tableRefCount.put(name, count);
+    //End logic 
+  }
+
+  public void printTableRefCount() {
+    System.out.println("");
+    System.out.println("===================================================");
+    System.out.printf("%10sTABLE REFERENCE COUNT:\n"," ");
+    System.out.printf("TABLENAME%21sHITS\n"," ");
+    System.out.println("");
+    for(Map.Entry<String,Integer> entry:tableRefCount.entrySet()) {
+      String key = entry.getKey();
+      Integer value = entry.getValue();
+      System.out.printf("%s",key);
+      int numberSpaces = 30-key.length();
+      int counter =0;
+      while(counter < numberSpaces){ System.out.print(" "); counter++;}
+      System.out.printf("%4d\n",value);
+    }
+    System.out.println("===================================================");
+    
+  }
+
+}

src/SQLAnalyser/visitor/MySelectItemVisitor.java

@@ -0,0 +1,36 @@
+package SQLAnalyser.visitor;
+
+import java.util.*;
+
+import net.sf.jsqlparser.*;
+import net.sf.jsqlparser.schema.*;
+import net.sf.jsqlparser.expression.*;
+import net.sf.jsqlparser.statement.select.*;
+
+public class MySelectItemVisitor implements SelectItemVisitor {
+
+  @Override
+  public void visit(AllColumns allColumns) {
+    System.out.println("IN ALL COLUMNS:"); 
+    System.out.println(allColumns.toString());
+  }
+
+  @Override
+  public void visit(AllTableColumns allTableColumns) {
+    System.out.println("IN ALL TABLE COLUMNS:"); 
+    System.out.println(allTableColumns.toString());
+    Table table = allTableColumns.getTable(); 
+    table.accept(new MyFromItemVisitor());
+  }
+
+  @Override
+  public void visit(SelectExpressionItem selectExpressionItem) {
+    System.out.println("IN SELECTEXPRESSION ITEM:"); 
+    System.out.println(selectExpressionItem.toString());
+    String alias = selectExpressionItem.getAlias();
+    if(alias != null) System.out.println("ALIAS: "+alias);
+    Expression expression = selectExpressionItem.getExpression();
+    expression.accept(new MyExpressionVisitor());
+  }
+
+}

src/SQLAnalyser/visitor/MySelectVisitor.java

@@ -0,0 +1,44 @@
+package SQLAnalyser.visitor;
+
+import java.util.*;
+
+import net.sf.jsqlparser.statement.*;
+import net.sf.jsqlparser.statement.select.*;
+
+public class MySelectVisitor implements SelectVisitor {
+
+  @Override
+  public void visit(PlainSelect plainSelect) {
+    System.out.println("IN PLAINSELECT:");
+    System.out.println(plainSelect.toString());
+
+    List<SelectItem> selectItems = plainSelect.getSelectItems();
+    if(selectItems != null) {
+      System.out.println("SELECT ITEMS:");
+      for(SelectItem selectItem:selectItems) {
+        selectItem.accept(new MySelectItemVisitor());
+      }
+    }
+
+    FromItem fromItem = plainSelect.getFromItem();
+    fromItem.accept(new MyFromItemVisitor());
+    List<Join> joins = plainSelect.getJoins();
+    if(joins != null) {
+      System.out.println("JOINS:");
+      for(Join join:joins){
+        System.out.print(join.toString());
+      }
+    }
+  }
+
+  @Override
+  public void visit(Union union) {
+    System.out.println("IN UNION:");
+    System.out.println(union.toString());
+    List<PlainSelect> plainSelects = union.getPlainSelects(); 
+    for(PlainSelect plainSelect:plainSelects) {
+      this.visit(plainSelect);
+    }
+  }
+
+}

src/SQLAnalyserExperiment/Parser.java

@@ -0,0 +1,38 @@
+package SQLAnalyserExperiment;
+
+import java.io.*;
+
+import net.sf.jsqlparser.*;
+import net.sf.jsqlparser.statement.*;
+import net.sf.jsqlparser.parser.CCJSqlParserManager;
+
+public class Parser {
+
+  private CCJSqlParserManager pm = new CCJSqlParserManager();
+
+  public void parse(String line) {
+    try {
+    Statement stmt = pm.parse(new StringReader(line));
+    } catch(JSQLParserException e) {
+      System.out.println(line);
+      System.out.println(e.getCause());
+      System.out.println("=====================================");
+      System.out.println("");
+    }
+  }
+
+  public static void main(String[] args) {
+    Parser parser = new Parser();
+    String filename = "data/unparsable.dat";
+    try {
+      FileReader inputFile = new FileReader(filename);
+      BufferedReader bufferReader = new BufferedReader(inputFile);
+      String line;
+      while((line = bufferReader.readLine()) != null){
+        parser.parse(line);
+      }
+    } catch(IOException e) {
+      System.out.println("Unable to open:"+ filename);
+    }
+  }
+}