diff --git a/.gitignore b/.gitignore index 7040aab..29ec5f8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ comment.cut *.aux *.fdb_latexmk *.fls -*.log +/*.log *.out *.synctex.gz *.bbl @@ -15,7 +15,7 @@ comment.cut ## Core latex/pdflatex auxiliary files: *.aux *.lof -*.log +/*.log *.lot *.fls *.out diff --git a/results/desktop-dataspread-varysize.log b/results/desktop-dataspread-varysize.log new file mode 100644 index 0000000..8fd833d --- /dev/null +++ b/results/desktop-dataspread-varysize.log @@ -0,0 +1,31 @@ +[info] DataspreadBenchmarkVizierSpec +[info] DataspreadBenchmarkVizierSpec should +[info] Perform Benchamrks consistent with those done with VizierDB +[test] @0: Init Spreadsheet: 21.147718712 s +[test] @0: Monitoring Overhead: 0.204571017 s +[test] @0: Init formulas: 5.258463694 s +[test] @0: Update one: 0.031132039 s +[test] @0: Update all: [not run] +[info] + Time Results @ 0 +[test] @60: Init Spreadsheet: 21.075780951 s +[test] @60: Monitoring Overhead: 0.121315652 s +[test] @60: Init formulas: 7.208747642 s +[test] @60: Update one: 0.020606538 s +[test] @60: Update all: [not run] +[info] + Time Results @ 60 +[test] @600: Init Spreadsheet: 21.059860767 s +[test] @600: Monitoring Overhead: 0.120807536 s +[test] @600: Init formulas: 24.264395221 s +[test] @600: Update one: 0.007306683 s +[test] @600: Update all: [not run] +[info] + Time Results @ 600 +[test] @6000: Init Spreadsheet: 21.056667272 s +[test] @6000: Monitoring Overhead: 0.11134842 s +[test] @6000: Init formulas: 245.847328681 s +[test] @6000: Update one: 0.033850361 s +[test] @6000: Update all: [not run] +[info] + Time Results @ 6000 +[info] o Time Results @ 60000 [not run - taking longer than 30 minutes] +[info] Total for specification DataspreadBenchmarkVizierSpec +[info] Finished in 6 minutes 16 seconds, 997 ms +[info] 4 examples, 6 expectations, 0 failure, 0 error diff --git a/results/desktop-dataspread-varystart.log b/results/desktop-dataspread-varystart.log new file mode 100644 index 0000000..72f2a2c --- /dev/null +++ b/results/desktop-dataspread-varystart.log @@ -0,0 +1,32 @@ +[info] DataspreadBenchmarkVizierSpec +[info] DataspreadBenchmarkVizierSpec should +[info] Perform Benchamrks consistent with those done with VizierDB +[info] + Warm up the cache +[test] @0: Init Spreadsheet: 21.144650182 s +[test] @0: Monitoring Overhead: 0.204167816 s +[test] @0: Init formulas: 5.258831865 s +[test] @0: Update one: 0.033579433 s +[test] @0: Update all: [not run] +[info] + Time Results @ 0 +[test] @60: Init Spreadsheet: 21.046036608 s +[test] @60: Monitoring Overhead: 0.116584287 s +[test] @60: Init formulas: 7.469271179 s +[test] @60: Update one: 0.013094508 s +[test] @60: Update all: [not run] +[info] + Time Results @ 60 +[test] @600: Init Spreadsheet: 21.068284745 s +[test] @600: Monitoring Overhead: 0.11571002 s +[test] @600: Init formulas: 21.723715773 s +[test] @600: Update one: 0.007254895 s +[test] @600: Update all: [not run] +[info] + Time Results @ 600 +[test] @6000: Init Spreadsheet: 21.072645619 s +[test] @6000: Monitoring Overhead: 0.11134726 s +[test] @6000: Init formulas: 244.810593821 s +[test] @6000: Update one: 0.021708022 s +[test] @6000: Update all: [not run] +[info] + Time Results @ 6000 +[info] o Time Results @ 60000 [not run - taking longer than 30 minutes] +[info] Total for specification DataspreadBenchmarkVizierSpec +[info] Finished in 6 minutes 15 seconds, 731 ms +[info] 5 examples, 7 expectations, 0 failure, 0 error diff --git a/results/desktop-dataspread-varystartandsize.log b/results/desktop-dataspread-varystartandsize.log new file mode 100644 index 0000000..b405e69 --- /dev/null +++ b/results/desktop-dataspread-varystartandsize.log @@ -0,0 +1,32 @@ +[info] DataspreadBenchmarkVizierSpec +[info] DataspreadBenchmarkVizierSpec should +[info] Perform Benchamrks consistent with those done with VizierDB +[test] @0: Init Spreadsheet: 21.159097527 s +[test] @0: Monitoring Overhead: 0.231722052 s +[test] @0: Init formulas: 5.259485356 s +[test] @0: Update one: 0.019440175 s +[test] @0: Update all: [not run] +[info] + Time Results @ 0 +[test] @60: Init Spreadsheet: 21.077188513 s +[test] @60: Monitoring Overhead: 0.119935001 s +[test] @60: Init formulas: 8.167940197 s +[test] @60: Update one: 0.024402629 s +[test] @60: Update all: [not run] +[info] + Time Results @ 60 +[test] @600: Init Spreadsheet: 0.030223373 s +[test] @600: Monitoring Overhead: 0.113077619 s +[test] @600: Init formulas: 32.570919298 s +[test] @600: Update one: 0.008316406 s +[test] @600: Update all: [not run] +[info] + Time Results @ 600 +[test] @6000: Init Spreadsheet: 0.024342633 s +[test] @6000: Monitoring Overhead: 0.111124168 s +[test] @6000: Init formulas: 193.063436155 s +[test] @6000: Update one: 0.018608992 s +[test] @6000: Update all: [not run] +[info] + Time Results @ 6000 +[info] o Time Results @ 60000 [not run - taking longer than 30 minutes] +[info] Total for specification DataspreadBenchmarkVizierSpec +[info] Finished in 4 minutes 51 seconds, 142 ms +[info] 4 examples, 6 expectations, 0 failure, 0 error + diff --git a/results/desktop-update_one.pdf b/results/desktop-init-varysize.pdf similarity index 70% rename from results/desktop-update_one.pdf rename to results/desktop-init-varysize.pdf index 4f622a0..2c3ca0a 100644 Binary files a/results/desktop-update_one.pdf and b/results/desktop-init-varysize.pdf differ diff --git a/results/desktop-init-varysize.png b/results/desktop-init-varysize.png new file mode 100644 index 0000000..994f2e9 Binary files /dev/null and b/results/desktop-init-varysize.png differ diff --git a/results/desktop-init_formulas.pdf b/results/desktop-init-varystart.pdf similarity index 71% rename from results/desktop-init_formulas.pdf rename to results/desktop-init-varystart.pdf index e14978b..ebcb975 100644 Binary files a/results/desktop-init_formulas.pdf and b/results/desktop-init-varystart.pdf differ diff --git a/results/desktop-init-varystart.png b/results/desktop-init-varystart.png new file mode 100644 index 0000000..6e1ae86 Binary files /dev/null and b/results/desktop-init-varystart.png differ diff --git a/results/desktop-init_spreadsheet.pdf b/results/desktop-init-varystartandsize.pdf similarity index 70% rename from results/desktop-init_spreadsheet.pdf rename to results/desktop-init-varystartandsize.pdf index 4efdb24..7b3796e 100644 Binary files a/results/desktop-init_spreadsheet.pdf and b/results/desktop-init-varystartandsize.pdf differ diff --git a/results/desktop-init-varystartandsize.png b/results/desktop-init-varystartandsize.png new file mode 100644 index 0000000..a786888 Binary files /dev/null and b/results/desktop-init-varystartandsize.png differ diff --git a/results/desktop-init_formulas-varysize.pdf b/results/desktop-init_formulas-varysize.pdf new file mode 100644 index 0000000..c67a760 Binary files /dev/null and b/results/desktop-init_formulas-varysize.pdf differ diff --git a/results/desktop-init_formulas-varysize.png b/results/desktop-init_formulas-varysize.png new file mode 100644 index 0000000..1c0ab8d Binary files /dev/null and b/results/desktop-init_formulas-varysize.png differ diff --git a/results/desktop-init_formulas-varystart.pdf b/results/desktop-init_formulas-varystart.pdf index f2b3409..779325f 100644 Binary files a/results/desktop-init_formulas-varystart.pdf and b/results/desktop-init_formulas-varystart.pdf differ diff --git a/results/desktop-init_formulas-varystart.png b/results/desktop-init_formulas-varystart.png index 560dc21..3d329ff 100644 Binary files a/results/desktop-init_formulas-varystart.png and b/results/desktop-init_formulas-varystart.png differ diff --git a/results/desktop-init_formulas-varystartandsize.pdf b/results/desktop-init_formulas-varystartandsize.pdf new file mode 100644 index 0000000..a3e24cd Binary files /dev/null and b/results/desktop-init_formulas-varystartandsize.pdf differ diff --git a/results/desktop-init_formulas-varystartandsize.png b/results/desktop-init_formulas-varystartandsize.png new file mode 100644 index 0000000..ed61dff Binary files /dev/null and b/results/desktop-init_formulas-varystartandsize.png differ diff --git a/results/desktop-init_formulas.png b/results/desktop-init_formulas.png deleted file mode 100644 index 66f7542..0000000 Binary files a/results/desktop-init_formulas.png and /dev/null differ diff --git a/results/desktop-init_spreadsheet-varysize.pdf b/results/desktop-init_spreadsheet-varysize.pdf new file mode 100644 index 0000000..ccf0788 Binary files /dev/null and b/results/desktop-init_spreadsheet-varysize.pdf differ diff --git a/results/desktop-init_spreadsheet-varysize.png b/results/desktop-init_spreadsheet-varysize.png new file mode 100644 index 0000000..1bf4936 Binary files /dev/null and b/results/desktop-init_spreadsheet-varysize.png differ diff --git a/results/desktop-init_spreadsheet-varystart.pdf b/results/desktop-init_spreadsheet-varystart.pdf index d6e6c31..bcfac03 100644 Binary files a/results/desktop-init_spreadsheet-varystart.pdf and b/results/desktop-init_spreadsheet-varystart.pdf differ diff --git a/results/desktop-init_spreadsheet-varystart.png b/results/desktop-init_spreadsheet-varystart.png index 24e7eeb..fd18202 100644 Binary files a/results/desktop-init_spreadsheet-varystart.png and b/results/desktop-init_spreadsheet-varystart.png differ diff --git a/results/desktop-init_spreadsheet-varystartandsize.pdf b/results/desktop-init_spreadsheet-varystartandsize.pdf new file mode 100644 index 0000000..587d8ba Binary files /dev/null and b/results/desktop-init_spreadsheet-varystartandsize.pdf differ diff --git a/results/desktop-init_spreadsheet-varystartandsize.png b/results/desktop-init_spreadsheet-varystartandsize.png new file mode 100644 index 0000000..6b37fd7 Binary files /dev/null and b/results/desktop-init_spreadsheet-varystartandsize.png differ diff --git a/results/desktop-init_spreadsheet.png b/results/desktop-init_spreadsheet.png deleted file mode 100644 index ae8cf51..0000000 Binary files a/results/desktop-init_spreadsheet.png and /dev/null differ diff --git a/results/desktop-update_one-varysize.pdf b/results/desktop-update_one-varysize.pdf new file mode 100644 index 0000000..07be12c Binary files /dev/null and b/results/desktop-update_one-varysize.pdf differ diff --git a/results/desktop-update_one-varysize.png b/results/desktop-update_one-varysize.png new file mode 100644 index 0000000..7336efa Binary files /dev/null and b/results/desktop-update_one-varysize.png differ diff --git a/results/desktop-update_one-varystart.pdf b/results/desktop-update_one-varystart.pdf index 1660111..6e0c131 100644 Binary files a/results/desktop-update_one-varystart.pdf and b/results/desktop-update_one-varystart.pdf differ diff --git a/results/desktop-update_one-varystart.png b/results/desktop-update_one-varystart.png index 40730b3..10acf78 100644 Binary files a/results/desktop-update_one-varystart.png and b/results/desktop-update_one-varystart.png differ diff --git a/results/desktop-update_one-varystartandsize.pdf b/results/desktop-update_one-varystartandsize.pdf new file mode 100644 index 0000000..6ef7a41 Binary files /dev/null and b/results/desktop-update_one-varystartandsize.pdf differ diff --git a/results/desktop-update_one-varystartandsize.png b/results/desktop-update_one-varystartandsize.png new file mode 100644 index 0000000..03d10ce Binary files /dev/null and b/results/desktop-update_one-varystartandsize.png differ diff --git a/results/desktop-update_one.png b/results/desktop-update_one.png deleted file mode 100644 index b9193a8..0000000 Binary files a/results/desktop-update_one.png and /dev/null differ diff --git a/results/desktop-vizier-varysize.log b/results/desktop-vizier-varysize.log new file mode 100644 index 0000000..e7a4705 --- /dev/null +++ b/results/desktop-vizier-varysize.log @@ -0,0 +1,56 @@ +SpreadsheetBenchmark + +@60/false: Init Spreadsheet: 0.502968873 s +@60/false: Monitoring Overhead: 0.006877202 s +@60/false: Init Formulas: 0.288059989 s +@60/false: Update one: 0.008191735 s +@60/false: Update all: 0.066918199 s ++ Time Results @ 60 +@60/true: Init Spreadsheet: 0.006720101 s +@60/true: Monitoring Overhead: 0.005177446 s +@60/true: Init Formulas: 0.224219809 s +@60/true: Update one: 0.008588053 s +@60/true: Update all: 0.059844046 s ++ Time Results @ 60 +@600/false: Init Spreadsheet: 0.010373324 s +@600/false: Monitoring Overhead: 0.008801294 s +@600/false: Init Formulas: 0.207266834 s +@600/false: Update one: 0.007608617 s +@600/false: Update all: 0.076818458 s ++ Time Results @ 600 +@600/true: Init Spreadsheet: 0.014502952 s +@600/true: Monitoring Overhead: 0.005923629 s +@600/true: Init Formulas: 0.191094545 s +@600/true: Update one: 0.007352669 s +@600/true: Update all: 0.057713751 s ++ Time Results @ 600 +@6000/false: Init Spreadsheet: 0.29348608 s +@6000/false: Monitoring Overhead: 0.010335879 s +@6000/false: Init Formulas: 0.225178793 s +@6000/false: Update one: 0.008125263 s +@6000/false: Update all: 0.061337873 s ++ Time Results @ 6000 +@6000/true: Init Spreadsheet: 0.005700485 s +@6000/true: Monitoring Overhead: 0.00465223 s +@6000/true: Init Formulas: 0.205236902 s +@6000/true: Update one: 0.007241557 s +@6000/true: Update all: 0.053460177 s ++ Time Results @ 6000 +@60000/false: Init Spreadsheet: 4.830022963 s +@60000/false: Monitoring Overhead: 0.004278716 s +@60000/false: Init Formulas: 0.160613413 s +@60000/false: Update one: 0.006726616 s +@60000/false: Update all: 0.057563017 s ++ Time Results @ 60000 +@60000/true: Init Spreadsheet: 0.007197506 s +@60000/true: Monitoring Overhead: 0.004691892 s +@60000/true: Init Formulas: 0.230308324 s +@60000/true: Update one: 0.007654771 s +@60000/true: Update all: 0.053793202 s ++ Time Results @ 60000 + + +Total for specification SpreadsheetBenchmark +Finished in 15 seconds, 208 ms +8 examples, 9 expectations, 0 failure, 0 error + diff --git a/results/desktop-vizier-varystart.log b/results/desktop-vizier-varystart.log new file mode 100644 index 0000000..77a6e34 --- /dev/null +++ b/results/desktop-vizier-varystart.log @@ -0,0 +1,56 @@ +SpreadsheetBenchmark + ++ Warm up the cache +@60/false: Init Spreadsheet: 26.140277045 s +@60/false: Monitoring Overhead: 0.014927718 s +@60/false: Init Formulas: 0.378240991 s +@60/false: Update one: 0.007560233 s +@60/false: Update all: 0.087895794 s ++ Time Results @ 60 +@60/true: Init Spreadsheet: 0.007541636 s +@60/true: Monitoring Overhead: 0.00600814 s +@60/true: Init Formulas: 0.309725093 s +@60/true: Update one: 0.00739941 s +@60/true: Update all: 0.080354155 s ++ Time Results @ 60 +@600/false: Init Spreadsheet: 0.00719922 s +@600/false: Monitoring Overhead: 0.005914293 s +@600/false: Init Formulas: 0.409290477 s +@600/false: Update one: 0.008420892 s +@600/false: Update all: 0.144015039 s ++ Time Results @ 600 +@600/true: Init Spreadsheet: 0.007104852 s +@600/true: Monitoring Overhead: 0.005998643 s +@600/true: Init Formulas: 0.284384031 s +@600/true: Update one: 0.008116714 s +@600/true: Update all: 0.077044715 s ++ Time Results @ 600 +@6000/false: Init Spreadsheet: 0.39814007 s +@6000/false: Monitoring Overhead: 0.005437453 s +@6000/false: Init Formulas: 4.291371661 s +@6000/false: Update one: 0.010826873 s +@6000/false: Update all: 0.697241208 s ++ Time Results @ 6000 +@6000/true: Init Spreadsheet: 0.006777436 s +@6000/true: Monitoring Overhead: 0.005843616 s +@6000/true: Init Formulas: 0.304945226 s +@6000/true: Update one: 0.007598998 s +@6000/true: Update all: 0.076540439 s ++ Time Results @ 6000 +@60000/false: Init Spreadsheet: 0.42241651 s +@60000/false: Monitoring Overhead: 0.005534486 s +@60000/false: Init Formulas: 47.899451675 s +@60000/false: Update one: 0.03709275 s +@60000/false: Update all: 29.091698828 s ++ Time Results @ 60000 +@60000/true: Init Spreadsheet: 0.006794719 s +@60000/true: Monitoring Overhead: 0.005888871 s +@60000/true: Init Formulas: 0.450570698 s +@60000/true: Update one: 0.007473072 s +@60000/true: Update all: 0.078396281 s ++ Time Results @ 60000 + + +Total for specification SpreadsheetBenchmark +Finished in 1 minute 58 seconds, 970 ms +9 examples, 10 expectations, 0 failure, 0 error diff --git a/results/desktop-vizier-varystartandsize.log b/results/desktop-vizier-varystartandsize.log new file mode 100644 index 0000000..e2ed729 --- /dev/null +++ b/results/desktop-vizier-varystartandsize.log @@ -0,0 +1,56 @@ +SpreadsheetBenchmark + +@60/false: Init Spreadsheet: 0.622319637 s +@60/false: Monitoring Overhead: 0.008267427 s +@60/false: Init Formulas: 0.379130661 s +@60/false: Update one: 0.008588219 s +@60/false: Update all: 0.096520684 s ++ Time Results @ 60 +@60/true: Init Spreadsheet: 0.007797491 s +@60/true: Monitoring Overhead: 0.006393537 s +@60/true: Init Formulas: 0.461880471 s +@60/true: Update one: 0.013939607 s +@60/true: Update all: 0.11355628 s ++ Time Results @ 60 +@600/false: Init Spreadsheet: 0.008746324 s +@600/false: Monitoring Overhead: 0.006425039 s +@600/false: Init Formulas: 0.551072941 s +@600/false: Update one: 0.01089435 s +@600/false: Update all: 0.158270664 s ++ Time Results @ 600 +@600/true: Init Spreadsheet: 0.007240454 s +@600/true: Monitoring Overhead: 0.006301443 s +@600/true: Init Formulas: 0.339588744 s +@600/true: Update one: 0.009767286 s +@600/true: Update all: 0.086571365 s ++ Time Results @ 600 +@6000/false: Init Spreadsheet: 0.443006796 s +@6000/false: Monitoring Overhead: 0.007127946 s +@6000/false: Init Formulas: 2.475917445 s +@6000/false: Update one: 0.01008147 s +@6000/false: Update all: 0.858393776 s ++ Time Results @ 6000 +@6000/true: Init Spreadsheet: 0.007298637 s +@6000/true: Monitoring Overhead: 0.005954614 s +@6000/true: Init Formulas: 0.342004032 s +@6000/true: Update one: 0.008022405 s +@6000/true: Update all: 0.08392664 s ++ Time Results @ 6000 +@60000/false: Init Spreadsheet: 5.584728068 s +@60000/false: Monitoring Overhead: 0.006156743 s +@60000/false: Init Formulas: 42.167866315 s +@60000/false: Update one: 0.0327693 s +@60000/false: Update all: 17.918641827 s ++ Time Results @ 60000 +@60000/true: Init Spreadsheet: 0.00672735 s +@60000/true: Monitoring Overhead: 0.005856171 s +@60000/true: Init Formulas: 0.339442519 s +@60000/true: Update one: 0.007649768 s +@60000/true: Update all: 0.076880671 s ++ Time Results @ 60000 + + +Total for specification SpreadsheetBenchmark +Finished in 1 minute 21 seconds, 988 ms +8 examples, 9 expectations, 0 failure, 0 error + diff --git a/results/gen_graph.py b/results/gen_graph.py index a4bf590..2dfbb2e 100644 --- a/results/gen_graph.py +++ b/results/gen_graph.py @@ -63,24 +63,68 @@ def read_vizier(testbed, experiment): # 'update one' - time to update one cell # 'update all' - time to update an entire column (not used) # 4. time: float (number of seconds) +# 5. experiment: 'varystart', 'varysize', 'varystartandsize' data = [ record for ds in [ read_vizier("desktop", "varystart"), - # read_vizier("laptop"), read_dataspread("desktop", "varystart"), - # read_dataspread("laptop") + read_vizier("desktop", "varysize"), + read_dataspread("desktop", "varysize"), + read_vizier("desktop", "varystartandsize"), + read_dataspread("desktop", "varystartandsize"), ] for record in ds ] stages = set(i[3] for i in data) +sizes = set(i[2] for i in data) + +experiment_xlabels = { + "varystart" : "First visible row", + "varysize" : "Number of rows", + "varystartandsize" : "Number of rows", +} + +system_labels = { + "vizier" : "Vizier", + "vizier-batch" : "Vizier (Simulated Batching)", + "dataspread" : "DataSpread" +} + + +init_costs = {} +init_fields = [ + "init spreadsheet", + "init formulas" +] +for record in data: + if record[3] in init_fields: + key = ( + record[0], + record[1], + record[2], + record[5] + ) + print(key) + init_costs[key] = init_costs.get(key, 0) + record[4] + +data += [ + ( + key[0], + key[1], + key[2], + "init", + init_costs[key], + key[3] + ) + for key in init_costs +] print(data) print(stages) - def plot_one(testbed, stage, experiment): global data fig, ax = plt.subplots( @@ -90,24 +134,25 @@ def plot_one(testbed, stage, experiment): # ax.set_title(f"{stage} ({testbed})") ax.set_ylabel(f"{stage} (s)") - ax.set_xlabel("Data Size (number of rows)") + ax.set_xlabel(experiment_xlabels[experiment]) ax.set_xscale("log") ax.set_yscale("log") - for system in ["vizier", "vizier-batch", "dataspread"]: + for system in system_labels: points = sorted([ (record[2], record[4]) for record in data if record[0] == testbed and record[1] == system and record[3] == stage + and record[5] == experiment ], key=lambda x: x[0]) ax.plot( [pt[0] for pt in points], [pt[1] for pt in points], - label=system + label=system_labels[system] ) ax.legend() stage = stage.replace(" ", "_") @@ -115,6 +160,19 @@ def plot_one(testbed, stage, experiment): fig.savefig(f"{testbed}-{stage}-{experiment}.png") + + plot_one("desktop", "init spreadsheet", "varystart") plot_one("desktop", "init formulas", "varystart") +plot_one("desktop", "init", "varystart") plot_one("desktop", "update one", "varystart") + +plot_one("desktop", "init spreadsheet", "varysize") +plot_one("desktop", "init formulas", "varysize") +plot_one("desktop", "init", "varysize") +plot_one("desktop", "update one", "varysize") + +plot_one("desktop", "init spreadsheet", "varystartandsize") +plot_one("desktop", "init formulas", "varystartandsize") +plot_one("desktop", "init", "varystartandsize") +plot_one("desktop", "update one", "varystartandsize")