WIP on the talk
parent
6fc43ddadb
commit
451d226376
GPG Key ID:
3E5F9B3ABD3FDB60
|
|
@ -3,6 +3,90 @@ template: templates/talk_slides_v1.erb
|
|||
title: Principled management of notebook state in Vizier
|
||||
---
|
||||
|
||||
<style type="text/css">
|
||||
.notebook {
|
||||
width: 100%;
|
||||
}
|
||||
.notebook .nbcell .nblabel {
|
||||
font-family: Courier;
|
||||
vertical-align: middle;
|
||||
margin-right: 20px;
|
||||
color: blue;
|
||||
font-weight: bold;
|
||||
}
|
||||
.notebook .nbcell pre {
|
||||
width: calc(100% - 100px);
|
||||
display: inline-block;
|
||||
vertical-align: middle;
|
||||
}
|
||||
.notebook .nbcell pre code {
|
||||
padding: 20px;
|
||||
}
|
||||
.notebook .nbcell pre.nbresult {
|
||||
width: calc(100% - 110px);
|
||||
padding-left: 20px;
|
||||
margin-left: 90px;
|
||||
padding-top: 10px;
|
||||
padding-bottom: 10px;
|
||||
}
|
||||
.notebook .nbcell.fragment.highlight-blue.current-fragment
|
||||
{
|
||||
border: solid 4px blue;
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
<%
|
||||
$cells = []
|
||||
def notebook()
|
||||
$cells = []
|
||||
ret = ""
|
||||
ret += "<div class='notebook'>"
|
||||
yield
|
||||
ret += $cells.join("")
|
||||
ret += "</div>"
|
||||
return ret
|
||||
end
|
||||
|
||||
def nbdiv(body, varargs={})
|
||||
hide = varargs.fetch(:hide, nil)
|
||||
show = varargs.fetch(:show, nil)
|
||||
highlight = varargs.fetch(:highlight, nil)
|
||||
css_class = varargs.fetch(:css, "nbcell")
|
||||
extra_attrs = ""
|
||||
unless show.nil?
|
||||
css_class += " fragment"
|
||||
extra_attrs += " data-fragment-index='#{show}'"
|
||||
end
|
||||
unless hide.nil?
|
||||
css_class += " fragment fade-out"
|
||||
extra_attrs += " data-fragment-index='#{hide}'"
|
||||
end
|
||||
unless highlight.nil?
|
||||
css_class += " fragment highlight-blue"
|
||||
extra_attrs += " data-fragment-index='#{highlight}'"
|
||||
end
|
||||
return "<div class='#{css_class}'#{extra_attrs}>#{body}</div>"
|
||||
end
|
||||
|
||||
def nbcell(text, varargs={})
|
||||
lang = varargs.fetch(:lang, "python")
|
||||
idx = varargs.fetch(:idx, nil)
|
||||
output = varargs.fetch(:output, nil)
|
||||
idx = $cells.size + 1 if idx.nil?
|
||||
cmd = "<span class='nblabel'>[#{idx}]</span><pre><code class='#{lang}'>#{text}</code></pre>"
|
||||
unless output.nil?
|
||||
cmd += "<br/><pre class='nbresult'>#{output}</pre>"
|
||||
end
|
||||
$cells += [nbdiv(cmd, varargs)]
|
||||
end
|
||||
|
||||
def nbnote(note, varargs={})
|
||||
varargs[:css_class] = "nbnote"
|
||||
$cells += [nbdiv(note, varargs)]
|
||||
end
|
||||
%>
|
||||
|
||||
<section>
|
||||
<h2><%= title %></h2>
|
||||
|
||||
|
|
@ -28,11 +112,222 @@ title: Principled management of notebook state in Vizier
|
|||
</section>
|
||||
|
||||
<section>
|
||||
|
||||
|
||||
<h3>High-Level Challenges</h3>
|
||||
<ul>
|
||||
<li>Not clear from context where a variable was written.</li>
|
||||
<li>A cell that runs may still be wrong (for the program).</li>
|
||||
<li>A state that was computed may be inconsistent.</li>
|
||||
</ul>
|
||||
|
||||
<p class="fragment takeaway">
|
||||
So why does everyone use this confusing state model?
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3>High-Level Challenges</h3>
|
||||
<svg data-src="graphics/2024-04-12/Dependencies.svg" height="400px"/>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<%=
|
||||
notebook() do
|
||||
nbcell("x = 3", idx: 1)
|
||||
nbcell("y = x + 2", idx: 2)
|
||||
nbcell("x = 4", idx: 3)
|
||||
nbcell("print(y)", idx: 4, output: "5")
|
||||
end
|
||||
%>
|
||||
</section>
|
||||
<section>
|
||||
<%=
|
||||
notebook() do
|
||||
nbcell("x = 3", idx: 1, highlight: 1)
|
||||
nbcell("y = x + 102", idx: 5)
|
||||
nbcell("x = 4", idx: 3, highlight: 1)
|
||||
nbcell("print(y)", idx: 4, output: "5", highlight: 2)
|
||||
end
|
||||
%>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3>Dependencies</h3>
|
||||
|
||||
<p class="fragment"><b>Reads: </b> <tt>x</tt></p>
|
||||
|
||||
<%=
|
||||
notebook() do
|
||||
nbcell("y = x + 2", idx: 2)
|
||||
end
|
||||
%>
|
||||
|
||||
<p class="fragment"><b>Writes: </b> <tt>y</tt></p>
|
||||
|
||||
<p class="fragment takeaway">
|
||||
<b>Question:</b> Which variables does the cell read/write?
|
||||
</p>
|
||||
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<%=
|
||||
notebook() do
|
||||
nbnote("$$\\{\\;\\;\\}$$", show: 1)
|
||||
nbcell("x = 3", idx: 1)
|
||||
nbnote("$$\\{\\;x \\rightarrow \\textbf{@1}\\;\\}$$", show: 2)
|
||||
nbcell("y = x + 2", idx: 2)
|
||||
nbnote("$$\\{\\;x \\rightarrow \\textbf{@1},\\;y \\rightarrow \\textbf{@2}\\;\\}$$", show: 3)
|
||||
nbcell("x = 4", idx: 3)
|
||||
nbnote("$$\\{\\;x \\rightarrow \\textbf{@3},\\;y \\rightarrow \\textbf{@2}\\;\\}$$", show: 4)
|
||||
nbcell("print(y)", idx: 4)
|
||||
end
|
||||
%>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<p>Interpreter State: $\{\;x \rightarrow \textbf{@3},\;y \rightarrow \textbf{@2}\;\}$</p>
|
||||
|
||||
<p>... but Cell 2 read $x \rightarrow \textbf{@1}$</p>
|
||||
|
||||
<p class="fragment takeaway">
|
||||
<b>Question:</b> How do we get the interpreter back to a known state?
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<%=
|
||||
notebook() do
|
||||
nbnote("$$\\{\\;x \\rightarrow \\textbf{@1}\\;\\}$$")
|
||||
nbcell("y = x + 102", idx: 5)
|
||||
nbnote("$$\\{\\;x \\rightarrow \\textbf{@1},\\;y \\rightarrow \\color{blue}{\\textbf{@4}}\\;\\}$$", show:1)
|
||||
nbcell("x = 4", idx: 3, highlight: 2)
|
||||
nbnote("$$\\{\\;x \\rightarrow \\textbf{@3},\\;y \\rightarrow \\color{blue}{\\textbf{@4}}\\;\\}$$", show:3)
|
||||
nbcell("print(y)", idx: 4, highlight: 4)
|
||||
end
|
||||
%>
|
||||
|
||||
<p class="fragment takeaway">
|
||||
<b><strike>Question</strike></b> A cell is stale if a value it read last time changed.
|
||||
</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3>Vizier Demo</h3>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3>Workflow-style execution</h3>
|
||||
|
||||
<ol>
|
||||
<li>Microkernel Notebooks</li>
|
||||
<li>Static Analysis</li>
|
||||
<li>Approximate Dependencies</li>
|
||||
<li>Inter-Kernel Interop <span style="color: grey;">[Work In Progress]</span></li>
|
||||
</ol>
|
||||
</section>
|
||||
|
||||
<!------------------------- Microkernel Notebooks -------------------------->
|
||||
<section>
|
||||
If we have to have the ability to recover a state, does it have to be the same interpreter?
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Not same interpreter means:
|
||||
- No worrying about crashes
|
||||
- Portability / Resume at any point
|
||||
- Parallel execution
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Outline the data model:
|
||||
|
||||
- Interpreter
|
||||
- "backend 'state database'"
|
||||
- Lazy-loading interpreter state
|
||||
</section>
|
||||
|
||||
<section>
|
||||
If we have to have the ability to recover a state, does it have to be the same interpreter *version*?
|
||||
</section>
|
||||
|
||||
<section>
|
||||
If we have to have the ability to recover a state, does it have to be the same language?
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Cool things we can do if we lift the "state lives in the kernel" model
|
||||
|
||||
- Deserialize program state into another interpreter
|
||||
- Graphical widgets for common tasks (data loading)
|
||||
- 1-3 slides on spreadsheets
|
||||
</section>
|
||||
|
||||
<!------------------------- Static Analysis -------------------------->
|
||||
<section>
|
||||
How to figure out dependencies
|
||||
|
||||
1. Run the code (exact, after the fact)
|
||||
2. Static analysis (imprecise, incomplete)
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Refer to Aditya's project w.r.t. static analysis
|
||||
</section>
|
||||
|
||||
<!------------------------- Approximate Dependencies -------------------------->
|
||||
<section>
|
||||
How to figure out dependencies
|
||||
|
||||
1. Run the code (exact, after the fact)
|
||||
2. Static analysis (imprecise, incomplete)
|
||||
3. Both!
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Idea: use static analysis to create a mask.
|
||||
|
||||
Cell state model:
|
||||
- stable
|
||||
- unknown
|
||||
- stale
|
||||
- runnable (revisit parallelism)
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Preliminary results: TAPP
|
||||
</section>
|
||||
|
||||
<!------------------------- Approximate Dependencies -------------------------->
|
||||
|
||||
<section>
|
||||
State model. Review:
|
||||
- State needs to come *out* of the cell that created it
|
||||
- State needs to go *into* the cell that is about to consume it
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Naive approach: Pickle
|
||||
|
||||
... but pickle doesn't allow interop
|
||||
... but pickle doesn't always work (e.g., for 'File' objects)
|
||||
</section>
|
||||
|
||||
<section>
|
||||
Interop: Define standards
|
||||
|
||||
- Primitive Values (int, float, date, etc...)
|
||||
- Collection Types (map, list, etc...)
|
||||
- Libraries
|
||||
- Function [Challenge: Chained Dependencies]
|
||||
- Dataframe/Series [Challenge: These are BIG]
|
||||
</section>
|
||||
|
||||
<section>
|
||||
|
||||
</section>
|
||||
|
||||
<!------------------------- Closing -------------------------->
|
||||
<%#
|
||||
<section>
|
||||
<a href="https://vizierdb.info">
|
||||
<img src="graphics/2022-06-20/vizier.svg" height="200px">
|
||||
|
|
@ -41,4 +336,4 @@ title: Principled management of notebook state in Vizier
|
|||
|
||||
<p style="font-size: 65%"><b>Mike Brachmann, Boris Glavic, Nachiket Deo</b>, Juliana Freire, Heiko Mueller, Sonia Castello, Munaf Arshad Qazi, William Spoth, Poonam Kumari, Soham Patel, and more...</p>
|
||||
</section>
|
||||
|
||||
%>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,225 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||
|
||||
<svg
|
||||
width="174.83727mm"
|
||||
height="60.998596mm"
|
||||
viewBox="0 0 174.83727 60.998596"
|
||||
version="1.1"
|
||||
id="svg1"
|
||||
sodipodi:docname="Dependencies.svg"
|
||||
inkscape:version="1.3.2 (091e20ef0f, 2023-11-25)"
|
||||
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
xmlns:svg="http://www.w3.org/2000/svg">
|
||||
<sodipodi:namedview
|
||||
id="namedview1"
|
||||
pagecolor="#ffffff"
|
||||
bordercolor="#666666"
|
||||
borderopacity="1.0"
|
||||
inkscape:showpageshadow="2"
|
||||
inkscape:pageopacity="0.0"
|
||||
inkscape:pagecheckerboard="0"
|
||||
inkscape:deskcolor="#d1d1d1"
|
||||
inkscape:document-units="mm"
|
||||
inkscape:zoom="1.4877245"
|
||||
inkscape:cx="245.00505"
|
||||
inkscape:cy="97.1282"
|
||||
inkscape:window-width="1912"
|
||||
inkscape:window-height="1003"
|
||||
inkscape:window-x="26"
|
||||
inkscape:window-y="23"
|
||||
inkscape:window-maximized="0"
|
||||
inkscape:current-layer="layer1" />
|
||||
<defs
|
||||
id="defs1">
|
||||
<marker
|
||||
style="overflow:visible"
|
||||
id="Triangle"
|
||||
refX="0"
|
||||
refY="0"
|
||||
orient="auto-start-reverse"
|
||||
inkscape:stockid="Triangle arrow"
|
||||
markerWidth="1"
|
||||
markerHeight="1"
|
||||
viewBox="0 0 1 1"
|
||||
inkscape:isstock="true"
|
||||
inkscape:collect="always"
|
||||
preserveAspectRatio="xMidYMid">
|
||||
<path
|
||||
transform="scale(0.5)"
|
||||
style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
|
||||
d="M 5.77,0 -2.88,5 V -5 Z"
|
||||
id="path135" />
|
||||
</marker>
|
||||
<linearGradient
|
||||
id="linearGradient4"
|
||||
inkscape:collect="always">
|
||||
<stop
|
||||
style="stop-color:#ffffff;stop-opacity:1"
|
||||
offset="0"
|
||||
id="stop4" />
|
||||
<stop
|
||||
style="stop-color:#ffffff;stop-opacity:1"
|
||||
offset="0.36767325"
|
||||
id="stop6" />
|
||||
<stop
|
||||
style="stop-color:#b3b3b3;stop-opacity:0"
|
||||
offset="1"
|
||||
id="stop5" />
|
||||
</linearGradient>
|
||||
<linearGradient
|
||||
inkscape:collect="always"
|
||||
xlink:href="#linearGradient4"
|
||||
id="linearGradient5"
|
||||
x1="8.422595"
|
||||
y1="74.255554"
|
||||
x2="48.759907"
|
||||
y2="74.394501"
|
||||
gradientUnits="userSpaceOnUse"
|
||||
gradientTransform="matrix(1,0,0,0.4234772,0,41.532591)" />
|
||||
<linearGradient
|
||||
inkscape:collect="always"
|
||||
xlink:href="#linearGradient4"
|
||||
id="linearGradient6"
|
||||
gradientUnits="userSpaceOnUse"
|
||||
gradientTransform="matrix(1,0,0,0.4234772,-186.98905,41.532591)"
|
||||
x1="8.422595"
|
||||
y1="74.255554"
|
||||
x2="48.759907"
|
||||
y2="74.394501" />
|
||||
</defs>
|
||||
<g
|
||||
inkscape:label="Layer 1"
|
||||
inkscape:groupmode="layer"
|
||||
id="layer1"
|
||||
transform="translate(-6.0758843,-41.482174)">
|
||||
<rect
|
||||
style="fill:#b3b3b3;stroke:#333333;stroke-width:1;stroke-linecap:round;stroke-linejoin:bevel;stroke-dasharray:none"
|
||||
id="rect1"
|
||||
width="42.189392"
|
||||
height="26.216751"
|
||||
x="74.883568"
|
||||
y="60.04607" />
|
||||
<rect
|
||||
style="fill:#b3b3b3;stroke:#333333;stroke-width:1;stroke-linecap:round;stroke-linejoin:bevel;stroke-dasharray:none"
|
||||
id="rect2"
|
||||
width="42.189392"
|
||||
height="26.216751"
|
||||
x="21.9669"
|
||||
y="60.04607" />
|
||||
<rect
|
||||
style="fill:#b3b3b3;stroke:#333333;stroke-width:1;stroke-linecap:round;stroke-linejoin:bevel;stroke-dasharray:none"
|
||||
id="rect3"
|
||||
width="42.189392"
|
||||
height="26.216751"
|
||||
x="127.8002"
|
||||
y="60.04607" />
|
||||
<rect
|
||||
style="fill:url(#linearGradient5);fill-opacity:1;stroke:none;stroke-width:0.650751;stroke-linecap:round;stroke-linejoin:bevel;stroke-dasharray:none"
|
||||
id="rect4"
|
||||
width="44.133179"
|
||||
height="27.703424"
|
||||
x="6.0758843"
|
||||
y="59.434734" />
|
||||
<rect
|
||||
style="fill:url(#linearGradient6);fill-opacity:1;stroke:none;stroke-width:0.650751;stroke-linecap:round;stroke-linejoin:bevel;stroke-dasharray:none"
|
||||
id="rect6"
|
||||
width="44.133179"
|
||||
height="27.703424"
|
||||
x="-180.91315"
|
||||
y="59.434734"
|
||||
transform="scale(-1,1)" />
|
||||
<path
|
||||
style="fill:none;stroke:#000000;stroke-width:0.7;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Triangle)"
|
||||
d="m 64.696622,73.154446 h 7.104642"
|
||||
id="path6"
|
||||
sodipodi:nodetypes="cc" />
|
||||
<path
|
||||
style="fill:none;stroke:#000000;stroke-width:0.7;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Triangle)"
|
||||
d="m 117.6133,73.154446 h 7.10465"
|
||||
id="path7"
|
||||
sodipodi:nodetypes="cc" />
|
||||
<g
|
||||
id="g13"
|
||||
class="fragment">
|
||||
<g
|
||||
id="g9"
|
||||
transform="translate(-11.405656,12.073267)">
|
||||
<circle
|
||||
style="opacity:1;fill:#ffffff;stroke:#000000;stroke-width:0.7;stroke-linecap:round;stroke-linejoin:bevel;stroke-dasharray:none"
|
||||
id="path9"
|
||||
cx="79.934235"
|
||||
cy="33.449871"
|
||||
r="3.690964" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-size:6.35px;line-height:1.25;font-family:Ubuntu;-inkscape-font-specification:Ubuntu;text-align:center;letter-spacing:0px;text-anchor:middle;stroke-width:0.264583"
|
||||
x="79.906754"
|
||||
y="35.650146"
|
||||
id="text9"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan9"
|
||||
style="text-align:center;text-anchor:middle;stroke-width:0.264583"
|
||||
x="79.906754"
|
||||
y="35.650146">1</tspan></text>
|
||||
</g>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-size:6.35px;line-height:1.25;font-family:Ubuntu;-inkscape-font-specification:Ubuntu;letter-spacing:0px;stroke-width:0.264583"
|
||||
x="75.052376"
|
||||
y="47.658699"
|
||||
id="text10"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan10"
|
||||
style="stroke-width:0.264583"
|
||||
x="75.052376"
|
||||
y="47.658699">Python state is mutable</tspan></text>
|
||||
<path
|
||||
style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Triangle)"
|
||||
d="M 27.060473,-15.336639 V 6.9043202"
|
||||
id="path12"
|
||||
transform="translate(41.511011,64.989938)" />
|
||||
</g>
|
||||
<g
|
||||
id="g11"
|
||||
transform="translate(41.511012,64.989937)"
|
||||
class="fragment">
|
||||
<circle
|
||||
style="opacity:1;fill:#ffffff;stroke:#000000;stroke-width:0.7;stroke-linecap:round;stroke-linejoin:bevel;stroke-dasharray:none"
|
||||
id="circle10"
|
||||
cx="79.934235"
|
||||
cy="33.449871"
|
||||
r="3.690964" />
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-size:6.35px;line-height:1.25;font-family:Ubuntu;-inkscape-font-specification:Ubuntu;text-align:center;letter-spacing:0px;text-anchor:middle;stroke-width:0.264583"
|
||||
x="79.906754"
|
||||
y="35.650146"
|
||||
id="text11"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan11"
|
||||
style="text-align:center;text-anchor:middle;stroke-width:0.264583"
|
||||
x="79.906754"
|
||||
y="35.650146">2</tspan></text>
|
||||
<text
|
||||
xml:space="preserve"
|
||||
style="font-size:6.35px;line-height:1.25;font-family:Ubuntu;-inkscape-font-specification:Ubuntu;letter-spacing:0px;stroke-width:0.264583"
|
||||
x="-7.6239524"
|
||||
y="35.563065"
|
||||
id="text12"><tspan
|
||||
sodipodi:role="line"
|
||||
id="tspan12"
|
||||
style="stroke-width:0.264583"
|
||||
x="-7.6239524"
|
||||
y="35.563065">Dependency tracking is hard</tspan></text>
|
||||
<path
|
||||
style="fill:none;stroke:#000000;stroke-width:0.264583px;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;marker-end:url(#Triangle)"
|
||||
d="M 79.977143,29.634024 V 9.5097307"
|
||||
id="path13"
|
||||
sodipodi:nodetypes="cc" />
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 7.9 KiB |
|
|
@ -406,6 +406,25 @@ body {
|
|||
.reveal .slides section .fragment.red-shadow-current.current-fragment {
|
||||
box-shadow: 0px 0px 12px red; }
|
||||
|
||||
.reveal .slides section .takeaway
|
||||
{
|
||||
border-radius: 8px;
|
||||
border: solid 2px black;
|
||||
background-color: #ccd4ff;
|
||||
color: #081669;
|
||||
padding: 10px;
|
||||
font-weight: bold;
|
||||
font-size: 80%;
|
||||
margin-top: 50px;
|
||||
}
|
||||
.reveal .slides section .takeaway b
|
||||
{
|
||||
font-weight: bolder;
|
||||
color: black;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*********************************************
|
||||
* CUSTOM TAGS
|
||||
*********************************************/
|
||||
|
|
|
|||
Loading…
Reference in New Issue