MAINT: prepare for experimentalist run (#14)

* MAINT: show errors on file upload

* TST: test errors propogated

* DOC: add to experimentalist docs

* TST: no repeats in targets

* ENH: implement key presses

* MAINT: small fixes
* Targets are always strings
* Response time is better configured
* Responding via arrow keys allowed

* Catch plotting errors gracefully

* Add joblib to requirements

docs/introduction.rst

@@ -1,45 +1,8 @@
 Getting started
 ===============
 
-Installation
-------------
-On your local machine? Run this following code in a terminal:
-
-.. code:: shell
-
-   $ git clone https://github.com/stsievert/salmon.git
-
-Launching Salmon
-----------------
-
-Developer
-^^^^^^^^^
-
-First, `install Docker`_ and `install Git`_. After that, run the following code:
-
-.. _install Docker: https://www.docker.com/products/docker-desktop
-.. _install Git: https://git-scm.com/downloads
-
-.. code:: shell
-
-   $ cd salmon
-   $ docker-compose build
-   $ docker-compose up
-   $ # visit http://localhost:8000/init_exp or http://localhost:8000/docs
-
-If you make changes to this code, use this to
-
-.. code:: shell
-
-	$ docker-compose stop
-	$ docker-compose build
-	$ docker-compose up
-
-Want to use AWS? See below.
-
-
 Experimentalist
-^^^^^^^^^^^^^^^
+---------------
 
 .. warning::
 
@@ -48,11 +11,12 @@ Experimentalist
 1. Sign into Amazon AWS (http://aws.amazon.com/)
 2. Select the "Oregon" region (or ``us-west-2``) in the upper right.
 3. Go to Amazon EC2
-4. Launch a new instance (the big blue button).
-5. Select AMI ``ami-00e64ce0cd129cb26`` titled "Salmon"
-   (appears in Community AMIs after searching "Salmon").
-6. On the rules page, make a custom TCP rule to open ports 1000-9999 from
-   anywhere. Here's a image description:
+4. Launch a new instance (the big blue button or square orange button).
+5. Select AMI ``ami-00e64ce0cd129cb26`` titled "Salmon" (it appears in
+   Community AMIs after searching "Salmon").
+6. Don't click the big blue button yet -- continue to the rules page. Make a
+   custom TCP rule to open ports 1000-9999 from anywhere. Here's a image
+   description:
 
 .. image:: imgs/networking-rule.png
 
@@ -79,7 +43,7 @@ Then after this AMI is finished launching and initializing, go to
 ``http://ec2-35-164-240-184.us-west-2.compute.amazonaws.com:8000/foo``.
 
 Experiment Launch
------------------
+^^^^^^^^^^^^^^^^^
 
 Visit ``[url]:8000/init_exp`` to initialize an experiment. This will ask for
 the following:
@@ -108,3 +72,35 @@ If you visit ``[url]``, you will see a query page:
 .. note::
 
    This image is almost certainly out of date.
+
+Local machine
+-------------
+On your local machine as a developer? Run this following code in a terminal:
+
+.. code:: shell
+
+   $ git clone https://github.com/stsievert/salmon.git
+
+First, `install Docker`_ and `install Git`_. After that, run the following code:
+
+.. _install Docker: https://www.docker.com/products/docker-desktop
+.. _install Git: https://git-scm.com/downloads
+
+.. code:: shell
+
+   $ cd salmon
+   $ docker-compose build
+   $ docker-compose up
+   $ # visit http://localhost:8000/init_exp or http://localhost:8000/docs
+
+Developer
+---------
+Follow the instructions for local machine launch.
+
+If you make changes to this code, follow these instructions:
+
+.. code:: shell
+
+	$ docker-compose stop
+	$ docker-compose build
+	$ docker-compose up

frontend/frontend/__init__.py

@@ -3,5 +3,5 @@
 """
 from .public import app
 from .private import *
-__version__ = "0.1.0-alpha"
+__version__ = "0.1.0"
 app.version = __version__

frontend/frontend/plotting.py

@@ -35,3 +35,29 @@ def _any_outliers(x, low=True, high=True):
         return _high or _low
     else:
         raise ValueError(f"high={high}, low={low}")
+
+async def time_human_delay(delay):
+    w = 3
+    fig, ax = plt.subplots(figsize=(w, w))
+    ax.hist(delay, bins="auto")
+    ax.set_xlabel("Response time (s)")
+    ax.set_ylabel("Count")
+    ax.grid(alpha=0.5)
+    ax.set_title("Human delay in answering")
+    ax.set_xlim(0, None)
+    if _any_outliers(delay, low=False):
+        upper = np.percentile(delay, 95)
+        ax.set_xlim(None, max(10, upper))
+    return fig, ax
+
+
+async def network_latency(latency):
+    w = 3
+    fig, ax = plt.subplots(figsize=(w, w))
+    ax.hist(latency, bins="auto")
+    ax.set_xlabel("Network latency (s)")
+    ax.set_ylabel("Count")
+    ax.grid(alpha=0.5)
+    ax.set_title("Network latency between questions")
+    ax.set_xlim(0, None)
+    return fig, ax

frontend/frontend/private.py

@@ -1,3 +1,4 @@
+import sys
 from typing import Dict, Any, List
 import os
 import yaml
@@ -30,7 +31,7 @@
 
 from .public import _ensure_initialized, app, templates
 from .utils import ServerException, get_logger, _extract_zipfile, _format_target
-from .plotting import time_histogram, _any_outliers
+from .plotting import time_histogram, _any_outliers, time_human_delay, network_latency
 
 security = HTTPBasic()
 
@@ -106,6 +107,7 @@ async def _get_config(exp: bytes, targets_file: bytes) -> Dict[str, Any]:
         "debrief": "Thanks!",
     }
     exp_config.update(config)
+    exp_config["targets"] = [str(x) for x in exp_config["targets"]]
 
     if targets_file:
         fnames = _extract_zipfile(targets_file)
@@ -164,6 +166,7 @@ async def process_form(
         - instructions: "Foobar!"
         - max_queries: 25
     """
+    logger.info("salmon.__version__ = %s", app.version)
     try:
         exp_config = await _get_config(exp, targets_file)
     except Exception as e:
@@ -291,38 +294,39 @@ async def get_dashboard(request: Request, authorized: bool = Depends(_authorize)
     )
 
     if len(responses) >= 2:
-        r = await time_histogram(df.time_received_since_start)
-        fig, ax = r
-        ax.set_title("Time responses received")
-        with StringIO() as f:
-            plt.savefig(f, format="svg", bbox_inches="tight")
-            hist_time_responses = f.getvalue()
-
-        w = 3
-        fig, ax = plt.subplots(figsize=(w, w))
-        ax.hist(df.response_time, bins="auto")
-        ax.set_xlabel("Response time (s)")
-        ax.set_ylabel("Count")
-        ax.grid(alpha=0.5)
-        ax.set_title("Human delay in answering")
-        ax.set_xlim(0, None)
-        if _any_outliers(df.response_time, low=False):
-            upper = np.percentile(df.response_time.values, 95)
-            ax.set_xlim(None, max(10, upper))
-        with StringIO() as f:
-            plt.savefig(f, format="svg", bbox_inches="tight")
-            hist_human_delay = f.getvalue()
-
-        fig, ax = plt.subplots(figsize=(w, w))
-        ax.hist(df.network_latency.values, bins="auto")
-        ax.set_xlabel("Network latency (s)")
-        ax.set_ylabel("Count")
-        ax.grid(alpha=0.5)
-        ax.set_title("Network latency between questions")
-        ax.set_xlim(0, None)
-        with StringIO() as f:
-            plt.savefig(f, format="svg", bbox_inches="tight")
-            hist_network_latency = f.getvalue()
+        try:
+            r = await time_histogram(df.time_received_since_start)
+        except:
+            name, descr, tr = sys.exc_info()
+            hist_time_responses = f"{name} exception: {descr}"
+        else:
+            fig, ax = r
+            ax.set_title("Time responses received")
+            with StringIO() as f:
+                fig.savefig(f, format="svg", bbox_inches="tight")
+                hist_time_responses = f.getvalue()
+            name, descr = "NameError", "because why?"
+            hist_time_responses = f"Time responses received:\n{name} exception: {descr}"
+        try:
+            r = await time_human_delay(df.response_time.to_numpy())
+        except:
+            name, descr, tr = sys.exc_info()
+            hist_human_delay = f"Histogram of human response time:\n{name} exception: {descr}"
+        else:
+            fig, ax = r
+            with StringIO() as f:
+                fig.savefig(f, format="svg", bbox_inches="tight")
+                hist_human_delay = f.getvalue()
+        try:
+            r = await network_latency(df.network_latency.to_numpy())
+        except Exception as e:
+            name, descr, traceback = sys.exc_info()
+            hist_network_latency = f"Network latency:\n{name} exception: {descr}"
+        else:
+            fig, ax = r
+            with StringIO() as f:
+                fig.savefig(f, format="svg", bbox_inches="tight")
+                hist_network_latency = f.getvalue()
     else:
         msg = (
             "Histogram of {} will appear here after at least 2 responses are collected"
@@ -363,6 +367,7 @@ async def get_logs(request: Request, authorized: bool = Depends(_authorize)):
 
 
 def _get_filename(html):
+    html = str(html)
     if "<img" in html or "<video" in html:
         i = html.find("src=")
         j = html[i:].find(" ")

frontend/requirements.txt

@@ -9,3 +9,4 @@ pytest
 aiofiles
 altair
 matplotlib
+joblib

frontend/templates/query_page.html

@@ -45,6 +45,11 @@
   border-style: solid;
   border-width: 2px;
 }
+.answer-sent {
+  border-color: #0c0;
+  border-style: solid;
+  border-width: 2px;
+}
 </style>
 </head>
 <body>
@@ -113,12 +118,14 @@
 
 var start = -1;
 var start_latency = -1;
+var response_start = -1;
 var end_latency = -1;
 var latency = -1;
 var max_queries = {{ max_queries }};
 var num_queries = 0;  // queries submitted
 
 async function send(winner, id) {
+  var response_time = getTime() - response_start;
   num_queries = num_queries + 1;
   $('#q-left').addClass("disabled");
   $('#q-right').addClass("disabled");
@@ -130,7 +137,7 @@
         "right": right,
         "winner": winner,
         "puid": puid,
-        "response_time": getTime() - start,
+        "response_time": response_time,
         "network_latency": latency,
     })
   );
@@ -147,6 +154,7 @@
       $("#q-head").html(targets[head]);
       end_latency = getTime();
       latency = end_latency - start_latency;
+      response_start = getTime();
       $('#q-left').removeClass("disabled");
       $('#q-right').removeClass("disabled");
   });
@@ -176,5 +184,39 @@
   start_latency = getTime();
   iterate(left, "#q-left");
 });
+
+
+document.onkeydown = function checkKey(e) {
+  e = e || window.event;
+  // console.log(e.keyCode);
+  // console.log(e.keyCode == "37");
+
+  left_disabled = $("#q-left").hasClass("answer-sent");
+  right_disabled = $("#q-right").hasClass("answer-sent");
+  console.log(left_disabled || right_disabled);
+  if (left_disabled || right_disabled){
+    return
+  }
+  start_latency = getTime();
+  if (e.keyCode == '37') { // left arrow
+    iterate(left, "#q-left");
+    winner_id = "#q-left";
+  }
+  else if (e.keyCode == '39') { // right arrow
+    iterate(right, "#q-right");
+    winner_id = "#q-right";
+  }
+  else if (e.keyCode == '38') { // up arrow
+  }
+  else if (e.keyCode == '40') { // down arrow
+  }
+
+  $(winner_id).addClass("answer-sent");
+  $(winner_id).addClass("disabled");
+  setTimeout(function(){
+    $(winner_id).removeClass("disabled");
+    $(winner_id).removeClass("answer-sent");
+  }, 150);  // 215ms is median reaction time for humans
+}
 </script>
 </html>