diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d17713cc..ca39883a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,6 +1,6 @@ name: Documentation build -# on: push + # on: push on: release: types: [published] @@ -22,19 +22,30 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.7 + - name: Install latex + run: | + sudo apt-get update + sudo apt-get install -y texmaker latexmk - name: Add conda to system path run: | # $CONDA is an environment variable pointing to the root of the miniconda directory echo $CONDA/bin >> $GITHUB_PATH - name: Install dependencies run: | - conda env update --file salmon.yml --name base + conda env update --file salmon.lock.yml --name base - name: Install Salmon - run: pip install -e . + run: | + pip install -e ".[dev,server]" - name: Build docs run: | cd docs make html + - name: Build backup docs + run: | + cd docs + sudo apt-get install -y texmaker latexmk + make latexpdf + mv build/latex/salmon.pdf build/html/salmon.pdf - name: Upload docs to gh-pages branch uses: peaceiris/actions-gh-pages@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 395e98f6..a913a9f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Tests (complete w/ conda) +name: Tests (install w/ pinned conda) on: push: @@ -42,9 +42,11 @@ jobs: sudo chown -R -H -L $USER:$USER salmon docs tests - name: Install dependencies run: | - conda env update --file salmon.yml --name base + conda env update --file salmon.lock.yml --name base - name: Install Salmon - run: pip install -e . + run: | + pip install . + pip install pytest - name: Run tests in salmon/salmon/ run: sudo /usr/share/miniconda/bin/pytest salmon/ - uses: docker/setup-buildx-action@v1 diff --git a/.github/workflows/test_conda.yml b/.github/workflows/test_conda.yml new file mode 100644 index 00000000..34a3823a --- /dev/null +++ b/.github/workflows/test_conda.yml @@ -0,0 +1,64 @@ +name: Tests (install w/ conda) + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + + build: + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Debug info + run: | + echo "Running pwd..." + pwd + echo "\nRunning echo $USER..." + echo $USER + echo "\nRunning chown..." + sudo chown -R -H -L $USER:$USER /home/runner/work/salmon/ + chown -R -H -L $USER:$USER /home/runner/work/salmon/ + echo "\nRunning chmod..." + sudo chmod -R 777 /home/runner/work/salmon/ + echo "\nRunning `ls`..." + ls + - name: Prepare for docker build... + run: | + chmod +x launch.sh + # chown -R -H -L $USER:$USER . + sudo chown -R -H -L $USER:$USER . + sudo chown -R -H -L $USER:$USER salmon docs tests + - name: Install dependencies + run: | + conda env update --file salmon.yml --name base + - name: Install Salmon + run: pip install -e . + - name: Run tests in salmon/salmon/ + run: sudo /usr/share/miniconda/bin/pytest salmon/ + - uses: docker/setup-buildx-action@v1 + with: + driver: docker + - name: Build Salmon server w/ Docker + env: + SALMON_NO_AUTH: 1 + run: | + sudo --preserve-env=SALMON_NO_AUTH docker-compose up & + until curl 127.0.0.1:8421 > /dev/null 2>&1; do :; done # wait for container to start + sudo docker ps + - name: Run all tests + run: | + # sudo docker-compose logs -f & # if debugging; shows logs + # sudo /usr/share/miniconda/bin/pytest -s + sudo /usr/share/miniconda/bin/pytest diff --git a/.github/workflows/test_offline.yml b/.github/workflows/test_offline.yml index 0f1bde12..400e0ab6 100644 --- a/.github/workflows/test_offline.yml +++ b/.github/workflows/test_offline.yml @@ -1,4 +1,4 @@ -name: Test (offline) +name: Offline tests (install w/ pip) on: push: @@ -45,7 +45,9 @@ jobs: sudo chown -R -H -L $USER:$USER . sudo chown -R -H -L $USER:$USER salmon docs tests - name: Install Salmon - run: pip install . + run: + pip install . + pip install pytest - name: change directories run: cd / - name: Debug info (salmon.__file__) @@ -54,5 +56,3 @@ jobs: python -c "import salmon; print(salmon.__file__)" - name: Run offline tests run: sudo /usr/share/miniconda/bin/pytest /home/runner/work/salmon/salmon/tests/test_offline.py - - name: Run tests in salmon/salmon/ - run: sudo /usr/share/miniconda/bin/pytest /home/runner/work/salmon/salmon/salmon diff --git a/.github/workflows/test_pip.yml b/.github/workflows/test_pip.yml index 236b0b98..994e7e3e 100644 --- a/.github/workflows/test_pip.yml +++ b/.github/workflows/test_pip.yml @@ -1,4 +1,4 @@ -name: Tests (complete w/ pip) +name: Tests (install w/ pip) on: push: @@ -42,7 +42,7 @@ jobs: sudo chown -R -H -L $USER:$USER salmon docs tests - name: Install dependencies run: | - pip install . + pip install ".[server,dev]" - name: Run tests in salmon/salmon/ run: sudo /usr/share/miniconda/bin/pytest salmon/ - uses: docker/setup-buildx-action@v1 diff --git a/Dockerfile b/Dockerfile index d9321995..5382ce1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,18 +4,18 @@ RUN apt-get update RUN apt-get install -y gcc cmake g++ RUN conda -V -COPY salmon.yml /salmon/salmon.yml -RUN conda env update -n base --file /salmon/salmon.yml --prune +COPY salmon.lock.yml /salmon/salmon.lock.yml +RUN conda env create -f /salmon/salmon.lock.yml VOLUME /salmon VOLUME /data COPY *.py *.cfg *.yml *.txt *.sh /salmon/ COPY ./salmon/ /salmon/salmon/ RUN ls /salmon -RUN pip install -e /salmon +RUN conda run -n salmon pip install -e /salmon[server] RUN chmod +x /salmon/launch.sh RUN chmod +rw /salmon # ENTRYPOINT bash launch.sh WORKDIR /salmon -CMD ["bash", "launch.sh"] +CMD ["conda", "run", "--no-capture-output", "-n", "salmon", "/bin/bash", "launch.sh"] diff --git a/LICENSE.txt b/LICENSE.txt index 1ad394ab..9f1ded08 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,21 +1,29 @@ -MIT License +BSD 3-Clause License -Copyright (c) 2020-2021 Scott Sievert and Salmon contributors. +Copyright (c) 2019, Scott Sievert and contributors +All rights reserved. -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 7dc6e0ef..6954e6be 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ + + + @@ -13,14 +16,30 @@ Salmon is a tool for efficiently generating ordinal embeddings. It relies on "active" machine learning algorithms to choose the most informative queries for humans to answer. -See the documentation for more detail: https://docs.stsievert.com/salmon/ +### Documentation + +This documentation is available at these locations: + +* **Primary source**: https://docs.stsievert.com/salmon/ +* Secondary source: as [a raw PDF][pdf] (and as a [slower loading PDF][blobpdf]). +* Secondary source: as [zipped HTML directory][ziphtml], which requires unzipping the directory + then opening up `index.html`. + +[pdf]:https://github.com/stsievert/salmon/raw/gh-pages/salmon.pdf +[blobpdf]:https://github.com/stsievert/salmon/blob/gh-pages/salmon.pdf +[ziphtml]:https://github.com/stsievert/salmon/archive/refs/heads/gh-pages.zip + +Please [file an issue][issue] if you can not access the documentation. + +[issue]:https://github.com/stsievert/salmon/issues/new ### Running Salmon offline Visit the documentation at https://docs.stsievert.com/salmon/offline.html. Briefly, this should work: ``` shell -$ conda env create -f salmon.yml +$ cd path/to/salmon +$ conda env create -f salmon.lock.yml $ conda activate salmon (salmon) $ pip install -e . ``` diff --git a/docker-compose.yml b/docker-compose.yml index 598c48db..4e926465 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: - "8421:8421" - "8400:8400" - "8787:8787" + depends_on: [redis] expose: - 8421 - 8400 @@ -30,31 +31,29 @@ services: volumes: - "./salmon/_out:/data:rw" command: /bin/sh -c "chmod -v 777 /data && redis-server --loadmodule /usr/lib/redis/modules/rejson.so" - # redis-server command from https://hub.docker.com/r/redislabs/rejson/dockerfile - # - # https://github.com/docker/compose/issues/3270 - # also https://github.com/boot2docker/boot2docker/issues/1083#issuecomment-151380687 - - # redismonitor: - # user: root - # build: docker/redismonitor - # ports: - # - "7381:7389" - # volumes: - # - "./out:/logs:rw" - # logging: - # driver: none prom: image: prom/prometheus + depends_on: [redis] expose: - 9090 volumes: - ./docker/prom:/etc/prometheus - # granfna: - # image: grafana/grafana - # ports: - # - "3000:3000" + + # redismonitor: + # user: root + # build: docker/redismonitor + # ports: + # - "7381:7389" + # volumes: + # - "./out:/logs:rw" + # logging: + # driver: none + +# redis-server command from https://hub.docker.com/r/redislabs/rejson/dockerfile +# +# https://github.com/docker/compose/issues/3270 +# also https://github.com/boot2docker/boot2docker/issues/1083#issuecomment-151380687 # To get bash on one of these machines: # docker exec -it bash diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..a2954e31 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1 @@ +numpydoc diff --git a/docs/source/adaptive.rst b/docs/source/adaptive.rst index 1c2f4242..b6daad7a 100644 --- a/docs/source/adaptive.rst +++ b/docs/source/adaptive.rst @@ -1,5 +1,5 @@ -Adaptive algorithms -=================== +Adaptive algorithm primer +========================= The API the must conform to below: diff --git a/docs/source/benchmarks/active.rst b/docs/source/benchmarks/active.rst index 4b11f750..8a7c04d0 100644 --- a/docs/source/benchmarks/active.rst +++ b/docs/source/benchmarks/active.rst @@ -116,7 +116,7 @@ accuracy. To start, let's assume that there's no clear relationship between items. Then, this visualization is most appropriate for the embeddings of particular accuracies: -.. image:: imgs/embeddings-n=30-colorless.svg +.. image:: imgs/embeddings-n=30-colorless.png :width: 90% :align: center @@ -124,7 +124,7 @@ These embeddings are remarkably simple, and have a clear and known relationship. Because of that, let's show the embeddings with colors from now on: -.. image:: imgs/embeddings-n=30.svg +.. image:: imgs/embeddings-n=30.png :width: 95% :align: center @@ -162,7 +162,7 @@ Embedding quality Here's the underlying embeddings for :math:`n = 180` for various accuracy levels on *simulated* human responses: -.. image:: imgs/embeddings-n=180.svg +.. image:: imgs/embeddings-n=180.png :width: 100% :align: center diff --git a/docs/source/benchmarks/imgs/embeddings-n=180.png b/docs/source/benchmarks/imgs/embeddings-n=180.png new file mode 100644 index 00000000..eeb0e1af Binary files /dev/null and b/docs/source/benchmarks/imgs/embeddings-n=180.png differ diff --git a/docs/source/benchmarks/imgs/embeddings-n=30-colorless.png b/docs/source/benchmarks/imgs/embeddings-n=30-colorless.png new file mode 100644 index 00000000..1ea6d033 Binary files /dev/null and b/docs/source/benchmarks/imgs/embeddings-n=30-colorless.png differ diff --git a/docs/source/benchmarks/imgs/embeddings-n=30.png b/docs/source/benchmarks/imgs/embeddings-n=30.png new file mode 100644 index 00000000..e741587d Binary files /dev/null and b/docs/source/benchmarks/imgs/embeddings-n=30.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 576a8960..903f19ca 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -24,9 +24,9 @@ author = "Scott Sievert" # The full version, including alpha/beta/rc tags -from salmon import __version__ - -release = __version__.replace(".dirty", "") +# from salmon import __version__ +# +# release = __version__.replace(".dirty", "") html_context = { "display_github": True, # Integrate GitHub diff --git a/docs/source/deploying.rst b/docs/source/deploying.rst new file mode 100644 index 00000000..aab62ee6 --- /dev/null +++ b/docs/source/deploying.rst @@ -0,0 +1,48 @@ +.. _deploying: + +Deploying +========= + +Most commonly, I've seen Salmon and interfaces like it deployed to +crowdsourcing services (e.g., Amazon's Mechanical Turk). With these services, +Salmon is typically deployed to these services by letting crowdsourcing +participants click on a single URL and go through various web pages before +entering a code into MTurk indicating the participant completed the study. + +I have a couple recommendations for this and similar processes: + +* **URL redirection.** Do not give Amazon AWS EC2 URLs/DNSs directly to + crowdsourcing participants. Instead, give them a URL that redirects to the + Amazon EC2 URL/DNS (e.g., via `GitHub Pages`_, ``foobar.github.io/mturk.html``). + + * This is a best practice because it avoids debugging production servers. If + something goes wrong with your machine (like it's overloaded with too many + users), having some redirection scheme allows redirectign crowdsourcing + participants away from your machine. + +* **Host detailed instructions/etc elsewhere.** This HTML files are typically + shown after the crowdsourcing clicks on a link and before they see Salmon. + (e.g., for an IRB notice). It is `technically` possible to include these + instructions by customizing the Salmon's query page with + :ref:`frontendcustomization`). + +Hosting HTML pages is possible and relatively straighforward with `GitHub +Pages`_, as are `URL Redirections`_. + +.. _URL Redirections: https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections#html_redirections + +In general, I've observed some crowdsourcing trends: + +* I've noticed two levels of fraud: crowdsourcing participants who submit bogus + codes (like their Amazon MTurk ID) and those who answer responses rather + quickly, too quickly for human response time. +* I've generally observed the responses from crowdsourcing participants to be + high quality. I have noticed some "bad" or "junk" responses, + and throw them out before I start my analysis. + +Please `file an issue`_ or reach out `via email`_ if you have any +more deployment questions. + +.. _file an issue: https://github.com/stsievert/salmon/issues/new/choose +.. _via email: mailto:stsievert@wisc.edu +.. _GitHub Pages: https://pages.github.com/ diff --git a/docs/source/deps.rst b/docs/source/deps.rst new file mode 100644 index 00000000..e1f2090c --- /dev/null +++ b/docs/source/deps.rst @@ -0,0 +1,1784 @@ +.. _deps: + +Dependencies +============ + +Salmon is licensed under the BSD License. Details are at +`LICENSE.txt `_. + +Salmon depends on the libraries below. Below this list, the current license of +each library at the time of Salmon v1.0 is included: + +* Python (`Python's license `_). +* NumPy (`NumPy's license `_). +* SciPy (`SciPy's license `_). +* Pandas (`Pandas's license `_). +* Cython (`Cython's license `_) +* Scikit-learn (`Scikit-learn's license `_) +* FastAPI (`FastAPI's license `_) +* PyTorch (`PyTorch's license `_) +* Skorch (`Skorch's license `_) +* Altair (`Altair's license `_) +* Bokeh (`Bokeh's license `_) +* Jinja2 (`Jinja2's license `_) +* Redis, ReJSON (`Redis's license `_) +* HTTPX (`HTTPX's license `_) +* Dask (`Dask's license `_). +* Dask-Distributed (`Dask-Distributed's license `_). +* Dask-ML (`Dask-ML's license `_). +* aiofiles (`aiofiles's license `_) +* click (`click's license `_) +* lz4 (`lz4's license `_) +* pythn-blosc (`python-block's license `_) +* cytoolz (`cytoolz's license `_) +* ujson (`ujson's license `_) +* Sphinx (`Sphinx's license `_) +* Sphinx RTD Theme (`Sphinx RTD Theme's license `_) +* PyYAML (`PyYAML's license `_) +* matplotlib (`matplotlib's license `_) +* python-multipart (`python-multipart's license `_) +* numpydoc (`numpydoc's license `_) +* pytest (`pytest's license `_) +* gunicorn (`gunicorn's license `_) +* cloudpickle (`cloudpickle's license `_) +* jupyter-server-proxy (`jupyter-server-proxy's license `_) +* autodoc pydantic (`autodoc's license `_) +* starlette_exporter (`starlette_exporter's license `_) + + +Python +------ + +.. code-block:: none + + 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and + the Individual or Organization ("Licensee") accessing and otherwise using Python + 3.11.0 software in source or binary form and its associated documentation. + + 2. Subject to the terms and conditions of this License Agreement, PSF hereby + grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, + analyze, test, perform and/or display publicly, prepare derivative works, + distribute, and otherwise use Python 3.11.0 alone or in any derivative + version, provided, however, that PSF's License Agreement and PSF's notice of + copyright, i.e., "Copyright © 2001-2022 Python Software Foundation; All Rights + Reserved" are retained in Python 3.11.0 alone or in any derivative version + prepared by Licensee. + + 3. In the event Licensee prepares a derivative work that is based on or + incorporates Python 3.11.0 or any part thereof, and wants to make the + derivative work available to others as provided herein, then Licensee hereby + agrees to include in any such work a brief summary of the changes made to Python + 3.11.0. + + 4. PSF is making Python 3.11.0 available to Licensee on an "AS IS" basis. + PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF + EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR + WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE + USE OF PYTHON 3.11.0 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + + 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 3.11.0 + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF + MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 3.11.0, OR ANY DERIVATIVE + THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + + 6. This License Agreement will automatically terminate upon a material breach of + its terms and conditions. + + 7. Nothing in this License Agreement shall be deemed to create any relationship + of agency, partnership, or joint venture between PSF and Licensee. This License + Agreement does not grant permission to use PSF trademarks or trade name in a + trademark sense to endorse or promote products or services of Licensee, or any + third party. + + 8. By copying, installing or otherwise using Python 3.11.0, Licensee agrees + to be bound by the terms and conditions of this License Agreement. + +NumPy +----- + +.. code-block:: none + + Copyright (c) 2005-2022, NumPy Developers. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the NumPy Developers nor the names of any + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +SciPy +----- + +.. code-block:: none + + Copyright © 2001, 2002 Enthought, Inc. + All rights reserved. + + Copyright © 2003-2019 SciPy Developers. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + Neither the name of Enthought nor the names of the SciPy Developers may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + + +Pandas +------ + +.. code-block:: none + + BSD 3-Clause License + + Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team + All rights reserved. + + Copyright (c) 2011-2022, Open source contributors. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Cython +------ + +.. code-block:: none + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + +Scikit-learn +------------ + +.. code-block:: none + + BSD 3-Clause License + + Copyright (c) 2007-2022 The scikit-learn developers. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FastAPI +------- + +.. code-block:: none + + The MIT License (MIT) + + Copyright (c) 2018 Sebastián Ramírez + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +PyTorch +------- + +.. code-block:: none + + From PyTorch: + + Copyright (c) 2016- Facebook, Inc (Adam Paszke) + Copyright (c) 2014- Facebook, Inc (Soumith Chintala) + Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) + Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) + Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) + Copyright (c) 2011-2013 NYU (Clement Farabet) + Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) + Copyright (c) 2006 Idiap Research Institute (Samy Bengio) + Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + + From Caffe2: + + Copyright (c) 2016-present, Facebook Inc. All rights reserved. + + All contributions by Facebook: + Copyright (c) 2016 Facebook Inc. + + All contributions by Google: + Copyright (c) 2015 Google Inc. + All rights reserved. + + All contributions by Yangqing Jia: + Copyright (c) 2015 Yangqing Jia + All rights reserved. + + All contributions by Kakao Brain: + Copyright 2019-2020 Kakao Brain + + All contributions by Cruise LLC: + Copyright (c) 2022 Cruise LLC. + All rights reserved. + + All contributions from Caffe: + Copyright(c) 2013, 2014, 2015, the respective contributors + All rights reserved. + + All other contributions: + Copyright(c) 2015, 2016 the respective contributors + All rights reserved. + + Caffe2 uses a copyright model similar to Caffe: each contributor holds + copyright over their contributions to Caffe2. The project versioning records + all such contribution and copyright details. If a contributor wants to further + mark their specific copyright on a particular contribution, they should + indicate their copyright solely in the commit message of the change when it is + committed. + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +Skorch +------ + +.. code-block:: none + + BSD 3-Clause License + + Copyright (c) 2017, Benjamin Bossan, Daniel Nouri, Marian Tietz + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Altair +------ + +.. code-block:: none + + Copyright (c) 2015-2022, Altair Developers + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of altair nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Bokeh +----- + +.. code-block:: none + + Copyright (c) 2012 - 2022, Anaconda, Inc., and Bokeh Contributors + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Anaconda nor the names of any contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +Jinja2 +------ + +.. code-block:: none + + Copyright 2007 Pallets + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +Redis, ReJSON +------------- + +.. code-block:: none + + Every file in the Redis distribution, with the exceptions of third party files + specified in the list below, contain the following license: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Redis nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +HTTPX +----- + +.. code-block:: none + + Copyright © 2019, [Encode OSS Ltd](https://www.encode.io/). + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Dask +---- + +.. code-block:: none + + BSD 3-Clause License + + Copyright (c) 2014, Anaconda, Inc. and contributors + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Dask Distributed +---------------- + +.. code-block:: none + + BSD 3-Clause License + + Copyright (c) 2015, Anaconda, Inc. and contributors + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Dask-ML +------- + +.. code-block:: none + + Copyright (c) 2017, Anaconda, Inc. and contributors + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + Neither the name of Anaconda nor the names of any contributors may be used to + endorse or promote products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +aiofiles +-------- + +.. code-block:: none + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +click +----- + +.. code-block:: none + + Copyright 2014 Pallets + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +lz4 +--- + +.. code-block:: none + + LZ4 Library + Copyright (c) 2011-2020, Yann Collet + All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +pythn-blosc +----------- + +.. code-block:: none + + BSD License + + For python-blosc - A Python wrapper for the Blosc compression library + + Copyright (C) 2010-2012 Francesc Alted (faltet@gmail.com) + Copyright (C) 2013-2019 Francesc Alted , Valentin Haenel + Copyright (C) 2019-present The Blosc Development Team + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Francesc Alted nor the names of its contributors may be used + to endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +cytoolz +------- + +.. code-block:: none + + Copyright (c) 2014-2021 Erik Welch + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + a. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + c. Neither the name of cytoolz nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + +ujson +----- + +.. code-block:: none + + Developed by ESN, an Electronic Arts Inc. studio. + Copyright (c) 2014, Electronic Arts Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of ESN, Electronic Arts Inc. nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL ELECTRONIC ARTS INC. BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + Portions of code from MODP_ASCII - Ascii transformations (upper/lower, etc) + https://github.com/client9/stringencoders + Copyright (c) 2007 Nick Galbreath -- nickg [at] modp [dot] com. All rights reserved. + + Numeric decoder derived from from TCL library + https://opensource.apple.com/source/tcl/tcl-14/tcl/license.terms + * Copyright (c) 1988-1993 The Regents of the University of California. + * Copyright (c) 1994 Sun Microsystems, Inc. + +Sphinx +------ + +.. code-block:: none + + Unless otherwise indicated, all code in the Sphinx project is licenced under + the two clause BSD licence below. + + Copyright (c) 2007-2022 by the Sphinx team (see AUTHORS file). + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +Sphinx RTD Theme +---------------- + +.. code-block:: none + + The MIT License (MIT) + + Copyright (c) 2013-2018 Dave Snider, Read the Docs, Inc. & contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy of + this software and associated documentation files (the "Software"), to deal in + the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + the Software, and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +PyYAML +------ + +.. code-block:: none + + Copyright (c) 2017-2021 Ingy döt Net + Copyright (c) 2006-2016 Kirill Simonov + + Permission is hereby granted, free of charge, to any person obtaining a copy of + this software and associated documentation files (the "Software"), to deal in + the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + of the Software, and to permit persons to whom the Software is furnished to do + so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +matplotlib +---------- + +.. code-block:: none + + + 1. This LICENSE AGREEMENT is between the Matplotlib Development Team + ("MDT"), and the Individual or Organization ("Licensee") accessing and + otherwise using matplotlib software in source or binary form and its + associated documentation. + + 2. Subject to the terms and conditions of this License Agreement, MDT + hereby grants Licensee a nonexclusive, royalty-free, world-wide license + to reproduce, analyze, test, perform and/or display publicly, prepare + derivative works, distribute, and otherwise use matplotlib + alone or in any derivative version, provided, however, that MDT's + License Agreement and MDT's notice of copyright, i.e., "Copyright (c) + 2012- Matplotlib Development Team; All Rights Reserved" are retained in + matplotlib alone or in any derivative version prepared by + Licensee. + + 3. In the event Licensee prepares a derivative work that is based on or + incorporates matplotlib or any part thereof, and wants to + make the derivative work available to others as provided herein, then + Licensee hereby agrees to include in any such work a brief summary of + the changes made to matplotlib . + + 4. MDT is making matplotlib available to Licensee on an "AS + IS" basis. MDT MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR + IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, MDT MAKES NO AND + DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS + FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF MATPLOTLIB + WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + + 5. MDT SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF MATPLOTLIB + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR + LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING + MATPLOTLIB , OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF + THE POSSIBILITY THEREOF. + + 6. This License Agreement will automatically terminate upon a material + breach of its terms and conditions. + + 7. Nothing in this License Agreement shall be deemed to create any + relationship of agency, partnership, or joint venture between MDT and + Licensee. This License Agreement does not grant permission to use MDT + trademarks or trade name in a trademark sense to endorse or promote + products or services of Licensee, or any third party. + + 8. By copying, installing or otherwise using matplotlib , + Licensee agrees to be bound by the terms and conditions of this License + Agreement. + + +python-multipart +---------------- + +.. code-block:: none + + Copyright 2012, Andrew Dunham + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +numpydoc +-------- + +.. code-block:: none + + Copyright (C) 2008-2022 Stefan van der Walt , Pauli Virtanen + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +pytest +------ + +.. code-block:: none + + The MIT License (MIT) + + Copyright (c) 2004 Holger Krekel and others + + Permission is hereby granted, free of charge, to any person obtaining a copy of + this software and associated documentation files (the "Software"), to deal in + the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + of the Software, and to permit persons to whom the Software is furnished to do + so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +gunicorn +-------- + +.. code-block:: none + + 2009-2018 (c) Benoît Chesneau + 2009-2015 (c) Paul J. Davis + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + +cloudpickle +----------- + +.. code-block:: none + + This module was extracted from the `cloud` package, developed by + PiCloud, Inc. + + Copyright (c) 2015, Cloudpickle contributors. + Copyright (c) 2012, Regents of the University of California. + Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +jupyter server proxy +-------------------- + +.. code-block:: none + + BSD 3-Clause License + + Copyright (c) 2017, Data Science 8 + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +autodoc pydantic +---------------- + +.. code-block:: none + + MIT License + + Copyright (c) 2021 Franz Wöllert + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +starlette_exporter +------------------ + +.. code-block:: none + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/docs/source/developers.rst b/docs/source/developers.rst index 89a3fb26..e1dab2ea 100644 --- a/docs/source/developers.rst +++ b/docs/source/developers.rst @@ -1,6 +1,68 @@ Developing algorithms ===================== +Install +------- + +This process is meant for developers. To launch, first download the code. It's +possible to download `a ZIP file of Salmon's source`_, or if Git is installed, +to run this command: + +.. _a ZIP file of Salmon's source: https://github.com/stsievert/salmon/archive/refs/heads/master.zip + +.. code:: shell + + $ git clone https://github.com/stsievert/salmon.git + +Then, to launch a local version of Salmon you'll need `Docker Compose`_. +After that dependency is intalled, run the following code: + +.. _install Docker: https://www.docker.com/products/docker-desktop +.. _install Git: https://git-scm.com/downloads + +.. code:: shell + + $ cd salmon + $ docker-compose build + $ docker-compose up + $ # visit http://localhost:8421/init or http://localhost:8421/docs + +.. _Docker Compose: https://docs.docker.com/compose/install/ + +If you make changes to this code, run these commands: + +.. code:: shell + + $ docker-compose stop + $ docker-compose build + $ docker-compose up + +If you want to log into the Docker container, execute these commands: + +.. code:: shell + + $ docker ps # to get list of running conatiners + CONTAINER ID IMAGE ... [more info] ... NAMES + 08b96fbcc4c3 salmon_server ... [more info] ... salmon_server_1 + 57cb3b7652d9 redislabs/rejson ... [more info] ... salmon_redis_1 + $ docker exec -it 08b96fbcc4c3 /bin/bash + (base) root@08b96fbcc4c3:/salmon# conda activate salmon + (salmon) root@08b96fbcc4c3:/salmon# + +.. note:: + + This is an alternative way to install Salmon's dependencies. If you create a + file in the Docker container in ``/salmon``, it will also be written to + ``/path/to/salmon`` on your local machine. + +If you run the command ``export SALMON_DEBUG=1``, the Salmon server will watch +for changes in the source and re-launch as necessary. This won't be perfect, +but it will reduce the number of times required to run ``docker-compose {stop, +build, up}``. + +If you run the command ``export SALMON_NO_AUTH=1``, the Salmon server will +not require a username/password. + Basics ------ diff --git a/docs/source/faq.rst b/docs/source/faq.rst index 059c9cae..3d0e145a 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -216,3 +216,13 @@ process is beyond scope for this project. [#f]_ https://nowak.ece.wisc.edu/ordinal_embedding.pdf .. [#f] though the package `mkcert`_ might help. + +The Docker machines aren't launching +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Are you using the command ``docker-compose up`` to launch Salmon? The command +``docker build .`` doesn't work. + +Salmon requires a Redis docker machine and certain directories/ports being +available. Technically, it's possible to build all the Docker machines +yourself (but it's not feasible). diff --git a/docs/source/frontend.rst b/docs/source/frontend.rst new file mode 100644 index 00000000..a8cd4ac0 --- /dev/null +++ b/docs/source/frontend.rst @@ -0,0 +1,67 @@ +.. _frontendcustomization: + +Frontend customization +====================== + +Custom HTML +----------- + +As mentioned in the :class:`~salmon.triplets.manager.HTML`, it's possible to +include custom HTML in the query page by customizing the ``element_top``, +``element_middle``, ``element_bottom`` and ``element_standalone`` fields. You'll probably have to look +at `query_page.html`_ to see exactly the custom HTML elements are inserted. + +.. _query_page.html: https://github.com/stsievert/salmon/blob/master/templates/query_page.html + +For example, if you wanted to wanted to include a button with the showing "Skip +this question" and didn't want to set ``html.skip_button = true``, you could +follow this example: + +.. code:: yaml + + html: + # skip_button: true # a much easier way to include the HTML code below! + element_bottom: > +
+
+ +
+
+ +A much easier way to get this is to set ``html.skip_button = true`` in +``init.yml``. + +CSS Styling +----------- + +As mentioned in the :class:`~salmon.triplets.manager.HTML`, it's possible to +customize the CSS by including a CSS field in the YAML file. For example, let's +say you were asking about the similarity of different colors, and wanted the +background to be dark to provide better contrast. This YAML block in +``init.yml`` would implement that: + +.. code:: yaml + + html: + instructions: Which of the comparison colors is most similar to the target color? + debrief: Thanks! Please enter this participant ID in the assignment. max_queries: 100 # about 2.94 minutes for 100 queries + arrow_keys: True + css: > + body { + background-color: #414141; + color: #ddd; + } + .answer { + background-color: #595959; + border-color: #bbb; + } + .head { + background-color: #505050; + border-color: #000; + } + .debrief { + color: #000; + } diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst index 83d5db0c..053ca41f 100644 --- a/docs/source/getting-started.rst +++ b/docs/source/getting-started.rst @@ -61,46 +61,33 @@ finished launching, you will see a couple links: * A link to the dashboard (``http://[url]:8421/dashboard``), an example of which is at ":ref:`exp-monitoring`." * A link to the query page to send to crowdsourcing users - (``http://[url]:8421/``). An (out-of-date) example: + (``http://[url]:8421/``). .. _YAML specification: https://yaml.org/ -.. image:: imgs/query_page.png - :align: center - :width: 500px Send the URL to participants ---------------------------- The URL to send to the crowdsourcing participants is ``http://[url]:8421/``. -For example, that may be +Typically, paid services like Mechantical Turk are used to recruit +crowdsourcing participants. Reddit and email have been used for unpaid +recruitment. In either case, that may involve a URL of this form: .. code:: - http://ec2-52-204-122-132.compute-1.amazonaws.com:8421/init + http://ec2-52-204-122-132.compute-1.amazonaws.com:8421/ -.. note:: +Opening this URL in the browser will show (a newer version) of this page: - Generally, I've found it useful to send out URLs I control to - crowdsourcing users via an `HTML redirect`_. Using an HTML redirect - means that I have more flexibility around the URL, and can handle many - use cases: launching a new machine and replacing it, restarting it, or - the machine dies unexpectedly. +.. image:: imgs/query_page.png + :align: center + :width: 500px - One method to do this is to use `GitHub Pages`_ (which allows creating a - URL like ``https://foo.github.io`` if your GitHub username is ``foo``) - then creating the HTML redirection file above (copy/pasting the shown - text into ``bar.html``). Then, users can visit - ``https://foo.github.io/bar.html`` to be redirected. +A couple notes: - As an example, the URL https://nextml.org/captioncontest currently - redirects to - https://s3.us-west-2.amazonaws.com/mlnow-newyorker/captioncontest_s3.html. - I would send out the URL ``https://nextml.org/captioncontest`` to users +* Customizing this page is possible and detailed in :ref:`frontendcustomization`. +* Tips on deploying this experiments can be found at :ref:`deploying`. .. _HTML redirect: https://www.w3docs.com/snippets/html/how-to-redirect-a-web-page-in-html.html .. _GitHub Pages: https://pages.github.com/ - -Typically, paid services like Mechantical Turk are used to recruit -crowdsourcing participants. Reddit and email have been used for unpaid -recruitment. diff --git a/docs/source/index.rst b/docs/source/index.rst index f54cbca9..a03f040b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -32,7 +32,9 @@ collecting responses will be reduced by a factor of 3 when compared with naive methods of collecting triplet queries. If you'd like to report bugs/issues, or improve Salmon please see `Salmon's -contribution guide`_. +contribution guide`_. The list of dependicies and their licenses is available +at :ref:`deps`. Salmon is licensed under the BSD License. Details are at +`LICENSE.txt `_. .. _Salmon's contribution guide: https://github.com/stsievert/salmon/blob/master/CONTRIBUTING.md @@ -51,6 +53,8 @@ Louisiana State University and Canada's Western University. getting-started init samplers + frontend + deploying monitoring offline faq @@ -64,12 +68,26 @@ Louisiana State University and Canada's Western University. benchmarks/active .. toctree:: - :maxdepth: 2 + :maxdepth: 1 :caption: Algorithm Developers adaptive developers + deps + +Other sources +------------- + +This documentation is available at these locations: + +* `https://docs.stsievert.com/salmon/ `_. +* On GitHub `as a raw PDF `_ (`alternate link `_). +* On GitHub as + `a zipped HTML directory `_, + which requires unzipping the directory then opening up ``index.html``. +Please `file an issue `_ if you +can not access the documentation above. Indices and tables ================== @@ -77,3 +95,6 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` + + + diff --git a/docs/source/init.rst b/docs/source/init.rst index ffeda772..4e2b090c 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -47,9 +47,11 @@ following: * Ask 50 questions before showing the participant ID. * Embed into :math:`d=2` dimensions if active samplers are specified. -The defaults for instructions/debrief can be found in -:class:`~salmon.triplets.manager.HTML`. To do anything fancier, additional -configuration is required. Here's a basic example: +More documentation on customizing these fields can be found in +:ref:`alg-config` and :ref:`frontendcustomization`, and the defaults for +instructions/debrief can be found in :class:`~salmon.triplets.manager.HTML`. To +do anything fancier, additional configuration is required. Here's a basic +example: .. code-block:: yaml @@ -67,7 +69,7 @@ configuration is required. Here's a basic example: The ``samplers`` controls which methods get to choose queries, and ``sampling`` controls how multiple samplers interact (i.e., how should a sampler be chosen?) -Some more details can be found at :ref:`alg-config`. +More documentation can be found at :ref:`alg-config`. Complete details on the YAML file are at at :class:`~salmon.triplets.manager.Config`. Examples of these files are in diff --git a/docs/source/installation.rst b/docs/source/installation.rst index aa65b8c7..4ed5a4e8 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -28,6 +28,7 @@ Experimentalist 6. Select an appropriate instance type. + * Salmon requires at least 2GB of memory and 1 CPU. * ``t3.large`` is recommended for passive algorithms (i.e, random sampling). * ``t3.xlarge`` is recommended for adaptive algorithms (e.g., ARR; see the @@ -120,49 +121,11 @@ To start using Salmon, these endpoints will be available: .. _local-install: -Local machine +Local install ------------- -This process is meant for developers. To launch, first download the code. It's -possible to download `a ZIP file of Salmon's source`_, or if Git is installed, -to run this command: +See :ref:`offlineinstall` for the process of installing locally. -.. _a ZIP file of Salmon's source: https://github.com/stsievert/salmon/archive/refs/heads/master.zip - -.. code:: shell - - $ git clone https://github.com/stsievert/salmon.git - -Then, to launch a local version of Salmon you'll need `Docker Compose`_. -After that dependency is intalled, run the following code: - -.. _install Docker: https://www.docker.com/products/docker-desktop -.. _install Git: https://git-scm.com/downloads - -.. code:: shell - - $ cd salmon - $ docker-compose build - $ docker-compose up - $ # visit http://localhost:8421/init or http://localhost:8421/docs - -.. _Docker Compose: https://docs.docker.com/compose/install/ - -If you make changes to this code, run these commands: - -.. code:: shell - - $ docker-compose stop - $ docker-compose build - $ docker-compose up - -If you run the command ``export SALMON_DEBUG=1``, the Salmon server will watch -for changes in the source and re-launch as necessary. This won't be perfect, -but it will reduce the number of times required to run ``docker-compose {stop, -build, up}``. - -If you run the command ``export SALMON_NO_AUTH=1``, the Salmon server will -not require a username/password. .. _troubleshooting: @@ -195,22 +158,38 @@ right of the Amazon EC2 interface. The Salmon AMI has been created in the ``us-west-2`` region, and EC2 AMIs are only available in the regions they're created in. - .. _restorefrombackupfaq: Restoring from a backup didn't work ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -That's perhaps expected depending on how much Salmon has changed. Launching -from EC2 always downloads the latest version of Salmon, which may not work with -your backup file. +This might happen if Salmon changed between when you downloaded and tried to +restore the experiment. Launching from EC2 always downloads the latest version +of Salmon, which may not work with your backup file. + +.. note:: + + Salmon follows `semantic software versioning`_. If the version string in the + .rdb file takes the form ``vA.B.C``, then: + + * The backup is guaranteed to work if `the latest release`_ has version + ``vA.B.C``. + * The backup will almost certainly work if `the latest release`_ has version + ``vA.B.*``. + * The backup `might` work if `the latest release`_ has version ``vA.*.*``. + + Uploading backup files when `relevant` "backwards incompatible" software + changes are made, which should be encoded in the release notes. -Let's follow this process to restore your backup with the correct version of Salmon: +So, it uploading your backup did not work (it should if the version numbers are +correct), let's launch the correct version of Salmon's server on your machine. That requires this process: 1. Get the correct version of Salmon. -2. Launch a Salmon server. -3. Restore. +2. Spin up a Salmon server. +3. Go to ``http://localhost:8421/init`` +4. Upload the ``.rdb`` file to restore. +This process is basically "launch Salmon's server on your machine." First, let's get the right version of Salmon: .. code-block:: shell @@ -220,45 +199,24 @@ First, let's get the right version of Salmon: $ cd salmon $ git checkout v0.7.0 # from .rdb filename; will take the form "vA.B.C" or "vA.B.CrcD" -Second, let's launch Salmon (following the same process as in -:ref:`local-install`). +Second, let's launch Salmon: .. code-block:: shell $ docker-compose up # takes a while - $ # visit http://[url]:8421/init and re-upload file + $ # visit http://localhost:8421/init -Finally, let's follow the instructions provided, which for Salmon v0.7.0 are -below: +**Now re-upload the file using the interface at the bottom of the screen.** + +Now Salmon will issue instructions to restart. Let's do that: .. code-block:: shell $ # Now, let's follow the directions Salmon gave: $ docker-compose stop; docker-compose start $ docker-compose logs -f - $ # visit http://[ur]:8421/dashboard - -Salmon follows `semantic software versioning`_. If the version string in the -.rdb file takes the form ``vA.B.C``, then: - -* The backup is guaranteed to work if `the latest release`_ has version - ``vA.B.C``. -* The backup will almost certainly work if `the latest release`_ has version - ``vA.B.*``. -* The backup `might` work if `the latest release`_ has version ``vA.*.*``. + $ # visit http://localhost:8421/dashboard -Uploading backup files when `relevant` "backwards incompatible" software -changes are made, which should be encoded in the release notes. .. _semantic software versioning: https://semver.org/ .. _the latest release: https://github.com/stsievert/salmon/releases - -The Docker machines aren't launching -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Are you using the command ``docker-compose up`` to launch Salmon? The command -``docker build .`` doesn't work. - -Salmon requires a Redis docker machine and certain directories/ports being -available. Technically, it's possible to build all the Docker machines -yourself (but it's not feasible). diff --git a/docs/source/offline.rst b/docs/source/offline.rst index 0c1e5ad8..ebd3f9a8 100644 --- a/docs/source/offline.rst +++ b/docs/source/offline.rst @@ -20,49 +20,32 @@ Downloading responses Download the responses, either by visiting ``http://[url]:8421/responses`` or clicking the link on the dashboard (as mentioned in :ref:`exp-monitoring`). +.. _offlineinstall: + Install Salmon -------------- -There are two options to install Salmon for offline embeddings. - -Option 1: Using pip -^^^^^^^^^^^^^^^^^^^ - -This option is recommended to generate embeddings offline. This option -requires ``pip``, a Python package manager. It's available through `Anaconda`_ -and `Miniconda`_. - - -1. Run the command ``pip install salmon-triplets``. -2. Run the command below in Python to verify that the installation worked successfully: - -.. code-block:: python - - from salmon.triplets.offline import OfflineEmbedding - -.. note:: - - This package named "salmon-triplets" on PyPI installs a Python package - named ``salmon``. +There are two options to install Salmon for offline embeddings. **Using +``conda`` is preferred** because it installs all the requirements (including +Python 3.8, which might not be installed) and has more sophisticated conflict +resolution than ``pip``. -Option 2: Using conda -^^^^^^^^^^^^^^^^^^^^^ +Using conda +^^^^^^^^^^^ This option is required for a complete installation. This option requires ``conda``, a Anaconda's Python package manager. It's available through `Anaconda`_ and `Miniconda`_. -1. Download `the latest release of Salmon`_, and unpack the `.zip` or `.tar.gz` - file. -2. Navigate to the directory in the shell/terminal and run these commands: +1. Download `the latest release of Salmon`_. +2. Unzip/unpack the `.zip` or `.tar.gz` file. +3. Navigate to the directory in the shell/terminal and run these commands: +4. Then run these commands: .. code-block:: shell - $ # download salmon-X.Y.Z.zip into ~/Downloads - $ # unzip/untar into directory `salmon` - $ # then navigate there in your shell/Terminal - $ cd ~/Downloads/salmon - $ conda env create -f salmon.yml + $ cd ~/Downloads/salmon # directory just downloaded and unzipped + $ conda env create -f salmon.lock.yml $ conda activate salmon (salmon) $ pip install . @@ -79,6 +62,27 @@ be Terminal.app. be your terminal prompt; leave it out when copy and pasting into the terminal. +Using pip +^^^^^^^^^ + +This option is recommended to generate embeddings offline. This option +requires ``pip``, a Python package manager. It's available through `Anaconda`_ +and `Miniconda`_. + +After you have the Python package manager ``pip``, run these commands: + +.. code-block:: shell + + $ pip install "salmon-triplets" + $ python -c "from salmon.triplets.offline import OfflineEmbedding" + +You have successfully installed Salmon if these commands complete successfully. + +.. note:: + + This package named "salmon-triplets" on PyPI installs a Python package + named ``salmon``. + Generate embeddings ------------------- @@ -86,25 +90,38 @@ This Python code will generate an embedding: .. code-block:: python + from pathlib import Path + + import pandas as pd + from sklearn.model_selection import train_test_split + from salmon.triplets.offline import OfflineEmbedding + import salmon.triplets.offline as offline + # Read in data df = pd.read_csv("responses.csv") # from dashboard - X = df[["head", "winner", "loser"]].to_numpy() + em = pd.read_csv("embedding.csv") # from dashboard; optional + config = yaml.loads(Path("config.yml").read_text()) # from dashboard - em = pd.read_csv("embedding.csv") # from dashboard + X = df[["head", "winner", "loser"]].to_numpy() + X_train, X_test = train_test_split(X, random_state=42, test_size=0.2) n = int(X.max() + 1) # number of targets d = 2 # embed into 2 dimensions - X_train, X_test = train_test_split(X, random_state=42, test_size=0.2) + # Fit the model model = OfflineEmbedding(n=n, d=d, max_epochs=500_000) - model.initialize(X_train, embedding=em.to_numpy()) + model.initialize(X_train, embedding=em.to_numpy()) # (optional) model.fit(X_train, X_test) + # Inspect the model model.embedding_ # embedding model.history_ # to view information on how well train/test performed + df_em = offline.join(model.embedding_, config["targets"]) + df_em.to_csv("final_embedding.csv") + Some customization can be done with ``model.history_``; it may not be necessary to train for 500,000 epochs. ``model.history_`` will include validation and training scores, which might help limit the number of epochs. diff --git a/examples/complete.yaml b/examples/complete.yaml index 759dd0a2..4eda9ef2 100644 --- a/examples/complete.yaml +++ b/examples/complete.yaml @@ -31,3 +31,8 @@ html: background-color: #cce; color: #220; } + element_top: "
" + element_middle: "
" + element_bottom: "
" + element_standalone: "
" + js: "console.log('custom js');" diff --git a/examples/default.yaml b/examples/default.yaml index 072a0bfe..94f6d2c5 100644 --- a/examples/default.yaml +++ b/examples/default.yaml @@ -8,6 +8,11 @@ html: skip_button: false title: "Similarity judgements" query_params: ['puid'] + element_top: "" + element_middle: "" + element_bottom: "" + element_standalone: "" + js: "" samplers: random: class: Random diff --git a/examples/zappos/_raw/README.md b/examples/zappos/_raw/README.md new file mode 100644 index 00000000..b8e095e0 --- /dev/null +++ b/examples/zappos/_raw/README.md @@ -0,0 +1,57 @@ +# Triplet dataset + +This dataset is mentioned in "Active Perceptual Similarity Modeling with +Auxiliary Information" by Eric Heim, Matthew Berger, Lee Seversky and Milos +Hauskrecht (https://arxiv.org/abs/1511.02254). + +Files: + +* `all_triplets.csv`: raw MTurk responses. + * Has header `head, b, c` (=> `head, close, far`) +* `deconflicted_triplets.csv`: deconflicted + * ordered `(head, b, c)` so that `b` closer to `head` than `c` + * => ordering should be `(head,close,far)` +* `cnn_feats.csv`: looks like feature for each shoe from a CNN. + + +From email with Matthew Berger: + +``` +From: Scott Sievert +Subject: Re: Zappos triplet dataset +To: Matthew Berger + +Hi Scott, + +Sure thing, you can find the data here: + +hdc.cs.arizona.edu/people/berger/triplet/zappos.zip + +- The deconflicted_triplets.csv file contains the triplet responses where we + have filtered triplets that resulted in inconsistencies, by analyzing the + graph formed by triplets and removing cycles. Each line is ordered as (a,b,c) + such that b is more similar to a than c is more similar to a. +- cnn_feats.csv contains (AlexNet) CNN features for each of the corresponding + images. The line numbers correspond to the indices in the triplets csv file. +- The images directory contains the images for each object, where the filename + corresponds to the index in the triplets csv file. + +If you are also interested in the raw triplet data that we collected through AMT please let us know. + +In addition, we have collected triplets through AMT for several other datasets that you might find of interest. First, we have collected triplets for the Animals with Attributes dataset, first popularized in the following paper: + +https://cvml.ist.ac.at/AwA/ + +Secondly, we have collected triplets for the CUB dataset: + +http://www.vision.caltech.edu/visipedia/CUB-200.html + +Let us know if you think triplets for these datasets would be useful for your research, and we’d be happy to share the data. + +Best, + +Matt + +``` + +This data was downloaded from http://hdc.cs.arizona.edu/people/berger/triplet/zappos.zip diff --git a/examples/zappos/_raw/all_triplets.csv.zip b/examples/zappos/_raw/all_triplets.csv.zip new file mode 100644 index 00000000..1d731e17 Binary files /dev/null and b/examples/zappos/_raw/all_triplets.csv.zip differ diff --git a/examples/zappos/_raw/cnn_feats.csv.zip b/examples/zappos/_raw/cnn_feats.csv.zip new file mode 100644 index 00000000..5ee63bf8 Binary files /dev/null and b/examples/zappos/_raw/cnn_feats.csv.zip differ diff --git a/examples/zappos/_raw/cnn_feats_read.py b/examples/zappos/_raw/cnn_feats_read.py new file mode 100644 index 00000000..2ff6f1a0 --- /dev/null +++ b/examples/zappos/_raw/cnn_feats_read.py @@ -0,0 +1,40 @@ +import numpy as np +from zipfile import ZipFile + +def _divide(n): + div = len(n) // 2 + idx = n.find(".", 2) + return n[:idx - 1], n[idx + 1:] + + +def _reduce(x): + out = [] + for xi in x: + if isinstance(xi, tuple): + for _ in xi: + out.append(float(_)) + else: + out.append(float(xi)) + return out + +def _get_features(filename: str = "cnn_feats.csv.zip"): + with ZipFile(filename) as zf: + with zf.open("cnn_feats.csv") as f: + lines = f.readlines() + assert len(lines) == 1 + txt = lines[0].decode("ascii") + raw = txt.split(",") + newlines = [i for i, n in enumerate(raw) if n.count(".") == 2] + mrare = [(n, ) if i not in newlines else _divide(n) for i, n in enumerate(raw)] + medium = _reduce(mrare) + + mwell = [medium[k * 4096 : (k + 1) * 4096] for k in range(n)] + well_done = np.array(mwell) + return well_done + +if __name__ == "__main__": + n = 85 + X = _get_features() + import pandas as pd + responses = pd.read_csv("all_triplets.csv.zip") + responses.columns = ["worker_id", "head", "close", "far"] diff --git a/examples/zappos/_raw/deconflicted_triplets.csv.zip b/examples/zappos/_raw/deconflicted_triplets.csv.zip new file mode 100644 index 00000000..5fd3765c Binary files /dev/null and b/examples/zappos/_raw/deconflicted_triplets.csv.zip differ diff --git a/examples/zappos/_raw/images/0.jpg b/examples/zappos/_raw/images/0.jpg new file mode 100644 index 00000000..0ac91478 Binary files /dev/null and b/examples/zappos/_raw/images/0.jpg differ diff --git a/examples/zappos/_raw/images/1.jpg b/examples/zappos/_raw/images/1.jpg new file mode 100644 index 00000000..4df458b8 Binary files /dev/null and b/examples/zappos/_raw/images/1.jpg differ diff --git a/examples/zappos/_raw/images/10.jpg b/examples/zappos/_raw/images/10.jpg new file mode 100644 index 00000000..3a3d5247 Binary files /dev/null and b/examples/zappos/_raw/images/10.jpg differ diff --git a/examples/zappos/_raw/images/11.jpg b/examples/zappos/_raw/images/11.jpg new file mode 100644 index 00000000..303cb7e7 Binary files /dev/null and b/examples/zappos/_raw/images/11.jpg differ diff --git a/examples/zappos/_raw/images/12.jpg b/examples/zappos/_raw/images/12.jpg new file mode 100644 index 00000000..195b563b Binary files /dev/null and b/examples/zappos/_raw/images/12.jpg differ diff --git a/examples/zappos/_raw/images/13.jpg b/examples/zappos/_raw/images/13.jpg new file mode 100644 index 00000000..77032232 Binary files /dev/null and b/examples/zappos/_raw/images/13.jpg differ diff --git a/examples/zappos/_raw/images/14.jpg b/examples/zappos/_raw/images/14.jpg new file mode 100644 index 00000000..c79f47cd Binary files /dev/null and b/examples/zappos/_raw/images/14.jpg differ diff --git a/examples/zappos/_raw/images/15.jpg b/examples/zappos/_raw/images/15.jpg new file mode 100644 index 00000000..bf6f159e Binary files /dev/null and b/examples/zappos/_raw/images/15.jpg differ diff --git a/examples/zappos/_raw/images/16.jpg b/examples/zappos/_raw/images/16.jpg new file mode 100644 index 00000000..ca84c315 Binary files /dev/null and b/examples/zappos/_raw/images/16.jpg differ diff --git a/examples/zappos/_raw/images/17.jpg b/examples/zappos/_raw/images/17.jpg new file mode 100644 index 00000000..bb157b60 Binary files /dev/null and b/examples/zappos/_raw/images/17.jpg differ diff --git a/examples/zappos/_raw/images/18.jpg b/examples/zappos/_raw/images/18.jpg new file mode 100644 index 00000000..e3a16167 Binary files /dev/null and b/examples/zappos/_raw/images/18.jpg differ diff --git a/examples/zappos/_raw/images/19.jpg b/examples/zappos/_raw/images/19.jpg new file mode 100644 index 00000000..209fe24f Binary files /dev/null and b/examples/zappos/_raw/images/19.jpg differ diff --git a/examples/zappos/_raw/images/2.jpg b/examples/zappos/_raw/images/2.jpg new file mode 100644 index 00000000..7162537f Binary files /dev/null and b/examples/zappos/_raw/images/2.jpg differ diff --git a/examples/zappos/_raw/images/20.jpg b/examples/zappos/_raw/images/20.jpg new file mode 100644 index 00000000..28ea1e28 Binary files /dev/null and b/examples/zappos/_raw/images/20.jpg differ diff --git a/examples/zappos/_raw/images/21.jpg b/examples/zappos/_raw/images/21.jpg new file mode 100644 index 00000000..b6771463 Binary files /dev/null and b/examples/zappos/_raw/images/21.jpg differ diff --git a/examples/zappos/_raw/images/22.jpg b/examples/zappos/_raw/images/22.jpg new file mode 100644 index 00000000..f207cff2 Binary files /dev/null and b/examples/zappos/_raw/images/22.jpg differ diff --git a/examples/zappos/_raw/images/23.jpg b/examples/zappos/_raw/images/23.jpg new file mode 100644 index 00000000..b519558a Binary files /dev/null and b/examples/zappos/_raw/images/23.jpg differ diff --git a/examples/zappos/_raw/images/24.jpg b/examples/zappos/_raw/images/24.jpg new file mode 100644 index 00000000..1835c969 Binary files /dev/null and b/examples/zappos/_raw/images/24.jpg differ diff --git a/examples/zappos/_raw/images/25.jpg b/examples/zappos/_raw/images/25.jpg new file mode 100644 index 00000000..0ab7eac8 Binary files /dev/null and b/examples/zappos/_raw/images/25.jpg differ diff --git a/examples/zappos/_raw/images/26.jpg b/examples/zappos/_raw/images/26.jpg new file mode 100644 index 00000000..1a719d7d Binary files /dev/null and b/examples/zappos/_raw/images/26.jpg differ diff --git a/examples/zappos/_raw/images/27.jpg b/examples/zappos/_raw/images/27.jpg new file mode 100644 index 00000000..37136478 Binary files /dev/null and b/examples/zappos/_raw/images/27.jpg differ diff --git a/examples/zappos/_raw/images/28.jpg b/examples/zappos/_raw/images/28.jpg new file mode 100644 index 00000000..9c3f7c28 Binary files /dev/null and b/examples/zappos/_raw/images/28.jpg differ diff --git a/examples/zappos/_raw/images/29.jpg b/examples/zappos/_raw/images/29.jpg new file mode 100644 index 00000000..6612d498 Binary files /dev/null and b/examples/zappos/_raw/images/29.jpg differ diff --git a/examples/zappos/_raw/images/3.jpg b/examples/zappos/_raw/images/3.jpg new file mode 100644 index 00000000..992729d4 Binary files /dev/null and b/examples/zappos/_raw/images/3.jpg differ diff --git a/examples/zappos/_raw/images/30.jpg b/examples/zappos/_raw/images/30.jpg new file mode 100644 index 00000000..6636cc5a Binary files /dev/null and b/examples/zappos/_raw/images/30.jpg differ diff --git a/examples/zappos/_raw/images/31.jpg b/examples/zappos/_raw/images/31.jpg new file mode 100644 index 00000000..346e0ac6 Binary files /dev/null and b/examples/zappos/_raw/images/31.jpg differ diff --git a/examples/zappos/_raw/images/32.jpg b/examples/zappos/_raw/images/32.jpg new file mode 100644 index 00000000..1d259204 Binary files /dev/null and b/examples/zappos/_raw/images/32.jpg differ diff --git a/examples/zappos/_raw/images/33.jpg b/examples/zappos/_raw/images/33.jpg new file mode 100644 index 00000000..7348a794 Binary files /dev/null and b/examples/zappos/_raw/images/33.jpg differ diff --git a/examples/zappos/_raw/images/34.jpg b/examples/zappos/_raw/images/34.jpg new file mode 100644 index 00000000..c27c9968 Binary files /dev/null and b/examples/zappos/_raw/images/34.jpg differ diff --git a/examples/zappos/_raw/images/35.jpg b/examples/zappos/_raw/images/35.jpg new file mode 100644 index 00000000..edca71dd Binary files /dev/null and b/examples/zappos/_raw/images/35.jpg differ diff --git a/examples/zappos/_raw/images/36.jpg b/examples/zappos/_raw/images/36.jpg new file mode 100644 index 00000000..8ee4f7b0 Binary files /dev/null and b/examples/zappos/_raw/images/36.jpg differ diff --git a/examples/zappos/_raw/images/37.jpg b/examples/zappos/_raw/images/37.jpg new file mode 100644 index 00000000..91ef58b9 Binary files /dev/null and b/examples/zappos/_raw/images/37.jpg differ diff --git a/examples/zappos/_raw/images/38.jpg b/examples/zappos/_raw/images/38.jpg new file mode 100644 index 00000000..3f8b84b6 Binary files /dev/null and b/examples/zappos/_raw/images/38.jpg differ diff --git a/examples/zappos/_raw/images/39.jpg b/examples/zappos/_raw/images/39.jpg new file mode 100644 index 00000000..68cfdbce Binary files /dev/null and b/examples/zappos/_raw/images/39.jpg differ diff --git a/examples/zappos/_raw/images/4.jpg b/examples/zappos/_raw/images/4.jpg new file mode 100644 index 00000000..a57ab6ef Binary files /dev/null and b/examples/zappos/_raw/images/4.jpg differ diff --git a/examples/zappos/_raw/images/40.jpg b/examples/zappos/_raw/images/40.jpg new file mode 100644 index 00000000..9952033c Binary files /dev/null and b/examples/zappos/_raw/images/40.jpg differ diff --git a/examples/zappos/_raw/images/41.jpg b/examples/zappos/_raw/images/41.jpg new file mode 100644 index 00000000..c3616a96 Binary files /dev/null and b/examples/zappos/_raw/images/41.jpg differ diff --git a/examples/zappos/_raw/images/42.jpg b/examples/zappos/_raw/images/42.jpg new file mode 100644 index 00000000..6a2ec8e4 Binary files /dev/null and b/examples/zappos/_raw/images/42.jpg differ diff --git a/examples/zappos/_raw/images/43.jpg b/examples/zappos/_raw/images/43.jpg new file mode 100644 index 00000000..edf3a7d8 Binary files /dev/null and b/examples/zappos/_raw/images/43.jpg differ diff --git a/examples/zappos/_raw/images/44.jpg b/examples/zappos/_raw/images/44.jpg new file mode 100644 index 00000000..ad6aa4bf Binary files /dev/null and b/examples/zappos/_raw/images/44.jpg differ diff --git a/examples/zappos/_raw/images/45.jpg b/examples/zappos/_raw/images/45.jpg new file mode 100644 index 00000000..61c23e28 Binary files /dev/null and b/examples/zappos/_raw/images/45.jpg differ diff --git a/examples/zappos/_raw/images/46.jpg b/examples/zappos/_raw/images/46.jpg new file mode 100644 index 00000000..67b00821 Binary files /dev/null and b/examples/zappos/_raw/images/46.jpg differ diff --git a/examples/zappos/_raw/images/47.jpg b/examples/zappos/_raw/images/47.jpg new file mode 100644 index 00000000..376a733d Binary files /dev/null and b/examples/zappos/_raw/images/47.jpg differ diff --git a/examples/zappos/_raw/images/48.jpg b/examples/zappos/_raw/images/48.jpg new file mode 100644 index 00000000..43eaf7e2 Binary files /dev/null and b/examples/zappos/_raw/images/48.jpg differ diff --git a/examples/zappos/_raw/images/49.jpg b/examples/zappos/_raw/images/49.jpg new file mode 100644 index 00000000..2bad90d9 Binary files /dev/null and b/examples/zappos/_raw/images/49.jpg differ diff --git a/examples/zappos/_raw/images/5.jpg b/examples/zappos/_raw/images/5.jpg new file mode 100644 index 00000000..3520a01a Binary files /dev/null and b/examples/zappos/_raw/images/5.jpg differ diff --git a/examples/zappos/_raw/images/50.jpg b/examples/zappos/_raw/images/50.jpg new file mode 100644 index 00000000..93cf0be4 Binary files /dev/null and b/examples/zappos/_raw/images/50.jpg differ diff --git a/examples/zappos/_raw/images/51.jpg b/examples/zappos/_raw/images/51.jpg new file mode 100644 index 00000000..7c33acd6 Binary files /dev/null and b/examples/zappos/_raw/images/51.jpg differ diff --git a/examples/zappos/_raw/images/52.jpg b/examples/zappos/_raw/images/52.jpg new file mode 100644 index 00000000..179453b9 Binary files /dev/null and b/examples/zappos/_raw/images/52.jpg differ diff --git a/examples/zappos/_raw/images/53.jpg b/examples/zappos/_raw/images/53.jpg new file mode 100644 index 00000000..7c36f2f1 Binary files /dev/null and b/examples/zappos/_raw/images/53.jpg differ diff --git a/examples/zappos/_raw/images/54.jpg b/examples/zappos/_raw/images/54.jpg new file mode 100644 index 00000000..71d33ea5 Binary files /dev/null and b/examples/zappos/_raw/images/54.jpg differ diff --git a/examples/zappos/_raw/images/55.jpg b/examples/zappos/_raw/images/55.jpg new file mode 100644 index 00000000..1325e4c6 Binary files /dev/null and b/examples/zappos/_raw/images/55.jpg differ diff --git a/examples/zappos/_raw/images/56.jpg b/examples/zappos/_raw/images/56.jpg new file mode 100644 index 00000000..d173e64b Binary files /dev/null and b/examples/zappos/_raw/images/56.jpg differ diff --git a/examples/zappos/_raw/images/57.jpg b/examples/zappos/_raw/images/57.jpg new file mode 100644 index 00000000..0401c323 Binary files /dev/null and b/examples/zappos/_raw/images/57.jpg differ diff --git a/examples/zappos/_raw/images/58.jpg b/examples/zappos/_raw/images/58.jpg new file mode 100644 index 00000000..d44e48aa Binary files /dev/null and b/examples/zappos/_raw/images/58.jpg differ diff --git a/examples/zappos/_raw/images/59.jpg b/examples/zappos/_raw/images/59.jpg new file mode 100644 index 00000000..b9d05fb7 Binary files /dev/null and b/examples/zappos/_raw/images/59.jpg differ diff --git a/examples/zappos/_raw/images/6.jpg b/examples/zappos/_raw/images/6.jpg new file mode 100644 index 00000000..f2647c92 Binary files /dev/null and b/examples/zappos/_raw/images/6.jpg differ diff --git a/examples/zappos/_raw/images/60.jpg b/examples/zappos/_raw/images/60.jpg new file mode 100644 index 00000000..1e313294 Binary files /dev/null and b/examples/zappos/_raw/images/60.jpg differ diff --git a/examples/zappos/_raw/images/61.jpg b/examples/zappos/_raw/images/61.jpg new file mode 100644 index 00000000..45b933ef Binary files /dev/null and b/examples/zappos/_raw/images/61.jpg differ diff --git a/examples/zappos/_raw/images/62.jpg b/examples/zappos/_raw/images/62.jpg new file mode 100644 index 00000000..87399a11 Binary files /dev/null and b/examples/zappos/_raw/images/62.jpg differ diff --git a/examples/zappos/_raw/images/63.jpg b/examples/zappos/_raw/images/63.jpg new file mode 100644 index 00000000..07642967 Binary files /dev/null and b/examples/zappos/_raw/images/63.jpg differ diff --git a/examples/zappos/_raw/images/64.jpg b/examples/zappos/_raw/images/64.jpg new file mode 100644 index 00000000..e3f0bdbc Binary files /dev/null and b/examples/zappos/_raw/images/64.jpg differ diff --git a/examples/zappos/_raw/images/65.jpg b/examples/zappos/_raw/images/65.jpg new file mode 100644 index 00000000..1942840f Binary files /dev/null and b/examples/zappos/_raw/images/65.jpg differ diff --git a/examples/zappos/_raw/images/66.jpg b/examples/zappos/_raw/images/66.jpg new file mode 100644 index 00000000..dc7f81bf Binary files /dev/null and b/examples/zappos/_raw/images/66.jpg differ diff --git a/examples/zappos/_raw/images/67.jpg b/examples/zappos/_raw/images/67.jpg new file mode 100644 index 00000000..df46b6b7 Binary files /dev/null and b/examples/zappos/_raw/images/67.jpg differ diff --git a/examples/zappos/_raw/images/68.jpg b/examples/zappos/_raw/images/68.jpg new file mode 100644 index 00000000..233f02ef Binary files /dev/null and b/examples/zappos/_raw/images/68.jpg differ diff --git a/examples/zappos/_raw/images/69.jpg b/examples/zappos/_raw/images/69.jpg new file mode 100644 index 00000000..1da20ca2 Binary files /dev/null and b/examples/zappos/_raw/images/69.jpg differ diff --git a/examples/zappos/_raw/images/7.jpg b/examples/zappos/_raw/images/7.jpg new file mode 100644 index 00000000..ce10a6b8 Binary files /dev/null and b/examples/zappos/_raw/images/7.jpg differ diff --git a/examples/zappos/_raw/images/70.jpg b/examples/zappos/_raw/images/70.jpg new file mode 100644 index 00000000..73fa6cee Binary files /dev/null and b/examples/zappos/_raw/images/70.jpg differ diff --git a/examples/zappos/_raw/images/71.jpg b/examples/zappos/_raw/images/71.jpg new file mode 100644 index 00000000..2a8c28b9 Binary files /dev/null and b/examples/zappos/_raw/images/71.jpg differ diff --git a/examples/zappos/_raw/images/72.jpg b/examples/zappos/_raw/images/72.jpg new file mode 100644 index 00000000..817d9674 Binary files /dev/null and b/examples/zappos/_raw/images/72.jpg differ diff --git a/examples/zappos/_raw/images/73.jpg b/examples/zappos/_raw/images/73.jpg new file mode 100644 index 00000000..601d6b51 Binary files /dev/null and b/examples/zappos/_raw/images/73.jpg differ diff --git a/examples/zappos/_raw/images/74.jpg b/examples/zappos/_raw/images/74.jpg new file mode 100644 index 00000000..0d9ebeaf Binary files /dev/null and b/examples/zappos/_raw/images/74.jpg differ diff --git a/examples/zappos/_raw/images/75.jpg b/examples/zappos/_raw/images/75.jpg new file mode 100644 index 00000000..a59b09de Binary files /dev/null and b/examples/zappos/_raw/images/75.jpg differ diff --git a/examples/zappos/_raw/images/76.jpg b/examples/zappos/_raw/images/76.jpg new file mode 100644 index 00000000..5c7c1140 Binary files /dev/null and b/examples/zappos/_raw/images/76.jpg differ diff --git a/examples/zappos/_raw/images/77.jpg b/examples/zappos/_raw/images/77.jpg new file mode 100644 index 00000000..7b037d35 Binary files /dev/null and b/examples/zappos/_raw/images/77.jpg differ diff --git a/examples/zappos/_raw/images/78.jpg b/examples/zappos/_raw/images/78.jpg new file mode 100644 index 00000000..c8c92a21 Binary files /dev/null and b/examples/zappos/_raw/images/78.jpg differ diff --git a/examples/zappos/_raw/images/79.jpg b/examples/zappos/_raw/images/79.jpg new file mode 100644 index 00000000..0407b3ff Binary files /dev/null and b/examples/zappos/_raw/images/79.jpg differ diff --git a/examples/zappos/_raw/images/8.jpg b/examples/zappos/_raw/images/8.jpg new file mode 100644 index 00000000..a0cc1987 Binary files /dev/null and b/examples/zappos/_raw/images/8.jpg differ diff --git a/examples/zappos/_raw/images/80.jpg b/examples/zappos/_raw/images/80.jpg new file mode 100644 index 00000000..a1320f97 Binary files /dev/null and b/examples/zappos/_raw/images/80.jpg differ diff --git a/examples/zappos/_raw/images/81.jpg b/examples/zappos/_raw/images/81.jpg new file mode 100644 index 00000000..ab92bc14 Binary files /dev/null and b/examples/zappos/_raw/images/81.jpg differ diff --git a/examples/zappos/_raw/images/82.jpg b/examples/zappos/_raw/images/82.jpg new file mode 100644 index 00000000..d751c824 Binary files /dev/null and b/examples/zappos/_raw/images/82.jpg differ diff --git a/examples/zappos/_raw/images/83.jpg b/examples/zappos/_raw/images/83.jpg new file mode 100644 index 00000000..2b68ef16 Binary files /dev/null and b/examples/zappos/_raw/images/83.jpg differ diff --git a/examples/zappos/_raw/images/84.jpg b/examples/zappos/_raw/images/84.jpg new file mode 100644 index 00000000..6d0a6598 Binary files /dev/null and b/examples/zappos/_raw/images/84.jpg differ diff --git a/examples/zappos/_raw/images/9.jpg b/examples/zappos/_raw/images/9.jpg new file mode 100644 index 00000000..78d66f47 Binary files /dev/null and b/examples/zappos/_raw/images/9.jpg differ diff --git a/paper/paper.bib b/paper/paper.bib index 83b55648..6c833702 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -126,12 +126,13 @@ @inproceedings{vehicles } @article{chem, - title={How to Model Implicit Knowledge? Similarity Learning Methods to Assess Perceptions of Visual Representations.}, - author={Rau, Martina A and Mason, Blake and Nowak, Robert}, - journal={International Educational Data Mining Society}, - year={2016}, - publisher={ERIC}, + title={Cognitive Task Analysis for Implicit Knowledge About Visual Representations With Similarity Learning Methods}, + author={Mason, Blake and Rau, Martina A and Nowak, Robert}, + journal={Cognitive science}, + year={2019}, doi={10.1111/cogs.12744}, + volume={43}, + issue={9}, } @inproceedings{next2, diff --git a/paper/paper.md b/paper/paper.md index 445f8d24..d99ef3e1 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -40,7 +40,7 @@ successfully to characterize human perceived similarity between faces Typically, experimentalists require an inordinate number of human responses (about 10,000) to produce an accurate embedding when making a similarity map in $d=2$ dimensions of $n = 50$ chemistry molecules [@chem]. -The number of human responses required will change like scale like +The number of human responses required will scale like $\mathcal{O}(nd\log n)$, which means that asking about $n=100$ molecules for $d=3$ dimensions will require about 35,000 responses. Many "active machine learning" methods have been proposed to reduce the number diff --git a/requirements.txt b/requirements.txt index 083dbac8..3c2b9df0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,10 +11,7 @@ cytoolz ujson torch # cpuonly pyyaml -ipykernel # to run this environ in jupyter (required for viz) -# nb_conda_kernels altair -seaborn fastapi[all] rejson httpx @@ -30,10 +27,11 @@ redis==3.5.* # https://github.com/RedisJSON/redisjson-py/issues/67 matplotlib gunicorn python-multipart # optional dep required by gunicorn -starlette-prometheus sphinx>=4.0.0 numpydoc sphinx_rtd_theme pytest jupyter-server-proxy # to view Dask dashboard autodoc_pydantic # to show config docs +ipywidgets # https://github.com/stsievert/salmon/issues/140 +starlette_exporter diff --git a/salmon.lock.yml b/salmon.lock.yml new file mode 100644 index 00000000..7b9bcee8 --- /dev/null +++ b/salmon.lock.yml @@ -0,0 +1,106 @@ +name: salmon +channels: + - conda-forge + - anaconda-fusion + - defaults +dependencies: + - bokeh=2.0.1 + - click=8.0.4 + - cloudpickle=2.2.0 + - cython=0.29.32 + - cytoolz=0.12.0 + - dask=2021.11.2 + - distributed=2021.11.2 + - jinja2=3.0.3 + - lz4=4.0.2 + - msgpack-python=1.0.4 + - numpy=1.23.5 + - pandas=1.5.2 + - pip=22.3.1 + - python=3.8.15 + - python-blosc=1.10.6 + - pyyaml=6.0 + - scikit-learn=1.1.3 + - scipy=1.9.3 + - snappy=1.1.9 + - sphinx=5.3.0 + - sphinx_rtd_theme=1.1.1 + - ujson=5.5.0 + - pip: + - aiofiles==22.1.0 + - aiohttp==3.8.3 + - aiosignal==1.3.1 + - anyio==3.6.2 + - argon2-cffi==21.3.0 + - argon2-cffi-bindings==21.2.0 + - async-timeout==4.0.2 + - autodoc-pydantic==1.8.0 + - beautifulsoup4==4.11.1 + - bleach==5.0.1 + - contourpy==1.0.6 + - cramjam==2.6.2 + - cycler==0.11.0 + - dask-glm==0.2.0 + - dask-ml==2022.5.27 + - defusedxml==0.7.1 + - dnspython==2.2.1 + - email-validator==1.3.0 + - exceptiongroup==1.0.4 + - fastapi==0.88.0 + - fastjsonschema==2.16.2 + - fastparquet==2022.11.0 + - fonttools==4.38.0 + - frozenlist==1.3.3 + - gunicorn==20.1.0 + - h11==0.14.0 + - httpcore==0.16.2 + - httptools==0.5.0 + - httpx==0.23.1 + - iniconfig==1.1.1 + - itsdangerous==2.1.2 + - jupyter-server==1.23.3 + - jupyter-server-proxy==3.2.2 + - jupyterlab-pygments==0.2.2 + - kiwisolver==1.4.4 + - llvmlite==0.39.1 + - matplotlib==3.6.2 + - mistune==2.0.4 + - multidict==6.0.3 + - multipledispatch==0.6.0 + - nbclient==0.7.2 + - nbconvert==7.2.5 + - nbformat==5.7.0 + - numba==0.56.4 + - numpydoc==1.5.0 + - orjson==3.8.3 + - pandocfilters==1.5.0 + - pluggy==1.0.0 + - prometheus-client==0.15.0 + - pyarrow==10.0.1 + - pydantic==1.10.2 + - pytest==7.2.0 + - python-dotenv==0.21.0 + - python-multipart==0.0.5 + - redis==3.5.3 + - rejson==0.5.6 + - rfc3986==1.5.0 + - send2trash==1.8.0 + - simpervisor==0.4 + - skorch==0.12.1 + - sniffio==1.3.0 + - soupsieve==2.3.2.post1 + - starlette==0.22.0 + - starlette-exporter==0.14.0 + - tabulate==0.9.0 + - terminado==0.17.0 + - tinycss2==1.2.1 + - tomli==2.0.1 + - torch==1.13.0 + - tqdm==4.64.1 + - uvicorn==0.20.0 + - uvloop==0.17.0 + - watchfiles==0.18.1 + - webencodings==0.5.1 + - websocket-client==1.4.2 + - websockets==10.4 + - yarl==1.8.2 diff --git a/salmon.yml b/salmon.yml index 0cff2ab0..a48ebd7b 100644 --- a/salmon.yml +++ b/salmon.yml @@ -3,6 +3,7 @@ channels: - conda-forge - defaults - anaconda + - bokeh dependencies: - python==3.8.* - pip @@ -14,20 +15,15 @@ dependencies: - dask>=2021.02.0 - distributed>=2021.02.0 - click >= 6.6, <= 8.0.4 # https://github.com/dask/distributed/pull/6014 - - lz4 - - python-blosc - - cytoolz + - lz4 # for dask + - python-blosc # for dask - ujson - - pyyaml - - ipykernel # to run this environ in jupyter (required for viz) - - nb_conda_kernels - - altair - bokeh==2.0.1 # because templates/dashboard.html requires this version - jinja2<3.1.0 - sphinx_rtd_theme - sphinx>=4.0.0 - pip: - - seaborn + - pyyaml - fastapi[all] - rejson - httpx @@ -41,9 +37,9 @@ dependencies: - matplotlib - gunicorn - python-multipart # optional dep required by gunicorn - - starlette-prometheus - numpydoc - pytest - jupyter-server-proxy # to view Dask dashboard - autodoc_pydantic # to show config docs - torch --extra-index-url https://download.pytorch.org/whl/cpu + - starlette_exporter diff --git a/salmon/__init__.py b/salmon/__init__.py index 37e1392b..5cb77747 100644 --- a/salmon/__init__.py +++ b/salmon/__init__.py @@ -1,6 +1,13 @@ from salmon._version import get_versions -from salmon.backend import app as app_algs -from salmon.frontend import app as app - -__version__ = app.version = app_algs.version = get_versions()["version"] +__version__ = get_versions()["version"] del get_versions + +try: + from salmon.backend import app as app_algs + from salmon.frontend import app as app + + app.version = app_algs.version = __version__ +except (ModuleNotFoundError, ImportError): + pass + +from salmon.triplets.offline import OfflineEmbedding diff --git a/salmon/backend/__init__.py b/salmon/backend/__init__.py index 06ba163f..5c04bfb8 100644 --- a/salmon/backend/__init__.py +++ b/salmon/backend/__init__.py @@ -1,2 +1,5 @@ -from salmon.backend.core import app +try: + from salmon.backend.core import app +except ModuleNotFoundError: + pass from salmon.backend.sampler import Sampler diff --git a/salmon/backend/core.py b/salmon/backend/core.py index f6656a99..5f80d424 100644 --- a/salmon/backend/core.py +++ b/salmon/backend/core.py @@ -189,14 +189,18 @@ async def get_model(sampler: str): async def get_timings(sampler: str): samplers = rj.jsonget("samplers") if sampler not in samplers: - raise ServerException( + msg = ( f"Can't find key for sampler='{sampler}'. " f"Valid choices for sampler are {samplers}" ) + logger.warning(msg) + raise ServerException(msg) keys = list(sorted(rj.keys())) if f"alg-perf-{sampler}" not in keys: logger.warning("rj.keys() = %s", keys) - raise ServerException( + msg = ( f"Performance data has not been created for sampler='{sampler}'. Database has keys {keys}" ) + logger.warning(msg) + raise ServerException(msg) return rj.jsonget(f"alg-perf-{sampler}") diff --git a/salmon/backend/sampler.py b/salmon/backend/sampler.py index 01fbf623..779ec34f 100644 --- a/salmon/backend/sampler.py +++ b/salmon/backend/sampler.py @@ -5,13 +5,7 @@ from time import sleep, time from typing import Any, Dict, List, Optional, Tuple, TypeVar -import cloudpickle -import dask.distributed as distributed import numpy as np -from dask.distributed import Client as DaskClient -from redis.exceptions import ResponseError -from rejson import Client as RedisClient -from rejson import Path from salmon.utils import flush_logger, get_logger @@ -19,7 +13,6 @@ Query = TypeVar("Query") Answer = TypeVar("Answer") -root = Path.rootPath() class Sampler: @@ -38,13 +31,16 @@ def __init__(self, ident: str = ""): self.ident = ident self.meta_ = [] - def redis_client(self, decode_responses=True) -> RedisClient: + def redis_client(self, decode_responses=True) -> "RedisClient": """ Get the database (/Redis client) """ + from rejson import Client as RedisClient + return RedisClient(host="redis", port=6379, decode_responses=decode_responses) - def run(self, client: DaskClient): + + def run(self, client: "DaskClient"): """ Run the algorithm. @@ -62,6 +58,11 @@ def run(self, client: DaskClient): ``"reset" in rj.keys() and rj.jsonget("reset")``. """ + import dask.distributed as distributed + from redis.exceptions import ResponseError + from rejson import Path + root = Path.rootPath() + rj = self.redis_client() answers: List = [] @@ -241,6 +242,8 @@ def save(self) -> bool: """ Save the sampler's state and current embedding to the database. """ + import cloudpickle + rj2 = self.redis_client(decode_responses=False) out = cloudpickle.dumps(self) @@ -259,6 +262,10 @@ def reset(self, client, rj, futures=None): Stop the algorithm. The algorithm will be deleted shortly after this function is called. """ + from rejson import Client as RedisClient + from rejson import Path + root = Path.rootPath() + reset = rj.jsonget("reset", root) logger.warning("reset=%s for %s", reset, self.ident) if not reset: @@ -367,7 +374,7 @@ def get_model(self) -> Dict[str, Any]: """ raise NotImplementedError - def clear_queries(self, rj: RedisClient) -> bool: + def clear_queries(self, rj: "RedisClient") -> bool: """ Clear all queries that this sampler has posted from the database. """ @@ -378,7 +385,7 @@ def post_queries( self, queries: List[Query], scores: List[float], - rj: Optional[RedisClient] = None, + rj: Optional["RedisClient"] = None, done=None, ) -> int: """ @@ -449,10 +456,13 @@ def serialize_query(self, q: Query) -> str: h, a, b = q return f"{h}-{a}-{b}" - def get_answers(self, rj: RedisClient, clear: bool = True) -> List[Answer]: + def get_answers(self, rj: "RedisClient", clear: bool = True) -> List[Answer]: """ Get all answers the frontend has received. """ + from rejson import Path + root = Path.rootPath() + if not clear: raise NotImplementedError key = f"alg-{self.ident}-answers" diff --git a/salmon/frontend/plotting.py b/salmon/frontend/plotting.py index e63581bd..9260b1cb 100644 --- a/salmon/frontend/plotting.py +++ b/salmon/frontend/plotting.py @@ -191,7 +191,7 @@ async def _get_server_metrics(): start = datetime.now() - timedelta(days=1) end = datetime.now() data = { - "query": "starlette_requests_processing_time_seconds_bucket", + "query": "starlette_request_duration_seconds_bucket", "start": start.isoformat(), "end": end.isoformat(), "step": 0.1, @@ -210,14 +210,13 @@ async def _get_server_metrics(): df["value"] = df["value"].astype(float) df["le"] = df["le"].astype(float) - cols = ["value", "le", "path_template"] + cols = ["value", "le", "path"] proc = df[cols] proc.columns = ["count", "le", "endpoint"] bad_endpoints = [ "/favicon.ico", "/metrics", - "/metrics", "/api/v1/query", "/static", "/init_exp", diff --git a/salmon/frontend/public.py b/salmon/frontend/public.py index 7f8c887f..a5b23622 100644 --- a/salmon/frontend/public.py +++ b/salmon/frontend/public.py @@ -15,7 +15,7 @@ from starlette.requests import Request from starlette.staticfiles import StaticFiles from starlette.templating import Jinja2Templates -from starlette_prometheus import PrometheusMiddleware, metrics +from starlette_exporter import PrometheusMiddleware, handle_metrics from salmon.frontend.utils import ServerException, image_url, sha256 from salmon.triplets import manager @@ -58,7 +58,7 @@ def stop_algs(): on_shutdown=[stop_algs], ) app.add_middleware(PrometheusMiddleware) -app.add_route("/metrics/", metrics) +app.add_route("/metrics", handle_metrics) pkg_dir = pathlib.Path(__file__).absolute().parent app.mount("/static", StaticFiles(directory=str(pkg_dir / "static")), name="static") diff --git a/salmon/triplets/manager.py b/salmon/triplets/manager.py index d1253d21..22fb053d 100644 --- a/salmon/triplets/manager.py +++ b/salmon/triplets/manager.py @@ -193,6 +193,30 @@ class Config: """, ) + js: str = Field( + "", + description="JavaScript to include in standalone `` - - - - - - + + + + + +
diff --git a/templates/query_page.html b/templates/query_page.html index 19c9e6ae..79ceb379 100644 --- a/templates/query_page.html +++ b/templates/query_page.html @@ -6,11 +6,11 @@ - - - - - + + + + + {{html['title']}} @@ -64,6 +64,7 @@
+ {{html['element_top'] | safe}}

{{html['instructions'] | safe}}

@@ -72,6 +73,7 @@

Target:

+ {{html['element_middle'] | safe}}
@@ -93,10 +95,12 @@
{% endif %} - + {{html['element_bottom'] | safe}}
+{{html['element_standalone'] | safe}} +
@@ -364,4 +368,9 @@ } {% endif %} + + + diff --git a/tests/test_active.py b/tests/test_active.py index baf1fb25..5f7cbf95 100644 --- a/tests/test_active.py +++ b/tests/test_active.py @@ -77,52 +77,48 @@ def test_active_bad_keys(server, logs): ) -@pytest.mark.parametrize("sampler", ["ARR", "CKL"]) +@pytest.mark.parametrize("sampler", ["ARR", "Random"]) def test_active_queries_generated(server, sampler, logs): # R=1 chosen because that determines when active sampling starts; this # test is designed to make sure no unexpected errors are thrown in # active portion (not that it generates a good embedding) + # tests ARR to make sure active scores are generated; + # tests Random to make sure that's not a false positive and + # random queries are properly identifies + n = 6 config = { "targets": [_ for _ in range(n)], "samplers": {sampler: {}}, - "sampling": {"common": {"d": 1, "R": 1}}, + "sampling": {}, } + if sampler != "Random": + config["sampling"]["common"] = {"d": 1, "R": 1} with logs: server.authorize() server.post("/init_exp", data={"exp": config}) - n_active_queries = 0 - for k in range(6 * n + 1): + active_queries_generated = False + for k in range(10 * n + 1): q = server.get("/query").json() + query = "random" if q["score"] == -9999 else "active" + if query == "active": + active_queries_generated = True + break + + sleep(200e-3) ans = random.choice([q["left"], q["right"]]) ans = {"winner": ans, "puid": "foo", **q} - print(q) server.post("/answer", json=ans) - if q["score"] != -9999: - # scored queries have been posted to the database - # now, only thing to test is popping off database - n_active_queries += 1 - if n_active_queries == n: - sleep(1) - break - - sleep(100e-3) if k % n == 0: sleep(1) - d = server.get("/responses").json() - - df = pd.DataFrame(d) - random_queries = df["score"] == -9999 - active_queries = ~random_queries - assert active_queries.sum() - assert random_queries.sum() - - samplers = set(df.sampler.unique()) - assert samplers == {sampler} + if sampler == "Random": + assert not active_queries_generated + else: + assert active_queries_generated def test_active_basics(server, logs): @@ -137,7 +133,7 @@ def test_active_basics(server, logs): with logs: server.authorize() server.post("/init_exp", data={"exp": exp.read_text()}) - for k in range(len(samplers) * 2): + for k in range(len(samplers) * 3): print(k) q = server.get("/query").json() @@ -154,3 +150,4 @@ def test_active_basics(server, logs): assert (df["score"] <= 1).all() algs = df.sampler.unique() assert set(algs) == {"TSTE", "ARR", "CKL", "tste2", "GNMDS"} + assert True # to see if a log error is caught in the traceback diff --git a/tests/test_allowabe_targets.py b/tests/test_allowabe_targets.py index 893fd3b1..209dbfff 100644 --- a/tests/test_allowabe_targets.py +++ b/tests/test_allowabe_targets.py @@ -14,7 +14,7 @@ def test_targets(sampler, server): server.authorize() server.post("/init_exp", data={"exp": config}) - for k in range(20): + for k in range(30): q = server.get("/query").json() ans = {"winner": random.choice([q["left"], q["right"]]), "puid": "foo", **q} server.post("/answer", json=ans) diff --git a/tests/test_offline.py b/tests/test_offline.py index ffd15949..b8dc0cc0 100644 --- a/tests/test_offline.py +++ b/tests/test_offline.py @@ -1,4 +1,5 @@ from pathlib import Path +import yaml import numpy as np import numpy.linalg as LA @@ -8,10 +9,11 @@ from salmon.triplets.offline import OfflineEmbedding from salmon.triplets.samplers import TSTE +import salmon.triplets.offline def test_salmon_import(): - """ This test makes sure that no errors are raised on import + """This test makes sure that no errors are raised on import (non-existant directories, etc)""" import salmon @@ -49,7 +51,7 @@ def test_score_accurate(): # Make sure the score has the expected value (winner has minimum distance) embed = alg.opt.embedding() * 1e3 y_hat2 = [] - for (head, left, right) in X: + for head, left, right in X: ldist = LA.norm(embed[head] - embed[left]) rdist = LA.norm(embed[head] - embed[right]) @@ -108,6 +110,26 @@ def test_offline_init(): assert not np.allclose(est.embedding_, em), "Embedding didn't change" +def test_offline_names_correct(): + DIR = Path(__file__).absolute().parent + _f = DIR / "data" / "active.yaml" + config = yaml.load(_f.read_text(), Loader=yaml.SafeLoader) + n = len(config["targets"]) + d = config["sampling"]["common"]["d"] + + X = np.random.choice(n, size=(100, 3)) + est = OfflineEmbedding(n=n, d=d) + est.partial_fit(X) + + import salmon.triplets.offline as offline + + em = offline.join(est.embedding_, config["targets"]) + assert isinstance(em, pd.DataFrame) + assert len(em) == len(config["targets"]) + assert set(em.columns) == {"x", "y", "target"} + assert (em["target"] == config["targets"]).all() + + if __name__ == "__main__": test_offline_init() test_offline_embedding_random_state()