diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 00000000..f3b5dda1 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,6 @@ +[bumpversion] +current_version = 2.4.0 +commit = True +tag = True + +[bumpversion:file:renderapi/__init__.py] diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..2660be86 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,21 @@ +[report] +exclude_lines = + pragma: no cover + + raise NotImplementedError + + # bad form, but annoying when reported + except Exception + + # raw input is low priority and may be removed + raw_input + + # __repr__ is usually only important in debugging + def __repr__ + + # ignore logger messages + logger.debug + logger.info + logger.warning + logger.error + logger.critical diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..53980b69 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +docker_setup.sh +docker_base_setup.sh +**/__pycache__ +**/*.pyc diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml new file mode 100644 index 00000000..a012f528 --- /dev/null +++ b/.github/workflows/release_publish.yml @@ -0,0 +1,29 @@ +# publish to pypi on github release (tagged as vX.Y.Z) +name: release_publish +on: + # workflow_dispatch: + release: + types: [published] +jobs: + pypi_publish: + name: Deploy to pypi + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v2 + + - name: Pixi setup + uses: prefix-dev/setup-pixi@v0.8.3 + with: + pixi-version: v0.41.4 + cache: false + run-install: false + + - name: Build with pixi env + run: pixi run pypi-build + + - name: Publish to Pypi + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c7196dd1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +**/*.pyc +*source_me.sh +build/ +.eggs* +*.egg-info +*.egg +dist/ +**/.DS_Store +.coverage +test-reports/ +docs/_build/ +htmlcov/ +.cache +.tox +.pytest_cache +.vscode +.idea/ +*.iml +.pixi/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..ab33e065 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,53 @@ +language: python +python: + - "2.7" + - "3.6" + - "3.7-dev" +services: + - docker +# command to install dependencies +addons: + apt: + packages: + - libblas-dev + - liblapack-dev + - libatlas-base-dev + - libopenblas-base + - libopenblas-dev + - gfortran + - openjdk-8-jdk + - maven +install: + - pip install codecov + - pip install -r requirements.txt + - pip install -r test_requirements.txt +before_install: + - git clone --depth 1 https://github.com/saalfeldlab/render.git render -b master + - export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/jre" + - mvn -version + - mvn package -pl render-ws-java-client -am -DskipTests -f render/pom.xml + - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` + - export RENDER_WS_JAVA_CLIENT_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/resources` + - export RENDER_APP_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/render/render-app/src/test/resources` + - export RENDER_JAVA_HOME=$JAVA_HOME + - mkdir -p /tmp/example_1 && cp -R $RENDER_WS_JAVA_CLIENT_EXAMPLE_DATA/example_1 /tmp/. + - cp -R $RENDER_APP_EXAMPLE_DATA/* /tmp/. + - export RENDER_EXAMPLE_DATA=/tmp + - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + - docker-compose up -d +env: + global: + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts + - secure: "AZCuswnHLEvDkfcmijSb4sy1E0vCg3nHn8gk9EvO7z9JeJG4FtHYBW3bLiN4WSjxszzaCK16lku0RY4OL2gCv0C87iHaXkDqAWwN0YIFGOkzDhZJRD5CENTsx1w/JFySqTBNiO82gIAnNP+J19mFm1ArtNhDIYAiUPo+CseG8S0cpbQqQe3tBd+XB/u32PUS1zA1mIg6qDXKupoDCUuWBxCZ01Jdv/YDUH3zxz6K9SQmbzJ4m5B1TCw8E6r/dj6mBSjJSh2XmZlOU0Ha2U9nEkkkr7nbNU1GYQYRt9uiqDH4juvFw6WePP+MVzFJCq0Hqdyb9lk/7jh4BX5ffN2o/vs14FuXZ57wblcHZkkEgD0ViIlGjqgPn0WuTJstkVlpvvwuweVfXXxJrYAZwhf14goKiWneDZMJ6g82Z7r9GVQFwEwl/mu5fdLP/qKZ8i3AAd9SgnGdn5eQ/xD5JVJj9JZFqfLUJs7JQreGYfW2r5URIL3Kd2laRucUD4IPwnzHT+VjgNHFtv07bmNxRGPdwC4pbWOMiyq2BvG3JFw/y8j3bNvLxK6XLhDgHtV13NinZx8uZcocb8yCNyBpez+Pfo3RzTcHKnF5foKoEEZIU5OuyGGgeHUup/vg2qsKGn3Wl1nH5fWSbdR2pPHLSD+mpZdLAA9JjqEfVrhZtoXLUpA=" + - secure: "tnw8YYrVoasmJ+3I3VjENfx6BIBG2qaJ4m0+rUR9dal7cmrGTrcyJ1d41Nm+K1XKRvLvBkEHKBzdEVlg2XaAOVBTam6u+zW/Dc16ZXgz7yK1qXtytwiXBe2P7c9HePXIDA/SCapKXmjF5/Zt0BdpXv4fPyK8kbdIjels9Xt8teDTfSxJ6AfXlfXXPiF6A1Lzgf+z9GRL9GXZrgic6itFdNfgErGThgefQSJKr3wKR/wRyx87QD5juzrZtcwtFzcvvO8X8n6D+oOdc9XfRsE1EgFO8RHla32c0BbcKmRP8vwwtfVUt1tqU3SQZP6+n9EUBiSUR9/B50hCOiX5O2RdtHIMncJT0n4ZQe2PmTmwpcY1bU1NJpUszxPXNZrEHRdGqbXfiQq8+sONAWZcrzbqLsP2cXpt+4FrHk+iepu0RBLpLq/UdcVLozELgkNxM1C9k3OF6OlAC//5jWi4hjeoc2L8LpR09Cxm7PCtBvqbCNO2mPhv+h4dsFzmlyN2JL34lQyfzD1v1Jqz4ifJHYupKKtIdA7UiU+RU40ylDzTKfvlI63MkUwxS2u4BY0Gc5WWy/NoJuF6F39c+peoLgcoHiKnXoIvTyWCNSfieXCnuiHrU7nLnijM9qzSZxM6a43KiSGRZAQInQUXzKX5a7cGHaQIFskd3TK71emnhdewM40=" + - secure: "UucVIVQCg9T2d9NiSdMirj37nlwWEyZi2mybzNNVJ8TMfC05Vz+E/b/1ByxOLpUYmXZTzU22Xx316ueIaIk05L+Q8q4Dy1S/spaKBNcJQ8kDxlcD7y0YBPX+QqyOpXCnB1Kt1oawy92g2qALFDMBoupr7xT8aTvCwDigrJYghMiVM1RVlQQAJBaGkB32qs934uQeknvLqSZV35BTFGHTaqJoWV8OyxNwOZFtf5Z8gdWxrAYHOQg9iW+5CP+zLhybROLrMuI5vhj/yvP2I07Scx+jMKOgGw7snPygHpwR0dNw/NvNdy+XIx3zqUlts5Bb1nqCeaNzNouXd1D5NHoqsw/0VLt/HcbCxUKXuaViJy5KfN/qi9U7lSJ0KC2qazwsQXviFYTrT2q1U2n6zlV/CC9e1sLjQmIF3GAysFD3Fcg88Qz/3aZKl6nkqP/T2Y1t/FOPjBSD5ytlpr8xbs6rpxZMU/k91agD5UeP4/lwbncfsJswPNajdTz5scWjHPACDLIxYTwEry64z808lHcc23c9MIeqrkXMI4nPYnLSPQyhXBOJncEugKernBh053Pu5+hDrg9bsoIjtsGmh+p9GnIjrtPAkc8g4tyU+v0PKMVUkqw+mvkVEgzRlKUzURuRTVy5lTHisu0mo6Xmtgw0qfVkTOa2Gnr9gsfmaTToyXY=" +# command to run tests +script: + - python setup.py test # or py.test for Python versions 3.5 and below +cache: + pip: true + directories: + - $HOME/.m2 +after_success: + - codecov -t 3f12d985-af62-455d-a11d-9669c039640d + - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=AllenInstitute/render-python .travis/merge_script.sh" diff --git a/.travis/merge_script.sh b/.travis/merge_script.sh new file mode 100755 index 00000000..16da047f --- /dev/null +++ b/.travis/merge_script.sh @@ -0,0 +1,33 @@ +#!/bin/bash -e + +: "${BRANCHES_TO_MERGE_REGEX?}" "${BRANCH_TO_MERGE_INTO?}" +: "${GITHUB_SECRET_TOKEN?}" "${GITHUB_REPO?}" + +export GIT_COMMITTER_EMAIL='travis@travis' +export GIT_COMMITTER_NAME='Travis CI' + +if ! grep -q "$BRANCHES_TO_MERGE_REGEX" <<< "$TRAVIS_BRANCH"; then + printf "Current branch %s doesn't match regex %s, exiting\\n" \ + "$TRAVIS_BRANCH" "$BRANCHES_TO_MERGE_REGEX" >&2 + exit 0 +fi + +# Since Travis does a partial checkout, we need to get the whole thing +repo_temp=$(mktemp -d) +git clone "https://github.com/$GITHUB_REPO" "$repo_temp" + +# shellcheck disable=SC2164 +cd "$repo_temp" + +printf 'Checking out %s\n' "$BRANCH_TO_MERGE_INTO" >&2 +git checkout "$BRANCH_TO_MERGE_INTO" + +printf 'Merging %s\n' "$TRAVIS_COMMIT" >&2 +git merge --ff-only "$TRAVIS_COMMIT" + +printf 'Pushing to %s\n' "$GITHUB_REPO" >&2 + +push_uri="https://$GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO" + +# Redirect to /dev/null to avoid secret leakage +git push "$push_uri" "$BRANCH_TO_MERGE_INTO" >/dev/null 2>&1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..e55867fc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Allen Institute Contribution Agreement + +This document describes the terms under which you may make “Contributions” — +which may include without limitation, software additions, revisions, bug fixes, configuration changes, +documentation, or any other materials — to any of the projects owned or managed by the Allen Institute. +If you have questions about these terms, please contact us at terms@alleninstitute.org. + +You certify that: + +• Your Contributions are either: + +1. Created in whole or in part by you and you have the right to submit them under the designated license +(described below); or +2. Based upon previous work that, to the best of your knowledge, is covered under an appropriate +open source license and you have the right under that license to submit that work with modifications, +whether created in whole or in part by you, under the designated license; or + +3. Provided directly to you by some other person who certified (1) or (2) and you have not modified them. + +• You are granting your Contributions to the Allen Institute under the terms of the [2-Clause BSD license](https://opensource.org/licenses/BSD-2-Clause) +(the “designated license”). + +• You understand and agree that the Allen Institute projects and your Contributions are public and that +a record of the Contributions (including all metadata and personal information you submit with them) is +maintained indefinitely and may be redistributed consistent with the Allen Institute’s mission and the +2-Clause BSD license. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5751a400 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +FROM fcollman/render-python-base:latest +MAINTAINER Forrest Collman (forrest.collman@gmail.com) + +WORKDIR /shared/render-python +COPY . /shared/render-python +RUN pip install /shared/render-python + +ENTRYPOINT [ "/usr/bin/tini", "--" ] +CMD [ "/bin/bash" ] diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 00000000..e8a9f1e6 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,77 @@ + +FROM fcollman/render:latest +MAINTAINER Forrest Collman (forrest.collman@gmail.com) + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ + libglib2.0-0 libxext6 libsm6 libxrender1 \ + git mercurial subversion + +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ + wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh + +RUN apt-get install -y curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb && \ + apt-get clean + +# RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ +# libglib2.0-0 libxext6 libsm6 libxrender1 \ +# git mercurial subversion + +# RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ +# wget --quiet https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh -O ~/anaconda.sh && \ +# /bin/bash ~/anaconda.sh -b -p /opt/conda && \ +# rm ~/anaconda.sh + +# RUN apt-get install -y curl grep sed dpkg && \ +# TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ +# curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ +# dpkg -i tini.deb && \ +# rm tini.deb && \ +# apt-get clean + +ENV PATH /opt/conda/bin:$PATH + +#install java +# auto validate license +#RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections +#RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list +#RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list +#RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 +#RUN apt-get update +#RUN apt-get install oracle-java8-installer -y +#ENV JAVA_HOME /www/var/render/deploy/jdkd + +#install pathos,multiprocess with gcc +RUN apt-get update +RUN apt-get install gcc build-essential libgeos-dev imagemagick -y +RUN apt-get install python-setuptools python-dev -y +#RUN apt-get install libblas-dev liblapack-dev -y +RUN apt-get clean + + +#install components for common render-python apps +#jupyter notebook, shapely with geos +# RUN /opt/conda/bin/conda install jupyter -y + + +#RUN easy_install pip +#RUN pip install -U pip setuptools +RUN pip install pillow xmltodict pathos==0.2.0 multiprocess==0.70.5 dill==0.2.6 opencv-python shapely==1.6b2 +RUN conda install scipy +RUN pip install setuptools --upgrade --disable-pip-version-check +RUN mkdir -p /usr/local/render-python +COPY . /usr/local/render-python +WORKDIR /usr/local/render-python +RUN pip install -r requirements.txt +RUN pip install -r test_requirements.txt +RUN pip install matplotlib pandas jupyter +#install render python using pip from github +#RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python + diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..00a05dad --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2022, Allen Institute +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..768217ce --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include requirements.txt +include test_requirements.txt \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index 9966b2c5..00000000 --- a/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# render-python - -This is a python API setup to interact via python with render databases and facilitate python scripting of tilespec creation -see -https://github.com/saalfeldlab/render -it presently interacts with render via a web-api, though a couple functions are presently implemented with the java client scripts that are part of render, ideally those would be cutout. - -tilespec.py is a set of python objects that facilitate serializing and deserializing objects to/from json. - -add_downsample_to_render_project.py is an example script that makes use of these files to read tilespecs from a render stack, -modify the tilespecs to include paths to downsampled images, and then write those tilespecs to disk, upload them back to render, and launch all the jobs to create those downsampled images in a parallel fashion. This program assumes a filestructure path that is unique to the Synapse Biology group at the Allen Institute, but nonetheless demonstrate the general utility of the above files. - -create_mipmaps.py is a simple python program for using Pillow to create downsampled images from a single image that is included simply for reference. - diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..e6c58cbb --- /dev/null +++ b/README.rst @@ -0,0 +1,39 @@ +.. image:: https://readthedocs.org/projects/render-python/badge/ + :target: http://render-python.readthedocs.io/en/latest/ + :alt: Documentation Status +.. image:: https://travis-ci.com/AllenInstitute/render-python.svg?branch=master + :target: https://travis-ci.com/AllenInstitute/render-python + :alt: Build Status +.. image:: https://codecov.io/gh/AllenInstitute/render-python/branch/master/graph/badge.svg + :target: https://codecov.io/gh/AllenInstitute/render-python + +render-python +############# + +This is a python API client to interact with `render `_ and facilitate python scripting of `tilespec `_ creation + +it presently interacts with render via a web-api, though the `client module `_ aims to interface by calling java client scripts to avoid server-side processing. + +Render connection objects created with `renderapi.connect()` can default to environment variables. Below is an example of the variables which can be sourced and added to, e.g., ~/.bashrc or ~/.bash_profile. +:: + + export RENDER_HOST="localhost" + export RENDER_PORT="8080" + export RENDER_PROJECT="YOURPROJECT" + export RENDER_OWNER="YOURNAME" + export RENDER_CLIENT_SCRIPTS=".../render/render-ws-java-client/src/main/scripts" + export RENDER_CLIENT_SCRIPT="$RENDER_CLIENT_SCRIPTS/run_ws_client.sh" + export RENDER_CLIENT_HEAP="1G" + + +`Usage examples for a development Array Tomography workflow `_ are available. + +Documentation +############# +http://render-python.readthedocs.io/en/latest/ + +Government Sponsorship +###################### +Supported by the Intelligence Advanced Research Projects Activity (IARPA) via Department of Interior / Interior Business Center (DoI/IBC) contract number D16PC00004. The U.S. Government is authorized to reproduce and distribute reprints for Governmental purposes notwithstanding any copyright annotation thereon. Disclaimer: The views and conclusions contained herein are those of the authors and should not be interpreted as necessarily representing the official policies or endorsements, either expressed or implied, of IARPA, DoI/IBC, or the U.S. Government. + +.. _render : diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..8f52a483 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: "2.0" +services: + renderservice: + image: fcollman/render-ws + ports: + - "8080:8080" + links: + - mongo + volumes: + - "${RENDER_EXAMPLE_DATA}/example_1:/tmp/example_1:ro" + - "${RENDER_EXAMPLE_DATA}/multichannel-test:/tmp/multichannel-test:ro" + environment: + - MONGO_HOST=mongo + mongo: + image: mongo:3.4.2 + expose: + - "27017" + security_opt: + - seccomp:unconfined \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..7b7dda3a --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = render-python +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/api/modules.rst b/docs/api/modules.rst new file mode 100644 index 00000000..0695f8b4 --- /dev/null +++ b/docs/api/modules.rst @@ -0,0 +1,7 @@ +renderapi +========= + +.. toctree:: + :maxdepth: 4 + + renderapi diff --git a/docs/api/renderapi.external.processpools.rst b/docs/api/renderapi.external.processpools.rst new file mode 100644 index 00000000..ff40f8c8 --- /dev/null +++ b/docs/api/renderapi.external.processpools.rst @@ -0,0 +1,22 @@ +renderapi\.external\.processpools package +========================================= + +Submodules +---------- + +renderapi\.external\.processpools\.pool\_pathos module +------------------------------------------------------ + +.. automodule:: renderapi.external.processpools.pool_pathos + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: renderapi.external.processpools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/renderapi.external.rst b/docs/api/renderapi.external.rst new file mode 100644 index 00000000..6756ed9d --- /dev/null +++ b/docs/api/renderapi.external.rst @@ -0,0 +1,17 @@ +renderapi\.external package +=========================== + +Subpackages +----------- + +.. toctree:: + + renderapi.external.processpools + +Module contents +--------------- + +.. automodule:: renderapi.external + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/renderapi.rst b/docs/api/renderapi.rst new file mode 100644 index 00000000..e99dcab8 --- /dev/null +++ b/docs/api/renderapi.rst @@ -0,0 +1,102 @@ +renderapi package +================= + +Submodules +---------- + +renderapi\.client module +------------------------ + +.. automodule:: renderapi.client + :members: + :undoc-members: + :show-inheritance: + +renderapi\.coordinate module +---------------------------- + +.. automodule:: renderapi.coordinate + :members: + :undoc-members: + :show-inheritance: + +renderapi\.errors module +------------------------ + +.. automodule:: renderapi.errors + :members: + :undoc-members: + :show-inheritance: + +renderapi\.image module +----------------------- + +.. automodule:: renderapi.image + :members: + :undoc-members: + :show-inheritance: + +renderapi\.pointmatch module +---------------------------- + +.. automodule:: renderapi.pointmatch + :members: + :undoc-members: + :show-inheritance: + +renderapi\.render module +------------------------ + +.. automodule:: renderapi.render + :members: + :undoc-members: + :show-inheritance: + +renderapi\.stack module +----------------------- + +.. automodule:: renderapi.stack + :members: + :undoc-members: + :show-inheritance: + +renderapi\.tilespec module +-------------------------- + +.. automodule:: renderapi.tilespec + :members: + :undoc-members: + :show-inheritance: + +renderapi\.resolvedtiles module +-------------------------- + +.. automodule:: renderapi.resolvedtiles + :members: + :undoc-members: + :show-inheritance: + +renderapi\.transform module +--------------------------- + +.. automodule:: renderapi.transform + :members: + :undoc-members: + :show-inheritance: + +renderapi\.utils module +----------------------- + +.. automodule:: renderapi.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: renderapi + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..acf0b2d1 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# +# render-python documentation build configuration file, created by +# sphinx-quickstart on Sat Jul 22 11:57:57 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'render-python' +copyright = u'2017, Forrest Collman, Russel Torres, Eric Perlman' +author = u'Forrest Collman, Russel Torres, Eric Perlman' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'1.0' +# The full version, including alpha/beta/rc tags. +release = u'1.0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'about.html', + 'navigation.html', + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + 'donate.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'render-pythondoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'render-python.tex', u'render-python Documentation', + u'Forrest Collman, Russel Torres, Eric Perlman', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'render-python', u'render-python Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'render-python', u'render-python Documentation', + author, 'render-python', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/add_downsample_to_render_project.py b/docs/examples/add_downsample_to_render_project.py similarity index 100% rename from add_downsample_to_render_project.py rename to docs/examples/add_downsample_to_render_project.py diff --git a/apply_TEM2_to_render_stack.py b/docs/examples/apply_TEM2_to_render_stack.py similarity index 100% rename from apply_TEM2_to_render_stack.py rename to docs/examples/apply_TEM2_to_render_stack.py diff --git a/apply_alignment_from_render_stack.py b/docs/examples/apply_alignment_from_render_stack.py similarity index 100% rename from apply_alignment_from_render_stack.py rename to docs/examples/apply_alignment_from_render_stack.py diff --git a/docs/examples/apply_alignment_transforms_to_otherStacks2.ipynb b/docs/examples/apply_alignment_transforms_to_otherStacks2.ipynb new file mode 100644 index 00000000..f7f00573 --- /dev/null +++ b/docs/examples/apply_alignment_transforms_to_otherStacks2.ipynb @@ -0,0 +1,775 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from tilespec import TileSpec,Transform,AffineModel,ResolvedTileSpecCollection, ResolvedTileSpecMap\n", + "from renderapi import Render\n", + "import os\n", + "import json\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "owner = 'Forrest'\n", + "project = 'M247514_Rorb_1'\n", + "\n", + "json_dir = '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/'\n", + "aligned_prefix = 'ALIGNED2_'\n", + "aligned_prefix_new = 'ALIGNED3_'\n", + "json_dir = '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/'\n", + "client_scripts = '/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/'\n", + "host = '192.168.0.150'\n", + "port = 8080\n", + "render = Render(host,port,owner,project,default_client_scripts=client_scripts)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_1 {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-41-14T14:41:36.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_1/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_1/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_DAPI_1', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_1.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_DAPI_1 /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_1.json\n", + "\n", + "\n", + "14:41:36.901 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:41:37.023 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_DAPI_1\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_1.json\" ]\n", + "}\n", + "14:41:37.235 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:41:37.235 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_1.json\n", + "14:41:37.237 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_1.json\n", + "14:41:37.659 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:41:42.661 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 344 out of 1836 tiles\n", + "14:41:47.668 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 711 out of 1836 tiles\n", + "14:41:52.680 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1079 out of 1836 tiles\n", + "14:41:57.682 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1446 out of 1836 tiles\n", + "14:42:02.694 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1814 out of 1836 tiles\n", + "14:42:02.995 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=25\n", + "14:42:03.435 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_1/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:42:04.728 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_1.json\n", + "14:42:04.729 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 27 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_1/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_2 {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-42-14T14:42:06.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_2/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_2/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_DAPI_2', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_2.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_DAPI_2 /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_2.json\n", + "\n", + "\n", + "14:42:06.233 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:42:06.360 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_DAPI_2\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_2.json\" ]\n", + "}\n", + "14:42:06.553 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:42:06.553 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_2.json\n", + "14:42:06.555 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_2.json\n", + "14:42:06.941 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:42:11.947 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 472 out of 1836 tiles\n", + "14:42:16.955 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 979 out of 1836 tiles\n", + "14:42:21.956 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1486 out of 1836 tiles\n", + "14:42:25.409 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=18\n", + "14:42:25.780 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_2/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:42:27.141 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_2.json\n", + "14:42:27.141 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 20 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_2/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_3 {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-42-14T14:42:28.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_3/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_3/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_DAPI_3', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_3.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_DAPI_3 /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_3.json\n", + "\n", + "\n", + "14:42:28.687 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:42:28.812 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_DAPI_3\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_3.json\" ]\n", + "}\n", + "14:42:29.010 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:42:29.010 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_3.json\n", + "14:42:29.012 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_3.json\n", + "14:42:29.371 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:42:34.373 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 345 out of 1836 tiles\n", + "14:42:39.377 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 712 out of 1836 tiles\n", + "14:42:44.384 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1080 out of 1836 tiles\n", + "14:42:49.391 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1448 out of 1836 tiles\n", + "14:42:54.398 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1816 out of 1836 tiles\n", + "14:42:54.670 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=25\n", + "14:42:55.075 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_3/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:42:56.264 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_DAPI_3.json\n", + "14:42:56.264 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 27 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_DAPI_3/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GABA {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-42-14T14:42:57.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GABA/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GABA/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_GABA', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GABA.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_GABA /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GABA.json\n", + "\n", + "\n", + "14:42:57.818 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:42:57.942 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_GABA\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GABA.json\" ]\n", + "}\n", + "14:42:58.157 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:42:58.157 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GABA.json\n", + "14:42:58.159 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GABA.json\n", + "14:42:58.521 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:43:03.527 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 341 out of 1836 tiles\n", + "14:43:08.540 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 704 out of 1836 tiles\n", + "14:43:13.550 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1059 out of 1836 tiles\n", + "14:43:18.551 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1421 out of 1836 tiles\n", + "14:43:23.558 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1783 out of 1836 tiles\n", + "14:43:24.291 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=25\n", + "14:43:24.713 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GABA/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:43:25.983 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GABA.json\n", + "14:43:25.983 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 28 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GABA/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GAD2 {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-43-14T14:43:27.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GAD2/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GAD2/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_GAD2', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GAD2.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_GAD2 /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GAD2.json\n", + "\n", + "\n", + "14:43:27.509 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:43:27.641 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_GAD2\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GAD2.json\" ]\n", + "}\n", + "14:43:27.859 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:43:27.859 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GAD2.json\n", + "14:43:27.861 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GAD2.json\n", + "14:43:28.326 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:43:33.333 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 437 out of 1836 tiles\n", + "14:43:38.341 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 947 out of 1836 tiles\n", + "14:43:43.347 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1458 out of 1836 tiles\n", + "14:43:47.051 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=18\n", + "14:43:47.494 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GAD2/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:43:48.764 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GAD2.json\n", + "14:43:48.764 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 21 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GAD2/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_Gephyrin {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-43-14T14:43:50.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_Gephyrin/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_Gephyrin/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_Gephyrin', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_Gephyrin.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_Gephyrin /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_Gephyrin.json\n", + "\n", + "\n", + "14:43:50.413 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:43:50.546 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_Gephyrin\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_Gephyrin.json\" ]\n", + "}\n", + "14:43:50.763 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:43:50.763 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_Gephyrin.json\n", + "14:43:50.765 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_Gephyrin.json\n", + "14:43:51.135 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:43:56.143 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 463 out of 1836 tiles\n", + "14:44:01.151 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 971 out of 1836 tiles\n", + "14:44:06.154 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1480 out of 1836 tiles\n", + "14:44:09.657 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=18\n", + "14:44:10.111 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_Gephyrin/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:44:11.355 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_Gephyrin.json\n", + "14:44:11.355 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 20 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_Gephyrin/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GluN1 {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-44-14T14:44:12.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GluN1/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GluN1/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_GluN1', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GluN1.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_GluN1 /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GluN1.json\n", + "\n", + "\n", + "14:44:12.902 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:44:13.025 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_GluN1\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GluN1.json\" ]\n", + "}\n", + "14:44:13.225 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:44:13.225 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GluN1.json\n", + "14:44:13.227 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GluN1.json\n", + "14:44:13.593 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:44:18.602 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 341 out of 1836 tiles\n", + "14:44:23.610 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 707 out of 1836 tiles\n", + "14:44:28.620 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1074 out of 1836 tiles\n", + "14:44:33.634 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1441 out of 1836 tiles\n", + "14:44:38.645 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1808 out of 1836 tiles\n", + "14:44:39.029 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=25\n", + "14:44:39.477 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GluN1/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:44:40.682 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_GluN1.json\n", + "14:44:40.683 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 27 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_GluN1/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_MBP {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-44-14T14:44:42.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_MBP/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_MBP/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_MBP', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_MBP.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_MBP /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_MBP.json\n", + "\n", + "\n", + "14:44:42.225 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:44:42.343 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_MBP\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_MBP.json\" ]\n", + "}\n", + "14:44:42.543 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:44:42.544 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_MBP.json\n", + "14:44:42.545 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_MBP.json\n", + "14:44:43.003 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:44:48.005 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 341 out of 1836 tiles\n", + "14:44:53.012 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 707 out of 1836 tiles\n", + "14:44:58.022 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1074 out of 1836 tiles\n", + "14:45:03.034 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1441 out of 1836 tiles\n", + "14:45:08.042 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1808 out of 1836 tiles\n", + "14:45:08.426 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=25\n", + "14:45:08.870 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_MBP/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:45:10.149 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_MBP.json\n", + "14:45:10.149 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 27 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_MBP/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_PSD95 {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-45-14T14:45:11.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_PSD95/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_PSD95/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_PSD95', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_PSD95.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_PSD95 /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_PSD95.json\n", + "\n", + "\n", + "14:45:11.711 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:45:11.835 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_PSD95\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_PSD95.json\" ]\n", + "}\n", + "14:45:12.036 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:45:12.036 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_PSD95.json\n", + "14:45:12.038 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_PSD95.json\n", + "14:45:12.532 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:45:17.544 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 343 out of 1836 tiles\n", + "14:45:22.550 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 707 out of 1836 tiles\n", + "14:45:27.554 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1070 out of 1836 tiles\n", + "14:45:32.564 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1433 out of 1836 tiles\n", + "14:45:37.575 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1796 out of 1836 tiles\n", + "14:45:38.128 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=25\n", + "14:45:38.570 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_PSD95/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:45:39.897 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_PSD95.json\n", + "14:45:39.897 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 28 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_PSD95/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_TdTomato {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-45-14T14:45:41.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_TdTomato/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_TdTomato/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_TdTomato', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_TdTomato.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_TdTomato /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_TdTomato.json\n", + "\n", + "\n", + "14:45:41.514 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:45:41.636 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_TdTomato\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_TdTomato.json\" ]\n", + "}\n", + "14:45:41.835 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:45:41.836 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_TdTomato.json\n", + "14:45:41.837 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_TdTomato.json\n", + "14:45:42.292 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:45:47.296 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 474 out of 1836 tiles\n", + "14:45:52.298 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 979 out of 1836 tiles\n", + "14:45:57.308 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1484 out of 1836 tiles\n", + "14:46:00.790 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=18\n", + "14:46:01.248 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_TdTomato/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:46:02.508 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_TdTomato.json\n", + "14:46:02.508 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 20 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_TdTomato/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_VGlut1 {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-46-14T14:46:03.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_VGlut1/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_VGlut1/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_VGlut1', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_VGlut1.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_VGlut1 /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_VGlut1.json\n", + "\n", + "\n", + "14:46:04.072 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:46:04.194 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_VGlut1\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_VGlut1.json\" ]\n", + "}\n", + "14:46:04.390 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:46:04.390 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_VGlut1.json\n", + "14:46:04.392 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_VGlut1.json\n", + "14:46:04.879 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:46:09.889 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 479 out of 1836 tiles\n", + "14:46:14.898 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 987 out of 1836 tiles\n", + "14:46:19.899 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1492 out of 1836 tiles\n", + "14:46:23.291 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=18\n", + "14:46:23.733 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_VGlut1/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:46:24.981 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_VGlut1.json\n", + "14:46:24.981 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 20 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_VGlut1/state/COMPLETE\n", + "stack version2 http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_synapsin {'stackResolutionZ': 1, 'mipmapPathBuilder': {'numberOfLevels': 0}, 'createTimestamp': '2016-46-14T14:46:26.00Z', 'stackResolutionX': 1, 'materializedBoxRootPath': 'string', 'cycleStepNumber': 1, 'cycleNumber': 1, 'stackResolutionY': 1, 'versionNotes': ''}\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_synapsin/state/LOADING\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_synapsin/state/LOADING\n", + "['/home/neurodata/Projects/render/render-ws-java-client/src/main/scripts/import_json.sh', '--baseDataUrl', 'http://192.168.0.150:8080/render-ws/v1', '--owner', 'Forrest', '--project', 'M247514_Rorb_1', '--stack', 'ALIGNED3_synapsin', '/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_synapsin.json']\n", + "\n", + " Running: /home/neurodata/Projects/render/deploy/jdk1.8.0_73/bin/java -cp /home/neurodata/Projects/render/render-ws-java-client/target/render-ws-java-client-0.3.0-SNAPSHOT-standalone.jar -Xms1G -Xmx1G -Djava.awt.headless=true -XX:+UseSerialGC org.janelia.render.client.ImportJsonClient --baseDataUrl http://192.168.0.150:8080/render-ws/v1 --owner Forrest --project M247514_Rorb_1 --stack ALIGNED3_synapsin /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_synapsin.json\n", + "\n", + "\n", + "14:46:26.571 [main] INFO [org.janelia.render.client.ClientRunner] run: entry\n", + "14:46:26.694 [main] INFO [org.janelia.render.client.ImportJsonClient] runClient: entry, parameters={\n", + " \"baseDataUrl\" : \"http://192.168.0.150:8080/render-ws/v1\",\n", + " \"owner\" : \"Forrest\",\n", + " \"project\" : \"M247514_Rorb_1\",\n", + " \"stack\" : \"ALIGNED3_synapsin\",\n", + " \"tileFiles\" : [ \"/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_synapsin.json\" ]\n", + "}\n", + "14:46:26.894 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTransformData: exit, loaded 0 transform specs\n", + "14:46:26.894 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: entry, tileFile=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_synapsin.json\n", + "14:46:26.896 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: entry, path=/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_synapsin.json\n", + "14:46:27.359 [main] INFO [org.janelia.render.client.ImportJsonClient] loadTileData: exit, loaded 1836 tile specs\n", + "14:46:32.361 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 344 out of 1836 tiles\n", + "14:46:37.368 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 708 out of 1836 tiles\n", + "14:46:42.381 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1074 out of 1836 tiles\n", + "14:46:47.387 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1439 out of 1836 tiles\n", + "14:46:52.400 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1806 out of 1836 tiles\n", + "14:46:52.811 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: derived bounding box for 1836 tiles, elapsedSeconds=25\n", + "14:46:53.242 [main] INFO [org.janelia.render.client.RenderDataClient] saveResolvedTiles: submitting PUT http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_synapsin/resolvedTiles for 0 transforms and 1836 tiles\n", + "14:46:54.512 [main] INFO [org.janelia.render.client.ImportJsonClient] importStackData: exit, saved tiles and transforms from /media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED3_synapsin.json\n", + "14:46:54.512 [main] INFO [org.janelia.render.client.ClientRunner] run: exit, processing completed in 0 hours, 0 minutes, 27 seconds\n", + "\n", + "http://192.168.0.150:8080/render-ws/v1/owner/Forrest/project/M247514_Rorb_1/stack/ALIGNED3_synapsin/state/COMPLETE\n" + ] + } + ], + "source": [ + "json_files = [os.path.join(json_dir,f) for f in os.listdir(json_dir) if aligned_prefix in f]\n", + "for f in json_files:\n", + " d=json.load(open(f,'r'))\n", + " tilespecs = [TileSpec(json=ts) for ts in d]\n", + " for ts in tilespecs:\n", + " ts.imageUrl=ts.imageUrl.replace('/nas/data/','/media/neurodata/6tbSFN/data/')\n", + " ts.maskUrl = None\n", + " fnew = f.replace(aligned_prefix,aligned_prefix_new)\n", + " json.dump([ts.to_dict() for ts in tilespecs],open(fnew,'w'))\n", + " #rts = ResolvedTileSpecMap(tilespecs=tilespecs,transforms=[])\n", + " inputStack=f[f.find(aligned_prefix):-5]\n", + " outputStack = inputStack.replace(aligned_prefix,aligned_prefix_new)\n", + " r=render.create_stack(outputStack)\n", + " render.set_stack_state(outputStack,'LOADING')\n", + " #r2=render.put_resolved_tilespecs(outputStack,json.dumps(rts.to_dict()))\n", + " \n", + " render.import_jsonfiles(outputStack,[fnew],verbose=True,client_scripts=client_scripts)\n", + " #break" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'/media/neurodata/6tbSFN/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_DAPI_1.json'" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "json_files[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "\n", + "js=rts.to_dict()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 93, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"transformIdToSpecMap\": {}, \"tileIdToSpecMap\": {\"101000004010000\": {\"minIntensity\": 1575.0, \"layout\n" + ] + } + ], + "source": [ + "print json.dumps(js)[0:100]" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['tileCount', 'transformSpecs', 'tileSpecs', 'transformCount']\n", + "[, , , ]\n", + "1836\n", + "\n", + "1836\n" + ] + } + ], + "source": [ + "print js.keys()\n", + "print [type(js[key]) for key in js.keys()]\n", + "print js['tileCount']\n", + "print type(js)\n", + "print len(js['tileSpecs'])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'inputOwner' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0minputStacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mst\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mst\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrender\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_stacks_by_owner_project\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputOwner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minputProject\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mst\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputPrefix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'inputOwner' is not defined" + ] + } + ], + "source": [ + "inputStacks = [st for st in render.get_stacks_by_owner_project(inputOwner,inputProject) if st.startswith(inputPrefix)]" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def define_aligned_tilespecs(prealignedStack,postalignedStack,inputStack):\n", + " zvalues = render.get_z_values_for_stack(inputStack)\n", + " #zvalues = [22.0]\n", + " finaltilespecs = []\n", + " for z in zvalues:\n", + " prealignedTS = render.get_tile_specs_from_z(prealignedStack,z,owner=prealignedOwner)[7]\n", + " postalignedTS = render.get_tile_specs_from_z(postalignedStack,z,owner=postalignedOwner)\n", + " postalignedTS =[ts for ts in postalignedTS if ts.tileId==prealignedTS.tileId][0]\n", + " tform_W_to_R = prealignedTS.tforms\n", + "\n", + " tform_R_to_W = list(tform_W_to_R)\n", + " tform_R_to_W.reverse()\n", + " tform_R_to_W = [tf.invert() for tf in tform_R_to_W]\n", + "\n", + " tform_W_to_A = postalignedTS.tforms\n", + "\n", + " tform_R_to_A = tform_R_to_W + tform_W_to_A\n", + " #print tform_R_to_A\n", + "\n", + " inputTS = render.get_tile_specs_from_z(inputStack,z,owner=inputOwner)\n", + " for ts in inputTS:\n", + " ts.tforms+=tform_R_to_A\n", + " finaltilespecs.append(ts)\n", + " return finaltilespecs" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "REGISTERED_FLATFIELD_FIX_MBP ALIGNED2_MBP\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_MBP/zValues/\n", + "ALIGNED2_MBP /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_MBP.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_MBP/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_MBP/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_DAPI_3 ALIGNED2_DAPI_3\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_DAPI_3/zValues/\n", + "ALIGNED2_DAPI_3 /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_DAPI_3.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_DAPI_3/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_DAPI_3/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_DAPI_1 ALIGNED2_DAPI_1\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_DAPI_1/zValues/\n", + "ALIGNED2_DAPI_1 /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_DAPI_1.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_DAPI_1/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_DAPI_1/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_DAPI_2 ALIGNED2_DAPI_2\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_DAPI_2/zValues/\n", + "ALIGNED2_DAPI_2 /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_DAPI_2.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_DAPI_2/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_DAPI_2/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_TdTomato ALIGNED2_TdTomato\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_TdTomato/zValues/\n", + "ALIGNED2_TdTomato /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_TdTomato.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_TdTomato/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_TdTomato/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_synapsin ALIGNED2_synapsin\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_synapsin/zValues/\n", + "ALIGNED2_synapsin /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_synapsin.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_synapsin/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_synapsin/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_VGlut1 ALIGNED2_VGlut1\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_VGlut1/zValues/\n", + "ALIGNED2_VGlut1 /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_VGlut1.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_VGlut1/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_VGlut1/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_PSD95 ALIGNED2_PSD95\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_PSD95/zValues/\n", + "ALIGNED2_PSD95 /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_PSD95.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_PSD95/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_PSD95/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_Gephyrin ALIGNED2_Gephyrin\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_Gephyrin/zValues/\n", + "ALIGNED2_Gephyrin /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_Gephyrin.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_Gephyrin/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_Gephyrin/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_GAD2 ALIGNED2_GAD2\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_GAD2/zValues/\n", + "ALIGNED2_GAD2 /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_GAD2.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_GAD2/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_GAD2/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_GABA ALIGNED2_GABA\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_GABA/zValues/\n", + "ALIGNED2_GABA /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_GABA.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_GABA/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_GABA/state/COMPLETE\n", + "REGISTERED_FLATFIELD_FIX_GluN1 ALIGNED2_GluN1\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/REGISTERED_FLATFIELD_FIX_GluN1/zValues/\n", + "ALIGNED2_GluN1 /nas/data/M247514_Rorb_1/processed/aligned_tilespecs/Sharmishtaas_M247514_Rorb_1_ALIGNED2_GluN1.json\n", + "\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_GluN1/state/LOADING\n", + "http://ibs-forrestc-ux1.corp.alleninstitute.org:8081/render-ws/v1/owner/Sharmishtaas/project/M247514_Rorb_1/stack/ALIGNED2_GluN1/state/COMPLETE\n" + ] + } + ], + "source": [ + "for inputStack in inputStacks:\n", + " inputchannel = inputStack.replace(inputPrefix,'')\n", + " outputStack = outputPrefix + inputchannel\n", + " print inputStack,outputStack\n", + " \n", + " finaltilespecs = define_aligned_tilespecs(prealignedStack,postalignedStack,inputStack)\n", + " json_tilespecs=[ts.to_dict() for ts in finaltilespecs]\n", + " if not os.path.isdir(outputDir):\n", + " os.makedirs(outputDir)\n", + " outjson = os.path.join(outputDir,'%s_%s_%s.json'%(inputOwner,inputProject,outputStack))\n", + " json.dump(json_tilespecs,open(outjson,'w'),indent=4)\n", + " print outputStack,outjson\n", + " render.delete_stack(outputStack,owner=outputOwner,project=outputProject)\n", + " render.create_stack(outputStack,owner=outputOwner,project=outputProject)\n", + " \n", + " render.import_jsonfiles(outputStack,[outjson],owner=outputOwner,project=outputProject)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "t=time.time()" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'2016-46-14T09:46:21.00Z'" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "time.strftime('%Y-%M-%dT%H:%M:%S.00Z')" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "time.strftime?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/apply_ztransforms_to_render_stack.py b/docs/examples/apply_ztransforms_to_render_stack.py similarity index 100% rename from apply_ztransforms_to_render_stack.py rename to docs/examples/apply_ztransforms_to_render_stack.py diff --git a/create_mipmaps.py b/docs/examples/create_mipmaps.py similarity index 100% rename from create_mipmaps.py rename to docs/examples/create_mipmaps.py diff --git a/detect_and_drop_stitching_mistakes.py b/docs/examples/detect_and_drop_stitching_mistakes.py similarity index 100% rename from detect_and_drop_stitching_mistakes.py rename to docs/examples/detect_and_drop_stitching_mistakes.py diff --git a/remove_outer_tiles.py b/docs/examples/remove_outer_tiles.py similarity index 100% rename from remove_outer_tiles.py rename to docs/examples/remove_outer_tiles.py diff --git a/docs/guide/index.rst b/docs/guide/index.rst new file mode 100644 index 00000000..1a9e3a08 --- /dev/null +++ b/docs/guide/index.rst @@ -0,0 +1,316 @@ +User Guide +========== + +Getting Started +--------------- + +First you must have a render server running and accessible to you. +For instructions on getting a render server up and running +http://www.github.com/saalfeldlab/render + +For the purposes of this tutorial i'm going to assume you have +a render-server installed and up and running on localhost and +your render dashboard is viewable at +http://localhost:8080/render-ws/view/index.html + +You also should have available on your local system a path to the +render-ws-java-client scripts with a compiled jar file. +One way to get these things, with render-python already installed +is to use get the render-python Docker image (fcollman\render-python), +where it will be installed at \usr\local\render\render-ws-java-client\src\scripts + +.. code-block:: bash + + $ docker pull fcollman\render-python + $ docker run -t fcollman\render-python \ + -v .:\scripts \ + python \scripts\my_script.py + +Making a new stack +------------------ + +First we have to setup our default connection properties for our render server +:: + + import renderapi + + #create a renderapi.connect.Render object + render_connect_params ={ + 'host':'localhost', + 'port':8080, + 'owner':'myowner', + 'project':'myproject', + 'client_scripts':'\usr\local\render\render-ws-java-client\src\scripts' + 'memGB':'2G' + } + render = renderapi.connect(**render_connect_params) + + +You can simplify your call to :func:`renderapi.connect` if you wish by setting up some or all of +the following environment variables +DEFAULT_RENDER_HOST, DEFAULT_RENDER_PORT, DEFAULT_RENDER_CLIENT_SCRIPTS, +DEFAULT_RENDER_PROJECT, DEFAULT_RENDER_OWNER, RENDER_CLIENT_HEAP. + +Now we can create a new stack on the render server + +:: + + #make a new stack + stack = 'mystack' + renderapi.stack.create_stack(stack,render=render) + +now you should be able to see your stack at http://localhost:8080/render-ws/view/stacks.html. +It is presently in the LOADING state as it is awaiting tiles to be loaded. +In order to give it some tiles you must fill out some metadata information about the tile. +Many fields are technically optional, but it's best to fill them out if you can. +In this example we will do it manually, of course most users will likely write code to +create this metadata from existing metadata. + +:: + + #define a tile layout + layout = Layout(sectionId='1', + scopeId='myscope', + cameraId='mycamera', + imageRow=0, + imageCol=0, + stageX=100.0, + stageY=300.0, + rotation=0.0, + pixelsize=3.0) + +Next you have to define the set of transformations that should be applied to the raw image. +In this example we will use :class:`renderapi.transform.AffineModel`. +However, you can use any of the transforms defined in :class:`renderapi.transform` +or any transformation in the mpicpg library (https://github.com/axtimwalde/mpicbg) +installed with your render server via the :class:`renderapi.transform.Transform` class, +provided you know its classname and dataString. + +:: + + #define a simple transformation, here a translation based upon layout + at = AffineTransform(B0=layout.stageX/layout.pixelsize, + B1=layout.stageY/layout.pixelsize) + +Now we can define the actual tile + +:: + + tilespec = TileSpec(tileId='000000000000', + z=0.0, + width=2048, + height=2048, + imageUrl='/data/images/0_0_0.tif', + maskUrl=None, + layout=layout, + tforms=[at]) + +Note that the path to the imageUrl needs to be a path that is readable by the render server +if you want server side rendering of images to function correctly. If you only want to use +render to store metadata information then this isn't strictly necessary, but other web-services +such as https://github.com/neurodata/ndviz will require this. + + +Importing tilespecs +------------------- +Now we can import the tile to our stack using :class:`renderapi.client` which uses the +render-ws-java-client scripts to perform client side validation and bounding box estimation +of your tiles before uploading them to the server. + +:: + + #use the simple non-parallelized upload option + renderapi.client.import_tilespecs(stack, + [tilespec], + render = render) + + #now close the stack + renderapi.stack.set_stack_state(stack, 'COMPLETE', render = render) + +If you have many tilespecs to import, it often makes sense to parallelize the client side +validation and bounding box estimation. So lets simulate the importing of many tiles + +:: + + rows = 10 + cols = 20 + sections = 500 + overlap = .8 #20% overlap + pix = 3.0 #nm + img_width = 2048 #pixels + img_height = img_width + + tilespecs = [] + for section in range(sections): + for r in range(rows): + for c in range(cols): + layout = Layout(sectionId='%05d'%section, + scopeId='myscope', + cameraId='mycamera', + imageRow=0, + imageCol=0, + stageX=c*img_width*overlap*pix, + stageY=r*img_height*overlap*pix, + rotation=0.0, + pixelsize=pix) + + + + #define a simple transformation, here a translation based upon layout + at = AffineTransform(B0=c*img_width*overlap, + B1=r*img_height*overlap) + + tileId = '%d_%d_%d'%(section,r,c) + + ts = TileSpec(tileId=, + z=section, + width=img_width, + height=img_height, + imageUrl='/data/images/%s.tif'%tileId, + maskUrl=None, + layout=layout, + tforms=[at]) + tilespecs.append(ts) + +This would of course would need to be adapted to suit the needs of your +specific situation, but assuming you have a large number of tilespecs, +they can be imported more efficently using +:func:`renderapi.client.import_tilespecs_parallel` +which will also close the stack for you if you'd like. +:: + + renderapi.client.import_tilespecs_parallel(stack, + tilespecs, + close_stack=True, + render = render) + + +When you are done, you should be able to see your stack on the render dashboard. + +Transformations +--------------- + +The idea behind render is that it serves as a central place to store the metadata data +about image tiles and how they should be tranformed. Central to that concept is what +transformations it supports. Render is written in java and uses the mpicbg library +(https://github.com/axtimwalde/mpicbg), the same library that backs TrakEM2, to +perform all server side image transformation. + +Render-python is client side library and can assist you in managing and setting +those tranformations, and performing some calculations using the +:class:`renderapi.transorm` module. We have focused our initial +efforts at supporting the most commonly used types of transformations. + +Some transformation types presently support `tform` and 'inverse_tform` methods +for calculating where numpy array sets of points map to and from these tranformations. +Some presently support `estimate` methods which given a set of source and destination +points, allow the estimation of a best fit transformation. + +:func:`renderapi.transform.estimate_dstpts` simplifies mapping points through an +ordered list of transformations. :func:`renderapi.transform.estimate_transformsum` +provides a general way to produce a single :class:`renderapi.transform.Polynomial2DTransform` +that approximates a list of tranforms which have implemented `.tform` methods. + +One aspect to keep in mind about render is that it supports +:class:`renderapi.transform.ReferenceTransform` which allows many tiles to share a common +transformation without having to store its parameters directly. This is a conveient way to +save on database storage and ensure many tiles are identically manipulated. +However, depending on what you want to do, it can make things slightly more complicated. + +For example, presently, the java client side scripts used to calculate bounding boxes and +validate transformations need to have access to the referenced transformations in order to +do their work, even if they already exist in the database. This is what the `sharedTransforms` +argument in import methods within :class:`renderapi.client` is for. + +The :func:`renderapi.transform.ReferenceTransform.tform` method does not exist because +the :class:`renderapi.transform.ReferenceTransform` transform doesn't have any data about what +kind the transform parameters are. Most render-python calls default to returning dereferenced +transforms, which avoids this issue. However, this will break the efficency gains if you simply +upload those dereferenced transforms. This is why :func:`renderapi.tilespec.get_tile_spec_raw` +exists, in order to give you referenced transforms if you so wish. Also, you can use the +:class:`renderapi.client.importTransformChangesClient` to accelerate and simplify many transform +modification tasks, as it won't do any client side validation or bounding box calculations. + + +Pointmatch database +------------------- + +Pointmatchs are locations between two images that correspond. Render has two groups +of web services that are both available on the same web interface, the 'tile' +based services and the 'pointmatch' services. + +They are loosely coupled in the sense that they are stored in distinct databases, +and make no assumptions about how the other is structured. This allows them to be +deployed and maintained seperately, but can make things confusing if you assume +that one knows more about what the other is doing than it does. To make things +less confusing there is a set of reccomendations that you can read at :doc:`pointmatchassumptions`. + +Pointmatches are stored by collection, group's and id's (see :ref:`reccomendation `) +and have a source 'p' and a destination 'q', thus each set of matches in a collection is specified by a +pGroupId, pId, qGroupId, qId combination. Functions in the :class:`renderapi.pointmatch` module +allow you to make queries on these point matches in various ways, and upload new matches. +You might place some point matches between tile 0_0_0 on section 0, and tile 1_0_0 on section 1, +using :func:`renderapi.pointmatch.import_matches` +and then retrieve them using +:func:`renderapi.pointmatch.get_matches_from_tile_to_tile`. + +:: + + matches_in={ + 'pGroupId':'0', + 'qGroupId':'1', + 'pId':'0_0_0', + 'qId':'1_0_0', + 'matches':{ + 'p':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'q':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'w':[1,1,1,1] + } + } + renderapi.pointmatch.import_matches('mycollection', + [matches_in], + render=render) + matches_out=renderapi.pointmatch.get_matches_from_tile_to_tile('mycollection' + '0', + '0_0_0, + '1', + '1_0_0', + render = render) + + print(matches_out) + >> [{ + 'pGroupId':'0', + 'qGroupId':'1', + 'pId':'0_0_0', + 'qId':'1_0_0', + 'matches':{ + 'p':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'q':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'w':[1,1,1,1] + } + }] + + + + + + + + + + + + + + + + +Installation +------------ +from source + +.. code-block:: bash + + $ git clone https://www.github.com/fcollman/render-python + $ cd render-python + $ python setup.py install \ No newline at end of file diff --git a/docs/guide/pointmatchassumptions.rst b/docs/guide/pointmatchassumptions.rst new file mode 100644 index 00000000..9b14b8a7 --- /dev/null +++ b/docs/guide/pointmatchassumptions.rst @@ -0,0 +1,70 @@ +Pointmatch Assumptions +====================== +Generally, I would reccomend adopting a set of conventions that constrain the +relationship between data in the 'tile' and 'pointmatch' databases. These conventions are informed +in large part based upon how the EMAligner https://github.com/khaledkhairy/EM_aligner +assumes they are related, when using them to solve alignment problems. + + +tile_owners = pointmatch_owners +------------------------------- +Particularly on deployments which have only a single render service, this will save you +from having to redefine the default owner in the :class:`renderapi.render.Render` or override +it at each step. Make all stacks related to a dataset owned by a single owner, and make all +pointmatch collections owned by that same owner. + +.. _group-section-explanation: + +groupId=sectionId and id=tileId +---------------------------- +Or more verbosely, groupId/qGroupId/pGroupId = sectionId and pId/qId/id = tileId. +Groups thus correspond to what section the point match is from and the id's correspond to what tile +they are from. +Technically, there is no need to make this association, and none of the render web services strictly require it. +However, if you want to use the pointmatch results in combination with the tile services, +it will be far easier if there is a strict mapping between these two databases. +In addition, tools like EMAligner and ndviz are presently written in a way that assumes this mapping is held, +so you need to make the same assumption if you want to use those services. + +Know your 'local' +------------------------------------------------------------------- +Write all pointmatches between tiles in a consistent 'local' coordinate system +and make that local coordinate system the raw image space given by rendering the tile +using :func:`renderapi.image.get_tile_image_data` with normalizeForMatching=True and scale=1.0. + +This would be conceptually simple, if the 'local' meant the same as +:class:`renderapi.coordinate` module defines local to mean, +namely the raw image space, with the upper left hand pixel at 0,0 and positive x to the right +and positive y down. + +However, in some deployments of render this is not the case, and you might find that +rendering a tile using normalizeForMatching=True does not produce a raw image tile. +In fact it might render blank data in some circumstances if you have more than 1 transformation. + +This is because, at Janelia the EMAligner was developed on TEM images that need a lens correction +transformation, and the pointmatches are defined on the 'local' coordinate system after lens correction. +This simplifies solving for the non-lens component of the transformation, as the EMAligner only +needs to specify the single transformation that brings 'local' pointmatches into 'global' alignment +and can safely disregard the non-linear effects of the lens correction. + +However, it produces the confusing result that mapping the 'local' point matches coordinates +through :func:`renderapi.coordinate.local_to_world_coordinates` does not give the correct result, +and you have to have a second stack with lens correction transformations removed in order to map +point match coordinates from local to world coordinates accurately using the coordinate mapping service. + +More discussion on this at https://github.com/saalfeldlab/render/issues/13, +https://github.com/saalfeldlab/render/issues/31. + +I implemented an alternative strategy at +https://github.com/saalfeldlab/render/pull/29 +which adds a removeAllOption=True to many render calls +which simply removes all transformations from the tilespec before returning or rendering. +In applications where non-linear lens corrections are minimal, this simplifies things. + +However for TEM or other applications with stereotyped non-linear transformations, +it will make using the EMAligner to solve alignment problems more difficult, +as the EMAligner doesn't know that it should map the pointmatches into the post non-linear correction +space before attemptign to solve and isn't presently written to do this. + + + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..656d6784 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,31 @@ +.. render-python documentation master file, created by + sphinx-quickstart on Sat Jul 22 11:57:57 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to render-python's documentation! +========================================= + + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +API +--- + +.. toctree:: + :maxdepth: 3 + + guide/index + guide/pointmatchassumptions + api/renderapi + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/explore_point_match_database-large.ipynb b/explore_point_match_database-large.ipynb deleted file mode 100644 index 3d8903c9..00000000 --- a/explore_point_match_database-large.ipynb +++ /dev/null @@ -1,8440 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "import renderapi\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from matplotlib.lines import Line2D\n", - "import time\n", - "\n", - "from matplotlib.patches import FancyArrowPatch, Circle, ConnectionStyle\n", - "\n", - "%matplotlib notebook" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "host = 'ibs-forrestc-ux1.corp.alleninstitute.org'\n", - "port = 8080\n", - "owner = 'Sharmishtaas'\n", - "project = 'M270907_Scnn1aTg2Tdt_13'\n", - "stack = 'ALIGNEDSTACK_JAN3_DAPI_1_NORM'\n", - "matchcollection = 'large_rigid_run'\n", - "render = renderapi.Render(host,port,owner,project)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[u'Forrest', u'Sharmishtaas']" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "render.get_matchcollection_owners()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[{u'collectionId': {u'name': u'rigid_sift_half_scale_fullrun',\n", - " u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 349937},\n", - " {u'collectionId': {u'name': u'rigid_sift_run2', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 3817},\n", - " {u'collectionId': {u'name': u'rigid_sift_half_scale_run2',\n", - " u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 193240},\n", - " {u'collectionId': {u'name': u'norm_sift_run1', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 384058},\n", - " {u'collectionId': {u'name': u'rigid_sift_run1', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 16679},\n", - " {u'collectionId': {u'name': u'tilepairtest', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 2},\n", - " {u'collectionId': {u'name': u'large_rigid_run', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 3508512},\n", - " {u'collectionId': {u'name': u'rigid_sift_half_scale_run1',\n", - " u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 15845},\n", - " {u'collectionId': {u'name': u'rigid_sift_run3', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 19313}]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "render.get_matchcollections()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "allmatches=render.get_matches_from_group_to_group(matchcollection,'0','1')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sections_per_row = 200\n", - "rows = int(np.ceil(maxz/sections_per_row))\n", - "\n", - "\n", - "f,ax = plt.subplots(rows,1,figsize=(12,2*rows))\n", - "first_section_indices=np.concatenate([np.array([0]),np.where(np.diff(groups%1000)<0)[0]+1])\n", - "first_sections=groups[first_section_indices]\n", - "first_section_zs = [zvalues[section] for section in first_sections]\n", - "\n", - "for row in range(rows):\n", - " startz = row*sections_per_row\n", - " endz = row*sections_per_row + sections_per_row\n", - " ax[row].imshow(match_matrix[startz:endz,:].T,interpolation='nearest',cmap=plt.cm.viridis,extent=(startz-.5,endz-.5,maxdz,-maxdz),vmin=0,vmax=200)\n", - " ax[row].autoscale(tight=True)\n", - " for z in first_section_zs:\n", - " if (z>=startz)&(z<=endz):\n", - " ax[row].plot([z-.5,z-.5],[-maxdz,maxdz],c='w',linewidth=2,linestyle='--')\n", - "\n", - "# img=ax.imshow(match_matrix,interpolation='nearest',extent =(-maxdz,maxdz,maxz+.5,-.5))\n", - "# plt.xlabel('dz')\n", - "# plt.ylabel('z values')\n", - "# ax.set_title('matches')\n", - "# plt.colorbar(img)\n", - "plt.tight_layout()\n", - "\n", - "\n", - "# ax.set_yticks(np.arange(0,maxz,100))\n", - "#ax.autoscale(tight=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 194, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 194, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "plt.figure(figsize=(12,6))\n", - "plt.plot(np.sum(match_matrix,axis=1))\n", - "plt.xlabel('section z')\n", - "plt.ylabel('num_matches')\n", - "plt.figure()\n", - "plt.errorbar(np.arange(-maxdz,maxdz+1).T,np.median(match_matrix,axis=0),np.std(match_matrix,axis=0))\n", - "plt.xlabel('dz')\n", - "plt.ylabel('median tile matches')" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "0.011663597298956415" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.sum(np.sum(match_matrix,axis=1)==0)*1.0/match_matrix.shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 198, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import networkx as nx\n", - "G=nx.Graph()\n", - "\n", - "\n", - "#subgroup =[group for i, group in enumerate(groups) if (zvalues[group]>1200) & (zvalues[group]<1300)]\n", - "#subanswers =[answers[i] for i, group in enumerate(groups) if (zvalues[group]>1200) & (zvalues[group]<1300)]\n", - "\n", - "G.add_nodes_from(groups)\n", - "for group in groups:\n", - " G.node[group]['z']=zvalues[group]\n", - " # if zvalues[group]<100:\n", - "# nx.set_node_attributes(G,'z',zvalues)\n", - "\n", - "for answer,group in zip(answers,groups):\n", - " for group2 in answer.keys():\n", - "\n", - " if G.has_edge(group,group2):\n", - " w = G[group][group2]['weight']\n", - " G[group][group2]['weight']=w+answer[group2]\n", - " else:\n", - " if group2=='z':\n", - " print group,group2\n", - " if group=='z':\n", - " print group,group2\n", - " G.add_edge(group,group2,weight=answer[group2])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 199, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 - 3145\n", - "3146 - 3257\n" - ] - } - ], - "source": [ - "subgraphs=nx.connected_components(G)\n", - "for sG in subgraphs:\n", - " zs = np.array([G.node[node]['z'] for node in sG])\n", - " print np.min(zs),'-',np.max(zs)" - ] - }, - { - "cell_type": "code", - "execution_count": 186, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{0,\n", - " 1,\n", - " 2,\n", - " 3,\n", - " 4,\n", - " 5,\n", - " 6,\n", - " 7,\n", - " 8,\n", - " 9,\n", - " 10,\n", - " 11,\n", - " 12,\n", - " 13,\n", - " 14,\n", - " 15,\n", - " 16,\n", - " 17,\n", - " 18,\n", - " 19,\n", - " 20,\n", - " 21,\n", - " 22,\n", - " 23,\n", - " 24,\n", - " 25,\n", - " 26,\n", - " 43013,\n", - " 45010,\n", - " 43014,\n", - " 41000,\n", - " 41001,\n", - " 41002,\n", - " 41003,\n", - " 41004,\n", - " 41005,\n", - " 41006,\n", - " 41007,\n", - " 41008,\n", - " 41009,\n", - " 41010,\n", - " 41011,\n", - " 41012,\n", - " 41013,\n", - " 41014,\n", - " 41015,\n", - " 41016,\n", - " 41017,\n", - " 41018,\n", - " 41019,\n", - " 41020,\n", - " 41021,\n", - " 41022,\n", - " 41023,\n", - " 41024,\n", - " 41025,\n", - " 41026,\n", - " 41027,\n", - " 43020,\n", - " 43021,\n", - " 43022,\n", - " 39000,\n", - " 39001,\n", - " 39002,\n", - " 39003,\n", - " 39004,\n", - " 39005,\n", - " 39006,\n", - " 39007,\n", - " 39008,\n", - " 39009,\n", - " 39010,\n", - " 39011,\n", - " 39012,\n", - " 39013,\n", - " 39014,\n", - " 39015,\n", - " 39016,\n", - " 39017,\n", - " 39018,\n", - " 39019,\n", - " 39020,\n", - " 39021,\n", - " 39022,\n", - " 39023,\n", - " 39024,\n", - " 39025,\n", - " 39026,\n", - " 39027,\n", - " 39028,\n", - " 39029,\n", - " 39030,\n", - " 45017,\n", - " 37000,\n", - " 37001,\n", - " 37002,\n", - " 37003,\n", - " 37004,\n", - " 37005,\n", - " 37006,\n", - " 37007,\n", - " 37008,\n", - " 37009,\n", - " 37010,\n", - " 37011,\n", - " 37012,\n", - " 37013,\n", - " 37014,\n", - " 37015,\n", - " 37016,\n", - " 37017,\n", - " 37018,\n", - " 37019,\n", - " 37020,\n", - " 37021,\n", - " 37022,\n", - " 37023,\n", - " 37024,\n", - " 37025,\n", - " 37026,\n", - " 35000,\n", - " 35001,\n", - " 35002,\n", - " 35003,\n", - " 35004,\n", - " 35005,\n", - " 35006,\n", - " 35007,\n", - " 35008,\n", - " 35009,\n", - " 35010,\n", - " 35011,\n", - " 35012,\n", - " 35013,\n", - " 35014,\n", - " 35015,\n", - " 35016,\n", - " 35017,\n", - " 35018,\n", - " 35019,\n", - " 35020,\n", - " 35021,\n", - " 35022,\n", - " 35023,\n", - " 35024,\n", - " 35025,\n", - " 35026,\n", - " 35027,\n", - " 45016,\n", - " 43011,\n", - " 33000,\n", - " 33001,\n", - " 33002,\n", - " 33003,\n", - " 33004,\n", - " 33005,\n", - " 33006,\n", - " 33007,\n", - " 33008,\n", - " 33009,\n", - " 33010,\n", - " 33011,\n", - " 33012,\n", - " 33013,\n", - " 33014,\n", - " 33015,\n", - " 33016,\n", - " 33017,\n", - " 33018,\n", - " 33019,\n", - " 33020,\n", - " 33021,\n", - " 33022,\n", - " 33023,\n", - " 33024,\n", - " 33025,\n", - " 33026,\n", - " 45009,\n", - " 31000,\n", - " 31001,\n", - " 31002,\n", - " 31003,\n", - " 31004,\n", - " 31005,\n", - " 31006,\n", - " 31007,\n", - " 31008,\n", - " 31009,\n", - " 31010,\n", - " 31011,\n", - " 31012,\n", - " 31013,\n", - " 31014,\n", - " 31015,\n", - " 31016,\n", - " 31017,\n", - " 31018,\n", - " 31019,\n", - " 31020,\n", - " 31021,\n", - " 31022,\n", - " 31023,\n", - " 31024,\n", - " 31025,\n", - " 31026,\n", - " 29000,\n", - " 29002,\n", - " 29003,\n", - " 29004,\n", - " 29005,\n", - " 29006,\n", - " 29007,\n", - " 29008,\n", - " 29009,\n", - " 29010,\n", - " 29011,\n", - " 29012,\n", - " 29013,\n", - " 29014,\n", - " 29015,\n", - " 29016,\n", - " 29017,\n", - " 29018,\n", - " 29019,\n", - " 29020,\n", - " 29021,\n", - " 29022,\n", - " 29023,\n", - " 29024,\n", - " 29025,\n", - " 29026,\n", - " 29027,\n", - " 27001,\n", - " 27002,\n", - " 27003,\n", - " 27004,\n", - " 27005,\n", - " 27006,\n", - " 27007,\n", - " 27008,\n", - " 27009,\n", - " 27010,\n", - " 27011,\n", - " 27012,\n", - " 27013,\n", - " 27014,\n", - " 27015,\n", - " 27016,\n", - " 27017,\n", - " 27018,\n", - " 27019,\n", - " 27020,\n", - " 27021,\n", - " 27022,\n", - " 27023,\n", - " 27024,\n", - " 27025,\n", - " 27026,\n", - " 27027,\n", - " 27028,\n", - " 45008,\n", - " 25000,\n", - " 25001,\n", - " 25002,\n", - " 25003,\n", - " 25004,\n", - " 25005,\n", - " 25006,\n", - " 25007,\n", - " 25008,\n", - " 25009,\n", - " 25010,\n", - " 25011,\n", - " 25012,\n", - " 25013,\n", - " 25014,\n", - " 25015,\n", - " 25016,\n", - " 25017,\n", - " 25018,\n", - " 25019,\n", - " 25020,\n", - " 25021,\n", - " 25022,\n", - " 25023,\n", - " 25024,\n", - " 25025,\n", - " 25026,\n", - " 45024,\n", - " 43024,\n", - " 23000,\n", - " 23001,\n", - " 23002,\n", - " 23003,\n", - " 23004,\n", - " 23005,\n", - " 23006,\n", - " 23007,\n", - " 23008,\n", - " 23009,\n", - " 23010,\n", - " 23011,\n", - " 23012,\n", - " 23013,\n", - " 23014,\n", - " 23015,\n", - " 23016,\n", - " 23017,\n", - " 23018,\n", - " 23019,\n", - " 23020,\n", - " 23021,\n", - " 23022,\n", - " 23023,\n", - " 23024,\n", - " 23025,\n", - " 23026,\n", - " 45012,\n", - " 21000,\n", - " 21001,\n", - " 21002,\n", - " 21003,\n", - " 21004,\n", - " 21005,\n", - " 21006,\n", - " 21007,\n", - " 21008,\n", - " 21009,\n", - " 21010,\n", - " 21011,\n", - " 21012,\n", - " 21013,\n", - " 21014,\n", - " 21015,\n", - " 21016,\n", - " 21017,\n", - " 21018,\n", - " 21019,\n", - " 21020,\n", - " 21021,\n", - " 21022,\n", - " 21023,\n", - " 21024,\n", - " 19000,\n", - " 19001,\n", - " 19002,\n", - " 19003,\n", - " 19004,\n", - " 19005,\n", - " 19006,\n", - " 19007,\n", - " 19008,\n", - " 19009,\n", - " 19010,\n", - " 19011,\n", - " 19012,\n", - " 19013,\n", - " 19014,\n", - " 19015,\n", - " 19016,\n", - " 19017,\n", - " 19018,\n", - " 19019,\n", - " 19020,\n", - " 19021,\n", - " 19022,\n", - " 19023,\n", - " 19024,\n", - " 19025,\n", - " 19026,\n", - " 19027,\n", - " 44017,\n", - " 17000,\n", - " 17001,\n", - " 17002,\n", - " 17003,\n", - " 17004,\n", - " 17005,\n", - " 17006,\n", - " 17007,\n", - " 17008,\n", - " 17009,\n", - " 17010,\n", - " 17011,\n", - " 17012,\n", - " 17013,\n", - " 17014,\n", - " 17015,\n", - " 17016,\n", - " 17017,\n", - " 17018,\n", - " 17019,\n", - " 17020,\n", - " 17021,\n", - " 17022,\n", - " 17023,\n", - " 17024,\n", - " 43008,\n", - " 44008,\n", - " 15000,\n", - " 15001,\n", - " 15002,\n", - " 15003,\n", - " 15004,\n", - " 15005,\n", - " 15006,\n", - " 15007,\n", - " 15008,\n", - " 15009,\n", - " 15010,\n", - " 15011,\n", - " 15012,\n", - " 15013,\n", - " 15014,\n", - " 15015,\n", - " 15016,\n", - " 15017,\n", - " 15018,\n", - " 15019,\n", - " 15020,\n", - " 15021,\n", - " 15022,\n", - " 15023,\n", - " 15024,\n", - " 15025,\n", - " 15026,\n", - " 44009,\n", - " 43019,\n", - " 13000,\n", - " 13001,\n", - " 13002,\n", - " 13003,\n", - " 13004,\n", - " 13005,\n", - " 13006,\n", - " 13007,\n", - " 13008,\n", - " 13009,\n", - " 13010,\n", - " 13011,\n", - " 13012,\n", - " 13013,\n", - " 13014,\n", - " 13015,\n", - " 13016,\n", - " 13017,\n", - " 13018,\n", - " 13019,\n", - " 13020,\n", - " 13021,\n", - " 13022,\n", - " 13023,\n", - " 13024,\n", - " 13025,\n", - " 13026,\n", - " 44011,\n", - " 44018,\n", - " 11000,\n", - " 11001,\n", - " 11002,\n", - " 11003,\n", - " 11004,\n", - " 11005,\n", - " 11006,\n", - " 11007,\n", - " 11008,\n", - " 11009,\n", - " 11010,\n", - " 11011,\n", - " 11012,\n", - " 11013,\n", - " 11014,\n", - " 11015,\n", - " 11016,\n", - " 11017,\n", - " 11018,\n", - " 11019,\n", - " 11020,\n", - " 11021,\n", - " 11022,\n", - " 11023,\n", - " 11024,\n", - " 11025,\n", - " 11026,\n", - " 11027,\n", - " 43009,\n", - " 45003,\n", - " 9000,\n", - " 9001,\n", - " 9002,\n", - " 9003,\n", - " 9004,\n", - " 9005,\n", - " 9006,\n", - " 9007,\n", - " 9008,\n", - " 9009,\n", - " 9010,\n", - " 9011,\n", - " 9012,\n", - " 9013,\n", - " 9014,\n", - " 9015,\n", - " 9016,\n", - " 9017,\n", - " 9018,\n", - " 9019,\n", - " 9020,\n", - " 9021,\n", - " 9022,\n", - " 9023,\n", - " 9024,\n", - " 9025,\n", - " 9026,\n", - " 7000,\n", - " 7001,\n", - " 7002,\n", - " 7003,\n", - " 7004,\n", - " 7005,\n", - " 7006,\n", - " 7007,\n", - " 7008,\n", - " 7009,\n", - " 7010,\n", - " 7011,\n", - " 7012,\n", - " 7013,\n", - " 7014,\n", - " 7015,\n", - " 7016,\n", - " 7017,\n", - " 7018,\n", - " 7019,\n", - " 7020,\n", - " 7021,\n", - " 7022,\n", - " 7023,\n", - " 7024,\n", - " 7025,\n", - " 7026,\n", - " 44016,\n", - " 5001,\n", - " 5002,\n", - " 5003,\n", - " 5004,\n", - " 5005,\n", - " 5006,\n", - " 5007,\n", - " 5008,\n", - " 5009,\n", - " 5010,\n", - " 5011,\n", - " 5012,\n", - " 5013,\n", - " 5014,\n", - " 5015,\n", - " 5016,\n", - " 5017,\n", - " 5018,\n", - " 5019,\n", - " 5020,\n", - " 5021,\n", - " 5022,\n", - " 5023,\n", - " 5024,\n", - " 5025,\n", - " 5026,\n", - " 5027,\n", - " 5028,\n", - " 3001,\n", - " 3002,\n", - " 3003,\n", - " 3004,\n", - " 3005,\n", - " 3006,\n", - " 3007,\n", - " 3008,\n", - " 3009,\n", - " 3010,\n", - " 3011,\n", - " 3012,\n", - " 3013,\n", - " 3014,\n", - " 3015,\n", - " 3016,\n", - " 3017,\n", - " 3018,\n", - " 3019,\n", - " 3020,\n", - " 3021,\n", - " 3022,\n", - " 3023,\n", - " 3024,\n", - " 3025,\n", - " 3026,\n", - " 3027,\n", - " 44019,\n", - " 44000,\n", - " 44001,\n", - " 44002,\n", - " 44003,\n", - " 44004,\n", - " 44005,\n", - " 44006,\n", - " 44007,\n", - " 1000,\n", - " 1001,\n", - " 1002,\n", - " 1003,\n", - " 1004,\n", - " 1005,\n", - " 1006,\n", - " 1007,\n", - " 1008,\n", - " 1009,\n", - " 1010,\n", - " 1011,\n", - " 1012,\n", - " 1013,\n", - " 1014,\n", - " 1015,\n", - " 1016,\n", - " 1017,\n", - " 1018,\n", - " 1019,\n", - " 1020,\n", - " 1021,\n", - " 1022,\n", - " 1023,\n", - " 1024,\n", - " 1025,\n", - " 1026,\n", - " 42000,\n", - " 42001,\n", - " 42002,\n", - " 42003,\n", - " 42004,\n", - " 42005,\n", - " 42006,\n", - " 42007,\n", - " 42008,\n", - " 42009,\n", - " 42010,\n", - " 42011,\n", - " 42012,\n", - " 42013,\n", - " 42014,\n", - " 42015,\n", - " 42016,\n", - " 42017,\n", - " 42018,\n", - " 42019,\n", - " 42020,\n", - " 42021,\n", - " 42022,\n", - " 42023,\n", - " 42024,\n", - " 42025,\n", - " 42026,\n", - " 40000,\n", - " 40001,\n", - " 40002,\n", - " 40003,\n", - " 40004,\n", - " 40005,\n", - " 40006,\n", - " 40007,\n", - " 40008,\n", - " 40009,\n", - " 40010,\n", - " 40011,\n", - " 40012,\n", - " 40013,\n", - " 40014,\n", - " 40015,\n", - " 40016,\n", - " 40017,\n", - " 40018,\n", - " 40019,\n", - " 40020,\n", - " 40021,\n", - " 40022,\n", - " 40023,\n", - " 40024,\n", - " 40025,\n", - " 40026,\n", - " 40027,\n", - " 38000,\n", - " 38001,\n", - " 38002,\n", - " 38003,\n", - " 38004,\n", - " 38005,\n", - " 38006,\n", - " 38007,\n", - " 38008,\n", - " 38009,\n", - " 38010,\n", - " 38011,\n", - " 38012,\n", - " 38013,\n", - " 38014,\n", - " 38015,\n", - " 38016,\n", - " 38017,\n", - " 38018,\n", - " 38019,\n", - " 38020,\n", - " 38021,\n", - " 38022,\n", - " 38023,\n", - " 38024,\n", - " 38025,\n", - " 38026,\n", - " 44020,\n", - " 44025,\n", - " 36000,\n", - " 36001,\n", - " 36002,\n", - " 36003,\n", - " 36004,\n", - " 36005,\n", - " 36006,\n", - " 36007,\n", - " 36008,\n", - " 36009,\n", - " 36010,\n", - " 36011,\n", - " 36012,\n", - " 36013,\n", - " 36014,\n", - " 36015,\n", - " 36016,\n", - " 36017,\n", - " 36018,\n", - " 36019,\n", - " 36020,\n", - " 36021,\n", - " 36022,\n", - " 36023,\n", - " 36024,\n", - " 36025,\n", - " 36026,\n", - " 44027,\n", - " 34000,\n", - " 34001,\n", - " 34002,\n", - " 34003,\n", - " 34004,\n", - " 34005,\n", - " 34006,\n", - " 34007,\n", - " 34008,\n", - " 34009,\n", - " 34010,\n", - " 34011,\n", - " 34012,\n", - " 34013,\n", - " 34014,\n", - " 34015,\n", - " 34016,\n", - " 34017,\n", - " 34018,\n", - " 34019,\n", - " 34020,\n", - " 34021,\n", - " 34022,\n", - " 34023,\n", - " 34024,\n", - " 34025,\n", - " 34026,\n", - " 32000,\n", - " 32001,\n", - " 32002,\n", - " 32003,\n", - " 32004,\n", - " 32005,\n", - " 32006,\n", - " 32007,\n", - " 32008,\n", - " 32009,\n", - " 32010,\n", - " 32011,\n", - " 32012,\n", - " 32013,\n", - " 32014,\n", - " 32015,\n", - " 32016,\n", - " 32017,\n", - " 32018,\n", - " 32019,\n", - " 32020,\n", - " 32021,\n", - " 32022,\n", - " 32023,\n", - " 32024,\n", - " 32025,\n", - " 32026,\n", - " 30000,\n", - " 30001,\n", - " 30002,\n", - " 30003,\n", - " 30004,\n", - " 30005,\n", - " 30006,\n", - " 30007,\n", - " 30008,\n", - " 30009,\n", - " 30010,\n", - " 30011,\n", - " 30012,\n", - " 30013,\n", - " 30014,\n", - " 30015,\n", - " 30016,\n", - " 30017,\n", - " 30018,\n", - " 30019,\n", - " 30020,\n", - " 30021,\n", - " 30022,\n", - " 30023,\n", - " 30024,\n", - " 30025,\n", - " 28000,\n", - " 28001,\n", - " 28002,\n", - " 28003,\n", - " 28004,\n", - " 28005,\n", - " 28006,\n", - " 28007,\n", - " 28008,\n", - " 28009,\n", - " 28010,\n", - " 28011,\n", - " 28012,\n", - " 28013,\n", - " 28014,\n", - " 28015,\n", - " 28016,\n", - " 28017,\n", - " 28018,\n", - " 28019,\n", - " 28020,\n", - " 28021,\n", - " 28022,\n", - " 28023,\n", - " 28024,\n", - " 28025,\n", - " 28026,\n", - " 45018,\n", - " 43010,\n", - " 26000,\n", - " 26001,\n", - " 26002,\n", - " 26003,\n", - " 26004,\n", - " 26005,\n", - " 26006,\n", - " 26007,\n", - " 26008,\n", - " 26009,\n", - " 26010,\n", - " 26011,\n", - " 26012,\n", - " 26013,\n", - " 26014,\n", - " 26015,\n", - " 26016,\n", - " 26017,\n", - " 26018,\n", - " 26019,\n", - " 26020,\n", - " 26021,\n", - " 26022,\n", - " 26023,\n", - " 26024,\n", - " 26025,\n", - " 26026,\n", - " 26027,\n", - " 43012,\n", - " 24000,\n", - " 24001,\n", - " 24002,\n", - " 24003,\n", - " 24004,\n", - " 24005,\n", - " 24006,\n", - " 24007,\n", - " 24008,\n", - " 24009,\n", - " 24010,\n", - " 24011,\n", - " 24012,\n", - " 24013,\n", - " 24014,\n", - " 24015,\n", - " 24016,\n", - " 24017,\n", - " 24018,\n", - " 24019,\n", - " 24020,\n", - " 24021,\n", - " 24022,\n", - " 24023,\n", - " 24024,\n", - " 24025,\n", - " 24026,\n", - " 22000,\n", - " 22001,\n", - " 22002,\n", - " 22003,\n", - " 22004,\n", - " 22005,\n", - " 22006,\n", - " 22007,\n", - " 22008,\n", - " 22009,\n", - " 22010,\n", - " 22011,\n", - " 22012,\n", - " 22013,\n", - " 22014,\n", - " 22015,\n", - " 22016,\n", - " 22017,\n", - " 22018,\n", - " 22019,\n", - " 22020,\n", - " 22021,\n", - " 22022,\n", - " 22023,\n", - " 22024,\n", - " 22025,\n", - " 22026,\n", - " 45019,\n", - " 43015,\n", - " 20000,\n", - " 20001,\n", - " 20002,\n", - " 20003,\n", - " 20004,\n", - " 20005,\n", - " 20006,\n", - " 20007,\n", - " 20008,\n", - " 20009,\n", - " 20010,\n", - " 20011,\n", - " 20012,\n", - " 20013,\n", - " 20014,\n", - " 20015,\n", - " 20016,\n", - " 20017,\n", - " 20018,\n", - " 20019,\n", - " 20020,\n", - " 43016,\n", - " 44010,\n", - " 18000,\n", - " 18001,\n", - " 18002,\n", - " 18003,\n", - " 18004,\n", - " 18005,\n", - " 18006,\n", - " 18007,\n", - " 18008,\n", - " 18009,\n", - " 18010,\n", - " 18011,\n", - " 18012,\n", - " 18013,\n", - " 18014,\n", - " 18015,\n", - " 18016,\n", - " 18017,\n", - " 18018,\n", - " 18019,\n", - " 18020,\n", - " 18021,\n", - " 18022,\n", - " 18023,\n", - " 18024,\n", - " 18025,\n", - " 43018,\n", - " 44024,\n", - " 43025,\n", - " 16000,\n", - " 16001,\n", - " 16002,\n", - " 16003,\n", - " 16004,\n", - " 16005,\n", - " 16006,\n", - " 16007,\n", - " 16008,\n", - " ...}" - ] - }, - "execution_count": 186, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sG" - ] - }, - { - "cell_type": "code", - "execution_count": 200, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "import networkx as nx\n", - "G=nx.Graph()\n", - "\n", - "subgroup =[group for i, group in enumerate(groups) if (zvalues[group]>3125) & (zvalues[group]<3175)]\n", - "subanswers =[answers[i] for i, group in enumerate(groups) if (zvalues[group]>3125) & (zvalues[group]<3175)]\n", - "\n", - "G.add_nodes_from(subgroup)\n", - "#for group in groups:\n", - "# if zvalues[group]<100:\n", - "# nx.set_node_attributes(G,'z',zvalues)\n", - "\n", - "for answer,group in zip(subanswers,subgroup):\n", - " for group2 in answer.keys():\n", - "\n", - " if G.has_edge(group,group2):\n", - " w = G[group][group2]['weight']\n", - " G[group][group2]['weight']=w+answer[group2]\n", - " else:\n", - " if group2=='z':\n", - " print group,group2\n", - " if group=='z':\n", - " print group,group2\n", - " G.add_edge(group,group2,weight=answer[group2])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 201, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "def draw_network(G,pos,ax,sg=None):\n", - " \n", - " \n", - " for n in G:\n", - " c=Circle(pos[n],radius=.25,alpha=0.5)\n", - " #ax.add_patch(c)\n", - " p=ax.plot(pos[n][0],pos[n][1],'bo')\n", - " G.node[n]['patch']=c\n", - " x,y=pos[n]\n", - " #print x,y\n", - " seen={}\n", - " #return\n", - " for (u,v,d) in G.edges(data=True):\n", - " x1,y1 = pos[u]\n", - " x2,y2 = pos[v]\n", - " n1=G.node[u]['patch']\n", - " if v=='z':\n", - " print u,v,d\n", - " n2=G.node[v]['patch']\n", - " dz=y2-y1\n", - " rad = np.sqrt(dz)\n", - " if (u,v) in seen:\n", - " rad=seen.get((u,v))\n", - " rad=(rad+np.sign(rad)*0.1)*-1\n", - " alpha=0.5\n", - " color='k'\n", - " \n", - " cs = ConnectionStyle('Angle',angleA=45*(-1)**(dz%2), angleB=-45*(-1)**(dz%2), rad=10)\n", - " \n", - " linew = d['weight']*.1\n", - " linew = min(linew,10)\n", - " e = FancyArrowPatch(n1.center,n2.center,patchA=n1,patchB=n2,\n", - " arrowstyle='-|>',\n", - "# connectionstyle='arc3,rad=%s'%rad,\n", - " connectionstyle=cs,\n", - " mutation_scale=10.0,\n", - " lw=linew,\n", - " alpha=alpha,\n", - " color=color)\n", - " seen[(u,v)]=rad\n", - " ax.add_patch(e)\n", - " return e" - ] - }, - { - "cell_type": "code", - "execution_count": 202, - "metadata": { - "collapsed": false, - "scrolled": false - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f,ax=plt.subplots(1,2,figsize=(12,12))\n", - "#imgjoin = np.concatenate([lowmag_tile_p,lowmag_tile_q],axis=1)\n", - "ax[0].imshow(lowmag_tile_p,extent=(bounds_p['minX'],bounds_p['maxX'],bounds_p['maxY'],bounds_p['minY']))\n", - "ax[1].imshow(lowmag_tile_q,extent=(bounds_q['minX'],bounds_q['maxX'],bounds_q['maxY'],bounds_q['minY']))\n", - "ax[0].scatter(all_points_global_p[:,0],all_points_global_p[:,1],c='r',zorder=8)\n", - "ax[1].scatter(all_points_global_q[:,0],all_points_global_q[:,1],c='r',zorder=8)\n", - "ax[0].autoscale(tight=True)\n", - "ax[1].autoscale(tight=True)\n", - "ax[0].set_title(z_p)\n", - "ax[1].set_title(z_q)\n", - "#transFigure = f.transFigure.inverted()\n", - "#coord1 = transFigure.transform(ax[0].transData.transform(all_points_global_p))\n", - "#coord2 = transFigure.transform(ax[1].transData.transform(all_points_global_q))\n", - "#coords=np.concatenate([coord1,coord2],axis=1)\n", - "#for coor in coords:\n", - "# line = Line2D(coor[0:5:2]+.019,coor[1:5:2],\n", - "# transform=f.transFigure,zorder=9)\n", - "# f.lines.append(line)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "z_p=0\n", - "z_q=1\n", - "section_p=[key for key in zvalues.keys() if zvalues[key]==z_p][0]\n", - "section_q=[key for key in zvalues.keys() if zvalues[key]==z_q][0]\n", - "bounds_p = render.get_bounds_from_z(stack,z_p)\n", - "bounds_q = render.get_bounds_from_z(stack,z_q)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "scale=.1\n", - "lowmag_tile_p = render.get_png_tile(stack,z_p,bounds_p['minX'],bounds_p['minY'],bounds_p['maxX']-bounds_p['minX'],bounds_p['maxY']-bounds_p['minY'],scale=scale)\n", - "lowmag_tile_q = render.get_png_tile(stack,z_q,bounds_p['minX'],bounds_p['minY'],bounds_p['maxX']-bounds_p['minX'],bounds_p['maxY']-bounds_p['minY'],scale=scale)\n", - "lowmag_rg = np.copy(lowmag_tile_p)\n", - "lowmag_rg[:,:,2]=lowmag_tile_q[:,:,1]\n", - "lowmag_rg[:,:,0]=0" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "allmatches = render.get_matches_from_group_to_group(matchcollection,section_p,section_q)\n", - "all_points_global_p = np.zeros((1,2))\n", - "all_points_global_q = np.zeros((1,2))\n", - "for matchobj in allmatches:\n", - " points_local_p = np.array(matchobj['matches']['p'])\n", - " points_local_q = np.array(matchobj['matches']['q'])\n", - " #print matchobj,section_p,section_q\n", - " \n", - " t_p = render.local_to_world_coordinates_array(stack,points_local_p.T,matchobj['pId'],z_p)\n", - " all_points_global_p=np.append(all_points_global_p,t_p,axis=0)\n", - " t_q = render.local_to_world_coordinates_array(stack,points_local_q.T,matchobj['qId'],z_q)\n", - " all_points_global_q=np.append(all_points_global_q,t_q,axis=0)\n", - "\n", - " #break\n", - "all_points_global_p = all_points_global_p[1:,:]\n", - "all_points_global_q = all_points_global_q[1:,:]\n" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "all_points=np.concatenate([all_points_global_p,all_points_global_q],axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(6347, 4)" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_points.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 206, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "f,ax=plt.subplots(1,1,figsize=(14,40))\n", - "ax.imshow(lowmag_rg,extent=(bounds_p['minX'],bounds_p['maxX'],bounds_p['maxY'],bounds_p['minY']))\n", - "ax.scatter(all_points[:,0],all_points[:,1],c='m',marker='o',s=5,linewidth=0)\n", - "ax.quiver(all_points[:,0].T,all_points[:,1].T,\n", - " all_points[:,2].T-all_points[:,0].T,\n", - " all_points[:,3].T-all_points[:,1].T,\n", - " color='m',\n", - " angles='xy', scale_units='xy', scale=1)\n", - "ax.set_xlim((bounds_p['minX'],bounds_p['maxX']))\n", - "ax.set_ylim((bounds_p['maxY'],bounds_p['minY']))\n", - "#ax.autoscale(tight=True)\n", - "plt.tight_layout()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "f,ax=plt.subplots(1,1,figsize=(10,30))\n", - "ax.imshow(lowmag_rg,extent=(bounds_p['minX'],bounds_p['maxX'],bounds_p['maxY'],bounds_p['minY']))\n", - "ax.plot(all_points[:,0:5:2].T,all_points[:,1:5:2].T,c='m')\n", - "ax.autoscale(tight=True)\n", - "plt.tight_layout()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "print np.where(np.sum(match_matrix,axis=1)==0)\n", - "plt.figure()\n", - "plt.plot(np.sum(match_matrix,axis=1))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "plt.scatter?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "ax[0].scatter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "f,ax = plt.subplots(1,1,figsize=(12,12))\n", - "\n", - "img=ax.imshow(match_matrix,interpolation='nearest')\n", - "plt.xlabel('section a')\n", - "plt.ylabel('section b')\n", - "ax.set_title('matches')\n", - "plt.colorbar(img)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "(z2-z1)" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "ts=render.get_tile_spec(stack,145000023044000)" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "1242.0" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ts.z" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(6347, 4)" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_points.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import json\n", - "jsonlist = []\n", - "for i in range(all_points.shape[0]):\n", - " jsonlist.append([(all_points[i,0],all_points[i,1],z_p),(all_points[i,2],all_points[i,3],z_q)])\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "outfile = 'z0_z1_point_matches_global.json'\n", - "json.dump(jsonlist,open(outfile,'w'))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py new file mode 100644 index 00000000..8e1dc551 --- /dev/null +++ b/integration_tests/test_client_integrated.py @@ -0,0 +1,442 @@ +import renderapi +import pytest +import tempfile +import os +import logging +import sys +import json +from PIL import Image +import numpy as np +from test_data import (render_host, render_port, + client_script_location, tilespec_file, tform_file, + test_pool_size) +import PIL +from renderapi.external.processpools.stdlib_pool import ( + WithThreadPool, WithDummyMapPool, WithMultiprocessingPool) + +root = logging.getLogger() +root.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +root.addHandler(ch) + + +@pytest.fixture(scope='module') +def render(): + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location} + return renderapi.render.connect(**render_test_parameters) + + +@pytest.fixture(scope='module') +def render_example_tilespec_and_transforms(): + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + root.debug(tforms) + return (tilespecs, tforms) + + +def render_example_json_files(render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + tfiles = [] + for ts in tilespecs: + tfile = renderapi.utils.renderdump_temp([ts]) + tfiles.append(tfile) + tfjson = renderapi.utils.renderdump_temp(tforms) + + return (tfiles, tfjson) + + +def validate_stack_import(render, stack, tilespecs): + stacks = renderapi.render.get_stacks_by_owner_project(render=render) + assert stack in stacks + ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) + assert len(ts) == len(tilespecs) + + +@pytest.mark.parametrize('call_mode', ( + 'call', 'check_call', 'check_output')) +def test_import_jsonfiles_validate_client( + render, render_example_tilespec_and_transforms, call_mode): + stack = 'test_import_jsonfiles_validate_client' + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files( + render_example_tilespec_and_transforms) + renderapi.client.import_jsonfiles_validate_client( + stack, tfiles, transformFile=transformFile, render=render, + subprocess_mode=call_mode) + validate_stack_import(render, stack, tilespecs) + renderapi.stack.delete_stack(stack, render=render) + + +@pytest.mark.parametrize('call_mode', ('check_call', 'check_output')) +def test_failed_jsonfiles_validate_client( + render, render_example_tilespec_and_transforms, call_mode): + stack = 'test_failed_import_jsonfiles_validate_client' + renderapi.stack.create_stack(stack, render=render) + with pytest.raises(renderapi.errors.ClientScriptError): + renderapi.client.import_jsonfiles_validate_client( + stack, ['not_a_file'], render=render, + subprocess_mode=call_mode) + + +@pytest.mark.parametrize('use_rest,stack', + [(True, 'test_import_jsonfiles_parallel'), + (False, 'test_import_jsonfiles_parallel_rest')]) +def test_import_jsonfiles_parallel( + render, render_example_tilespec_and_transforms, + stack, use_rest, + poolsize=test_pool_size, **kwargs): + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files( + render_example_tilespec_and_transforms) + renderapi.client.import_jsonfiles_parallel( + stack, tfiles, transformFile=transformFile, + render=render, poolsize=poolsize, use_rest=use_rest, **kwargs) + validate_stack_import(render, stack, tilespecs) + renderapi.stack.delete_stack(stack, render=render) + + +def test_bbox_transformed(render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + ts = tilespecs[0] + xy = ts.bbox_transformed(ndiv_inner=0, tf_limit=0) + assert xy.shape == (5, 2) + assert np.abs((xy[2, :] - np.array([ts.width, ts.height])).sum()) < 1e-10 + xy = ts.bbox_transformed(ndiv_inner=1, tf_limit=0) + assert xy.shape == (9, 2) + + +def square(x): + return x**2 + + +# this test was added in order to validate that multiple WithPools would work +# pathos was breaking when we did this before. Should now be not relevant, +# but who ever deletes a test if you don't have to. +def test_import_jsonfiles_parallel_multiple( + render, render_example_tilespec_and_transforms, + poolsize=test_pool_size): + stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] + mylist = range(10) + for stack in stacks: + with renderapi.client.WithPool(poolsize) as pool: + results = pool.map(square, mylist) # noqa: F841 + test_import_jsonfiles_parallel( + render, render_example_tilespec_and_transforms, stack, + use_rest=False, poolsize=poolsize) + + +def test_import_tilespecs_parallel(render, + render_example_tilespec_and_transforms, + stack='test_import_tilespecs_parallel', + **kwargs): + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + renderapi.client.import_tilespecs_parallel( + stack, tilespecs, sharedTransforms=tforms, + poolsize=test_pool_size, render=render, **kwargs) + validate_stack_import(render, stack, tilespecs) + + +def test_import_jsonfiles(render, render_example_tilespec_and_transforms, + stack='test_import_jsonfiles'): + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files( + render_example_tilespec_and_transforms) + + renderapi.client.import_jsonfiles( + stack, tfiles, transformFile=transformFile, poolsize=test_pool_size, + render=render) + validate_stack_import(render, stack, tilespecs) + + +@pytest.fixture(scope="module") +def teststack(render, render_example_tilespec_and_transforms): + stack = 'teststack' + test_import_jsonfiles(render, render_example_tilespec_and_transforms, + stack=stack) + yield stack + renderapi.stack.delete_stack(stack, render=render) + +@pytest.fixture(scope="module") +def teststack2(render, render_example_tilespec_and_transforms): + #copy of teststack for the purpose of testing pointmatchClient with two stacks + stack = 'teststack2' + test_import_jsonfiles(render, render_example_tilespec_and_transforms, + stack=stack) + yield stack + renderapi.stack.delete_stack(stack, render=render) + + +def test_tile_pair_client(render, teststack, **kwargs): + zvalues = np.array(renderapi.stack.get_z_values_for_stack( + teststack, render=render)) + outjson = kwargs.pop('outjson', None) + tilepairjson = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, + render=render, **kwargs) + assert isinstance(tilepairjson, dict) + assert len(tilepairjson['neighborPairs']) > 3 + + multitpjson_tmp, multitpfiles_tmp = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=None, + render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) + + with tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) as f: + outjson_specified = f.name + + multitpjson, multitpfiles = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=outjson_specified, + render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) + + assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) == len(multitpjson_tmp["neighborPairs"]) + assert len(multitpfiles) == len(multitpfiles_tmp) == len(tilepairjson["neighborPairs"]) // 2 + + +@pytest.mark.parametrize("bounds,raises", [ + ({}, True), + ({'maxX': 1000, 'minX': 2000, 'minY': 1000, 'maxY': 2000}, True), + ({'maxX': 2000, 'minX': 1000, 'minY': 2000, 'maxY': 1000}, True), + ({'maxX': 2000, 'minX': 1000, 'minY': 1000, 'maxY': 2000}, False), + (None, False) +]) +def test_renderSectionClient(render, teststack, bounds, raises, scale=.05): + root_directory = tempfile.mkdtemp() + root.debug('section_directory:{}'.format(root_directory)) + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + + if raises: + with pytest.raises(renderapi.client.ClientScriptError): + renderapi.client.renderSectionClient(teststack, + root_directory, + zvalues, + scale=scale, + render=render, + bounds=bounds, + format='png') + else: + renderapi.client.renderSectionClient(teststack, + root_directory, + zvalues, + scale=scale, + render=render, + bounds=bounds, + format='png') + pngfiles = [] + for (dirpath, dirname, filenames) in os.walk(root_directory): + pngfiles += [os.path.join(dirpath, f) for f in filenames + if f.endswith('png')] + assert len(pngfiles) == len(zvalues) + if bounds is not None: + for f in pngfiles: + img = PIL.Image.open(f) + width, height = img.size + assert( + np.abs( + width - (bounds['maxX'] - bounds['minX']) * scale) < 1) + assert( + np.abs( + height - + (bounds['maxY'] - bounds['minY']) * scale) < 1 + ) + + +def test_importTransformChangesClient(render, teststack): + deststack = 'test_stack_TCC' + + tform_to_append = renderapi.transform.AffineModel() + + TCCjson = renderapi.utils.renderdump_temp( + [{'tileId': tileId, 'transform': tform_to_append} + for tileId in renderapi.stack.get_stack_tileIds( + teststack, render=render)]) + renderapi.client.importTransformChangesClient( + teststack, deststack, TCCjson, changeMode='APPEND', render=render) + renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) + os.remove(TCCjson) + + output_ts = renderapi.tilespec.get_tile_specs_from_stack( + deststack, render=render) + + assert all([ts.tforms[-1].to_dict() == tform_to_append.to_dict() + for ts in output_ts]) + renderapi.stack.delete_stack(deststack, render=render) + + +def test_transformSectionClient(render, teststack, + render_example_tilespec_and_transforms): + deststack = 'test_stack_TSC' + transformId = 'TSC_testtransform' + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + tform = renderapi.transform.AffineModel(transformId=transformId) + + renderapi.client.transformSectionClient( + teststack, transformId, tform.className, + tform.dataString.replace(" ", ","), zvalues, targetStack=deststack, + render=render) + renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) + + output_ts = renderapi.tilespec.get_tile_specs_from_stack( + deststack, render=render) + root.debug(output_ts[0].tforms[-1].to_dict()) + root.debug(output_ts[-1].tforms[-1].to_dict()) + root.debug(tform.to_dict()) + assert all([ts.tforms[-1].to_dict() == tform.to_dict() + for ts in output_ts]) + renderapi.stack.delete_stack(deststack, render=render) + + +def test_point_match_client(teststack, render, tmpdir): + collection = 'test_client_collection' + zvalues = np.array(renderapi.stack.get_z_values_for_stack( + teststack, render=render)) + tilepairjson = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), render=render) + + tile_pairs = [(tp['p']['id'], tp['q']['id']) for tp + in tilepairjson['neighborPairs'][0:1]] + sift_options = renderapi.client.SiftPointMatchOptions(renderScale=.25) + renderapi.client.pointMatchClient(teststack, + collection, + tile_pairs, + debugDirectory=tmpdir, + sift_options=sift_options, + render=render) + tp = tilepairjson['neighborPairs'][0] + pms = renderapi.pointmatch.get_matches_involving_tile( + collection, tp['p']['groupId'], tp['p']['id'], render=render) + assert(len(pms) > 0) + + +def test_point_match_client_2args(teststack, teststack2, render, tmpdir): + collection = 'test_client_collection' + zvalues = np.array(renderapi.stack.get_z_values_for_stack( + teststack, render=render)) + tilepairjson = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), render=render) + + tile_pairs = [(tp['p']['id'], tp['q']['id']) for tp + in tilepairjson['neighborPairs'][0:1]] # tile pairs are the same because the two stacks have the same information + sift_options = renderapi.client.SiftPointMatchOptions(renderScale=.25) + renderapi.client.pointMatchClient(teststack, + collection, + tile_pairs, stack2 = teststack2, + debugDirectory=tmpdir, + sift_options=sift_options, + render=render) + tp = tilepairjson['neighborPairs'][0] + pms = renderapi.pointmatch.get_matches_involving_tile( + collection, tp['p']['groupId'], tp['p']['id'], render=render) + assert(len(pms) > 0) + + +def test_call_run_ws_client_renderclient(render, teststack): + # class for this test should be something relatively lightweight.... + test_class = 'org.janelia.render.client.ValidateTilesClient' + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + args = renderapi.stack.make_stack_params( + render.DEFAULT_HOST, render.DEFAULT_PORT, render.DEFAULT_OWNER, + render.DEFAULT_PROJECT, teststack) + [zvalues[0]] + assert not renderapi.client.call_run_ws_client( + test_class, add_args=args, subprocess_mode='call', renderclient=render) + + +@pytest.mark.parametrize("stackbase,poolclass", [ + ("ThreadPooltest", WithThreadPool), + ("DummyMapPoolTest", WithDummyMapPool), + ("MultiprocessingPoolTest", WithMultiprocessingPool)]) +def test_processpools_parallelfuncs( + render, render_example_tilespec_and_transforms, + stackbase, poolclass, poolsize=test_pool_size): + test_import_tilespecs_parallel( + render, render_example_tilespec_and_transforms, + "{}_tilespecs".format(stackbase), mpPool=poolclass) + test_import_jsonfiles_parallel( + render, render_example_tilespec_and_transforms, + "{}_jsonfiles".format(stackbase), False, poolsize=poolsize, + mpPool=poolclass) + + +@pytest.fixture(scope='module') +def example_renderparameters(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + width = (bounds['maxX'] - bounds['minX']) / 2 + height = (bounds['maxY'] - bounds['minY']) / 2 + x = bounds['minX'] + width / 4 + y = bounds['minY'] + width / 4 + renderparams = renderapi.image.get_bb_renderparams( + teststack, z, x, y, width, height, + scale=0.25, render=render) + yield renderparams + + +@pytest.fixture(scope='module') +def renderedimg_renderparams(render, example_renderparameters): + # TODO is there a "render from renderparams" api? + arr = renderapi.image.get_renderparameters_image( + example_renderparameters, render=render) + yield arr, example_renderparameters + + +@pytest.fixture(scope='module') +def tile_tilespec(): + tile_dims = (256, 256) + with tempfile.NamedTemporaryFile(suffix='.tif', mode='w') as imgf: + arr = np.random.randint(0, 256, size=tile_dims, dtype='uint8') + img = Image.fromarray(arr) + img.save(imgf.name) + + ts = renderapi.tilespec.TileSpec( + tileId='myTestTile', + z=1., + sectionId="z1.0", + width=tile_dims[0], + height=tile_dims[1], + minint=0, + maxint=255, + imageUrl="file://{}".format(imgf.name)) + ts.minX = 0 + ts.maxX = tile_dims[0] + ts.minY = 0 + ts.maxY = tile_dims[1] + + yield imgf.name, ts + + +def test_ARGBrenderclient(render, tile_tilespec): + tile, tspec = tile_tilespec + arr = renderapi.client.render_tilespec(tspec, memGB='512M', render=render) + with Image.open(tile) as tileimg: + tilearr = np.array(tileimg) + assert arr.shape[:-1] == (tspec.width, tspec.height) == tilearr.shape + assert np.all(arr[:, :, 0] == tilearr) + + +def testRendererClient(render, renderedimg_renderparams): + renderedarr, renderparams = renderedimg_renderparams + arr = renderapi.client.render_renderparameters(renderparams, render=render) + assert arr.shape[:-1] == ( + int(renderparams['height'] * renderparams['scale']), + int(renderparams['width'] * renderparams['scale']) + ) == renderedarr.shape[:-1] + assert np.all(arr == renderedarr) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py new file mode 100644 index 00000000..c515347b --- /dev/null +++ b/integration_tests/test_coordinate_integrated.py @@ -0,0 +1,182 @@ +import renderapi +import pytest +import logging +import sys +import json +import numpy as np +from test_data import render_host, render_port, \ + client_script_location, tilespec_file, tform_file + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) + +logger.addHandler(ch) + + +@pytest.fixture(scope='module') +def render(): + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test_coordinate', + 'project': 'test_coordinate_project', + 'client_scripts': client_script_location + } + return renderapi.render.connect(**render_test_parameters) + + +@pytest.fixture(scope='module') +def teststack_tilespec(render): + ''' + render_test_parameters = { + 'host': render_host, + 'port': 8080, + 'owner': 'test_coordinate', + 'project': 'test_coordinate_project', + 'client_scripts': client_script_location + } + render = renderapi.render.connect(**render_test_parameters) + ''' + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + + stack = 'test_coordinate_stack' + render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run( + renderapi.client.import_tilespecs, stack, tilespecs, + sharedTransforms=tforms) + render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') + yield (stack, tilespecs[0]) + render.run(renderapi.stack.delete_stack, stack) + + +@pytest.fixture(scope='module') +def local_corners_json(teststack_tilespec): + (stack, ts) = teststack_tilespec + corners = [[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]] + batch = [] + for corner in corners: + d = { + 'tileId': ts.tileId, + 'visible': True + } + d['local'] = corner + batch.append(d) + return batch + + +@pytest.fixture(scope='module') +def world_corners_json(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + corners = [[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]] + world_corners = [] + for corner in corners: + world_corners.append(renderapi.coordinate.local_to_world_coordinates( + stack, ts.tileId, corner[0], corner[1], render=render)) + return world_corners + + +def test_world_to_local_coordinates(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local = render.run(renderapi.coordinate.world_to_local_coordinates, + stack, ts.z, 1500, 1500) + logger.debug(local) + for tile in local: + assert('error' not in tile.keys()) + assert(tile['tileId'] == ts.tileId) + assert(len(tile['local']) >= 2) + + +def test_local_to_world_coordinates(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + world = render.run(renderapi.coordinate.local_to_world_coordinates, + stack, ts.tileId, 0, 0) + logger.debug(world) + assert('error' not in world.keys()) + assert(world['tileId'] == ts.tileId) + assert(len(world['world']) >= 2) + + +def test_world_to_local_coordinates_batch(render, teststack_tilespec, + world_corners_json): + (stack, ts) = teststack_tilespec + local = renderapi.coordinate.world_to_local_coordinates_batch( + stack, world_corners_json, ts.z, execute_local=False, render=render) + logger.debug(local) + assert(len(local) == len(world_corners_json)) + for ans in local: + for tile in ans: + assert('error' not in tile.keys()) + + +def test_local_to_world_coordinates_batch(render, teststack_tilespec, + local_corners_json): + (stack, ts) = teststack_tilespec + world = renderapi.coordinate.local_to_world_coordinates_batch( + stack, local_corners_json, ts.z, execute_local=False, render=render) + assert(len(local_corners_json) == len(world)) + for ans in world: + assert('error' not in ans.keys()) + + +def test_world_to_local_coordinates_array(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, render=render) + local_corners2 = renderapi.coordinate.world_to_local_coordinates_array( + stack, world_corners, ts.tileId, ts.z, render=render) + logger.debug('local corners2: {}'.format(local_corners2)) + for pt, ptafter in zip(local_corners, local_corners2): + assert(np.sum(np.abs(pt-ptafter)) < .1) + + +def test_local_to_world_coordinates_array(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, render=render) + logger.debug('world corners:{}'.format(world_corners)) + assert(world_corners.shape[0] == local_corners.shape[0]) + + +def test_world_to_local_coordinates_clientside(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, + render=render, doClientSide=True) + local_corners2 = renderapi.coordinate.world_to_local_coordinates_array( + stack, world_corners, ts.tileId, ts.z, + render=render, doClientSide=True) + logger.debug('local corners2: {}'.format(local_corners2)) + for pt, ptafter in zip(local_corners, local_corners2): + assert(np.sum(np.abs(pt-ptafter)) < .1) + + +def test_local_to_world_coordinates_clientside(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, + doClientSide=True, render=render) + logger.debug('world corners:{}'.format(world_corners)) + assert(world_corners.shape[0] == local_corners.shape[0]) diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py new file mode 100644 index 00000000..7015a638 --- /dev/null +++ b/integration_tests/test_data.py @@ -0,0 +1,44 @@ +import os +from jinja2 import Environment, FileSystemLoader +import json + + +def render_json_template(env, template_file, **kwargs): + template = env.get_template(template_file) + d = json.loads(template.render(**kwargs)) + return d + + +example_dir = os.environ.get('RENDER_EXAMPLE_DATA', + '/var/www/render/examples/') +renderapp_example_dir = os.environ.get('RENDERAPP_EXAMPLE_DATA', example_dir) + +test_files_dir = os.path.join(os.path.dirname(__file__), 'test_files') +example_env = Environment(loader=FileSystemLoader(test_files_dir)) + +render_host = os.environ.get('RENDER_HOST', 'renderservice') +render_port = os.environ.get('RENDER_PORT', 8080) +client_script_location = os.environ.get( + 'RENDER_CLIENT_SCRIPTS', + '/var/www/render/render-ws-java-client/src/main/scripts/') + +render_params = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location +} + +tilespec_file = os.path.join( + example_dir, 'example_1', 'cycle1_step1_acquire_tiles.json') +tform_file = os.path.join( + example_dir, 'example_1', 'cycle1_step1_acquire_transforms.json') +test_pool_size = os.environ.get('RENDER_PYTHON_TEST_POOL_SIZE', 3) + +multi_channel_dir = os.path.join(renderapp_example_dir, 'multichannel-test') + +test_2_channels_d = render_json_template( + example_env, + 'test_2_channels.json', + multi_channel_example_dir=multi_channel_dir) diff --git a/integration_tests/test_files/test_2_channels.json b/integration_tests/test_files/test_2_channels.json new file mode 100755 index 00000000..b00cce8c --- /dev/null +++ b/integration_tests/test_files/test_2_channels.json @@ -0,0 +1,92 @@ +[ + { + "tileId": "100000001003000", + "z": 1.0, + "minX": 662.0, + "minY": 1659.0, + "maxX": 2709.0, + "maxY": 3707.0, + "width": 2048.0, + "height": 2048.0, + "channels": [ + { + "name": "DAPI", + "minIntensity":100.0, + "maxIntensity":6000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/DAPI_1_S0001_F0003_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + }, + { + "name": "TdTomato", + "minIntensity": 0.0, + "maxIntensity": 10000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/TdTomato_S0001_F0003_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + } + ], + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.000000 0.000000 0.000000 1.000000 662.6192164270324 1659.706576042783 " + } + ] + }, + "meshCellSize": 64.0 + }, + { + "tileId": "100000001004000", + "z": 1.0, + "minX":2296.0, + "minY":1648.0, + "maxX":4343.0, + "maxY":3695.0, + "width": 2048.0, + "height": 2048.0, + "channels": [ + { + "name": "DAPI", + "minIntensity":100.0, + "maxIntensity":6000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/DAPI_1_S0001_F0004_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + }, + { + "name": "TdTomato", + "minIntensity": 0.0, + "maxIntensity": 10000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/TdTomato_S0001_F0004_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + } + ], + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.000000 0.000000 0.000000 1.000000 2296.8235136181947 1648.2548941942687 " + } + ] + }, + "meshCellSize": 64.0 + } +] diff --git a/integration_tests/test_multichannel.py b/integration_tests/test_multichannel.py new file mode 100644 index 00000000..12d6d109 --- /dev/null +++ b/integration_tests/test_multichannel.py @@ -0,0 +1,114 @@ +import renderapi +from test_data import render_params, test_2_channels_d +import pytest +import logging +import sys +import numpy as np + +root = logging.getLogger() +root.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +root.addHandler(ch) + +render_params['project'] = 'multi_channel_test' + + +@pytest.fixture(scope='module') +def render(): + return renderapi.connect(**render_params) + + +@pytest.fixture(scope='module') +def multichannel_test_stack(render): + stack = 'multichannel_test' + tilespecs = [renderapi.tilespec.TileSpec(json=d) + for d in test_2_channels_d] + renderapi.stack.create_stack(stack, render=render) + renderapi.client.import_tilespecs(stack, tilespecs, render=render) + renderapi.stack.set_stack_state(stack, 'COMPLETE', render=render) + return stack + + +@pytest.fixture(scope='module') +def multichannel_test_stack_channelflip(render): + stack = 'multichannel_test_channelflip' + tilespecs = [renderapi.tilespec.TileSpec(json=d) + for d in test_2_channels_d] + for tilespec in tilespecs: + tilespec.channels.reverse() + renderapi.stack.create_stack(stack, render=render) + renderapi.client.import_tilespecs(stack, tilespecs, render=render) + renderapi.stack.set_stack_state(stack, 'COMPLETE', render=render) + return stack + + +@pytest.fixture(scope='module') +def test_pm_collection(render): + collection = 'test_multichan_collection' + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) + return collection + + +def test_section_image_channels(render, multichannel_test_stack): + section_image = renderapi.image.get_section_image(multichannel_test_stack, + 1.0, channel='DAPI', + render=render) + print(section_image.shape) + + +def test_section_image_same_channel_different_channel_order(render, multichannel_test_stack, multichannel_test_stack_channelflip): + section_image = renderapi.image.get_section_image(multichannel_test_stack, + 1.0, channel='DAPI', + render=render) + section_image2 = renderapi.image.get_section_image(multichannel_test_stack_channelflip, + 1.0, channel='DAPI', + render=render) + assert(np.array_equal(section_image, section_image2)) + + +def test_section_image_different_channel_order(render, multichannel_test_stack, multichannel_test_stack_channelflip): + section_image = renderapi.image.get_section_image(multichannel_test_stack, + 1.0, + render=render) + section_image2 = renderapi.image.get_section_image(multichannel_test_stack_channelflip, + 1.0, + render=render) + assert(not np.array_equal(section_image, section_image2)) + + +def test_multichannel_pointmatches(render, multichannel_test_stack): + collection = 'test_multichannel_pointmatch_same_channel' + sift_options = renderapi.client.SiftPointMatchOptions(renderScale=0.5) + tile_pairs = [['100000001003000', '100000001004000']] + renderapi.client.pointMatchClient(multichannel_test_stack, + collection, + tile_pairs, + filter=True, + excludeAllTransforms=True, + stackChannels='DAPI', + sift_options=sift_options, + render=render) + pms1 = renderapi.pointmatch.get_matches_involving_tile( + collection, collection, '100000001003000', render=render) + assert(len(pms1) > 0) + + collection2 = 'test_multichannel_pointmatch_different_channel' + renderapi.client.pointMatchClient(multichannel_test_stack, + collection2, + tile_pairs, + stack2=multichannel_test_stack, + filter=True, + excludeAllTransforms=True, + stackChannels='DAPI', + stack2Channels='TdTomato', + sift_options=sift_options, + render=render) + with pytest.raises(renderapi.errors.RenderError): + pms2 = renderapi.pointmatch.get_matches_involving_tile( + collection2, collection2, '100000001003000', render=render) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py new file mode 100644 index 00000000..0981b40e --- /dev/null +++ b/integration_tests/test_pointmatch_integrated.py @@ -0,0 +1,194 @@ +import renderapi +import pytest +import logging +import sys +from test_data import render_host, render_port, client_script_location + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +# +logger.addHandler(ch) + +test_matches = [ + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "1", + "qId": "1-1", + "matches": { + "p": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "q": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "w": [1, 1, 1, 1] + } + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "2", + "qId": "2-1", + "matches": { + "p": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "q": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "w": [1, 1, 1, 1] + } + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "0", + "qId": "0-2", + "matches": { + "p": [ + [100, 100], + [100, 0] + ], + "q": [ + [0, 100], + [0, 0] + ], + "w": [1, 1] + } + } +] + + +@pytest.fixture(scope='module') +def render(): + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_pointmatch_project', + 'client_scripts': client_script_location + } + return renderapi.render.connect(**render_test_parameters) + + +@pytest.fixture(scope='module') +def test_pm_collection(render): + collection = 'test_collection' + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) + return collection + + +def test_get_matchcollection_owners(render, test_pm_collection): + owners = renderapi.pointmatch.get_matchcollection_owners(render=render) + assert 'test' in owners + + +def test_get_matchcollections(render, test_pm_collection): + collections = renderapi.pointmatch.get_matchcollections(render=render) + matched_collection = next( + coljson for coljson in collections + if coljson['collectionId']['name'] == test_pm_collection) + assert matched_collection is not None + + +def test_get_matches_involving_tile(render, test_pm_collection): + matches = renderapi.pointmatch.get_matches_involving_tile( + test_pm_collection, "0", "0-1", render=render) + assert len(matches) == 3 + + +def test_get_match_groupIds(render, test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds( + test_pm_collection, render=render) + assert len(groups) == 3 + + +def test_get_matches_outside_group(render, test_pm_collection): + matches = renderapi.pointmatch.get_matches_outside_group( + test_pm_collection, "0", render=render) + assert test_matches[0] in matches + assert test_matches[1] in matches + + +def test_get_matches_within_group(render, test_pm_collection): + matches = renderapi.pointmatch.get_matches_within_group( + test_pm_collection, "0", render=render) + assert matches[0] == test_matches[2] + + +def test_get_matches_from_group_to_group(render, test_pm_collection): + group1 = "0" + group2 = "1" + matches = renderapi.pointmatch.get_matches_from_group_to_group( + test_pm_collection, group1, group2, render=render) + assert matches[0] == test_matches[0] + + +def test_get_matches_from_tile_to_tile(render, test_pm_collection): + group1 = "0" + group2 = "1" + tile1 = "0-1" + tile2 = "1-1" + matches = renderapi.pointmatch.get_matches_from_tile_to_tile( + test_pm_collection, group1, tile1, group2, tile2, render=render) + assert matches[0] == test_matches[0] + + +def test_get_matches_with_group(render, test_pm_collection): + group1 = "0" + matches = renderapi.pointmatch.get_matches_with_group( + test_pm_collection, group1, render=render) + assert len(matches) == 3 + + +def test_get_match_groupIds_from_only(render, test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds_from_only( + test_pm_collection, render=render) + assert len(groups) == 1 + + +def test_get_match_groupIds_to_only(render, test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds_to_only( + test_pm_collection, render=render) + assert len(groups) == 3 + + +def test_delete_point_matches_between_groups(render): + collection = 'test_delete' + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) + renderapi.pointmatch.delete_point_matches_between_groups( + collection, '0', '1', render=render) + groups = renderapi.pointmatch.get_match_groupIds(collection, render=render) + assert len(groups) == 2 + + +def test_delete_collection(render): + collection = 'test_delete_collection' + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) + + renderapi.pointmatch.delete_collection(collection, render=render) + collections = renderapi.pointmatch.get_matchcollections(render=render) + assert(collection not in collections) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py new file mode 100644 index 00000000..61345553 --- /dev/null +++ b/integration_tests/test_stack_integrated.py @@ -0,0 +1,576 @@ +import renderapi +import pytest +import logging +import sys +import json +import numpy as np +from test_data import (render_host, render_port, + client_script_location, tilespec_file, + tform_file) + +root = logging.getLogger() +root.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +root.addHandler(ch) + + +@pytest.fixture(scope='module') +def render(): + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location} + return renderapi.render.connect(**render_test_parameters) + + +@pytest.fixture +def simpletilespec(): + mm = renderapi.image_pyramid.MipMap(imageUrl='file://not/a/path.jpg') + ip = renderapi.image_pyramid.ImagePyramid({0: mm}) + tform = renderapi.transform.AffineModel(labels=['simple']) + layout = renderapi.tilespec.Layout(sectionId="section0", + scopeId="testscope", + cameraId="testcamera", + imageRow=1, + imageCol=1, + stageX=40.0, + stageY=50.0, + rotation=0) + ts = renderapi.tilespec.TileSpec(tileId="1000", + width=2048, + height=2048, + imagePyramid=ip, + z=0, + tforms=[tform], + layout=layout) + return ts + + +@pytest.fixture(scope='module') +def render_example_tilespec_and_transforms(): + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + root.debug(tforms) + return (tilespecs, tforms) + + +def test_stack_creation_deletion(render): + test_stack = 'test_stack1' + r = render.run(renderapi.stack.create_stack, + test_stack, force_resolution=True) + assert (r.status_code == 201) + + sv = render.run(renderapi.stack.get_stack_metadata, test_stack) + assert (sv is not None) + + assert(sv.stackResolutionX == 1.0) + assert(sv.stackResolutionY == 1.0) + assert(sv.stackResolutionZ == 1.0) + + owners = render.run(renderapi.render.get_owners) + assert('test' in owners) + + projects = render.run(renderapi.render.get_projects_by_owner) + assert('test_project' in projects) + + stacks = render.run(renderapi.render.get_stacks_by_owner_project) + assert(test_stack in stacks) + + r = render.run(renderapi.stack.delete_stack, test_stack) + + assert (r.status_code != 400) + + +def test_failed_metadata(render): + with pytest.raises(renderapi.errors.RenderError): + render.run(renderapi.stack.get_stack_metadata, 'NOTASTACKNAME') + + +def test_set_stack_metadata(render): + test_stack = 'test_stack2' + r = render.run(renderapi.stack.create_stack, + test_stack, force_resolution=True) + assert (r.status_code == 201) + + sv = render.run(renderapi.stack.get_stack_metadata, test_stack) + sv.stackResolutionX = 2.0 + sv.stackResolutionY = 3.0 + sv.stackResolutionZ = 4.0 + + r = render.run(renderapi.stack.set_stack_metadata, test_stack, sv) + assert r.status_code == 201 + sv = render.run(renderapi.stack.get_stack_metadata, test_stack) + assert sv.stackResolutionX == 2.0 + assert sv.stackResolutionY == 3.0 + assert sv.stackResolutionZ == 4.0 + + +def test_simple_import(render, simpletilespec, tmpdir): + # open a temporary file + tfile = tmpdir.join('testfile.json') + fp = tfile.open('w') + + # write the file to disk + renderapi.utils.renderdump([simpletilespec], fp) + fp.close() + + r = render.run(renderapi.stack.create_stack, + 'test_insert', force_resolution=True) + render.run(renderapi.client.import_single_json_file, + 'test_insert', str(tfile)) + r = render.run(renderapi.stack.set_stack_state, + 'test_insert', 'COMPLETE') + assert (r.status_code == 201) + ts_out = render.run(renderapi.tilespec.get_tile_spec, + 'test_insert', simpletilespec.tileId) + assert (ts_out.z == simpletilespec.z) + render.run(renderapi.stack.delete_stack, 'test_insert') + + +def test_simple_import_with_transforms( + render, render_example_tilespec_and_transforms, tmpdir): + (tilespecs, tforms) = render_example_tilespec_and_transforms + + # open a temporary file + tilespecfile = tmpdir.join('tilespecs.json') + fp = tilespecfile.open('w') + # write the file to disk + renderapi.utils.renderdump(tilespecs, fp) + fp.close() + + transformfile = tmpdir.join('transforms.json') + fp = transformfile.open('w') + # write the file to disk + renderapi.utils.renderdump(tforms, fp) + fp.close() + + r = render.run(renderapi.stack.create_stack, + 'test_insert_tform', force_resolution=True) + render.run(renderapi.client.import_single_json_file, 'test_insert_tform', + str(tilespecfile), transformFile=str(transformfile)) + r = render.run(renderapi.stack.set_stack_state, + 'test_insert_tform', 'COMPLETE') + assert r.status_code == 201 + ts_out = render.run(renderapi.tilespec.get_tile_spec, + 'test_insert_tform', tilespecs[0].tileId) + assert ts_out.z == tilespecs[0].z + render.run(renderapi.stack.delete_stack, 'test_insert_tform') + + +def test_import_tilespecs(render, simpletilespec): + stack = 'test_insert2' + render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.client.import_tilespecs, stack, [simpletilespec]) + response = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') + assert response.status_code == 201 + ts_out = render.run(renderapi.tilespec.get_tile_spec, + stack, simpletilespec.tileId) + assert ts_out.z == simpletilespec.z + render.run(renderapi.stack.delete_stack, stack) + + +def test_remove_section(render, simpletilespec, tmpdir): + # open a temporary file + tfile = tmpdir.join('testfile.json') + fp = tfile.open('w') + + # write the file to disk + renderapi.utils.renderdump([simpletilespec], fp) + fp.close() + + r = render.run(renderapi.stack.create_stack, + 'test_insert', force_resolution=True) + render.run(renderapi.client.import_single_json_file, + 'test_insert', str(tfile)) + r = render.run(renderapi.stack.set_stack_state, + 'test_insert', 'COMPLETE') + stack_zs_before = render.run(renderapi.stack.get_z_values_for_stack, + 'test_insert') + assert simpletilespec.z in stack_zs_before + r = render.run(renderapi.stack.set_stack_state, + 'test_insert', 'LOADING') + r = renderapi.stack.delete_section( # noqa: F841 + 'test_insert', simpletilespec.z, render=render) + stack_zs_after = render.run(renderapi.stack.get_z_values_for_stack, + 'test_insert') + assert len(stack_zs_after) == (len(stack_zs_before) - 1) + assert simpletilespec.z not in stack_zs_after + render.run(renderapi.stack.delete_stack, 'test_insert') + + +@pytest.fixture(scope="module") +def teststack(request, render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + stack = 'test_insert3' + r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.client.import_tilespecs, stack, tilespecs, + sharedTransforms=tforms) + r = render.run( # noqa: F841 + renderapi.stack.set_stack_state, stack, 'COMPLETE') + yield stack + render.run(renderapi.stack.delete_stack, stack) + +@pytest.fixture(scope="module") +def teststack_loading(request, render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + stack = 'test_insert_loading' + r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.client.import_tilespecs, stack, tilespecs, + sharedTransforms=tforms) + yield stack + render.run(renderapi.stack.delete_stack, stack) + +def test_stack_bounds(render, teststack): + # check the stack bounds + stack_bounds = render.run(renderapi.stack.get_stack_bounds, teststack) + expected_bounds = {u'maxZ': 3408.0, u'maxX': 5103.0, u'maxY': 5386.0, + u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} + + for key in stack_bounds.keys(): + assert np.abs(stack_bounds[key]-expected_bounds[key]) < 1.0 + + +def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + # check a single z stack bounds + zbounds = render.run(renderapi.stack.get_bounds_from_z, + teststack, tilespecs[0].z) + + expected_bounds = {u'maxZ': 3407.0, u'maxX': 4918.0, u'maxY': 4507.0, + u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} + for key in zbounds.keys(): + assert np.abs(zbounds[key]-expected_bounds[key]) < 1.0 + + +def test_get_section_z(render, teststack): + # check getting section Z + z = render.run(renderapi.stack.get_section_z_value, + teststack, "3407.0") + assert z == 3407 + + +def test_clone_stack(render, teststack): + stack2 = 'cloned_stack' + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + renderapi.stack.clone_stack(teststack, stack2, render=render) + zvalues2 = renderapi.stack.get_z_values_for_stack(stack2, render=render) + renderapi.stack.delete_stack(stack2, render=render) + assert zvalues == zvalues2 + + +def test_clone_stack_subset(render, teststack): + stack2 = 'cloned_stack_subset' + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + renderapi.stack.clone_stack( + teststack, stack2, zs=zvalues[0:1], render=render) + zvalues2 = renderapi.stack.get_z_values_for_stack(stack2, render=render) + renderapi.stack.delete_stack(stack2, render=render) + assert zvalues[0:1] == zvalues2 + + +def test_get_stack_sectionData(render, teststack): + sectionData = renderapi.stack.get_stack_sectionData( + teststack, render=render) + assert len(sectionData) == 2 + + +def test_get_z_values(render, teststack): + # check get z values + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + assert zvalues == [3407.0, 3408.0] + + +def test_uniq_value(render): + # check likelyUniqueId + uniq = render.run(renderapi.stack.likelyUniqueId) + assert len(uniq) >= len('58ceebb7a7b11b0001dc4e32') + + +def test_bb_image(render, teststack, **kwargs): + formats = renderapi.image.IMAGE_FORMATS.keys() + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + width = (bounds['maxX'] - bounds['minX']) / 2 + height = (bounds['maxY'] - bounds['minY']) / 2 + x = bounds['minX'] + width / 4 + y = bounds['minY'] + width / 4 + + for fmt in formats: + data = render.run(renderapi.image.get_bb_image, + teststack, z, x, y, width, height, + scale=.25, img_format=fmt, **kwargs) + assert data.shape[0] == (np.floor(height*.25)) + assert data.shape[1] == (np.floor(width*.25)) + if len(data.shape) > 2: + assert data.shape[2] >= 3 + + +def test_bb_image_options(render, teststack): + test_bb_image(render, teststack, filter=True, + binaryMask=True, maxTileSpecsToRender=20, minIntensity=0, + maxIntensity=255) + + +def test_bb_renderparams(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + width = (bounds['maxX'] - bounds['minX']) / 2 + height = (bounds['maxY'] - bounds['minY']) / 2 + x = bounds['minX'] + width / 4 + y = bounds['minY'] + width / 4 + scale = 0.25 + + rp = renderapi.image.get_bb_renderparams( + teststack, z, x, y, width, height, scale=scale, render=render) + assert int(rp['width'] * rp['scale']) == int(width * scale) + assert int(rp['height'] * rp['scale']) == int(height * scale) + assert int(rp['x']) == int(x) + assert int(rp['y']) == int(y) + assert len(rp['tileSpecs']) + + +def test_tile_image(render, teststack, render_example_tilespec_and_transforms, + **kwargs): + (tilespecs, tforms) = render_example_tilespec_and_transforms + data = render.run(renderapi.image.get_tile_image_data, + teststack, tilespecs[0].tileId, **kwargs) + if kwargs.get('scale') is None: + testscale = 1. + else: + testscale = kwargs['scale'] + + assert len(data.shape) == 3 + assert data.shape[0] >= np.floor(tilespecs[0].height * testscale) + assert data.shape[1] >= np.floor(tilespecs[0].width * testscale) + + +def test_tile_image_options(render, teststack, + render_example_tilespec_and_transforms): + testscale = 0.5 + test_tile_image( + render, teststack, render_example_tilespec_and_transforms, + scale=testscale, filter=True, normalizeForMatching=False) + + +def test_tile_renderparams(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + + tspecs = renderapi.tilespec.get_tile_specs_from_z( + teststack, z, render=render) + ts = tspecs[0] + + rp = renderapi.image.get_tile_renderparams( + teststack, ts.tileId, render=render) + assert int(rp['x']) == int(ts.minX) + assert int(rp['y']) == int(ts.minY) + assert len(rp['tileSpecs']) == 1 + assert rp['height'] - 1 == int(ts.maxY - ts.minY) + assert rp['width'] - 1 == int(ts.maxX - ts.minX) + + +def test_section_image(render, teststack, **kwargs): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + + width = bounds['maxX'] - bounds['minX'] + height = bounds['maxY'] - bounds['minY'] + fmt = 'png' + scalefactor = 0.05 + data = render.run(renderapi.image.get_section_image, teststack, z, + scale=scalefactor, img_format=fmt, **kwargs) + assert data.shape[0] == (np.floor(height * scalefactor)) + assert data.shape[1] == (np.floor(width * scalefactor)) + assert data.shape[2] >= 3 + + +def test_section_image_options(render, teststack): + test_section_image(render, teststack, filter=True, + maxTileSpecsToRender=50) + + +def test_section_renderparams(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + tspecs = renderapi.tilespec.get_tile_specs_from_z( + teststack, z, render=render) + + scalefactor = 0.05 + rp = renderapi.image.get_section_renderparams( + teststack, z, scale=scalefactor, render=render) + assert len(rp['tileSpecs']) == len(tspecs) + assert rp['scale'] == scalefactor + + +def fail_image_get(render, teststack, render_example_tilespec_and_transforms): + with pytest.raises(KeyError): + render.run(renderapi.image.get_tile_image_data, teststack, + 'test', img_format='JUNK') + + +def test_get_tilespecs_from_z(render, teststack, + render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + tiles = renderapi.tilespec.get_tile_specs_from_z( + teststack, tilespecs[0].z, render=render) + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] + assert len(tiles) == len(tsz) + + +def test_get_tilespec_raw( + render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + ts = renderapi.tilespec.get_tile_spec_raw( + teststack, tilespecs[0].tileId, render=render) + assert ts.to_dict() == tilespecs[0].to_dict() + + +def test_get_tile_specs_from_minmax_box( + render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + z = tilespecs[0].z + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] + zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) + ts = renderapi.tilespec.get_tile_specs_from_minmax_box( + teststack, z, zbounds['minX'], zbounds['maxX'], + zbounds['minY'], zbounds['maxY'], render=render) + assert len(ts) == len(tsz) + + +def test_get_tile_specs_from_box(render, teststack, + render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + z = tilespecs[0].z + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] + zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) + width = zbounds['maxX']-zbounds['minX'] + height = zbounds['maxY']-zbounds['minY'] + + ts = renderapi.tilespec.get_tile_specs_from_box( + teststack, z, zbounds['minX'], + zbounds['minY'], width, height, render=render) + assert len(ts) == len(tsz) + + +def test_get_tile_specs_from_stack(render, teststack, + render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + ts = renderapi.tilespec.get_tile_specs_from_stack(teststack, render=render) + assert len(ts) == len(tilespecs) + + +def test_get_sectionId_for_z(render, teststack, teststack_loading, + render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + sectionId = render.run( + renderapi.stack.get_sectionId_for_z, teststack, tilespecs[0].z) + assert (sectionId == tilespecs[0].layout.sectionId) + sectionId = render.run( + renderapi.stack.get_sectionId_for_z, teststack_loading, tilespecs[0].z) + assert (sectionId == tilespecs[0].layout.sectionId) + + +def test_get_resolvedtiles_from_z(render, teststack, + render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + resolved_tiles = renderapi.resolvedtiles.get_resolved_tiles_from_z( + teststack, tilespecs[0].z, render=render) + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] + assert(len(tsz) == len(resolved_tiles.tilespecs)) + matching_ts = next(ts for ts in resolved_tiles.tilespecs + if ts.tileId == tsz[0].tileId) + assert (len(matching_ts.tforms) == len(tsz[0].tforms)) + + +def test_put_resolved_tiles_scratch( + render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + out_stack = 'resolved_test_stack' + renderapi.stack.create_stack(out_stack, render=render) + resolved_tilespecs = renderapi.resolvedtiles.ResolvedTiles( + tilespecs, tforms) + r = renderapi.resolvedtiles.put_tilespecs( # noqa: F841 + out_stack, resolved_tilespecs, render=render) + tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack( + out_stack, render=render) + assert(len(tilespecs_out) == len(resolved_tilespecs.tilespecs)) + + +def test_put_tilespecs_and_tforms( + render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + out_stack = 'resolved_test_stack2' + renderapi.stack.create_stack(out_stack, render=render) + r = renderapi.resolvedtiles.put_tilespecs( # noqa: F841 + out_stack, tilespecs=tilespecs, + shared_transforms=tforms, render=render) + tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack( + out_stack, render=render) + assert(len(tilespecs_out) == len(tilespecs)) + + +def test_put_tilespecs_fail(render): + with pytest.raises(renderapi.errors.RenderError): + r = renderapi.resolvedtiles.put_tilespecs( # noqa: F841 + 'fail_stack', render=render) + + +def test_bad_delete(render): + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.delete_section('not_a_stack', 0.0, render=render) + + +def test_bad_get(render): + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.get_stack_metadata('not_a_stack', render=render) + + +def test_bad_post(render): + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.create_stack('not_a___stack', render=render) + + +def test_bad_put(render): + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.clone_stack( + 'not_a_stack', 'not_getting_here', render=render) + + +def test_rename_stack(render): + from_stack = 'from' + from_owner = 'from_owner' + from_project = 'from_project' + to_stack = 'to' + to_owner = 'to_owner' + to_project = 'to_project' + + renderapi.stack.create_stack(from_stack, owner=from_owner, + project=from_project, render=render) + renderapi.stack.rename_stack(from_stack, to_stack, to_owner=to_owner, + to_project=to_project, owner=from_owner, + project=from_project, render=render) + stacks_out = renderapi.render.get_stacks_by_owner_project( + project=to_project, owner=to_owner, render=render) + stacks_in = renderapi.render.get_stacks_by_owner_project( + project=from_project, owner=from_owner, render=render) + + assert(to_stack in stacks_out) + assert(from_stack not in stacks_in) + + renderapi.stack.delete_stack(to_stack, owner=to_owner, project=to_owner, + render=render) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..b87d73ff --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,103 @@ +[project] +name = "render-python" +description = "python API client to interact with render databases (see https://github.com/saalfeldlab/render)" +readme = "README.rst" +license = {file = "LICENSE"} +authors = [ + {name = "Forrest Collman"}, + {name = "Russel Torres"}, + {name = "Eric Perlman"}, + {name = "Sharmi Seshamani"} +] +maintainers = [ + {name = "Russel Torres"} +] +requires-python = ">3.8,<3.14" +dynamic = ["version"] +dependencies = [ + "requests", + "scipy", + "numpy", + "pillow", + "sphinxcontrib-napoleon", + "decorator", + "six" +] + +[project.urls] +Repository = "https://github.com/AllenInstitute/render-python.git" + +[project.optional-dependencies] +test = [ + "coverage>=4.1", + "mock>=2.0.0", + "pep8>=1.7.0", + # "pytest>4.6,<5.0", + "pytest", + "pytest-cov>=2.2.1", + "pytest-pep8>=1.0.6", + "pytest-xdist>=1.14", + "flake8>=3.0.4", + "pylint>=1.5.4", + "jinja2", + "ujson" +] +pypi-build = [ + "build", + "twine" +] + +[build-system] +requires = ["setuptools>=64", "setuptools_scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = [ + "renderapi" +] + +[tool.setuptools_scm] +version_file = "renderapi/_version.py" + +# some downgrades from pixi's expectations +[tool.pixi.system-requirements] +linux = "5.0.0" +libc = "2.27" + +[tool.pixi.project] +channels = ["conda-forge"] +platforms = ["linux-64"] + +[tool.pixi.pypi-dependencies] +render-python = { path = ".", editable = true } + +# conda-enabled features +[tool.pixi.feature.pixi-base.dependencies] +numpy = "*" +scipy = "*" + +[tool.pixi.feature.py312.dependencies] +python = "3.12.*" +[tool.pixi.feature.py313.dependencies] +python = "3.13.*" + + + +[tool.pixi.environments] +test = ["test"] +conda = ["pixi-base"] +conda-test = ["pixi-base", "test"] +build = ["pypi-build"] +test312 = ["py312", "test"] +test313 = ["py313", "test"] + +[tool.coverage.run] +omit = ["integration_tests/*", "test/*"] + +[tool.pixi.feature.test.tasks] +test = "pytest --cov --cov-report=xml --junitxml=test-reports/test.xml" + +[tool.pixi.feature.pypi-build.tasks] +pypi-build = "python -m build" +pypi-test-upload = "python -m twine upload --repository testpypi dist/*" +pypi-test-deploy = "python -m build && python -m twine upload --repository testpypi dist/*" diff --git a/renderapi.py b/renderapi.py deleted file mode 100755 index 055aaf0e..00000000 --- a/renderapi.py +++ /dev/null @@ -1,689 +0,0 @@ -import tempfile -import os -import json -import subprocess -import sys -import requests -import numpy as np -from tilespec import TileSpec,StackVersion -this = sys.modules[__name__] -from io import BytesIO -import numpy as np -from PIL import Image -import pathos.multiprocessing as mp -import time -from functools import partial - - -# DEFAULT_HOST = "renderer.int.janelia.org" -this.DEFAULT_HOST = "ibs-forrestc-ux1.corp.alleninstitute.org" -this.DEFAULT_PORT = 8080 -this.DEFAULT_OWNER = "Forrest" -this.DEFAULT_PROJECT = "M246930_Scnn1a_4" -this.DEFAULT_CLIENT_SCRIPTS = "/pipeline/render/render-ws-java-client/src/main/scripts" - -# GET http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/world-to-local-coordinates/{x},{y} -# curl "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates/40000,40000" -# returns: -# [ -# { -# "tileId": "140422184419060139", -# "visible": true, -# "local": [ -# 1238.9023, -# 1044.9727, -# 2239.0 -# ] -# } -# ] - -class Render(object): - DEFAULT_HOST = "ibs-forrestc-ux1.corp.alleninstitute.org" - DEFAULT_PORT = 8080 - DEFAULT_OWNER = "Forrest" - DEFAULT_PROJECT = "M246930_Scnn1a_4" - DEFAULT_CLIENT_SCRIPTS = "/pipeline/render/render-ws-java-client/src/main/scripts" - - def __init__(self,default_host,default_port,default_owner,default_project,default_client_scripts=this.DEFAULT_CLIENT_SCRIPTS): - self.DEFAULT_HOST = default_host - self.DEFAULT_PORT = default_port - self.DEFAULT_PROJECT = default_project - self.DEFAULT_OWNER = default_owner - self.DEFAULT_CLIENT_SCRIPTS = default_client_scripts - - def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): - #def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): - #utility function which will convert arguments to default arguments if they are None - #allows Render object to be used with defaults if lazy, but allows projects/hosts/owners to be changed - #from call to call if desired. - #used by many functions convert default None arguments to default values. - if host is None: - host=self.DEFAULT_HOST - if port is None: - port = self.DEFAULT_PORT - if owner is None: - owner = self.DEFAULT_OWNER - if project is None: - project = self.DEFAULT_PROJECT - if client_scripts is None: - client_scripts = self.DEFAULT_CLIENT_SCRIPTS - return (host,port,owner,project,client_scripts) - - def make_stack_params(self,host,port,owner,project,stack): - #utility function to turn host,port,owner,project,stack combinations - #to java CLI based argument list for subprocess calling - #returns [--baseDataUrl,self.format_baseurl(host,port),--owner - baseurl = self.format_baseurl(host,port) - project_params = ['--baseDataUrl', baseurl, '--owner', owner, '--project', project] - stack_params= project_params + ['--stack', stack] - return stack_params - - def delete_stack(self,stack,host=None,port=None,owner=None,project=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack) - if verbose: - print request_url - r=session.delete(request_url) - print r.text - return r - - def create_stack(self,stack,cycleNumber=1,cycleStepNumber=1, - client_scripts = None,host = None,port = None,owner = None, - project = None,verbose=False,session=requests.session()): - - - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project,client_scripts) - sv = StackVersion(cycleNumber=cycleNumber,cycleStepNumber=cycleStepNumber) - request_url = self.format_preamble(host,port,owner,project,stack) - if verbose: - print "stack version2",request_url,sv.to_dict() - payload = json.dumps(sv.to_dict()) - r = session.post(request_url,data=payload,headers={"content-type":"application/json","Accept":"application/json"}) - try: - return r - except: - print r.text - return None - - # import subprocess - # my_env = os.environ.copy() - # stack_params = self.make_stack_params(host,port,owner,project,stack) - # cmd = [os.path.join(client_scripts, 'manage_stacks.sh')] + \ - # stack_params + \ - # ['--action', 'CREATE', '--cycleNumber', '%d'%cycleNumber, '--cycleStepNumber', '%d'%cycleStepNumber] - # if verbose: - # print cmd - # proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - # proc.wait() - # if verbose: - # print proc.stdout.read() - - def import_single_json_file(self,stack,jsonfile,transformFile=None, - client_scripts=DEFAULT_CLIENT_SCRIPTS,host=None,port=None,owner=None,project=None,verbose=False): - - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) - - if transformFile is None: - transform_params =[] - else: - transform_params = ['--transformFile',transformFile] - import subprocess - my_env= os.environ.copy() - stack_params = self.make_stack_params(host,port,owner,project,stack) - cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - [jsonfile] - if verbose: - print cmd - proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - proc.wait() - if verbose: - print proc.stdout.read() - - def import_jsonfiles_and_transforms_parallel_by_z(self,stack,jsonfiles,transformfiles,poolsize=20,client_scripts=DEFAULT_CLIENT_SCRIPTS, - host=None,port=None,owner=None,project=None,close_stack=True,verbose=False): - - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) - self.set_stack_state(stack,'LOADING',host,port,owner,project) - - - pool = mp.ProcessingPool(poolsize) - - partial_import = partial(self.import_single_json_file,stack, - client_scripts=client_scripts,host=host,port=port,owner=owner,project=project,verbose=verbose) - - rs = pool.amap(partial_import, jsonfiles,transformfiles) - rs.wait() - - if close_stack: - self.set_stack_state(stack,'COMPLETE',host,port,owner,project) - - def import_jsonfiles_parallel(self,stack,jsonfiles,poolsize=20,transformFile=None,client_scripts=DEFAULT_CLIENT_SCRIPTS, - host=None,port=None,owner=None,project=None,close_stack=True,verbose=False): - - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) - self.set_stack_state(stack,'LOADING',host,port,owner,project) - - - pool = mp.ProcessingPool(poolsize) - - partial_import = partial(self.import_single_json_file,stack,transformFile=transformFile, - client_scripts=client_scripts,host=host,port=port,owner=owner,project=project,verbose=verbose) - partial_import(jsonfiles[0]) - rs = pool.amap(partial_import, jsonfiles) - rs.wait() - - if close_stack: - self.set_stack_state(stack,'COMPLETE',host,port,owner,project) - - - def import_jsonfiles(self,stack,jsonfiles,transformFile = None, - client_scripts=DEFAULT_CLIENT_SCRIPTS,host = None,port = None, - owner = None,project = None,close_stack=True,verbose=False): - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) - - self.set_stack_state(stack,'LOADING',host,port,owner,project) - - if transformFile is None: - transform_params = [] - else: - transform_params = ['--transformFile',transformFile] - import subprocess - my_env = os.environ.copy() - stack_params = self.make_stack_params(host,port,owner,project,stack) - cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - jsonfiles - if verbose: - print cmd - proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - proc.wait() - if verbose: - print proc.stdout.read() - if close_stack: - self.set_stack_state(stack,'COMPLETE',host,port,owner,project) - - - def world_to_local_coordinates(self,stack, z, x, y, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) - - r = session.get(request_url) - try: - #print(r.json()) - return r.json() - except: - print(r.text) - return None - - # GET http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/tile/{tileId}/local-to-world-coordinates/{x},{y} - # curl "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/tile/140422184419063136/local-to-world-coordinates/1244.0508,1433.8711" - # returns: - # { - # "tileId": "140422184419063136", - # "world": [ - # 40000.0, - # 40000.004, - # 2239.0 - # ] - # } - def local_to_world_coordinates(self,stack, tileId, x, y, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - - request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) - - r = session.get(request_url) - try: - #print(r.json()) - return r.json() - except: - print(r.text) - return None - - def get_owners(self,host=None,port=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,None,None) - request_url = "%s/owners/"%self.format_baseurl(host,port) - return self.process_simple_url_request(request_url,session) - - def get_projects_by_owner(self,owner=None,host=None,port=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - metadata = self.get_stack_metadata_by_owner(owner) - projects = list(set([m['stackId']['project'] for m in metadata])) - return projects - - def get_stacks_by_owner_project(self,owner=None,project=None,host=None,port=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - metadata = self.get_stack_metadata_by_owner(owner) - stacks =([m['stackId']['stack'] for m in metadata if m['stackId']['project']==project]) - return stacks - - def get_stack_metadata_by_owner(self,owner=None,host=None,port=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = "%s/owner/%s/stacks/"%(self.format_baseurl(host,port),owner) - if verbose: - request_url - return self.process_simple_url_request(request_url,session) - - # PUT http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/local-to-world-coordinates - # with request body containing JSON array of local coordinate elements - # curl -H "Content-Type: application/json" -X PUT --data @coordinate-local.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/local-to-world-coordinates" - # [ - # { - # "tileId": "140422184419063136", - # "world": [ - # 40000.0, - # 40000.004, - # 2239.0 - # ] - # } - # ] - def world_to_local_coordinates_batch(self,stack, z, data, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/world-to-local-coordinates" % (z) - r = session.put(request_url, data=data, headers={"content-type":"application/json"}) - - - #print r.text - return r.json() - - # curl -H "Content-Type: application/json" -X PUT --data @coordinate-world.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates" - # [ - # [ - # { - # "tileId": "140422184419060139", - # "visible": true, - # "local": [ - # 1238.9023, - # 1044.9727, - # 2239.0 - # ] - # } - # ] - # ] - def get_z_values_for_stack(self,stack,project = None, - host = None,port = None,owner = None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/zValues/" - if verbose: - print request_url - r = session.get(request_url) - try: - return r.json() - except: - print(r.text) - return None - def get_sectionData_for_stack(self,stack,project = None, - host = None,port = None,owner = None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/sectionData" - if verbose: - print(request_url) - r = session.get(request_url) - try: - return r.json() - except: - print(r.text) - return None - - - def get_z_value_for_section(self,stack,sectionId,project = None, - host = None,port = None,owner = None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/section/%s/z"%(sectionId) - r = session.get(request_url) - try: - #print(r.json()) - return r.json() - except: - print(r.text) - return None - - def get_section_image_data(self,stack,z,scale=0.1,maxIntensity=None,minIntensity=None,host=None,port=None,owner=None,project=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%s/png-image?" % (z) - args = ['scale=%f'%scale] - if maxIntensity is not None: - args.append('maxIntensity=%f'%maxIntensity) - if minIntensity is not None: - args.append('minIntensity=%f'%minIntensity) - request_url+='&'.join(args) - r = session.get(request_url) - try: - img = Image.open(BytesIO(r.content)) - array = np.asarray(img) - return array - except: - print (r.text) - return None - - def get_tile_image_data(self,stack,tileId,normalizeForMatching=True,host=None,port=None,owner=None,project=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/png-image" % (tileId) - if normalizeForMatching: - request_url+="?normalizeForMatching=true" - if verbose: - print request_url - r = session.get(request_url) - try: - img = Image.open(BytesIO(r.content)) - array = np.asarray(img) - return array - except: - print (r.text) - return None - - def world_to_local_coordinates_array(self,stack, dataarray, tileId,z,host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/world-to-local-coordinates" % (z) - dlist =[] - for i in range(dataarray.shape[0]): - d ={} - d['tileId']=tileId - d['world']=[dataarray[i,0],dataarray[i,1]] - dlist.append(d) - jsondata=json.dumps(dlist) - - r = session.put(request_url, data=jsondata, headers={"content-type":"application/json"}) - - json_answer = r.json() - #print json_answer - try: - answer = np.zeros(dataarray.shape) - - for i,matches in enumerate(json_answer): - if len(matches)>0: - for coord in matches: - if coord['tileId']==tileId: - c = coord['local'] - answer[i,0]=c[0] - answer[i,1]=c[1] - return answer - - except: - print json_answer - return json_answer - - def local_to_world_coordinates_array(self,stack, dataarray, tileId, z=0,host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/local-to-world-coordinates" % (z) - dlist =[] - for i in range(dataarray.shape[0]): - d ={} - d['tileId']=tileId - d['local']=[dataarray[i,0],dataarray[i,1]] - dlist.append(d) - jsondata=json.dumps(dlist) - - r = session.put(request_url, data=jsondata, headers={"content-type":"application/json"}) - - json_answer = r.json() - #print json_answer - try: - answer = np.zeros(dataarray.shape) - - for i,coord in enumerate(json_answer): - - c = coord['world'] - answer[i,0]=c[0] - answer[i,1]=c[1] - return answer - - except: - print json_answer - return None - - - def local_to_world_coordinates_batch(self,stack, data, z, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/local-to-world-coordinates" % (z) - - - r = session.put(request_url, data=data, headers={"content-type":"application/json"}) - - #print r.text() - return r.json() - - - def format_baseurl(self,host,port): - return 'http://%s:%d/render-ws/v1'%(host,port) - - def format_preamble(self,host,port,owner,project,stack): - preamble = "%s/owner/%s/project/%s/stack/%s"%(self.format_baseurl(host,port),owner,project,stack) - return preamble - - def process_simple_url_request(self,request_url,session): - r = session.get(request_url) - try: - #print(r.json()) - return r.json() - except: - #print e - print(r.text) - return None - def put_resolved_tilespecs(self,stack,data,host = None,port = None,owner = None,project = None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/resolvedTiles" - if verbose: - print request_url - - r = session.put(request_url, data=data, headers={"content-type":"application/json","Accept":"text/plain"}) - return r - - - # http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/tile/140422184419060139 - def get_tile_spec(self,stack, tile, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/render-parameters"%(tile) - - tilespec_json= self.process_simple_url_request(request_url,session) - - return TileSpec(json=tilespec_json['tileSpecs'][0]) - - - def get_tile_specs_from_minmax_box(self,stack,z,xmin,xmax,ymin,ymax,scale=1.0,host=None,port=None,owner=None, - project=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - x = xmin - y = ymin - width = xmax-xmin - height = ymax -ymin - return self.get_tile_specs_from_box(stack, z, x, y, width, height,scale,host,port,owner,project,session,verbose) - - def get_tile_specs_from_box(self,stack,z,x,y,width,height,scale=1.0,host=None, port=None,owner=None,project=None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters"%(z,x,y,width,height,scale) - if verbose: - print request_url - tilespecs_json=self.process_simple_url_request(request_url,session)['tileSpecs'] - #return tilespecs_json - return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json] - - - def get_tile_specs_from_z(self,stack,z,host = None,port = None,owner=None,project=None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+'/z/%f/tile-specs'%(z) - if verbose: - print request_url - tilespecs_json=self.process_simple_url_request(request_url,session) - if len(tilespecs_json)==0: - return None - else: - return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json] - - def get_bounds_from_z(self,stack,z,host = None,port = None,owner = None,project = None, - session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+'/z/%f/bounds'%(z) - return self.process_simple_url_request(request_url,session) - - # - # API for doing the bulk requests locally (i.e., to be run on the cluster) - # Full documentation here: http://wiki.int.janelia.org/wiki/display/flyTEM/Coordinate+Mapping+Tools - # - - MAP_COORD_SCRIPT = "/groups/flyTEM/flyTEM/render/bin/map-coord.sh" - - def set_stack_state(self,stack,state='LOADING',host = None,port = None,owner = None,project = None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - assert state in ['LOADING','COMPLETE','OFFLINE'] - request_url = self.format_preamble(host,port,owner,project,stack)+"/state/%s"%state - if verbose: - request_url - r=session.put(request_url,data=None,headers={"content-type":"application/json"}) - return r - - def batch_local_work(self,stack, z, data, host = None, port = None, owner = None, project = None, localToWorld=False, deleteTemp=True, threads=16): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - fromJson = tempfile.NamedTemporaryFile(suffix=".json", mode='w', delete=False) - fromJson.write(data) - fromJson.flush() - fromJson.close() - - toJson = tempfile.NamedTemporaryFile(suffix=".json", mode='r', delete=False) - toJson.close() - - #cmd = "%s --owner %s --project %s --stack %s --z %d --fromJson %s --toJson %s --baseDataUrl http://tem-services.int.janelia.org:8080/render-ws/v1 --numberOfThreads %d" % (MAP_COORD_SCRIPT, owner, project, stack, z, fromJson.name, toJson.name, threads) - cmd = "%s --owner %s --project %s --stack %s --z %d --fromJson %s --toJson %s --baseDataUrl http://10.40.3.162:8080/render-ws/v1 --numberOfThreads %d" % (MAP_COORD_SCRIPT, owner, project, stack, z, fromJson.name, toJson.name, threads) - if localToWorld: - cmd = cmd + " --localToWorld" - #print(cmd) - - try: - rc = subprocess.call(cmd, shell="True") - if rc != 0: - raise Exception("Invalid return code (%d): %s" % (rc, cmd)) - - with open(toJson.name) as f: - outdata = json.load(f) - except: - print("Unexpected error:", sys.exc_info()[0]) - return json.loads("{}") - - if deleteTemp: - os.unlink(fromJson.name) - os.unlink(toJson.name) - - return outdata - - def get_png_tile(self,stack,z,x,y,width,height,scale=1.0,host=None,port=None,owner=None,project=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/box/%d,%d,%d,%d,%3.2f/png-image"%(z,x,y,width,height,scale) - #print request_url - r = session.get(request_url) - import io - from PIL import Image - from array import array - try: - image = np.asarray(Image.open(io.BytesIO(r.content))) - except: - print r.text - return image - - def world_to_local_coordinates_batch_local(self,stack, z, data, host = None, port = None, owner = None, project = None): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - return batch_local_work(stack, z, data, host, port, owner, project, localToWorld=False) - - def local_to_world_coordinates_batch_local(self,stack, z, data, host = None, port = None, owner = None, project = None): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - return batch_local_work(stack, z, data, host, port, owner, project, localToWorld=True) - - def get_matchcollection_owners(self,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,None,None) - request_url=self.format_baseurl(host,port)+"/matchCollectionOwners" - return self.process_simple_url_request(request_url, session) - - def get_matchcollections(self,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollections"%owner - return self.process_simple_url_request(request_url, session) - - def get_match_groupIds(self,matchCollection,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/groupIds"%(owner,matchCollection) - return self.process_simple_url_request(request_url, session) - - def get_section_z_value(self,stack,sectionId,host=None,port=None,owner=None,project=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host, port, owner, project, stack)+"/section/%s/z"%sectionId - return float(self.process_simple_url_request(request_url, session)) - - def get_matches_outside_group(self,matchCollection,groupId,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup"%\ - (owner,matchCollection,groupId) - return self.process_simple_url_request(request_url, session) - - def get_matches_within_group(self,matchCollection,groupId,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup"%\ - (owner,matchCollection,groupId) - return self.process_simple_url_request(request_url, session) - - def delete_matches_from_group_to_group(self,matchCollection,pgroup,qgroup,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/matchesWith/%s"%\ - (owner,matchCollection,pgroup,qgroup) - r=session.delete(request_url) - return r - - def get_matches_from_group_to_group(self,matchCollection,pgroup,qgroup,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/matchesWith/%s"%\ - (owner,matchCollection,pgroup,qgroup) - return self.process_simple_url_request(request_url, session) - - def delete_matches_from_tile_to_tile(self,matchCollection,pgroup,pid,qgroup,qid,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/id/%s/matchesWith/%s/id/%s"%\ - (owner,matchCollection,pgroup,pid,qgroup,qid) - r=session.delete(request_url) - return r - - def get_matches_from_tile_to_tile(self,matchCollection,pgroup,pid,qgroup,qid,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/id/%s/matchesWith/%s/id/%s"%\ - (owner,matchCollection,pgroup,pid,qgroup,qid) - return self.process_simple_url_request(request_url, session) - def get_matches_involving_tile(self,matchCollection,group,id,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/id/%s"%\ - (owner,matchCollection,group,id) - return self.process_simple_url_request(request_url, session) - - def get_matches_with_group(self,matchCollection,pgroup,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/pGroup/%s/matches/"%\ - (owner,matchCollection,pgroup) - return self.process_simple_url_request(request_url, session) - - def get_match_groupIds_from_only(self,matchCollection,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/pGroupIds"%\ - (owner,matchCollection) - return self.process_simple_url_request(request_url, session) - - def get_match_groupIds_to_only(self,matchCollection,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/qGroupIds"%\ - (owner,matchCollection) - return self.process_simple_url_request(request_url, session) - - def import_matches(self,matchCollection,data,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url =self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/matches"%(owner,matchCollection) - if verbose: - print request_url - r = session.put(request_url, data=data, headers={"content-type":"application/json","Accept":"application/json"}) - return r - diff --git a/renderapi/__init__.py b/renderapi/__init__.py new file mode 100644 index 00000000..a639e30f --- /dev/null +++ b/renderapi/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python + +from . import render +from . import tilespec +from . import errors +from . import stack +from . import client +from . import image +from . import transform +from . import pointmatch +from . import coordinate +from . import resolvedtiles +from .render import connect +from .render import Render + +__version__ = "2.4.0" +__all__ = ['render', 'client', 'tilespec', 'errors', + 'stack', 'image', 'pointmatch', 'coordinate', + 'connect', 'transform', 'resolvedtiles', 'Render'] diff --git a/renderapi/channel.py b/renderapi/channel.py new file mode 100644 index 00000000..5b456269 --- /dev/null +++ b/renderapi/channel.py @@ -0,0 +1,54 @@ +from .image_pyramid import ImagePyramid + + +class Channel: + '''class for storing channels of different mipmapsources''' + + def __init__(self, name=None, maxIntensity=None, minIntensity=None, + ip=None, json=None): + ''' + Parameters + ========== + name: str + name of channel + maxIntensity: int + maximum intensity to display (optional) + minIntensity: int + minimum default intensity to display (optional) + ip: ImagePyramid + set of mipmaplevel images for this channel + json: dict + json representation of this channel + ''' + + if json is not None: + self.from_dict(json) + else: + self.name = name + self.maxIntensity = maxIntensity + self.minIntensity = minIntensity + self.ip = ip + + def to_dict(self): + '''method for serializing this class to a json compatible dictionary''' + d = {} + d['name'] = self.name + if self.minIntensity is not None: + d['minIntensity'] = self.minIntensity + if self.maxIntensity is not None: + d['maxIntensity'] = self.maxIntensity + d['mipmapLevels'] = self.ip.to_dict() + return d + + def from_dict(self, d): + '''method for deserializing this class from a json compatible dictionary + + Parameters + ========== + d: dict + json compatible dictionary representation of this channel + ''' + self.name = d['name'] + self.minIntensity = d['minIntensity'] + self.maxIntensity = d['maxIntensity'] + self.ip = ImagePyramid.from_dict(d['mipmapLevels']) diff --git a/renderapi/client/__init__.py b/renderapi/client/__init__.py new file mode 100644 index 00000000..ffa7ec61 --- /dev/null +++ b/renderapi/client/__init__.py @@ -0,0 +1,4 @@ +from .client import * # noqa: F403, F401 +from .client_calls import * # noqa: F403, F401 +from .utils import * # noqa: F403, F401 +from .params import * # noqa: F403, F401 diff --git a/renderapi/client/client.py b/renderapi/client/client.py new file mode 100644 index 00000000..b49209a6 --- /dev/null +++ b/renderapi/client/client.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python +''' +render functions relying on render-ws client scripts +''' +import os +from functools import partial +import logging +import tempfile + +import numpy +from PIL import Image + +from renderapi.utils import NullHandler, renderdump_temp +from renderapi.render import renderaccess +from renderapi.stack import set_stack_state, make_stack_params +from renderapi.resolvedtiles import put_tilespecs +from renderapi.external.processpools.stdlib_pool import WithMultiprocessingPool + +from .utils import renderclientaccess +from .client_calls import importJsonClient, call_run_ws_client, renderClient, rendererClient + +# setup logger +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + +WithPool = WithMultiprocessingPool + + +@renderclientaccess +def import_single_json_file(stack, jsonfile, transformFile=None, + subprocess_mode=None, client_script=None, + memGB=None, host=None, port=None, + owner=None, project=None, render=None, **kwargs): + """calls client script to import given jsonfile + + Parameters + ---------- + stack : str + stack to import into + jsonfile : str + path to jsonfile to import + transformFile : str + path to a file that contains shared + transform references if necessary + render : renderapi.render.RenderClient + render connect object + """ + importJsonClient(stack, [jsonfile], transformFile, + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, client_script=client_script, + memGB=memGB, **kwargs) + + +@renderclientaccess +def import_jsonfiles_and_transforms_parallel_by_z( + stack, jsonfiles, transformfiles, poolsize=20, mpPool=WithPool, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, render=None, **kwargs): + """imports json files and transform files in parallel + + Parameters + ---------- + stack : str + the stack to import within + jsonfiles : :obj:`list` of :obj:`str` + "list of tilespec" json paths to import + transformfiles : :obj:`list` of :obj:`str` + "list of transform files" paths which matches + in a 1-1 way with jsonfiles, so referenced transforms + are shared only within a single element of these matched lists. + Useful cases where there is as single z transforms shared + by all tiles within a single z, but not across z's + poolsize : int, optional + number of processes for multiprocessing pool + close_stack : bool, optional + whether to mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + **kwargs + arbitrary keyword arguments + + """ + set_stack_state(stack, 'LOADING', host, port, owner, project) + partial_import = partial(import_single_json_file, stack, render=render, + client_scripts=client_scripts, host=host, + port=port, owner=owner, project=project) + with mpPool(poolsize) as pool: + pool.map(partial_import, jsonfiles, transformfiles) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderclientaccess +def import_jsonfiles_parallel( + stack, jsonfiles, poolsize=20, transformFile=None, mpPool=WithPool, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, render=None, **kwargs): + """import jsons using client script in parallel + + Parameters + ---------- + stack : str + the stack to upload into + jsonfiles : :obj:`list` of :obj:`str` + list of jsonfile paths to upload + poolsize : int + number of upload processes spawned by multiprocessing pool + transformFile : str + a single json file path containing transforms referenced + in the jsonfiles + close_stack : bool + whether to mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + **kwargs + arbitrary keyword arguments + + """ + set_stack_state(stack, 'LOADING', host, port, owner, project) + + partial_import = partial(import_single_json_file, stack, render=render, + transformFile=transformFile, + client_scripts=client_scripts, + host=host, port=port, owner=owner, + project=project, **kwargs) + with mpPool(poolsize) as pool: + pool.map(partial_import, jsonfiles) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderaccess +def import_jsonfiles(stack, jsonfiles, transformFile=None, + subprocess_mode=None, client_script=None, memGB=None, + host=None, port=None, owner=None, project=None, + close_stack=True, render=None, **kwargs): + """import jsons using client script serially + + Parameters + ---------- + jsonfiles : :obj:`list` of :obj:`str` + iterator of filenames to be uploaded + transformFile : str + path to a jsonfile that contains shared + transform references (if necessary) + close_stack : bool + mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + + """ + + set_stack_state(stack, 'LOADING', host, port, owner, project) + importJsonClient(stack, jsonfiles, transformFile, + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, client_script=client_script, + memGB=memGB, **kwargs) + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderclientaccess +def import_jsonfiles_validate_client(stack, jsonfiles, + transformFile=None, client_script=None, + host=None, port=None, owner=None, + project=None, close_stack=True, mem=6, + validator=None, subprocess_mode=None, + memGB=None, + render=None, **kwargs): + """Uses java client for parallelization and validation + + Parameters + ---------- + stack: str + stack to which jsonfiles should be uploaded + jsonfiles: :obj:`list` of :obj:`str` + tilespecs in json files + transformFile: str, optional + json file listing transformspecs with ids which are referenced + in tilespecs contained in jsonfiles + + """ + transform_params = (['--transformFile', transformFile] + if transformFile is not None else []) + if validator is None: + validator_params = [ + '--validatorClass', + 'org.janelia.alignment.spec.validator.TemTileSpecValidator', + '--validatorData', + 'minCoordinate:-500,maxCoordinate:100000,' + 'minSize:500,maxSize:10000'] + else: + raise NotImplementedError('No custom validation handling!') + + stack_params = make_stack_params(host, port, owner, project, stack) + set_stack_state(stack, 'LOADING', host, port, owner, project) + + call_run_ws_client('org.janelia.render.client.ImportJsonClient', + stack_params + + validator_params + + transform_params + + jsonfiles, client_script=client_script, + memGB=memGB, subprocess_mode=subprocess_mode, + **kwargs) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderclientaccess +def import_tilespecs(stack, tilespecs, sharedTransforms=None, + use_rest=False, deriveData=True, + subprocess_mode=None, host=None, port=None, + owner=None, project=None, client_script=None, + memGB=None, render=None, **kwargs): + """method to import tilesepcs directly from + :class:`renderapi.tilespec.TileSpec` objects + + Parameters + ---------- + stack : str + stack to which tilespecs will be added + tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` + list of tilespecs to import + sharedTransforms : :obj:`list` of :class:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional + list of shared referenced transforms to be ingested + use_rest: bool + whether to import the tilespecs using the post method directly with deriveData=True + deriveData: bool + if doing use_rest, will determine whether to have the server calculate bounds (default=True) + render : renderapi.render.Render + render connect object + + """ # noqa: E501 + if use_rest: + put_tilespecs(stack, + deriveData=deriveData, + tilespecs=tilespecs, + shared_transforms=sharedTransforms, + host=host, port=port, owner=owner, + project=project, **kwargs) + else: + tsjson = renderdump_temp(tilespecs) + + if sharedTransforms is not None: + trjson = renderdump_temp(sharedTransforms) + + importJsonClient(stack, tileFiles=[tsjson], transformFile=( + trjson if sharedTransforms is not None else None), + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs) + + os.remove(tsjson) + if sharedTransforms is not None: + os.remove(trjson) + + +@renderclientaccess +def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, + subprocess_mode=None, poolsize=20, + mpPool=WithPool, + close_stack=True, max_tilespecs_per_group=None, + host=None, port=None, + owner=None, project=None, + client_script=None, memGB=None, render=None, + **kwargs): + """method to import tilesepcs directly from + :class:`renderapi.tilespec.TileSpec` objects using + pathos.multiprocessing to parallelize + + Parameters + ---------- + stack : str + stack to which tilespecs will be added + tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` + list of tilespecs to import + sharedTransforms : obj:`list` of :obj:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional + list of shared referenced transforms to be ingested + poolsize : int + degree of parallelism to use + subprocess_mode : str + subprocess mode used when calling client side java + close_stack : bool + mark render stack as COMPLETE after successful import + max_tilespecs_per_group: int + maximum tilespecs per import process, default to len(tilespecs)/poolsize + render : :class:renderapi.render.Render + render connect object + kwargs: dict .. all other kwargs to pass on to renderapi.client.import_tilespecs + """ # noqa: E501 + tslists = ( + max((len(tilespecs) // max_tilespecs_per_group) + 1, poolsize) if + max_tilespecs_per_group is not None else poolsize) + + set_stack_state(stack, 'LOADING', host, port, owner, project) + partial_import = partial( + import_tilespecs, stack, sharedTransforms=sharedTransforms, + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, client_script=client_script, + render=render, memGB=memGB, **kwargs) + + # TODO this is a weird way to do splits.... is that okay? + tilespec_groups = [g for g in + (tilespecs[i::tslists] for i in range(tslists)) if g] + with mpPool(poolsize) as pool: + pool.map(partial_import, tilespec_groups) + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +# TODO handle fromJson and toJson persistence in these calls +@renderclientaccess +def local_to_world_array(stack, points, tileId, subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + """placeholder function for coordinateClient localtoworld + + Parameters + ---------- + stack : str + stack to which world coordinates are mapped + points : dict + local points to map to world + tileId : str + tileId to which points correspond + subprocess_mode : str + subprocess mode used when calling + clientside java client + Returns + ------- + list + points in world coordinates corresponding to local points + """ + raise NotImplementedError('Whoops') + + +@renderclientaccess +def world_to_local_array(stack, points, subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + """placeholder function for coordinateClient worldtolocal + + Parameters + ---------- + stack : str + stack to which world coordinates are mapped + points : dict + local points to map to world + subprocess_mode : str + subprocess mode used when calling client side java + render : :class:`renderapi.render.Render` + render connect object + + Returns + ------- + :obj:`list` of :obj:`list` + dictionaries defining local coordinates + and tileIds corresponding to world point + """ + raise NotImplementedError('Whoops.') + + +def _defaultval(v, default=None): + return default if v is None else v + + +@renderclientaccess +def materialize_tilespec_image( + tilespec, out_fn=None, height=None, width=None, + x=None, y=None, res=32, + subprocess_mode=None, + client_script=None, memGB=None, + render=None, **kwargs): + tspecfile = renderdump_temp([tilespec]) + + x = _defaultval(x, tilespec.minX) + y = _defaultval(y, tilespec.minY) + width = _defaultval(width, int(float((tilespec.maxX - tilespec.minX)))) + height = _defaultval(height, int(float((tilespec.maxY - tilespec.minY)))) + renderClient(tile_spec_url=tspecfile, out_fn=out_fn, + height=height, width=width, x=x, y=y, res=res, + subprocess_mode=subprocess_mode, + client_script=client_script, memGB=memGB, **kwargs) + + os.remove(tspecfile) + + +def render_tilespec(*args, **kwargs): + with tempfile.NamedTemporaryFile(suffix='.tif') as f: + materialize_tilespec_image(*args, out_fn=f.name, **kwargs) + arr = numpy.array(Image.open(f.name)) + return arr + + +@renderclientaccess +def materialize_renderparameters_image( + obj, out_fn=None, subprocess_mode=None, client_script=None, memGB=None, + render=None, **kwargs): + tfile = renderdump_temp(obj) + rendererClient(parameters_url=tfile, out_fn=out_fn, + subprocess_mode=subprocess_mode, + client_script=client_script, memGB=memGB, **kwargs) + os.remove(tfile) + + +def render_renderparameters(*args, **kwargs): + # NOTE this is python2 compatible hack for keyword-only args + image_ext = kwargs.get('image_ext', '.png') + with tempfile.NamedTemporaryFile(suffix=image_ext) as f: + materialize_renderparameters_image(*args, out_fn=f.name, **kwargs) + arr = numpy.array(Image.open(f.name)) + return arr + + +__all__ = [ + "import_single_json_file", + "import_jsonfiles_and_transforms_parallel_by_z", + "import_jsonfiles_parallel", "import_jsonfiles", + "import_jsonfiles_validate_client", "import_tilespecs", + "import_tilespecs_parallel", "local_to_world_array", + "world_to_local_array", "WithPool", + "render_tilespec", "materialize_tilespec_image", + "materialize_renderparameters_image", "render_renderparameters"] diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py new file mode 100644 index 00000000..60bb4f93 --- /dev/null +++ b/renderapi/client/client_calls.py @@ -0,0 +1,825 @@ +import logging +import json +import os +import re +import subprocess +import tempfile + +from renderapi.utils import NullHandler +from renderapi.errors import ClientScriptError +from renderapi.render import (format_baseurl, format_preamble, + RenderClient, Render) +from renderapi.stack import make_stack_params, set_stack_state + +from .utils import renderclientaccess +from .params import ArgumentParameters, SiftPointMatchOptions + + +# setup logger +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +def run_subprocess_mode(args, subprocess_mode=None, **kwargs): + subprocess_options = ['bufsize', 'executable', 'stdin', 'stdout', + 'stderr', 'preexec_fn', 'close_fds', 'shell', + 'cwd', 'env', 'universal_newlines', 'startupinfo', + 'creationflags'] + subprocess_kwargs = {k: v for k, v in kwargs.items() + if k in subprocess_options} + subprocess_modes = {'call': subprocess.call, + 'check_call': subprocess.check_call, + 'check_output': subprocess.check_output, + None: subprocess.check_call} + if subprocess_mode not in subprocess_modes: + logger.warning( + 'Unknown subprocess mode {} specified -- ' + 'using default subprocess.check_call'.format(subprocess_mode)) + sub_mode = subprocess_modes.get(subprocess_mode, subprocess.check_call) + return sub_mode(args, **subprocess_kwargs) + + +def call_run_ws_client(className, add_args=[], renderclient=None, + memGB=None, client_script=None, + **kwargs): + """simple call for run_ws_client.sh -- all arguments set in add_args + + Parameters + ---------- + className : str + Render java client classname to call as first argv for + Render's call_run_ws_client.sh wrapper script + add_args : :obj:`list` of :obj:`str`, optional + command line arguments + renderclient : :class:`renderapi.render.RenderClient`, optional + render client connection object + memGB : str, optional + GB memory for this java process + (defaults to '1G' or value defined in renderclient) + client_script : str, optional + client script to be used as the Render library's call_run_ws_client.sh + wrapper script (this option overrides value in renderclient) + subprocess_mode: str, optional + subprocess mode 'call', 'check_call', 'check_output' (default 'call') + + + Returns + ------- + obj + result of subprocess_mode call + """ + logger.debug('call_run_ws_client -- classname:{} add_args:{} ' + 'client_script:{} memGB:{}'.format( + className, add_args, client_script, memGB)) + + if renderclient is not None: + if isinstance(renderclient, RenderClient): + return call_run_ws_client(className, add_args=add_args, + **renderclient.make_kwargs( + memGB=memGB, + client_script=client_script, + **kwargs)) + if memGB is None: + logger.warning('call_run_ws_client requires memory specification -- ' + 'defaulting to 1G') + memGB = '1G' + args = list(map(str, [client_script, memGB, className] + add_args)) + try: + ret_val = run_subprocess_mode(args, **kwargs) + except subprocess.CalledProcessError: + raise ClientScriptError('client_script call {} failed'.format(args)) + + return ret_val + + +def get_param(var, flag): + return ([flag, var] if var is not None else []) + + +@renderclientaccess +def importJsonClient(stack, tileFiles=None, transformFile=None, + subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + """run ImportJsonClient.java + see render documentation (add link here) + + Parameters + ---------- + stack : str + stack to which tilespecs in tileFiles will be imported + tileFiles : :obj:`list` of :obj:`str` + json files containing tilespecs to import + transformFile : str, optional + json file containing transform specs which are + referenced by tilespecs in tileFiles + render : :class:`renderapi.render.Render` + render connection object + """ + argvs = (make_stack_params(host, port, owner, project, stack) + + (['--transformFile', transformFile] if transformFile else []) + + (tileFiles if isinstance(tileFiles, list) + else [tileFiles])) + call_run_ws_client('org.janelia.render.client.ImportJsonClient', + add_args=argvs, subprocess_mode=subprocess_mode, + client_script=client_script, memGB=memGB, **kwargs) + + +@renderclientaccess +def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, + baseowner=None, baseproject=None, basestack=None, + xyNeighborFactor=None, zNeighborDistance=None, + excludeCornerNeighbors=None, + excludeCompletelyObscuredTiles=None, + excludeSameLayerNeighbors=None, + excludeSameSectionNeighbors=None, + excludePairsInMatchCollection=None, + minx=None, maxx=None, miny=None, maxy=None, + useRowColPositions=None, existingMatchOwner=None, + minExistingMatchCount=None, onlyIncludeTilesFromStack=None, + onlyIncludeTilesNearTileIdsJson=None, maxPairsPerFile=None, + return_jsondata=True, + return_jsonfiles=False, + subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + """run TilePairClient.java + see render documentation (#add link here) + + This client selects a set of tiles 'p' based on its position in + a stack and then searches for nearby 'q' tiles using geometric parameters + + Parameters + ---------- + stack : str + stack from which tilepairs should be considered + minz : str + minimum z bound from which tile 'p' is selected + maxz : str + maximum z bound from which tile 'p' is selected + outjson : str or None + json to which tile pair file should be written + (defaults to using temporary file and deleting after completion) + delete_json : bool + whether to delete outjson on function exit (True if outjson is None) + baseowner : str + owner of stack from which stack was derived + baseproject : str + project of stack from which stack was derived + basestack : str + stack from which stack was derived + xyNeighborFactor : float + factor to multiply by max(width, height) of tile 'p' in order + to generate search radius in z (0.9 if None) + zNeighborDistance : int + number of z sections defining the half-height of search cylinder + for tile 'p' (2 if None) + excludeCornerNeighbors : bool + whether to exclude potential 'q' tiles based on center points + falling outside search (True if None) + excludeCompletelyObscuredTiles : bool + whether to exclude potential 'q' tiles that are obscured by other tiles + based on Render's sorting (True if None) + excludeSameLayerNeighbors : bool + whether to exclude potential 'q' tiles in the same z layer as 'p' + excludeSameSectionNeighbors : bool + whether to exclude potential 'q' tiles with the same sectionId as 'p' + excludePairsInMatchCollection : str + a matchCollection whose 'p' and 'q' pairs will be ignored + if generated using this client + minx : float + minimum x bound from which tile 'p' is selected + maxx : float + maximum x bound from wich tile 'p' is selected + miny : float + minimum y bound from which tile 'p' is selected + maxy : float + maximum y bound from wich tile 'p' is selected + useRowColPositions : bool + whether to use raster positions for neighbor analysis rather + than radial cutoff + existingMatchOwner : str + owner for the excludePairsInMatchCollection collection + minExistingMatchCount : int + minimum match threshold to exclude pairs + present in excludePairsInMatchCollection collection + onlyIncludeTilesFromStack : str + only include tiles which exist in this stack + onlyIncludeTilesNearTileIdsJson : str + path to json listing tileIds which should be paired + maxPairsPerFile : int + maximum neighborPairs per file. Generating more pairs than + this number (default 100000) will generate additional json + files with a p[0-9]+ suffix on the root. + + Returns + ------- + :obj:`list` of :obj:`dict`, optional + list of tilepairs + :obj:`list` of string, optional + list of json files containing tilepairs + """ + if outjson is None: + with tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) as f: + outjson = f.name + delete_json = True + + argvs = (make_stack_params(host, port, owner, project, stack) + + get_param(baseowner, '--baseOwner') + + get_param(baseproject, '--baseProject') + + get_param(basestack, '--baseStack') + + ['--minZ', minz, '--maxZ', maxz] + + get_param(xyNeighborFactor, '--xyNeighborFactor') + + get_param(zNeighborDistance, '--zNeighborDistance') + + get_param(excludeCornerNeighbors, '--excludeCornerNeighbors') + + get_param(excludeCompletelyObscuredTiles, + '--excludeCompletelyObscuredTiles') + + get_param(excludeSameLayerNeighbors, + '--excludeSameLayerNeighbors') + + get_param(excludeSameSectionNeighbors, + '--excludeSameSectionNeighbors') + + get_param(excludePairsInMatchCollection, + '--excludePairsInMatchCollection') + + (['--useRowColPositions'] if useRowColPositions else []) + + get_param(existingMatchOwner, + '--existingMatchOwner') + + get_param(minExistingMatchCount, + '--minExistingMatchCount') + + get_param(onlyIncludeTilesFromStack, + "--onlyIncludeTilesFromStack") + + get_param(onlyIncludeTilesNearTileIdsJson, + '--onlyIncludeTilesNearTileIdsJson') + + get_param(maxPairsPerFile, + '--maxPairsPerFile') + + ['--toJson', outjson] + + get_param(minx, '--minX') + get_param(maxx, '--maxX') + + get_param(miny, '--minY') + get_param(maxy, '--maxY')) + + call_run_ws_client('org.janelia.render.client.TilePairClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, + add_args=argvs, **kwargs) + + # We create the jsonfile, so if it is empty it could be multiple + if not os.path.isfile(outjson) or os.stat(outjson).st_size == 0: + if os.path.isfile(outjson): + # outjson we created is empty, so remove it + os.remove(outjson) + + outbn_root, outbn_ext = os.path.splitext(os.path.basename(outjson)) + outbn_pattern = "{root}_p[0-9]+{ext}".format( + root=outbn_root, ext=outbn_ext) + jsonfiles = [ + os.path.join(os.path.dirname(outjson), bn) + for bn in os.listdir(os.path.dirname(outjson)) + if re.match(outbn_pattern, bn)] + else: + jsonfiles = [outjson] + + if return_jsondata: + jsondata_list = [] + for jsonfile in jsonfiles: + with open(jsonfile, 'r') as f: + jsondata_list.append(json.load(f)) + if len({d["renderParametersUrlTemplate"] for d in jsondata_list}) != 1: # pragma: no cover + raise ValueError( + "Found tilepair files with disparate " + "renderParametersUrlTemplate values. " + "Maybe there are additional files " + "matching the outjson pattern?") + pairdata = [i for l in (d["neighborPairs"] for d in jsondata_list) + for i in l] + jsondata = dict(jsondata_list[0], **{"neighborPairs": pairdata}) + + if delete_json: + for jsonfile in jsonfiles: + os.remove(jsonfile) + + return ((jsondata, jsonfiles) if return_jsonfiles and return_jsondata + else jsondata if return_jsondata + else jsonfiles if return_jsonfiles + else None) + + +@renderclientaccess +def importTransformChangesClient(stack, targetStack, transformFile, + targetOwner=None, targetProject=None, + changeMode=None, close_stack=True, + subprocess_mode=None, + host=None, port=None, owner=None, + project=None, client_script=None, memGB=None, + render=None, **kwargs): + """run ImportTransformChangesClient.java + + Parameters + ---------- + stack : str + stack from which tiles will be transformed + targetStack : str + stack that will hold results of transforms + transformFile : str + locaiton of json file in format defined below + :: + [{{"tileId": , + "transform": }}, + {{"tileId": ...}}, + ... + ] + targetOwner : str + owner of target stack + targetProject : str + project of target stack + changeMode : str + method to apply transform to tiles. Options are: + 'APPEND' -- add transform to tilespec's list of transforms + 'REPLACE_LAST' -- change last transform in tilespec's + list of transforms to match transform + 'REPLACE_ALL' -- overwrite tilespec's transforms field to match + transform + + Raises + ------ + ClientScriptError + if changeMode is not valid + """ + if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: + raise ClientScriptError( + 'changeMode {} is not valid!'.format(changeMode)) + argvs = (make_stack_params(host, port, owner, project, stack) + + ['--targetStack', targetStack] + + ['--transformFile', transformFile] + + get_param(targetOwner, '--targetOwner') + + get_param(targetProject, '--targetProject') + + get_param(changeMode, '--changeMode')) + call_run_ws_client( + 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, + client_script=client_script, subprocess_mode=subprocess_mode, + add_args=argvs, **kwargs) + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderclientaccess +def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, + numberOfThreads=None, subprocess_mode=None, + host=None, port=None, owner=None, + project=None, client_script=None, memGB=None, + render=None, **kwargs): + """run CoordinateClient.java + + map coordinates between local and world systems + + Parameters + ---------- + stack : str + stack representing the world coordinates + z : str + z value of the section containing the tiles to map + fromJson : str + input json file in format defined by + list of coordinate dictionaries (for world to local) + or list of list of coordinate dictionaries (local to world) + toJson : str + json to save results of mapping coordinates + localToWorld : bool + whether to transform form local to world coordinates (False if None) + numberOfThreads : int + number of threads for java process (1 if None) + + Returns + ------- + :obj:`list` of :obj:`dict` for local to world or :obj:`list` of :obj:`list` of :obj:`dict` for world to local + list representing mapped coordinates + """ # noqa: E501 + argvs = (make_stack_params(host, port, owner, project, stack) + + ['--z', z, '--fromJson', fromJson, '--toJson', toJson] + + (['--localToWorld'] if localToWorld else []) + + get_param(numberOfThreads, '--numberOfThreads')) + call_run_ws_client('org.janelia.render.client.CoordinateClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + with open(toJson, 'r') as f: + jsondata = json.load(f) + + return jsondata + + +@renderclientaccess +def renderSectionClient(stack, rootDirectory, zs, scale=None, + maxIntensity=None, minIntensity=None, bounds=None, + format=None, channel=None, customOutputFolder=None, + customSubFolder=None, padFileNamesWithZeros=None, + resolutionUnit=None, doFilter=None, fillWithNoise=None, + convertToGray=None, subprocess_mode=None, host=None, + port=None, owner=None, project=None, + client_script=None, memGB=None, render=None, + **kwargs): + """run RenderSectionClient.java + + Parameters + ---------- + stack : str + stack to which zs to render belong + rootDirectory : str + directory to which rendered sections should be generated + zs : :obj:`list` of :obj:`str` + z indices of sections to render + scale : float + factor by which section image should be scaled + (this materialization is 32-bit limited) + maxIntensity : int + value todisplay as white on a linear colormap + minIntensity : int + value to display as black on a linear colormap + bounds: dict + dictionary with keys of minX maxX minY maxY + format : str + output image format in 'PNG', 'TIFF', 'JPEG' + channel : str + channel to render out (use on multichannel stack) + customOutputFolder : str + folder to save all images in (overrides default of sections_at_%scale) + customSubFolder : str + folder to save all images in under outputFolder (overrides default of none) + padFileNamesWithZeros: bool + whether to pad file names with zeros to make sortable + resolutionUnit: str + if format is tiff and unit is specified (e.g. as 'nm'), include resolution data + in rendered tiff headers. + convertToGray: str + string representing java boolean for whether to save output as 8bit uint + doFilter : str + string representing java boolean for whether to render image + with default filter (varies with render version) + fillWithNoise : str + string representing java boolean for whether to replace saturated + image values with uniform noise + + """ # noqa: E501 + if bounds is not None: + try: + if bounds['maxX'] < bounds['minX']: + raise ClientScriptError('maxX:{} is less than minX:{}'.format( + bounds['maxX'], bounds['minX'])) + if bounds['maxY'] < bounds['minY']: + raise ClientScriptError('maxY:{} is less than minY:{}'.format( + bounds['maxY'], bounds['minY'])) + bound_list = ','.join(map(lambda x: str(int(x)), + [bounds['minX'], bounds['maxX'], + bounds['minY'], bounds['maxY']])) + bound_param = ['--bounds', bound_list] + except KeyError as e: + raise ClientScriptError( + 'bounds does not contain correct keys {}. Missing {}'.format( + bounds, e)) + else: + bound_param = [] + + argvs = (make_stack_params(host, port, owner, project, stack) + + ['--rootDirectory', rootDirectory] + + get_param(scale, '--scale') + get_param(format, '--format') + + get_param(doFilter, '--doFilter') + + get_param(minIntensity, '--minIntensity') + + get_param(maxIntensity, '--maxIntensity') + + get_param(fillWithNoise, '--fillWithNoise') + + get_param(customOutputFolder, '--customOutputFolder') + + (['--convertToGray'] if convertToGray else []) + + get_param(channel, '--channels') + + get_param(customSubFolder, '--customSubFolder') + + get_param(padFileNamesWithZeros, '--padFileNamesWithZeros') + + get_param(resolutionUnit, '--resolutionUnit') + + bound_param + zs) + + call_run_ws_client('org.janelia.render.client.RenderSectionClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + +@renderclientaccess +def transformSectionClient(stack, transformId, transformClass, transformData, + zValues, targetProject=None, targetStack=None, + replaceLast=None, subprocess_mode=None, + host=None, port=None, + owner=None, project=None, client_script=None, + memGB=None, render=None, **kwargs): + """run TranformSectionClient.java + + Parameters + ---------- + stack : str + stack containing section to transform + transformId : str + unique transform identifier + transformClass : str + transform className defined by the java mpicbg library + transformData : str + mpicbg datastring delimited by "," instead of " " + zValues : list + z values to which transform should be applied + targetProject : str, optional + project to which transformed sections should be added + targetStack : str, optional + stack to which transformed sections should be added + replaceLast : bool, optional + whether to replace the last transform in the section + with this transform + + """ + argvs = (make_stack_params(host, port, owner, project, stack) + + (['--replaceLast'] if replaceLast else []) + + get_param(targetProject, '--targetProject') + + get_param(targetStack, '--targetStack') + + ['--transformId', transformId, '--transformClass', transformClass, + '--transformData', transformData] + zValues) + call_run_ws_client('org.janelia.render.client.TransformSectionClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + +@renderclientaccess +def get_canvas_url_template( + stack, filter=False, renderWithoutMask=False, + normalizeForMatching=True, excludeTransformsAfterLast=None, + excludeFirstTransformAndAllAfter=None, excludeAllTransforms=False, + channels=None, + host=None, port=None, owner=None, project=None, client_script=None, + render=None, **kwargs): + """function for making a render-parameters url template for point matching + + Parameters + ---------- + stack: str + render stack name + filter: bool + whether to apply default filtering to tile (default=False) + renderWithoutMask: bool + whether to exclude the mask when rendering tile (default=False) + normalizeForMatching: bool + whether to apply traditional 'normalizeForMatching' transform manipulation to image + this removes the last transform from the transformList, then if there are more than 3 transforms + continues to remove transforms until there are exactly 3. Then assumes the image will be near 0,0 + with a width/height that is about equal to the raw image width/height. This is true for Janelia's + conventions for transformation alignment, but use at your own risk. (default=True) + excludeTransformsAfterLast: str or None + alternative to normalizeForMatching, which uses transformLabels. Will remove all transformations + after the last transformation with this transform label. i.e. if all lens corrections have a 'lens' + label. Then this will remove all non-lens transformations from the list. + This is more general than normalizeForMatching=true, but requires you have transform labels applied. + default = None + excludeFirstTransformAndAllAfter: str + alternative to normalizeForMatching which finds the first transform in the list with a given label + and then removes that transform and all transforms that follow it. i.e. if you had a compound list + of transformations, and you had labelled the first non-local transform 'montage' then setting + excludeFirstTransformAndAllAfter='montage' would remove that montage transform and any other + transforms that you had applied after it. default= None. + excludeAllTransforms: bool + alternative to normalizeForMatching which simply removes all transforms from the list. + default=False + channels: str + list of channels and weights to render in the format [channel name], [channel name]__[weight] or + channel one name]__[weight]__ ... [channel n name]__[weight]. + e.g., "DAPI" or "DAPI__0.9__TdTomato__0.1" + default = None + """ # noqa: E501 + request_url = format_preamble(host, port, owner, project, stack) + tile_base_url = request_url + "/tile" + url_suffix = "render-parameters" + if filter: + url_suffix += '?filter=true' + else: + url_suffix += '?filter=false' + + if normalizeForMatching: + url_suffix += '&normalizeForMatching=true' + else: + url_suffix += '&normalizeForMatching=false' + + if renderWithoutMask: + url_suffix += '&renderWithoutMask=true' + else: + url_suffix += '&renderWithoutMask=false' + + if excludeTransformsAfterLast is not None: + url_suffix += '&excludeTransformsAfterLast={}'.format( + excludeTransformsAfterLast) + if excludeFirstTransformAndAllAfter is not None: + url_suffix += '&excludeFirstTransformAndAllAfter={}'.format( + excludeFirstTransformAndAllAfter) + if excludeAllTransforms: + url_suffix += '&excludeAllTransforms=true' + if channels: + url_suffix += '&channels={}'.format(channels) + + canvas_url_template = "%s/{}/%s" % (tile_base_url, + url_suffix) + return canvas_url_template + + +@renderclientaccess +def pointMatchClient(stack, collection, tile_pairs, + stack2=None, + sift_options=None, + pointMatchRender=None, + debugDirectory=None, + filter=False, + renderWithoutMask=False, normalizeForMatching=True, + excludeTransformsAfterLast=None, + excludeAllTransforms=None, + excludeFirstTransformAndAllAfter=None, + stackChannels=None, + stack2Channels=None, + subprocess_mode=None, + host=None, port=None, + owner=None, project=None, client_script=None, + memGB=None, render=None, **kwargs): + """run SiftPointMatchClient.java + + Parameters + ---------- + stack : str + stack containing the tiles + stack2 : str + second optional stack containing tiles (if stack2 is not none, then tile_pair['p'] comes from stack and tile_pair['q'] comes from stack2) + collection : str + point match collection to save results into + tile_pairs : iterable + list of iterables of length 2 containing tileIds to calculate point matches between + sift_options: SiftOptions + options for running point matching + pointMatchRender : renderapi.render.renderaccess + renderaccess object specifying the render server to store point matches in + defaults to values specified by render and its keyword argument overrides + debugDirectory : str + directory to store debug results (optional) + filter: bool + whether to apply default filtering to tile (default=False) + renderWithoutMask: bool + whether to exclude the mask when rendering tile (default=False) + normalizeForMatching: bool + whether to apply traditional 'normalizeForMatching' transform manipulation to image + this removes the last transform from the transformList, then if there are more than 3 transforms + continues to remove transforms until there are exactly 3. Then assumes the image will be near 0,0 + with a width/height that is about equal to the raw image width/height. This is true for Janelia's + conventions for transformation alignment, but use at your own risk. (default=True) + excludeTransformsAfterLast: str or None + alternative to normalizeForMatching, which uses transformLabels. Will remove all transformations + after the last transformation with this transform label. i.e. if all lens corrections have a 'lens' + label. Then this will remove all non-lens transformations from the list. + This is more general than normalizeForMatching=true, but requires you have transform labels applied. + default = None + excludeFirstTransformAndAllAfter: str + alternative to normalizeForMatching which finds the first transform in the list with a given label + and then removes that transform and all transforms that follow it. i.e. if you had a compound list + of transformations, and you had labelled the first non-local transform 'montage' then setting + excludeFirstTransformAndAllAfter='montage' would remove that montage transform and any other + transforms that you had applied after it. default= None. + excludeAllTransforms: bool + alternative to normalizeForMatching which simply removes all transforms from the list. + default = False + stackChannels: str or None + If specified, option to select which channel is used for the stack. + default = None + stack2Channels: str or None + If specified, option to select which channel is used for stack2, if specified. + default = None + + """ # noqa: E501 + sift_options = (SiftPointMatchOptions(**kwargs) if sift_options is None + else sift_options) + + if pointMatchRender is None: + pointMatchRender = Render(host, port, owner, project, client_script) + + baseDataUrl = format_baseurl(pointMatchRender.DEFAULT_KWARGS['host'], + pointMatchRender.DEFAULT_KWARGS['port']) + argvs = [] + argvs += ['--baseDataUrl', baseDataUrl] + argvs += ['--owner', pointMatchRender.DEFAULT_KWARGS['owner']] + argvs += ['--collection', collection] + if debugDirectory is not None: + argvs += ['--debugDirectory', debugDirectory] + argvs += sift_options.to_java_args() + + canvas_url_template = get_canvas_url_template( + stack, + filter, + renderWithoutMask, + normalizeForMatching, + excludeTransformsAfterLast, + excludeFirstTransformAndAllAfter, + excludeAllTransforms, + channels=stackChannels, + host=host, + port=port, + owner=owner, + project=project, + client_script=client_script) + + if stack2 is not None: + canvas_url_template2 = get_canvas_url_template( + stack2, + filter, + renderWithoutMask, + normalizeForMatching, + excludeTransformsAfterLast, + excludeFirstTransformAndAllAfter, + excludeAllTransforms, + channels=stack2Channels, + host=host, + port=port, + owner=owner, + project=project, + client_script=client_script) + else: + canvas_url_template2 = canvas_url_template + for tile1, tile2 in tile_pairs: + argvs += [canvas_url_template.format(tile1), + canvas_url_template2.format(tile2)] + + call_run_ws_client('org.janelia.render.client.PointMatchClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + +@renderclientaccess +def renderClient(tile_spec_url=None, height=None, width=None, in_fn=None, + out_fn=None, x=None, y=None, res=None, + subprocess_mode=None, client_script=None, + memGB=None, render=None, **kwargs): + """call render + """ + get_cmd_opt = ArgumentParameters.get_cmd_opt + + argvs = (get_cmd_opt(tile_spec_url, '--tile_spec_url') + + get_cmd_opt(height, '--height') + + get_cmd_opt(width, '--width') + + get_cmd_opt(in_fn, '--in') + + get_cmd_opt(out_fn, '--out') + + get_cmd_opt(x, '--x') + + get_cmd_opt(y, '--y') + + get_cmd_opt(res, '--res')) + + call_run_ws_client('org.janelia.alignment.Render', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + +@renderclientaccess +def rendererClient(tile_spec_url=None, meshCellSize=None, minMeshCellSize=None, + in_fn=None, out_fn=None, x=None, y=None, width=None, + height=None, scale=None, area_offset=None, + minIntensity=None, maxIntensity=None, gray=None, + quality=None, threads=None, skip_interpolation=None, + binary_mask=None, exclude_mask=None, parameters_url=None, + do_filter=None, background_color=None, fill_with_noise=None, + channels=None, subprocess_mode=None, client_script=None, + memGB=None, render=None, **kwargs): + get_cmd_opt = ArgumentParameters.get_cmd_opt + + argvs = (get_cmd_opt(tile_spec_url, '--tile_spec_url') + + get_cmd_opt(height, '--height') + + get_cmd_opt(width, '--width') + + get_cmd_opt(in_fn, '--in') + + get_cmd_opt(out_fn, '--out') + + get_cmd_opt(x, '--x') + + get_cmd_opt(y, '--y') + + get_cmd_opt(meshCellSize, '--meshCellSize') + + get_cmd_opt(minMeshCellSize, '--minMeshCellSize') + + get_cmd_opt(scale, '--scale') + + get_cmd_opt(area_offset, '--area_offset') + + get_cmd_opt(minIntensity, '--minIntensity') + + get_cmd_opt(maxIntensity, '--maxIntensity') + + get_cmd_opt(gray, '--gray') + + get_cmd_opt(quality, '--quality') + + get_cmd_opt(threads, '--threads') + + get_cmd_opt(skip_interpolation, '--skip_interpolation') + + get_cmd_opt(binary_mask, '--binary_mask') + + get_cmd_opt(exclude_mask, '--exclude_mask') + + get_cmd_opt(parameters_url, '--parameters_url') + + get_cmd_opt(do_filter, '--do_filter') + + get_cmd_opt(background_color, '--background_color') + + get_cmd_opt(fill_with_noise, '--fill_with_noise') + + get_cmd_opt(channels, '--channels')) + + call_run_ws_client('org.janelia.alignment.Render', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + +__all__ = [ + "call_run_ws_client", "run_subprocess_mode", + "importJsonClient", "tilePairClient", + "importTransformChangesClient", "coordinateClient", + "renderSectionClient", "transformSectionClient", + "get_canvas_url_template", "pointMatchClient", "renderClient", + "rendererClient"] diff --git a/renderapi/client/params.py b/renderapi/client/params.py new file mode 100644 index 00000000..ce2d6b64 --- /dev/null +++ b/renderapi/client/params.py @@ -0,0 +1,73 @@ +from renderapi.errors import ClientScriptError + + +class ArgumentParameters(object): + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def sanitize_cmd(cmd): + def jbool_str(c): + return str(c) if type(c) is not bool else "true" if c else "false" + if any([i is None for i in cmd]): + raise ClientScriptError( + 'missing argument in command "{}"'.format(map(str, cmd))) + return map(jbool_str, cmd) + + @staticmethod + def get_cmd_opt(v, flag=None): + return [] if v is None else [v] if flag is None else [flag, v] + + @staticmethod + def get_flag_cmd(v, flag=None): + # for arity 0 + return [flag] if v else [] + + def to_java_args(self): + args = [] + for key, value in self.__dict__.items(): + if (value is not None) and not (key == 'kwargs'): + args += self.get_cmd_opt(value, "--{}".format(key)) + return self.sanitize_cmd(args) + + +class FeatureExtractionParameters(ArgumentParameters): + def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, + SIFTminScale=None, SIFTsteps=None, **kwargs): + super(FeatureExtractionParameters, self).__init__(**kwargs) + self.SIFTfdSize = SIFTfdSize + self.SIFTmaxScale = SIFTmaxScale + self.SIFTminScale = SIFTminScale + self.SIFTsteps = SIFTsteps + + +class MatchDerivationParameters(ArgumentParameters): + def __init__(self, matchIterations=None, + matchMaxEpsilon=None, matchMaxNumInliers=None, + matchMaxTrust=None, matchMinInlierRatio=None, + matchMinNumInliers=None, + matchModelType=None, matchRod=None, **kwargs): + super(MatchDerivationParameters, self).__init__(**kwargs) + self.matchIterations = matchIterations + self.matchMaxEpsilon = matchMaxEpsilon + self.matchMaxNumInliers = matchMaxNumInliers + self.matchMaxTrust = matchMaxTrust + self.matchMinInlierRatio = matchMinInlierRatio + self.matchMinNumInliers = matchMinNumInliers + self.matchMinNumInliers = matchMinNumInliers + self.matchModelType = matchModelType + self.matchRod = matchRod + + +class SiftPointMatchOptions(MatchDerivationParameters, + FeatureExtractionParameters): + def __init__(self, renderScale=None, fillWithNoise=None, **kwargs): + # TODO add missing parameters + super(SiftPointMatchOptions, self).__init__(**kwargs) + self.renderScale = renderScale + self.fillWithNoise = fillWithNoise + + +__all__ = [ + "ArgumentParameters", "FeatureExtractionParameters", + "MatchDerivationParameters", "SiftPointMatchOptions"] diff --git a/renderapi/client/utils.py b/renderapi/client/utils.py new file mode 100644 index 00000000..e2985a10 --- /dev/null +++ b/renderapi/client/utils.py @@ -0,0 +1,65 @@ +from decorator import decorator +import os + +from renderapi.errors import ClientScriptError +from renderapi.utils import fitargspec +from renderapi.render import RenderClient, Render + + +@decorator +def renderclientaccess(f, *args, **kwargs): + """Decorator allowing functions asking for host, port, owner, project, + client_script to default to a connection defined by :class:`RenderClient` + object using its :func:`RenderClient.make_kwargs` method. + Will also attempt to derive a :class:`RenderClient` from an input + :class:`Render` object and fail if client scripts cannot be reached. + + Parameters + ---------- + f : func + function to decorate + Returns + ------- + obj + output of decorated function + """ + args, kwargs = fitargspec(f, args, kwargs) + render = kwargs.get('render') + if render is not None: + if not isinstance(render, RenderClient): + if isinstance(render, Render): + render = RenderClient(**render.make_kwargs(**kwargs)) + else: + raise ValueError( + 'invalid RenderClient object type {} specified!'.format( + type(render))) + return f(*args, **render.make_kwargs(**kwargs)) + else: + try: + client_script = kwargs.get('client_script') + cs_valid = os.path.isfile(client_script) + except TypeError: + try: + client_scripts = kwargs.get('client_scripts') + if os.path.isdir(client_scripts): + client_script = os.path.join(client_scripts, + 'run_ws_client.sh') + cs_valid = os.path.isfile(client_script) + else: + raise ClientScriptError( + 'invalid client_scripts directory {}'.format( + client_scripts)) + except TypeError: + raise ClientScriptError( + 'No client script information specified: ' + 'client_scripts={} client_script={}'.format( + kwargs.get('client_scripts'), + kwargs.get('client_script'))) + if not cs_valid: + # TODO should also check for executability + raise ClientScriptError( + 'invalid client script: {} not a file'.format(client_script)) + return f(*args, **kwargs) + + +__all__ = ["renderclientaccess", "ClientScriptError"] diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py new file mode 100644 index 00000000..beeeced6 --- /dev/null +++ b/renderapi/coordinate.py @@ -0,0 +1,610 @@ +#!/usr/bin/env python +''' +coordinate mapping functions for render api +''' +from .render import format_preamble, renderaccess +from .utils import NullHandler, renderdumps, renderdump, get_json +from .client import coordinateClient +from .errors import RenderError +import json +import numpy as np +import logging +import tempfile +import os + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +@renderaccess +def world_to_local_coordinates(stack, z, x, y, host=None, + port=None, owner=None, project=None, + session=None, + render=None, **kwargs): + """maps an world x,y,z coordinate in stack to a local coordinate + Parameters + ---------- + stack : str + render stack to map coordinates through + z : float + z coordinate to map + x : float + x coordinate to map + y : float + y coordinate to map + session : requests.session.Session + session object used in request + render : renderapi.render.Render + render connect object + Returns + ------- + json + list of dictionaries of local coordinates following this pattern + :: + + [ + { + "tileId": "string", + "visible": false, + "local": [ + [0,0], + [1,0]... + ], + "error": "string" + } + ] + """ + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) + return get_json(session, request_url) + + +@renderaccess +def local_to_world_coordinates(stack, tileId, x, y, + host=None, port=None, owner=None, project=None, + session=None, + render=None, **kwargs): + """convert coordinate from local to world with webservice request + + Parameters + ---------- + stack : str + render stack to map coordinates through + z : float + z coordinate to map + x : float + x coordinate to map + y : float + y coordinate to map + session : requests.session.Session + session object used in request + render : renderapi.render.Render + render connect object + + Returns + ------- + dict + dictionary of world coordinates following this pattern + :: + + { + "tileId": "string", + "visible": false, + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + } + + """ + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) + return get_json(session, request_url) + + +@renderaccess +def world_to_local_coordinates_batch(stack, d, z, host=None, + port=None, owner=None, project=None, + execute_local=False, + session=None, + render=None, **kwargs): + + """convert coordinate parameters from world to local + + Parameters + ---------- + stack : str + stack to map coordinates + d : list[dict] + list of dictionary of world coordinates to map following this schema + :: + + [ { + "tileId": "string", + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + z : float + z coordinate to map + execute_local : boolean + (Default value = False) + session : requests.session.Session + session object used in request + render : renderapi.render.Render + render connect object + + Returns + ------- + list[list[dict]] + list of lists of dictionaries containing local positions + that overlap with this point, (one world point may map + to multiple local points) following.. + :: + + [[ { + "tileId": "string", + "visible": True,False, + "local": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + ] + + """ + if (execute_local is True): + raise NotImplementedError("local execution not yet implemented") + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%s/world-to-local-coordinates" % (str(z)) + r = session.put(request_url, data=renderdumps(d), + headers={"content-type": "application/json"}) + return r.json() + + +@renderaccess +def local_to_world_coordinates_batch(stack, d, z, host=None, + port=None, owner=None, project=None, + session=None, + render=None, **kwargs): + """convert coordinate parameters from local to world + + Parameters + ---------- + stack : str + + d : list[dict] + list of dictionary of local coordinates to map + :: + [ { + "tileId": "string", + "local": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + + z : float + z coordinate to map from + session : + (Default value = requests.session() + render : renderapi.render.Render + render connect object + + Returns + ------- + list[dict] + list of dictionaries containing world coordinates + + :: + + [ { + "tileId": "string", + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + """ + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%s/local-to-world-coordinates" % (str(z)) + r = session.put(request_url, data=renderdumps(d), + headers={"content-type": "application/json"}) + try: + return r.json() + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError(r.text) + + +def package_point_match_data_into_json(dataarray, tileId, + local_or_world='local'): + """Convert a set of points defined by a numpy array and a tileId to a json + for use in the renderapi + + Parameters + ---------- + dataarray : numpy.array + a Nx2 array of points + + tileId : str + a tileId to package them into + local_or_world : + whether this should be represented as a local or world coordinate + (Default value = 'local') + + Returns + ------- + dict + dictionary representation of those points and tileId + following + + :: + + { + "tileId": "string", + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + } + """ + dlist = [] + for i in range(dataarray.shape[0]): + d = {} + d['tileId'] = tileId + d[local_or_world] = [dataarray[i, 0], dataarray[i, 1]] + dlist.append(d) + return dlist + + +def unpackage_world_to_local_point_match_from_json(json_answer, tileId): + """Converts a dictionary answer from a world>local + coordinates call from a dictionary to numpy array format + + Parameters + ---------- + json_answer : list[dict] + json reponse from a world>local call (N long) + + tileId : str + tileId to extract, usually the world tileId passed in + + Returns + ------- + numpy.array + Nx2 array of local points + """ + answer = np.zeros((len(json_answer), 2)) + for i, local_answer in enumerate(json_answer): + coord = next(ans for ans in local_answer if ans['tileId'] == tileId) + c = coord['local'] + answer[i, 0] = c[0] + answer[i, 1] = c[1] + return answer + + +# @renderaccess +# def old_world_to_local_coordinates_array(stack, dataarray, tileId, z=0, +# host=None, port=None, +# owner=None, project=None, +# session=None, +# render=None, **kwargs): +# '''''' + +# request_url = format_preamble( +# host, port, owner, project, stack) + \ +# "/z/%d/world-to-local-coordinates" % (z) +# dlist = [] +# for i in range(dataarray.shape[0]): +# d = {} +# d['tileId'] = tileId +# d['world'] = [dataarray[i, 0], dataarray[i, 1]] +# dlist.append(d) +# jsondata = json.dumps(dlist) +# r = session.put(request_url, data=jsondata, +# headers={"content-type": "application/json"}) +# json_answer = r.json() +# try: +# answer = np.zeros(dataarray.shape) +# for i, coord in enumerate(json_answer): +# c = coord['local'] +# answer[i, 0] = c[0] +# answer[i, 1] = c[1] +# return answer +# except Exception as e: +# logger.error(e) +# logger.error(json_answer) + + +def unpackage_local_to_world_point_match_from_json(json_answer): + """converts a local>world call json response into a numpy array + + Parameters + ---------- + json_answer : list[dict] + response from a local>world call (N long) + + Returns + ------- + numpy.array + Nx2 numpy array of coordinates + """ + logger.debug("json_answer_length %d" % len(json_answer)) + answer = np.zeros((len(json_answer), 2)) + for i, coord in enumerate(json_answer): + c = coord['world'] + answer[i, 0] = c[0] + answer[i, 1] = c[1] + return answer + + +@renderaccess +def world_to_local_coordinates_array(stack, dataarray, tileId, z, + render=None, host=None, port=None, + owner=None, project=None, + client_script=None, + doClientSide=False, number_of_threads=20, + session=None, **kwargs): + """map world to local coordinates using numpy array + + Parameters + ---------- + stack : str + render stack to map + dataarray : numpy.array + Nx2 numpy array of points to world points to map + tileId : str + tileId to map from and to + z : float + z coordinate to map + render : renderapi.render.Render + render connect object + doClientSide : boolean + (Default value = False) + number_of_threads : int + (Default value = 20) + session : requests.session.Session + session object used in request + + Returns + ------- + numpy.array: + Nx2 numpy array of points in local coordinates + """ + jsondata = package_point_match_data_into_json(dataarray, tileId, 'world') + if doClientSide: + json_answer = world_to_local_coordinates_clientside( + stack, jsondata, z, host=host, port=port, owner=owner, + project=project, client_script=client_script, + number_of_threads=number_of_threads) + else: + json_answer = world_to_local_coordinates_batch( + stack, jsondata, z, host=host, port=port, owner=owner, + project=project, session=session) + return unpackage_world_to_local_point_match_from_json(json_answer, tileId) + + +# @renderaccess +# def old_local_to_world_coordinates_array(stack, dataarray, tileId, z=0, +# host=None, port=None, +# owner=None, project=None, +# session=None, +# render=None, **kwargs): +# '''''' +# request_url = format_preamble( +# host, port, owner, project, stack) + \ +# "/z/%d/local-to-world-coordinates" % (z) +# dlist = [] +# for i in range(dataarray.shape[0]): +# d = {} +# d['tileId'] = tileId +# d['local'] = [dataarray[i, 0], dataarray[i, 1]] +# dlist.append(d) +# jsondata = json.dumps(dlist) +# r = session.put(request_url, data=jsondata, +# headers={"content-type": "application/json"}) +# json_answer = r.json() +# try: +# answer = np.zeros(dataarray.shape) +# logger.debug('shape {}'.format(dataarray.shape)) +# logger.debug('length of json_answer {}'.format(len(json_answer))) +# for i, coord in enumerate(json_answer): +# c = coord['world'] +# answer[i, 0] = c[0] +# answer[i, 1] = c[1] +# return answer +# except Exception as e: +# logger.error(e) +# logger.error(json_answer) + + +@renderaccess +def local_to_world_coordinates_array(stack, dataarray, tileId, z, + render=None, host=None, port=None, + owner=None, project=None, + client_script=None, + doClientSide=False, number_of_threads=20, + session=None, **kwargs): + """map local to world coordinates using numpy array + + Parameters + ---------- + stack : str + render stack to map + dataarray : numpy.array + Nx2 array of points in local coordinates + tileId : str + tile to map points from + z : float + z position to map + render : renderapi.render.Render + render connect object + doClientSide : boolean + (Default value = False) + number_of_threads : int + (Default value = 20) + session : requests.session.Session + session object used in request + render : renderapi.render.Render + render connect object + + Returns + ------- + numpy.array + Nx2 numpy array in world coordinates + + """ + jsondata = package_point_match_data_into_json(dataarray, tileId, 'local') + if doClientSide: + json_answer = local_to_world_coordinates_clientside( + stack, [[lp] for lp in jsondata], z, host=host, port=port, + owner=owner, project=project, client_script=client_script, + number_of_threads=number_of_threads) + else: + json_answer = local_to_world_coordinates_batch( + stack, jsondata, z, host=host, port=port, owner=owner, + project=project, session=session) + return unpackage_local_to_world_point_match_from_json(json_answer) + + +def map_coordinates_clientside(stack, jsondata, z, host, port, owner, + project, client_script, isLocalToWorld=False, + store_injson=False, store_outjson=False, + number_of_threads=20, memGB='1G'): + """map coordinates using the java client library + + Parameters + ---------- + stack : str + stack to map + jsondata : dict + json dictionary to map following the pattern of local>world or world>local + z : float + z position to map + isLocalToWorld : boolean + whether transform is local to world (False implies world to local) + store_injson : boolean + whether to store input json file (created with tempfile) + store_outjson : boolean + whether to store output json file (created with tempfile) + number_of_threads : int + threads to execute clientside computation + render : renderapi.render.Render + render connect object + + Returns + ------- + json + json data as would be returned by client calls + of local>world or world>local + """ # noqa: E501 + # write point match json to temp file on disk + with tempfile.NamedTemporaryFile( + prefix='render_coordinates_in_', suffix='.json', + mode='w', delete=False) as f: + logger.debug('jsondata:{}'.format(jsondata)) + json_inpath = f.name + renderdump(jsondata, f) + + # get a temporary location for the output + with tempfile.NamedTemporaryFile( + prefix='render_coordinates_out_', suffix='.json', + delete=False) as f: + json_outpath = f.name + # call the java client + coordinateClient(stack, z, fromJson=json_inpath, toJson=json_outpath, + localToWorld=isLocalToWorld, + numberOfThreads=number_of_threads, + host=host, port=port, owner=owner, project=project, + client_script=client_script, memGB=memGB) + + # return the json results + with open(json_outpath, 'r') as f: + j = json.load(f) + if not store_injson: + os.remove(json_inpath) + if not store_outjson: + os.remove(json_outpath) + + return j + + +@renderaccess +def world_to_local_coordinates_clientside(stack, jsondata, z, + host=None, port=None, owner=None, + project=None, client_script=None, + number_of_threads=20, + render=None, **kwargs): + """map_coordinates_clientside for mapping world to local + + Parameters + ---------- + stack : str + render stack to map + jsondata : dict + world coordinates in dictionary format + z : float + z coordinate to map + number_of_threads : int + number of threads to use when doing parallelization + render : renderapi.render.Render + render connect object + + Returns + ------- + json + local coordinates in dictionary format + """ + + return map_coordinates_clientside(stack, jsondata, z, + host=host, port=port, owner=owner, + project=project, + client_script=client_script, + isLocalToWorld=False, + number_of_threads=number_of_threads) + + +@renderaccess +def local_to_world_coordinates_clientside(stack, jsondata, z, + host=None, port=None, owner=None, + project=None, client_script=None, + number_of_threads=20, + render=None, **kwargs): + """map_coordinates_clientside for mapping local to world + + Parameters + ---------- + stack : str + render stack to map + jsondata : list[dict] + local coordinates in dictionary format + z : float + z position to map + number_of_threads : int + threads for java client script to use during mapping + + Returns + ------- + dict + world coordinates in dictionary format + """ + return map_coordinates_clientside(stack, jsondata, z, + host=host, port=port, owner=owner, + project=project, + client_script=client_script, + isLocalToWorld=True, + number_of_threads=number_of_threads) diff --git a/renderapi/errors.py b/renderapi/errors.py new file mode 100644 index 00000000..e519793f --- /dev/null +++ b/renderapi/errors.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +''' +Custom errors for render api +''' + + +class RenderError(Exception): + pass + + +class ClientScriptError(RenderError): + pass + + +class ConversionError(RenderError): + pass + + +class EstimationError(RenderError): + pass + + +class SpecError(RenderError): + pass diff --git a/renderapi/external/__init__.py b/renderapi/external/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/renderapi/external/processpools/__init__.py b/renderapi/external/processpools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/renderapi/external/processpools/pool_pathos.py b/renderapi/external/processpools/pool_pathos.py new file mode 100644 index 00000000..f5344c42 --- /dev/null +++ b/renderapi/external/processpools/pool_pathos.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +""" +external processing pool support for Pathos (legacy mode) +""" +from pathos.multiprocessing import ProcessingPool as Pool + + +class PathosWithPool(Pool): + def __init__(self, *args, **kwargs): + super(PathosWithPool, self).__init__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + super(PathosWithPool, self)._clear() + + +WithPool = PathosWithPool + +__all__ = ['PathosWithPool', 'WithPool'] diff --git a/renderapi/external/processpools/stdlib_pool.py b/renderapi/external/processpools/stdlib_pool.py new file mode 100644 index 00000000..2fd2d54d --- /dev/null +++ b/renderapi/external/processpools/stdlib_pool.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +""" +WithPool style helper functions using python's standard library +""" +from multiprocessing.pool import Pool, ThreadPool + + +class WithThreadPool(ThreadPool): + def __init__(self, *args, **kwargs): + super(WithThreadPool, self).__init__(*args, **kwargs) + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + self.join() + + +class WithDummyMapPool: + @staticmethod + def map(*args, **kwargs): + return list(map(*args, **kwargs)) + + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass + + +class WithMultiprocessingPool(Pool): + """Multiprocessing.pool.Pool with functioning __exit__ call + + Parameters + ---------- + *args + variable length argument list matching input + to multiprocessing.pool.Pool + **kwargs + keyword argument input matching multiprocessing.pool.Pool + + Examples + -------- + >>> with WithMultiprocessingPool(number_processes) as pool: + >>> pool.map(myfunc, myInput) + """ + + def __init__(self, *args, **kwargs): + super(WithMultiprocessingPool, self).__init__(*args, **kwargs) + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + self.join() diff --git a/renderapi/image.py b/renderapi/image.py new file mode 100644 index 00000000..76253b32 --- /dev/null +++ b/renderapi/image.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python + +import io +from PIL import Image +import numpy as np +import logging +from .render import format_preamble, format_baseurl, renderaccess +from .errors import RenderError +from .utils import NullHandler, jbool, get_json, put_json + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + +# define acceptable image formats -- currently render generates png, jpeg, tiff +IMAGE_FORMATS = {'png': 'png-image', + '.png': 'png-image', + 'jpg': 'jpeg-image', + 'jpeg': 'jpeg-image', + '.jpg': 'jpeg-image', + 'tif': 'tiff-image', + '.tif': 'tiff-image', + 'tiff': 'tiff-image', + 'tiff16': 'tiff16-image', + None: 'png-image'} # Default to png + + +def _strip_None_value_dictitems(d, exclude_keys=[]): + return {k: v for k, v in d.items() + if v is not None and k not in exclude_keys} + + +@renderaccess +def get_bb_renderparams(stack, z, x, y, width, height, scale=1.0, + channel=None, minIntensity=None, maxIntensity=None, + binaryMask=None, filter=None, filterListName=None, + convertToGray=None, excludeMask=None, + host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%f/render-parameters" % ( + z, x, y, width, height, scale) + + qparams = _strip_None_value_dictitems({ + "minIntensity": minIntensity, + "maxIntensity": maxIntensity, + "binaryMask": binaryMask, + "filter": filter, + "filterListName": filterListName, + "convertToGray": convertToGray, + "excludeMask": excludeMask, + "channels": channel}) + + return get_json(session, request_url, params=qparams) + + +@renderaccess +def get_bb_image(stack, z, x, y, width, height, scale=1.0, + channel=None, + minIntensity=None, maxIntensity=None, binaryMask=None, + filter=None, maxTileSpecsToRender=None, + host=None, port=None, owner=None, project=None, + img_format=None, session=None, + render=None, **kwargs): + """render image from a bounding box defined in xy and return numpy array: + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + name of render stack to get image from + z : float + z value to render + x : int + leftmost point of bounding rectangle + y : int + topmost pont of bounding rectangle + width : int + number of units @scale=1.0 to right (+x() of bounding box to render + height : int + number of units @scale=1.0 down (+y) of bounding box to render + scale : float + scale to render image at (default 1.0) + channel : str + channel name to render, (e.g. 'DAPI') or a weighted average of channels of the format + e.g 'DAPI___.8___GFP___.2' + binaryMask : bool + whether to treat maskimage as binary + maxTileSpecsToRender : int + max number of tilespecs to render + filter : bool + whether to use server side filtering + render : :class:`renderapi.render.Render` + render connect object + session : :class:`requests.sessions.Session` + sessions object to connect with + + Returns + ------- + numpy.array + [N,M,:] array of image data from render + + Raises + ------ + RenderError + """ # noqa: E501 + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: # pragma: no cover + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%f/%s" % ( + z, x, y, width, height, scale, image_ext) + qparams = {} + if minIntensity is not None: + qparams['minIntensity'] = minIntensity + if maxIntensity is not None: + qparams['maxIntensity'] = maxIntensity + if binaryMask is not None: + qparams['binaryMask'] = jbool(binaryMask) + if filter is not None: + qparams['filter'] = jbool(filter) + if maxTileSpecsToRender is not None: + qparams['maxTileSpecsToRender'] = maxTileSpecsToRender + if channel is not None: + qparams.update({'channels': channel}) + + r = session.get(request_url, params=qparams) + try: + image = np.asarray(Image.open(io.BytesIO(r.content))) + return image + except Exception as e: + logger.error(e) + logger.error(r.text) + return RenderError(r.text) + + +@renderaccess +def get_tile_renderparams( + stack, tileId, channel=None, normalizeForMatching=None, + excludeAllTransforms=None, excludeTransformsAfterLast=None, + excludeFirstTransformAndAllAfter=None, scale=None, + width=None, height=None, minIntensity=None, maxIntensity=None, + filter=None, filterListName=None, excludeMask=None, convertToGray=None, + binaryMask=None, host=None, port=None, owner=None, + project=None, img_format=None, + session=None, render=None, **kwargs): + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/render-parameters" % ( + tileId) + + qparams = _strip_None_value_dictitems({ + "normalizeForMatching": normalizeForMatching, + "excludeAllTransforms": excludeAllTransforms, + "excludeTransformsAfterLast": excludeTransformsAfterLast, + "excludeFirstTransformAndAllAfter": excludeFirstTransformAndAllAfter, + "scale": scale, + "width": width, + "height": height, + "minIntensity": minIntensity, + "maxIntensity": maxIntensity, + "binaryMask": binaryMask, + "filter": filter, + "filterListName": filterListName, + "convertToGray": convertToGray, + "excludeMask": excludeMask, + "channels": channel}) + + return get_json(session, request_url, params=qparams) + + +@renderaccess +def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, + excludeAllTransforms=False, scale=None, + minIntensity=None, maxIntensity=None, + filter=None, host=None, port=None, owner=None, + project=None, img_format=None, + session=None, render=None, **kwargs): + """render image from a tile with all transforms and return numpy array + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + name of render stack to get tile from + tileId : str + tileId of tile to render + channel : str + channel name to render, (e.g. 'DAPI') or a weighted average of channels of the format + e.g 'DAPI___.8___GFP___.2' + normalizeForMatching : bool + whether to render the tile with transformations + removed ('local' coordinates) + removeAllOption : bool + whether to remove all transforms from image when + doing normalizeForMatching some versions of render + only remove the last transform from list. + (or remove till there are max 3 transforms) + scale : float + force scale of image + minIntensity : int + Minimum pixel value to rescale image + maxIntensity : int + Maximum pixel value to rescale image + filter : bool + whether to apply server side filtering to image + img_format : str + image format: one of IMAGE_FORMATS = 'png','.png','jpg', + 'jpeg','.jpg','tif','.tif','tiff' + render : :obj:`renderapi.render.Render` + render connect object + session : :obj:`requests.sessions.Session` + sessions object to connect with + + Returns + ------- + numpy.array + [N,M,:] array of image data from render + + Raises + ------ + RenderError + + """ # noqa: E501 + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: # pragma: no cover + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/%s" % (tileId, image_ext) + + qparams = {} + if normalizeForMatching: + qparams['normalizeForMatching'] = jbool(normalizeForMatching) + if scale is not None: + qparams['scale'] = scale + if filter is not None: + qparams['filter'] = jbool(filter) + if excludeAllTransforms is not None: + qparams['excludeAllTransforms'] = jbool(excludeAllTransforms) + if channel is not None: + qparams.update({'channels': channel}) + if minIntensity is not None: + qparams['minIntensity'] = minIntensity + if maxIntensity is not None: + qparams['maxIntensity'] = maxIntensity + logger.debug(request_url) + + r = session.get(request_url, params=qparams) + try: + img = Image.open(io.BytesIO(r.content)) + array = np.asarray(img) + return array + except Exception as e: + logger.error(e) + logger.error(r.text) + return RenderError(r.text) + + +@renderaccess +def get_section_renderparams(stack, z, binaryMask=None, channel=None, + convertToGray=None, excludeMask=None, filter=None, + filterListName=None, minIntensity=None, + maxIntensity=None, scale=None, + host=None, port=None, owner=None, project=None, + session=None, + render=None, **kwargs): + request_url = format_preamble( + host, port, owner, project, stack) + "/z/{}/render-parameters".format( + z) + + qparams = _strip_None_value_dictitems({ + "scale": scale, + "minIntensity": minIntensity, + "maxIntensity": maxIntensity, + "binaryMask": binaryMask, + "filter": filter, + "filterListName": filterListName, + "convertToGray": convertToGray, + "excludeMask": excludeMask, + "channels": channel}) + + return get_json(session, request_url, params=qparams) + + +@renderaccess +def get_section_image(stack, z, scale=1.0, channel=None, + minIntensity=None, maxIntensity=None, + filter=False, + maxTileSpecsToRender=None, img_format=None, + host=None, port=None, owner=None, project=None, + session=None, + render=None, **kwargs): + """render an section of image + + :func:`renderapi.render.renderaccess` decorated function + + + Parameters + ---------- + stack : str + name of render stack to render image from + z : float + layer Z + scale : float + linear scale at which to render image (e.g. 0.5) + channel: str + channel name to render, (e.g. 'DAPI') or a weighted average of channels of the format + e.g 'DAPI___.8___GFP___.2' + minIntensity : int + Minimum pixel value to rescale image + maxIntensity : int + Maximum pixel value to rescale image + filter : bool + whether or not to apply server side filtering + maxTileSpecsToRender : int + maximum number of tile specs in rendering + img_format : str + one of IMAGE_FORMATS 'png','.png','jpg','jpeg', + '.jpg','tif','.tif','tiff' + render : :obj:`renderapi.render.Render` + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + numpy.array + [N,M,:] array of image data of section from render + + Examples + -------- + :: + + >>> import renderapi + >>> render = renderapi.render.connect('server',8080,'me','myproject') + >>> img = render.run(renderapi.stack.get_section_image,'mystack',3.0) + + """ # noqa: E501 + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: # pragma: no cover + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = format_preamble( + host, port, owner, project, stack) + '/z/{}/{}'.format(z, image_ext) + qparams = {'scale': scale, 'filter': jbool(filter)} + if maxTileSpecsToRender is not None: + qparams.update({'maxTileSpecsToRender': maxTileSpecsToRender}) + if channel is not None: + qparams.update({'channels': channel}) + if minIntensity is not None: + qparams['minIntensity'] = minIntensity + if maxIntensity is not None: + qparams['maxIntensity'] = maxIntensity + + r = session.get(request_url, params=qparams) + return np.asarray(Image.open(io.BytesIO(r.content))) + + +@renderaccess +def get_renderparameters_image(renderparams, img_format=None, + host=None, port=None, owner=None, + session=None, + render=None, **kwargs): + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: # pragma: no cover + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = format_baseurl(host, port) + '/owner/{owner}/{ext}'.format( + owner=owner, ext=image_ext) + + r = put_json(session, request_url, renderparams) + return np.array(Image.open(io.BytesIO(r.content))) diff --git a/renderapi/image_pyramid.py b/renderapi/image_pyramid.py new file mode 100644 index 00000000..e92b3e96 --- /dev/null +++ b/renderapi/image_pyramid.py @@ -0,0 +1,179 @@ +from collections.abc import MutableMapping +from .errors import RenderError +import logging +from .utils import NullHandler +import warnings + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class MipMap: + """MipMap class to represent a image and its mask + + Attributes + ---------- + imageUrl : str or None + uri corresponding to image + maskUrl : str or None + uri corresponding to mask + + """ + + def __init__(self, imageUrl=None, maskUrl=None): + self.imageUrl = imageUrl + self.maskUrl = maskUrl + + def to_dict(self): + """ + Returns + ------- + dict + json compatible dictionary representaton + """ + return dict(self.__iter__()) + + def _formatUrls(self): + d = {} + if self.imageUrl is not None: + d.update({'imageUrl': self.imageUrl}) + if self.maskUrl is not None: + d.update({'maskUrl': self.maskUrl}) + return d + + def __setitem__(self, key, value): + if key == 'imageUrl': + self.imageUrl = value + elif key == 'maskUrl': + self.maskUrl = value + else: + raise KeyError('{} not a valid mipmap attribute'.format(key)) + + def __getitem__(self, key): + if key == 'imageUrl': + return self.imageUrl + if key == 'maskUrl': + return self.maskUrl + else: + raise KeyError( + '{} is not a valid attribute of a mipmapLevel'.format(key)) + + def __iter__(self): + return iter(self._formatUrls().items()) + + def __eq__(self, b): + try: + return all([self.imageUrl == b.imageUrl, + self.maskUrl == b.maskUrl]) + except AttributeError as e: + return all([self.imageUrl == b.get('imageUrl'), + self.maskUrl == b.get('maskUrl')]) + + +class MipMapLevel: + """MipMapLevel class to represent a level of an image pyramid. + Can be put in dictionary formatting using dict(mML) + + Attributes + ---------- + level : int + level of 2x downsampling represented by mipmaplevel + imageUrl : str or None + uri corresponding to image + maskUrl : str or None + uri corresponding to mask + + """ + + def __init__(self, level, imageUrl=None, maskUrl=None): + warnings.warn( + "use of mipmaplevels deprecated, use MipMap and ImagePyramid", + DeprecationWarning) + self.level = level + self.mipmap = MipMap(imageUrl, maskUrl) + + def to_dict(self): + """ + Returns + ------- + dict + json compatible dictionary representaton + """ + return dict(self.mipmap) + + def __getitem__(self, key): + if key == 'imageUrl': + return self.mipmap.imageUrl + if key == 'maskUrl': + return self.mipmap.maskUrl + else: + raise KeyError( + '{} is not a valid attribute of a mipmapLevel'.format(key)) + + def __iter__(self): + return iter(self.to_dict().items()) + + def __eq__(self, b): + return self.mipmap == b.mipmap + + +class TransformedDict(MutableMapping): + """A dictionary that applies an arbitrary key-altering + function before accessing the keys""" + + def __init__(self, *args, **kwargs): + self.store = dict() + self.update(dict(*args, **kwargs)) # use the free update to set keys + + def __getitem__(self, key): + return self.store[self.__keytransform__(key)] + + def __setitem__(self, key, value): + self.store[self.__keytransform__(key)] = value + + def __delitem__(self, key): + del self.store[self.__keytransform__(key)] + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) + + def __keytransform__(self, key): + return key + + +class ImagePyramid(TransformedDict): + '''Image Pyramid class representing a set of MipMapLevels which correspond + to mipmapped (continuously downsmapled by 2x) representations + of an image at level 0 + Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) + ''' + + def __keytransform__(self, key): + try: + level = int(key) + except ValueError as e: + raise RenderError("{} is not a valid mipmap level".format(key)) + if level < 0: + raise RenderError( + "{} is not a valid mipmap level (less than 0)".format(key)) + return "{}".format(level) + + def to_dict(self): + return {k: v.to_dict() for k, v in self.items()} + + @classmethod + def from_dict(cls, d): + return cls({l: MipMap(v.get('imageUrl', None), + v.get('maskUrl', None)) + for l, v in d.items()}) + + def __iter__(self): + return iter(sorted(self.store)) + + @property + def levels(self): + """list of MipMapLevels in this ImagePyramid""" + return self.store.keys() diff --git a/renderapi/layout.py b/renderapi/layout.py new file mode 100644 index 00000000..83a1b35a --- /dev/null +++ b/renderapi/layout.py @@ -0,0 +1,113 @@ +class Layout: + """Layout class to describe acquisition settings + + Attributes + ---------- + sectionId : str + sectionId this tile was taken from + scopeId : str + what microscope this came from + cameraId : str + camera this was taken with + imageRow : int + what row from a row,col layout this was taken + imageCol : int + column from a row,col layout this was taken + stageX : float + X stage coordinates for where this was taken + stageY : float + Y stage coordinates for where this taken + rotation : float + angle of camera when this was taken + pixelsize : float + effective size of pixels (in units of choice) + distanceZ : float + distance (in units of choice) from prior layer + + """ + def __init__(self, sectionId=None, scopeId=None, cameraId=None, + imageRow=None, imageCol=None, stageX=None, stageY=None, + rotation=None, pixelsize=None, + force_pixelsize=True, distanceZ=None, **kwargs): + """Initialize Layout + + Parameters + ---------- + sectionId : str + sectionId this tile was taken from + scopeId : str + what microscope this came from + cameraId : str + camera this was taken with + imageRow : int + what row from a row,col layout this was taken + imageCol : int + column from a row,col layout this was taken + stageX : float + X stage coordinates for where this was taken + stageY : float + Y stage coordinates for where this taken + rotation : float + angle of camera when this was taken + pixelsize : float + effective size of pixels (in units of choice) + force_pixelsize : bool + whether to default pixelsize to 0.1 + distanceZ : float + distance (in units of choice) from prior layer + + """ + self.sectionId = sectionId + self.scopeId = scopeId + self.cameraId = cameraId + self.imageRow = imageRow + self.imageCol = imageCol + self.stageX = stageX + self.stageY = stageY + self.rotation = rotation + if force_pixelsize: + pixelsize = 0.100 if pixelsize is None else pixelsize + self.pixelsize = pixelsize + self.distanceZ = distanceZ + + def to_dict(self): + """return a dictionary representation of this object + + Returns + ------- + dict + json compatible dictionary of this object + """ + d = {} + d['sectionId'] = self.sectionId + d['temca'] = self.scopeId + d['camera'] = self.cameraId + d['imageRow'] = self.imageRow + d['imageCol'] = self.imageCol + d['stageX'] = self.stageX + d['stageY'] = self.stageY + d['rotation'] = self.rotation + d['pixelsize'] = self.pixelsize + d['distanceZ'] = self.distanceZ + d = {k: v for k, v in d.items() if v is not None} + return d + + def from_dict(self, d): + """set this object equal to the fields found in dictionary + + Parameters + ---------- + d : dict + dictionary to use to update + """ + if d is not None: + self.sectionId = d.get('sectionId') + self.cameraId = d.get('camera') + self.scopeId = d.get('temca') + self.imageRow = d.get('imageRow') + self.imageCol = d.get('imageCol') + self.stageX = d.get('stageX') + self.stageY = d.get('stageY') + self.rotation = d.get('rotation') + self.pixelsize = d.get('pixelsize') + self.distanceZ = d.get('distanceZ') diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py new file mode 100644 index 00000000..6e478c28 --- /dev/null +++ b/renderapi/pointmatch.py @@ -0,0 +1,697 @@ +#!/usr/bin/env python +''' +Point Match APIs +''' +import logging +from .render import format_baseurl, renderaccess +from .utils import NullHandler, get_json, put_json, rest_delete + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +def copy_match_explicit(match): + """create independent match dictionary equivalent to the input match + significantly faster than e.g. copy.deepcopy or json serialization + + Parameters + ---------- + match : dict + match dictionary + + Returns + ------- + new_match : dict + match dictionary equivalent to input match + + """ + # explicitly copy match dictionary since it contains lists + new_matchd = { + "p": [i[:] for i in match["matches"]["p"]], + "q": [i[:] for i in match["matches"]["q"]], + "w": match["matches"]["w"][:] + } + + new_match = {k: (match[k] if k != "matches" else new_matchd) + for k in match.keys()} + return new_match + + +def copy_matches_explicit(matches): + """create independent match dictionaries equivalent to the input matches + significantly faster than e.g. copy.deepcopy or json serialization + + Parameters + ---------- + matches : list[dict] + list of match dictionaries to copy + + Returns + ------- + new_matches : list[dict] + list of match dictionaries equivalent to input matches + + """ + return [copy_match_explicit(match) for match in matches] + + +def swap_matchpair(match, copy=True): + """ + + Parameters + ---------- + match : dict + match dictionary to swap p->q,q->p + copy : bool + whether to return a copy which, when modified, does not change original + + Returns + ------- + new_match : dict + match dictionary with "p" and "q" swapped + """ + updated_d = { + "pId": match["qId"], + "qId": match["pId"], + "qGroupId": match["pGroupId"], + "pGroupId": match["qGroupId"], + "matches": { + "p": match["matches"]["q"], + "q": match["matches"]["p"], + "w": match["matches"]["w"] # include weight because shallow swap + } + } + + new_match = {k: updated_d.get(k, v) for k, v in match.items()} + + return (copy_match_explicit(new_match) if copy else new_match) + + +@renderaccess +def get_matchcollection_owners(host=None, port=None, + session=None, + render=None, **kwargs): + + """get all the matchCollection owners + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + render : renderapi.render.Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`unicode` + matchCollection owners + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/matchCollectionOwners" + return get_json(session, request_url) + + +@renderaccess +def get_matchcollections(owner=None, host=None, port=None, + session=None, render=None, **kwargs): + """get all the matchCollections owned by owner + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`unicode` + matchcollections owned by owner + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollections" % owner + return get_json(session, request_url) + + +@renderaccess +def get_match_groupIds(matchCollection, owner=None, host=None, + port=None, session=None, + render=None, **kwargs): + """get all the groupIds in a matchCollection + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + owner : str + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`str` + groupIds in matchCollection + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) + return get_json(session, request_url) + + +@renderaccess +def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, + stream=True, + owner=None, host=None, + port=None, session=None, + render=None, **kwargs): + """get all the matches outside a groupId in a matchCollection + returns all matches where pGroupId == groupId and qGroupId != groupId + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + groupId : str + groupId to query + mergeCollections : :obj:`list` of :obj:`str` + other matchCollections to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( + owner, matchCollection, groupId) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url, stream=stream) + + +@renderaccess +def get_matches_within_group(matchCollection, groupId, mergeCollections=None, + stream=True, + owner=None, host=None, port=None, + session=None, + render=None, **kwargs): + """get all the matches within a groupId in a matchCollection + returns all matches where pGroupId == groupId and qGroupId == groupId + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + groupId : str + groupId to query + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections + to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( + owner, matchCollection, groupId) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url, stream=stream) + + +@renderaccess +def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, + mergeCollections=None, stream=True, + render=None, owner=None, host=None, + port=None, + session=None, **kwargs): + """get all the matches between two specific groups + returns all matches where pgroup == pGroupId and qgroup == qGroupId + OR pgroup == qGroupId and qgroup == pGroupId + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + first group + qgroup : str + second group + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections + to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( + owner, matchCollection, pgroup, qgroup) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url, stream=stream) + + +def add_merge_collections(request_url, mcs): + """utility function to add mergeCollections to request_url + + Parameters + ---------- + request_url : str + request url + mcs : :obj:`list` of :obj:`str` + list of mergeCollections to add + Returns + ------- + str + request_url with ?mergeCollection=mc[0]&mergeCollection=mc[1]... + appended + """ + if mcs is not None: + if type(mcs) is list: + request_url += "?"+"&".join( + ['mergeCollection=%s' % mc for mc in mcs]) + return request_url + + +@renderaccess +def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, + qgroup, qid, mergeCollections=None, + render=None, owner=None, + host=None, port=None, + session=None, **kwargs): + """get all the matches between two specific tiles + returns all matches where + pgroup == pGroupId and pid=pId and qgroup == qGroupId and qid == qId + OR + qgroup == pGroupId and Qid=pId and Pgroup == qGroupId and pid == qId + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + first group + pid : str + first id + qgroup : str + second group + qid : str + second id + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + ("/owner/%s/matchCollection/%s/group/%s/id/%s/" + "matchesWith/%s/id/%s" % ( + owner, matchCollection, pgroup, pid, qgroup, qid)) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url) + + +@renderaccess +def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, + stream=True, + render=None, owner=None, + host=None, port=None, + session=None, **kwargs): + """get all the matches from a specific groups + returns all matches where pgroup == pGroupId + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + source group to query + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections to aggregate into answer + stream : bool + whether to invoke streaming (default=True) + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( + owner, matchCollection, pgroup) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url, stream=stream) + + +@renderaccess +def get_match_groupIds_from_only(matchCollection, mergeCollections=None, + render=None, owner=None, + host=None, port=None, + session=None, **kwargs): + """get all the source pGroupIds in a matchCollection + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`str` + pGroupIds in matchCollection + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url) + + +@renderaccess +def get_match_groupIds_to_only(matchCollection, mergeCollections=None, + render=None, owner=None, + host=None, port=None, + session=None, **kwargs): + """get all the destination qGroupIds in a matchCollection + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`str` + qGroupIds in matchCollection + + Raises + ------ + RenderError + if cannot get a reponse from server + + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url) + + +@renderaccess +def get_matches_involving_tile(matchCollection, groupId, id, + mergeCollections=None, stream=True, + owner=None, host=None, port=None, + session=None, **kwargs): + """get all the matches involving a specific tile + returns all matches where groupId == pGroupId and id == pId + OR groupId == qGroupId and id == qId + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + groupId : str + groupId to query + id : str + id to query + mergeCollections : :obj:`list` of :obj:`str`, optional + other matchCollections to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ + request_url = format_baseurl(host, port) + \ + "/owner/{}/matchCollection/{}/group/{}/id/{}/".format( + owner, matchCollection, groupId, id) + request_url = add_merge_collections(request_url, mergeCollections) + + return get_json(session, request_url, stream=stream) + + +@renderaccess +def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, + render=None, owner=None, host=None, + port=None, session=None, + **kwargs): + """delete all the matches between two specific groups + deletes all matches where (pgroup == pGroupId and qgroup == qGroupId) + OR (pgroup == qGroupId and qgroup == pGroupId() + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + first group + qgroup : str + second group + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + + """ + request_url = format_baseurl(host, port) + \ + "/owner/{}/matchCollection/{}/group/{}/matchesWith/{}".format( + owner, matchCollection, pGroupId, qGroupId) + r = rest_delete(session, request_url) # noqa: F841 + + +@renderaccess +def import_matches(matchCollection, data, owner=None, host=None, port=None, + session=None, render=None, **kwargs): + """import matches into render database + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name + data : :obj:`list` of :obj:`dict` + list of matches to import (see matches definition) + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + requests.response.Reponse + server response + + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/matches" % (owner, matchCollection) + logger.debug(request_url) + return put_json(session, request_url, data) + + +@renderaccess +def delete_collection(matchCollection, owner=None, host=None, port=None, + session=None, render=None, **kwargs): + """delete match collection from render database + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name to delete + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + requests.response.Reponse + server response + + Raises + ------ + RenderError + if cannot get a proper reponse from server + + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s" % (owner, matchCollection) + logger.debug(request_url) + r = rest_delete(session, request_url) # noqa: F841 diff --git a/renderapi/render.py b/renderapi/render.py new file mode 100755 index 00000000..f0e7598d --- /dev/null +++ b/renderapi/render.py @@ -0,0 +1,609 @@ +#!/usr/bin/env python +import logging +import os +import requests +from .utils import defaultifNone, NullHandler, fitargspec, get_json +from .errors import ClientScriptError +from decorator import decorator +from six.moves import input as raw_input + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class Render(object): + """Render object to store connection settings for render server. + Baseclass that doesn't require client_scripts definition + for client side java processing. + + See :func:`connect` for parameter definitions. + + Attributes + ---------- + DEFAULT_HOST : str + render host to which make_kwargs will default + DEFAULT_PORT : int + render port to which make_kwargs will default + DEFAULT_OWNER : str + render owner to which make_kwargs will default + DEFAULT_PROJECT : str + render project to which make_kwargs will default + DEFAULT_CLIENT_SCRIPTS : str + render client scripts path to which make_kwargs will default + session : requests.sessions.Session + session object to which make_kwargs will default + create_session : bool + create a new session for this instance of render if no session was + provided, allowing the same session to be reused for requests and + increasing performance, defaults to True + + """ + + def __init__(self, host=None, port=None, owner=None, project=None, + client_scripts=None, session=None, create_session=True, + **kwargs): + self.DEFAULT_HOST = host + self.DEFAULT_PORT = port + self.DEFAULT_PROJECT = project + self.DEFAULT_OWNER = owner + self.DEFAULT_CLIENT_SCRIPTS = client_scripts + if create_session and session is None: + self.session = requests.Session() + else: + self.session = session + + logger.debug('Render object created with ' + 'host={h}, port={p}, project={pr}, ' + 'owner={o}, scripts={s}'.format( + h=self.DEFAULT_HOST, p=self.DEFAULT_PORT, + pr=self.DEFAULT_PROJECT, o=self.DEFAULT_OWNER, + s=self.DEFAULT_CLIENT_SCRIPTS)) + + @property + def DEFAULT_KWARGS(self): + """"kwargs to which the render object falls back. Depends on: + self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, + self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS, + self.session + + Returns + ------- + dict + default keyword arguments + """ + return self.make_kwargs() + + def make_kwargs(self, host=None, port=None, owner=None, project=None, + client_scripts=None, session=None, **kwargs): + """make kwargs using this render object's defaults and any + designated kwargs passed in + + Parameters + ---------- + host : str or None + render webservice host + port : int or None + render webservice port + owner : str or None + render webservice owner + project : str or None + render webservice project + client_scripts : str or None + render java client script location + session : requests.sessions.Session + http session to use + **kwargs + all other keyword arguments passed through + + Returns + ------- + dict + keyword arguments with missing + host,port,owner,project,client_scripts,session filled in with + defaults + """ + processed_kwargs = { + 'host': self.DEFAULT_HOST if host is None else host, + 'port': self.DEFAULT_PORT if port is None else port, + 'owner': self.DEFAULT_OWNER if owner is None else owner, + 'project': self.DEFAULT_PROJECT if project is None else project, + 'client_scripts': (self.DEFAULT_CLIENT_SCRIPTS if client_scripts + is None else client_scripts), + 'session': self.session if session is None else session, + } + processed_kwargs.update(kwargs) + return processed_kwargs + + def run(self, f, *args, **kwargs): + """run function from object + technically shorter than adding render=Render to kwargs + + Parameters + ---------- + f : func + renderapi function you want to call + *args + args passed to that function + **kwargs + kwargs passed to that function + + Returns + ------- + func + function with this :class:`Render` instance in + keyword arguments as render= + + Examples + -------- + >>> render = Render('server',8080) + >>> metadata = render.run(renderapi.render.get_stack_metadata_by_owner, 'myowner') + + """ # noqa: E501 + # FIXME WARNING I think renderaccess can default to + # another render if defined in args (test/squash) + kwargs['render'] = self + return f(*args, **kwargs) + + +class RenderClient(Render): + """Render object to run java client commands via a wrapped client script. + Should use :func:`connect` to create and for documentation of parameters. + + Attributes + ---------- + DEFAULT_HOST : str + render host to which make_kwargs will default + DEFAULT_PORT : int + render port to which make_kwargs will default + DEFAULT_OWNER : str + render owner to which make_kwargs will default + DEFAULT_PROJECT : str + render project to which make_kwargs will default + DEFAULT_CLIENT_SCRIPTS : str + render client scripts path to which make_kwargs will default + session : requests.sessions.Session + session object to which make_kwargs will default + create_session : bool + create a new session for this instance of render if no session was + provided, allowing the same session to be reused for requests and + increasing performance, defaults to True + client_script : str + location of wrapper script for java client with input same as Render + java client's run_ws_client.sh + memGB : str + string defining heap in GB to be utilized by + java clients (default '1G' for 1 GB) + """ + + client_script_wrapper = 'run_ws_client.sh' + + def __init__(self, client_script=None, memGB=None, validate_client=True, + *args, **kwargs): + """Initialize RenderClient object extending Render to + running java client scripts + + Parameters + ---------- + client_script : str + path to script with same inputs as + Render Java Client 'run_ws_client.sh' + memGB : str + string defining heap to be utilized by java clients in GB + (defaults to '1G' for 1GB) + validate_client : bool + whether to validate that client script is a file + + Raises + ------ + ClientScriptError + if render client script cannot be found + """ + super(RenderClient, self).__init__(**kwargs) + if validate_client: + if client_script is None: + if self.DEFAULT_CLIENT_SCRIPTS is None: + raise ClientScriptError( + 'No RenderClient script specified!') + else: + logger.debug("Attempting to derive client script " + "from client_scripts variable {}".format( + self.DEFAULT_CLIENT_SCRIPTS)) + client_script = self.clientscript_from_clientscripts( + self.DEFAULT_CLIENT_SCRIPTS) + + if not os.path.isfile(client_script): + raise ClientScriptError('Client script {} not found!'.format( + client_script)) + if self.client_script_wrapper not in os.path.basename(client_script): + logger.warning( + 'Unrecognized client script {}!'.format(client_script)) + self.client_script = client_script + + if memGB is None: + logger.warning( + 'No default Java heap specified -- defaulting to 1G') + memGB = '1G' + self.memGB = memGB + + @classmethod + def clientscript_from_clientscripts(cls, client_scripts): + return os.path.join(client_scripts, cls.client_script_wrapper) + + def make_kwargs(self, *args, **kwargs): + """method to fill in default properties of RenderClient object + + Parameters + ---------- + *args + args used to initialize RenderClient + **kwargs + kwargs used to initialize RenderClient + + Returns + ------- + dict + keyword arguments with missing + host,port,owner,project,client_scripts,session,client_script,memGB + filled in with defaults + """ + # hack to get dictionary defaults to work + client_script = defaultifNone( + kwargs.pop('client_script', None), self.client_script) + memGB = defaultifNone(kwargs.pop('memGB', None), self.memGB) + return super(RenderClient, self).make_kwargs( + client_script=client_script, + memGB=memGB, + *args, **kwargs) + + +def connect(host=None, port=None, owner=None, project=None, + client_scripts=None, client_script=None, memGB=None, + force_http=True, validate_client=True, web_only=False, + session=None, **kwargs): + """helper function to create a :class:`Render` instance, or + :class:`RenderClient` if sufficent parameters are provided. + Will default to using environment variables if not specified in call, + and prompt user for any parameters that are not given. + + Parameters + ---------- + host : str + hostname for target render server -- will prepend + "http://" if host does not begin with 'http' and + force_http keyword evaluates True. + Can be set by environment variable RENDER_HOST. + port : str, int, or None + port for target render server. + Optional as in 'http://hostname[:port]'. + Can be set by environment variable RENDER_PORT. + owner : str + owner for render-ws. + Can be set by environment variable RENDER_OWNER. + project : str + project for render webservice. + Can be set by environment variable RENDER_PROJECT. + client_scripts : str + directory path for render-ws-java-client scripts. + Can be set by environment variable RENDER_CLIENT_SCRIPTS. + client_script : str, optional + path to a wrapper for java client classes. + Used only in RenderClient. + Can be set by environment variable RENDER_CLIENT_SCRIPT. + memGB : str + heap size in GB for java client scripts, + example for 1 GB: '1G'. Used only in RenderClient. + Can be set by environment variable RENDER_CLIENT_HEAP. + force_http : bool + whether to prepend + 'http://' to render host if it does not begin with 'http' + validate_client : bool + whether to validate existence of RenderClient run_ws_client.sh script + web_only : bool + whether to check environment variables/prompt user + for client_scripts directory if not in arguments + session : requests.sessions.Session + http session to use + + Returns + ------- + Render + a connect object to simplify specifying what render + server to connect to + (returns :class:`RenderClient` if sufficent parameters are passed) + + Raises + ------ + ValueError + if empty user input is given for required field + + """ + if host is None: + if 'RENDER_HOST' not in os.environ: + host = str(raw_input("Enter Render Host: ")) + if host == '': # pragma: no cover + logger.critical('Render Host must not be empty!') + raise ValueError('Render Host must not be empty!') + else: + host = os.environ['RENDER_HOST'] + if force_http: + host = (host if host.startswith('http') + else 'http://{}'.format(host)) + + if port is None: + if 'RENDER_PORT' not in os.environ: + port = str(int(raw_input("Enter Render Port: "))) + if port == '': # pragma: no cover + # TODO better (no) port handling + logger.critical('Render Port must not be empty!') + raise ValueError('Render Port must not be empty!') + else: + port = int(os.environ['RENDER_PORT']) + + if project is None: + if 'RENDER_PROJECT' not in os.environ: + project = str(raw_input("Enter Render Project: ")) + else: + project = str(os.environ['RENDER_PROJECT']) + if project == '': # pragma: no cover + logger.critical('Render Project must not be empty!') + raise ValueError('Render Project must not be empty!') + + if owner is None: + if 'RENDER_OWNER' not in os.environ: + owner = str(raw_input("Enter Render Owner: ")) + else: + owner = str(os.environ['RENDER_OWNER']) + if owner == '': # pragma: no cover + logger.critical('Render Owner must not be empty!') + raise ValueError('Render Owner must not be empty!') + + if client_scripts is None and not web_only: + if 'RENDER_CLIENT_SCRIPTS' not in os.environ: + client_scripts = str(raw_input( + "Enter Render Client Scripts location: ")) + else: + client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) + if client_scripts == '': # pragma: no cover + logger.critical('Render Client Scripts must ' + 'not be empty!') + raise ValueError('Render Client Scripts must ' + 'not be empty!') + if client_script is None: + if 'RENDER_CLIENT_SCRIPT' not in os.environ: + # client_script = str(raw_input("Enter Render Client Script: ")) + client_script = RenderClient.clientscript_from_clientscripts( + client_scripts) + else: + client_script = str(os.environ['RENDER_CLIENT_SCRIPT']) + + if memGB is None: + if 'RENDER_CLIENT_HEAP' not in os.environ: + pass + else: + memGB = str(os.environ['RENDER_CLIENT_HEAP']) + + try: + return RenderClient(client_script=client_script, memGB=memGB, + host=host, port=port, + owner=owner, project=project, + client_scripts=client_scripts, + validate_client=validate_client, + session=session) + except ClientScriptError as e: + logger.info(e) + logger.warning( + 'Could not initiate render Client -- falling back to web') + return Render(host=host, port=port, owner=owner, project=project, + client_scripts=client_scripts, session=session) + + +@decorator +def renderaccess(f, *args, **kwargs): + """Decorator allowing functions asking for host, port, owner, project, + and/or session to default to a connection defined by a :class:`Render` + object using its :func:`RenderClient.make_kwargs` method. + + You can if you wish specify any of the arguments, in which case they + will not be filled in by the default values, but you don't have to. + + The default value for session is None, which will be replaced with a newly + created requests.Session object. + + As such, the documentation omits describing the parameters which are + natural to expect will be filled in by the renderaccess decorator. + + Parameters + ---------- + f : func + function to decorate + Returns + ------- + func + decorated function + + Examples + -------- + >>> render = renderapi.render.connect('server',8080,'me','my_project') + >>> stacks = renderapi.render.get_stacks_by_owner_project(render=render) + """ + args, kwargs = fitargspec(f, args, kwargs) + render = kwargs.get('render') + if render is not None: + if isinstance(render, Render): + kwargs = render.make_kwargs(**kwargs) + else: + raise ValueError( + 'invalid Render object type {} specified!'.format( + type(render))) + + session = kwargs.get("session") + if session is None: + kwargs["session"] = requests.Session() + + return f(*args, **kwargs) + + +def format_baseurl(host, port): + """format host and port to a standard template render-ws url + + Parameters + ---------- + host : str + host of render server + port : int or None + port of render server + + Returns + ------- + str + a url to the render endpoint at that + host/port combination (append render-ws/v1) + """ + # return 'http://%s:%d/render-ws/v1' % (host, port) + server = '{}{}'.format(host, ('' if port is None else ':{}'.format(port))) + return '{}/render-ws/v1'.format(server) + + +def format_preamble(host, port, owner, project, stack): + """format host, port, owner, project, and stack parameters + to the access point to stack-based apis + + Parameters + ---------- + host : str + render host + port : int + render host port + owner : str + render owner + project : str + render project + stack : str + render stack + + Returns + ------- + str + a url to the endpoint for that host, port, + owner, project, stack combination + """ + preamble = "%s/owner/%s/project/%s/stack/%s" % ( + format_baseurl(host, port), owner, project, stack) + return preamble + + +@renderaccess +def get_owners(host=None, port=None, session=None, + render=None, **kwargs): + """return list of owners across all Projects and Stacks for a render server + + :func:`renderaccess` decorated function + + Parameters + ---------- + host : str + render host (defaults to host from render) + port : int + render port (default to port from render) + session : requests.sessions.Session + requests session + render : RenderClient + RenderClient connection object + + Returns + ------- + list + list of strings containing all render owners + + """ + request_url = "%s/owners/" % format_baseurl(host, port) + return get_json(session, request_url) + + +@renderaccess +def get_stack_metadata_by_owner(owner=None, host=None, port=None, + session=None, + render=None, **kwargs): + """return metadata for all stacks belonging to particular + owner on render server + + :func:`renderaccess` decorated function + + Parameters + ---------- + owner : str + render owner + render : RenderClient + render connect object + session : requests.sessions.Session + http session to use + + Returns + ------- + dict + stackInfo metadata, TODO example + """ + request_url = "%s/owner/%s/stacks/" % ( + format_baseurl(host, port), owner) + logger.debug(request_url) + return get_json(session, request_url) + + +@renderaccess +def get_projects_by_owner(owner=None, host=None, port=None, + session=None, render=None, **kwargs): + """return list of projects belonging to a single owner for render stack + + :func:`renderaccess` decorated function + + Parameters + ---------- + owner : str + render owner + render : RenderClient + render connect object + session : requests.sessions.Session + http session to use + + Returns + ------- + :obj:`list` of :obj:`unicode` + render projects by this owner + + """ + metadata = get_stack_metadata_by_owner(owner=owner, host=host, + port=port, session=session) + projects = list(set([m['stackId']['project'] for m in metadata])) + return projects + + +@renderaccess +def get_stacks_by_owner_project(owner=None, project=None, host=None, + port=None, session=None, + render=None, **kwargs): + """return list of stacks belonging to an owner's project on render server + + :func:`renderaccess` decorated function + + Parameters + ---------- + owner : str + render owner + project : str + render project + render : RenderClient + render connect object + session : requests.sessions.Session + http session to use + + Returns + ------- + :obj:`list` of :obj:`str` + render stacks by this owner in this project + + """ + metadata = get_stack_metadata_by_owner(owner=owner, host=host, + port=port, session=session) + stacks = ([m['stackId']['stack'] for m in metadata + if m['stackId']['project'] == project]) + return stacks diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py new file mode 100644 index 00000000..41ef7f43 --- /dev/null +++ b/renderapi/resolvedtiles.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +from .tilespec import TileSpec +from .transform import load_transform_json +from .utils import NullHandler, put_json, jbool, get_json +from .render import format_preamble, renderaccess +from .errors import RenderError +import logging + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class ResolvedTiles: + def __init__(self, tilespecs=None, transformList=None, json=None): + if json is None: + if tilespecs is None: + self.tilespecs = [] + else: + self.tilespecs = tilespecs + if transformList is None: + self.transforms = [] + else: + self.transforms = transformList + else: + self.from_dict(json) + + def to_dict(self): + d = { + 'transformIdToSpecMap': {tf.transformId: tf.to_dict() + for tf in self.transforms}, + 'tileIdToSpecMap': {ts.tileId: ts.to_dict() + for ts in self.tilespecs} + } + return d + + def from_dict(self, d): + self.tilespecs = [] + self.transforms = [] + for ts in d['tileIdToSpecMap'].values(): + self.tilespecs.append(TileSpec(json=ts)) + for transformId, tform_json in d['transformIdToSpecMap'].items(): + tform_json['transformId'] = transformId + self.transforms.append(load_transform_json(tform_json)) + + # def get_tilespecs(): + """return a set of TileSpecs that include resolved tilespecs + + Returns + ------- + List(renderapi.tilespec.TileSpec) + A list of tilespecs stored in this ResolvedTiles with the transformations dereferenced + """ # noqa: E501 + + +def combine_resolvedtiles(rts_l): + """combine multiple ResolvedTiles objects into a single ResolvedTiles + + Like render, this has an implicit expectation that transformIds and tileIds + are unique among all input objects + + Parameters + ---------- + rts_l : list[renderapi.resolvedtiles.ResolvedTiles] + list of ResolvedTiles objects to combine + + Returns + ------- + rts : renderapi.resolvedtiles.ResolvedTiles + combined ResolvedTiles object + """ + tforms = list({tform.transformId: tform for l in ( + rts.transforms for rts in rts_l) + for tform in l}.values()) + tspecs = list({ts.tileId: ts for l in ( + rts.tilespecs for rts in rts_l) + for ts in l}.values()) + + return ResolvedTiles(transformList=tforms, tilespecs=tspecs) + + +@renderaccess +def put_tilespecs(stack, resolved_tiles=None, deriveData=True, + tilespecs=None, shared_transforms=None, + host=None, port=None, owner=None, project=None, + session=None, render=None, **kwargs): + """upload resolved tiles to the server + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack + resolved_tiles: renderapi.resolvedtiles.ResolvedTiles + resolved tiles to upload + deriveData: bool + whether or not to calculate bounding boxes serverside + tilespecs: list[renderapi.tilespec.Tilespec] + list of tilespecs to upload + sharedTransforms: list[renderapi.transform.Transform] + list of shared transforms to upload + render: renderapi.render.Render + render connect object + + Returns + ------- + requests.response.Reponse + server response + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/resolvedTiles' + qparams = {} if deriveData is None else {'deriveData': jbool(deriveData)} + logger.debug(request_url) + if resolved_tiles is None: + if (tilespecs is None): + raise RenderError("need to pass resolved_tiles or tilespecs") + resolved_tiles = ResolvedTiles(tilespecs=tilespecs, + transformList=shared_transforms) + r = put_json(session, request_url, resolved_tiles, qparams) + logger.debug(r) + return r + + +@renderaccess +def get_resolved_tiles_from_z(stack, z, host=None, port=None, + owner=None, project=None, + session=None, + render=None, **kwargs): + """Get a set of ResolvedTiles from a specific z value. + Returns a tuple of tilespecs and referenced transforms. + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack + z : float + render z + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`ResolvedTiles` + ResolvedTiles object containing tilespecs and transforms + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/z/%f/resolvedTiles' % (z) + logger.debug(request_url) + d = get_json(session, request_url) + return ResolvedTiles(json=d) diff --git a/renderapi/stack.py b/renderapi/stack.py new file mode 100644 index 00000000..c7e91ab2 --- /dev/null +++ b/renderapi/stack.py @@ -0,0 +1,833 @@ +#!/usr/bin/env python +import logging +from time import strftime +from .errors import RenderError +from .utils import jbool, NullHandler, post_json, put_json, rest_delete +from .render import (format_baseurl, format_preamble, + renderaccess) +from .utils import get_json +import json + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class StackVersion: + """StackVersion, metadata about a stack + + Attributes + ---------- + cycleNumber : int + cycleNumber, use as you wish to track versions (default None) + cycleStepNumber : int + cycleStepNumber, use as you with to track versions (default None) + stackResolutionX : float + stackResolutionX, resolution of scale = 1.0 in nm + stackResolutionY : float + stackResolutionY, resolution of scale = 1.0 in nm + stackResolutionZ : float + stackResolutionZ, resolution of scale = 1.0 in nm + mipmapPathBuilder : str + path to mipmap builder (?) + materializedBoxRootPath : str + path to materializer (?) + createTimeStamp : str + time stamp of stack creation (default to now) + versionNotes : str + notes about this stack (optional) + """ + def __init__(self, cycleNumber=None, cycleStepNumber=None, + stackResolutionX=None, stackResolutionY=None, + stackResolutionZ=None, + materializedBoxRootPath=None, mipmapPathBuilder=None, + versionNotes=None, + createTimestamp=None, **kwargs): + self.cycleNumber = cycleNumber + self.cycleStepNumber = cycleStepNumber + self.stackResolutionX = stackResolutionX + self.stackResolutionY = stackResolutionY + self.stackResolutionZ = stackResolutionZ + self.mipmapPathBuilder = mipmapPathBuilder + self.materializedBoxRootPath = materializedBoxRootPath + self.createTimestamp = (strftime('%Y-%m-%dT%H:%M:%S.00Z') if + createTimestamp is None else createTimestamp) + self.versionNotes = versionNotes + + def to_dict(self): + """serialization function + + Returns + ------- + dict + json compatible verson of this object + """ + d = {} + d.update(({'cycleNumber': self.cycleNumber} + if self.cycleNumber is not None else {})) + d.update(({'cycleStepNumber': self.cycleStepNumber} + if self.cycleStepNumber is not None else {})) + d.update(({'stackResolutionX': self.stackResolutionX} + if self.stackResolutionX is not None else {})) + d.update(({'stackResolutionY': self.stackResolutionY} + if self.stackResolutionY is not None else {})) + d.update(({'stackResolutionZ': self.stackResolutionZ} + if self.stackResolutionZ is not None else {})) + d.update(({'createTimestamp': self.createTimestamp} + if self.createTimestamp is not None else {})) + d.update(({'mipmapPathBuilder': self.mipmapPathBuilder} + if self.mipmapPathBuilder is not None else {})) + d.update(({'versionNotes': self.versionNotes} + if self.versionNotes is not None else {})) + d.update(({'materializedBoxRootPath': self.materializedBoxRootPath} + if self.materializedBoxRootPath is not None else {})) + return d + + def from_dict(self, d): + """deserialization function + + Parameters + ---------- + d : dict + dictionary to update the properties of this object + """ + self.__dict__.update({k: v for k, v in d.items()}) + + +@renderaccess +def set_stack_metadata(stack, sv, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """sets the stack metadata for a stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to set the metadata for + sv : StackVersion + metadata for the stack + render : renderapi.render.RenderClient + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.response + response from server + + """ + request_url = format_preamble(host, port, owner, project, stack) + logger.debug(request_url) + return post_json(session, request_url, sv.to_dict()) + + +@renderaccess +def get_full_stack_metadata(stack, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """get stack metadata for stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to get the metadata for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + dict + metadata of the stack + + Raises + ------ + RenderError + """ + request_url = format_preamble(host, port, owner, project, stack) + + logger.debug(request_url) + return get_json(session, request_url) + + +def get_stack_metadata(*args, **kwargs): + """get the stack version metadata for a stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to get the metadata for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + StackVersion + metadata of the stack + + Raises + ------ + RenderError + + """ + j = get_full_stack_metadata(*args, **kwargs) + try: + sv = StackVersion() + sv.from_dict(j['currentVersion']) + return sv + except Exception as e: + logger.error(e) + raise RenderError(e) + + +@renderaccess +def set_stack_state(stack, state='LOADING', host=None, port=None, + owner=None, project=None, + session=None, render=None, **kwargs): + """ + set state of selected stack. + + TODO there is a limited direction in which these stack changes can go + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to set state for + state : str + state of stack, one of ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'] + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + Raises + ------ + RenderError + """ + if state not in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY']: + raise RenderError('state {} not in known states {}'.format( + state, ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'])) + request_url = format_preamble( + host, port, owner, project, stack) + "/state/%s" % state + logger.debug(request_url) + r = session.put(request_url, data=None, + headers={"content-type": "application/json"}) + if (r.status_code != 201): + logger.error(r.text) + raise RenderError(r.text) + return r + + +@renderaccess +def likelyUniqueId(host=None, port=None, + session=None, render=None, **kwargs): + """return hex-code nearly-unique id from render server + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + str + string representation of hex-code + """ + request_url = '{}/likelyUniqueId'.format(format_baseurl(host, port)) + r = session.get(request_url, data=None, + headers={"content-type": "text/plain"}) + return r.text + + +def make_stack_params(host, port, owner, project, stack): + """utility function to turn host,port,owner,project,stack combinations + to java CLI based argument list for subprocess calling + + Parameters + ---------- + host : str + render server + port : int + render server port + owner : str + render owner + project : str + render project + stack : str + render stack + + Returns + ------- + :obj:`list` of :obj:`str` + java CLI list of arguments for subprocess calling + + """ + baseurl = format_baseurl(host, port) + project_params = ['--baseDataUrl', baseurl, + '--owner', owner, '--project', project] + stack_params = project_params + ['--stack', stack] + return stack_params + + +@renderaccess +def delete_stack(stack, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """deletes a stack from render server + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to delete + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + """ + request_url = format_preamble(host, port, owner, project, stack) + r = rest_delete(session, request_url) + logger.debug(r.text) + return r + + +@renderaccess +def delete_section(stack, z, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """removes a single z from a stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to delete section from + z : int or float or str + z value to delete + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + """ + request_url = '{}/z/{}'.format( + format_preamble(host, port, owner, project, stack), z) + r = rest_delete(session, request_url) + logger.debug(r.text) + return r + + +@renderaccess +def delete_tile(stack, tileId, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """ + removes a tile from a stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to delete tile from + tileId : str + tileId of tilespec to remove from stack + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + """ + request_url = '{}/tile/{}'.format( + format_preamble(host, port, owner, project, stack), tileId) + r = rest_delete(session, request_url) + logger.debug(r.text) + return r + + +@renderaccess +def create_stack(stack, cycleNumber=None, cycleStepNumber=None, + stackResolutionX=None, stackResolutionY=None, + stackResolutionZ=None, force_resolution=True, + host=None, port=None, owner=None, project=None, + session=None, render=None, **kwargs): + """creates a new stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack name to create + cycleNumber : int + cycleNumber to use to track stages + cycleStepNumber : int + cycleStepNumber to use to track stages + stackResolutionX : float + resolution of x pixels at scale=1.0 + stackResolutionY : float + resolution of y pixels at scale=1.0 + stackResoluiontZ : float + resolution of z sections at scale=1.0 + force_resolution : bool + fill in resolution of 1.0 for missing resolutions + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + Raises + ------ + RenderError + """ + if force_resolution: + stackResolutionX, stackResolutionY, stackResolutionZ = [ + (1.0 if res is None else res) + for res in [stackResolutionX, stackResolutionY, stackResolutionZ]] + logger.debug('forcing resolution x:{}, y:{}, z:{}'.format( + stackResolutionX, stackResolutionY, stackResolutionZ)) + + sv = StackVersion( + cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, + stackResolutionX=stackResolutionX, stackResolutionY=stackResolutionY, + stackResolutionZ=stackResolutionZ) + request_url = format_preamble(host, port, owner, project, stack) + logger.debug("stack version {} {}".format(request_url, sv.to_dict())) + r = post_json(session, request_url, sv.to_dict()) + try: + return r + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError(r.text) + + +@renderaccess +def rename_stack(stack, to_stack, to_project=None, to_owner=None, + host=None, port=None, owner=None, project=None, + session=None, render=None, **kwargs): + """ + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + inputstack : str + name of input stack to clone + to_stack : str + name of destination stack. if exists, must be LOADING + to_project : str + name of project to rename stack to (default = leave the same as inputstack) + outputOwner: str + name of owner to rename stack to (default = leave the same as inputstack) + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + """ # noqa: E501 + + request_url = format_preamble( + host, port, owner, project, stack) + "/stackId" + d = { + "owner": owner if to_owner is None else to_owner, + "project": project if to_project is None else to_project, + "stack": stack if to_stack is None else to_stack + } + return put_json(session, request_url, d) + + +@renderaccess +def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, + zs=None, close_stack=True, host=None, port=None, + owner=None, project=None, session=None, render=None, **kwargs): + """clone a stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + inputstack : str + name of input stack to clone + outputstack : str + name of destination stack. if exists, must be LOADING + skipTransforms : bool + boolean whether to strip transformations in new stack (default=False) + toProject : str + string name of destination project (default same as inputstack) + zs : :obj:`list` of :obj:`float` or None + list of selected z values to clone into stack (optional) + close_stack : bool + whether to set stack to COMPLETE when finished + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + """ + sv = StackVersion(**kwargs) + newstack_project = project + qparams = {} + if zs is not None: + qparams['z'] = [float(i) for i in zs] + if skipTransforms is not None: + qparams['skipTransforms'] = jbool(skipTransforms) + if toProject is not None: + qparams['toProject'] = toProject + newstack_project = toProject + + request_url = '{}/cloneTo/{}'.format(format_preamble( + host, port, owner, project, inputstack), outputstack) + + logger.debug(request_url) + r = put_json(session, request_url, sv.to_dict(), params=qparams) + + if close_stack: + set_stack_state(outputstack, 'COMPLETE', host, port, owner, + newstack_project) + return r + + +@renderaccess +def get_z_values_for_stack(stack, project=None, host=None, port=None, + owner=None, session=None, + render=None, **kwargs): + """get a list of z values for which there are tiles in the stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to get z values for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + :obj:`list` of :obj:`float` + z values in stack + + Raises + ------ + RenderError + """ + request_url = format_preamble( + host, port, owner, project, stack) + "/zValues/" + logger.debug(request_url) + return get_json(session, request_url) + + +# haven't fully supported this yet +# @renderaccess +# def put_resolved_tilespecs(stack, json_dict, host=None, port=None, +# owner=None, project=None, +# session=None, +# render=None, **kwargs): +# request_url = format_preamble( +# host, port, owner, project, stack) + "/resolvedTiles" +# r = post_json(session, request_url, json_dict) +# return r + + +@renderaccess +def get_bounds_from_z(stack, z, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """get a bounds dictionary for a specific z + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to get bounds from + z : int or float or str + z value to get bounds for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + dict + bounds with keys minY,minY,maxX,maxY,minZ,maxZ + + Raises + ------ + RenderError + + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/z/%f/bounds' % (z) + + return get_json(session, request_url) + + +@renderaccess +def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, + session=None, render=None, **kwargs): + """get bounds of a whole stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to get bounds from + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + dict + bounds with keys minY,minY,maxX,maxY,minZ,maxZ + + Raises + ------ + RenderError + + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/bounds' + return get_json(session, request_url) + + +@renderaccess +def get_tilebounds_for_z(stack, z, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """returns the bounds for each tile associated with a particular z value + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to look within + z : int or float or str + z value for which to get tile bounds + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + list + list of dictionaries with tilebounds + + Raises + ------ + RenderError + + """ + + request_url = format_preamble( + host, port, owner, project, stack) + '/z/{}/tileBounds'.format(z) + return get_json(session, request_url) + + +@renderaccess +def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """returns the sectionId associated with a particular z value + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to look within + z : int or float or str + section z value + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + str + value of sectionId + + Raises + ------ + RenderError + + """ + + bounds = get_tilebounds_for_z( + stack, z, host, port, owner, project, session) + + try: + return bounds[0]['sectionId'] + except Exception as e: + logger.error(e) + raise RenderError('Could not find z value %f in stack %s' % (z, stack)) + + +@renderaccess +def get_stack_sectionData(stack, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """returns information about the sectionIds of each slice in stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + name of stack to get data about + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + dict + sectionData as below + :: + [{ + "sectionId": "string", + "z": 0, + "tileCount": 0, + "minX": 0, + "maxX": 0, + "minY": 0, + "maxY": 0 + }] + Raises + ------ + RenderError + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/sectionData' + return get_json(session, request_url) + + +@renderaccess +def get_section_z_value(stack, sectionId, host=None, port=None, + owner=None, project=None, session=None, + render=None, **kwargs): + """get the z value for a specific sectionId (string) + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack string to look within + sectionId : str + sectionId to find z value + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + float + z value of section + + Raises + ------ + RenderError + """ + request_url = format_preamble( + host, port, owner, project, stack) + "/section/%s/z" % sectionId + return get_json(session, request_url) + + +@renderaccess +def get_stack_tileIds(stack, host=None, port=None, owner=None, project=None, + session=None, render=None, **kwargs): + """get tileIds for a stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to get tileIds + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + :obj:`list` of :obj:`str` + list of tileIds in stack + + Raises + ------ + RenderError + """ + request_url = '{}/tileIds'.format( + format_preamble(host, port, owner, project, stack)) + r = session.get(request_url) + try: + # FIXME render bug return non-json formatted answer + # return r.json() + return json.loads(r.text.replace("'", '"')) + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError(r.text) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py new file mode 100644 index 00000000..e2fabe43 --- /dev/null +++ b/renderapi/tilespec.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python +import logging +import numpy as np +from .render import format_preamble, renderaccess +from .utils import NullHandler, get_json +from .stack import get_z_values_for_stack +from .transform import TransformList, estimate_dstpts +from .image_pyramid import MipMap, ImagePyramid +from .layout import Layout +from .channel import Channel + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class TileSpec: + '''Fundamental class of render that store image tiles + and their transformations + + Attributes + ---------- + tileId : str + unique string specifying a tile's identity + z : float + z values this tile exists within + width : int + width in pixels of the raw tile + height : int + height in pixels of the raw tile + imageUrl : str + an image path URI that can be accessed by the render server, + with an ImageJ.open command. Preceded by , e.g. 'file://' for files + or s3:// for s3 + maskUrl : str + an image path that can be accessed by the render server + which can be interpreted as an + alpha mask for the image (same as spec imageUrl) + minint : int + pixel intensity value to display as black in a + linear colormap (default 0) + maxint : int + pixel intensity value to display as white in a + linear colormap (default 65535) + layout : :class:`Layout` + a :class:`Layout` object for this tile + tforms : :obj:`list` of :obj:`Transform` + or :obj:`list` of :obj:`TransformList` + or :obj:`list` of :obj:`InterpolatedTransform` + Transform objects + (see :class:`.transform.AffineModel`, + :class:`.transform.TransformList`, + :class:`.transform.Polynomial2DTransform`, + :class:`.transform.Transform`, + :class:`.transform.ReferenceTransform`) to apply to this tile + inputfilters : list + a list of filters to apply to this tile (not yet implemented) + mipMapLevels :obj:`list` of :obj:`MipMapLevel` + :class:`MipMapLevel` objects for this tile + (DEPRECATED, use imagePyramid instead) + imagePyramid :obj:`ImagePyramid` + :class:`ImagePyramid` for this tile + json : dict or None + dictionary to initialize this object with + (if not None overrides and ignores all keyword arguments) + ''' + + def __init__(self, tileId=None, z=None, width=None, height=None, + imageUrl=None, maskUrl=None, + minint=0, maxint=65535, layout=None, tforms=None, + labels=None, groupId=None, inputfilters=None, json=None, + channels=None, mipMapLevels=None, imagePyramid=None, + **kwargs): + if json is not None: + self.from_dict(json) + else: + self.tileId = tileId + self.z = z + self.width = width + self.height = height + self.layout = layout + self.minint = minint + self.maxint = maxint + self.tforms = tforms if tforms else [] + self.labels = labels if labels else [] + self.groupId = groupId + self.inputfilters = inputfilters if inputfilters else [] + self.layout = Layout(**kwargs) if layout is None else layout + + if imagePyramid is not None: + self.ip = imagePyramid + else: + if mipMapLevels is not None: + self.ip = ImagePyramid({m.level: m.mipmap + for m in mipMapLevels}) + else: + self.ip = ImagePyramid() + + # legacy scaleXUrl + self.maskUrl = maskUrl + self.imageUrl = imageUrl + + self.channels = channels + if imageUrl is not None: + self.ip[0] = MipMap( + imageUrl=imageUrl, + maskUrl=maskUrl) + + @property + def bbox(self): + """bbox defined to fit shapely call""" + box = (self.minX, self.minY, self.maxX, self.maxY) + if any([v is None for v in box]): + logger.error( + 'undefined bounding box for tile {}'.format(self.tileId)) + return box + + def bbox_transformed(self, ndiv_inner=0, + tf_limit=None, reference_tforms=None): + """method to return Nx2 transformed coordinates of bounding box + Paramters + --------- + ndiv_inner : starting with just corner points, add intermediate + points to the boundary, recursively, ndiv_inner times + tf_limit : + 0 returns the raw bounding box + 1 returns the bounding box with the first transform applied + ... + None all transforms are applied + + Returns + ------- + Nx2 array ready for input to shapely.Polygon() + """ + # start with closed Nx2 array of corners + xy = np.zeros((5, 2)).astype('float') + xy[0, :] = [0, 0] + xy[1, :] = [0, self.height] + xy[2, :] = [self.width, self.height] + xy[3, :] = [self.width, 0] + xy[4, :] = [0, 0] + + # recursively add points to the boundary + while ndiv_inner > 0: + sz = 2 * xy.shape[0] - 1 + newxy = np.zeros((sz, 2)).astype('float') + newxy[0::2, :] = xy[:, :] + newxy[1:sz:2, :] = 0.5 * \ + (newxy[0:(sz - 2):2, :] + newxy[2:sz:2, :]) + xy = newxy + ndiv_inner -= 1 + + xy = estimate_dstpts(self.tforms[0:tf_limit], + src=xy, reference_tforms=reference_tforms) + + return xy + + def to_dict(self): + """method to produce a json tilespec for this tile + returns a json compatible dictionary + + Returns + ------- + dict + json compatible dictionary representation of this object + """ + thedict = {} + thedict['tileId'] = self.tileId + thedict['z'] = self.z + thedict['width'] = self.width + thedict['height'] = self.height + thedict['minIntensity'] = self.minint + thedict['maxIntensity'] = self.maxint + if self.layout is not None: + thedict['layout'] = self.layout.to_dict() + thedict['mipmapLevels'] = self.ip.to_dict() + thedict['transforms'] = {} + thedict['transforms']['type'] = 'list' + # thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] + thedict['transforms']['specList'] = [] + if self.channels is not None: + thedict['channels'] = [ch.to_dict() for ch in self.channels] + for t in self.tforms: + strlist = {} + # added by sharmi - if your speclist contains a speclist (can + # happen if you run the optimization more than once) + if isinstance(t, list): + strlist['type'] = 'list' + strlist['specList'] = [tt.to_dict() for tt in t] + thedict['transforms']['specList'].append(strlist) + else: + thedict['transforms']['specList'].append(t.to_dict()) + + if len(self.labels) > 0: + thedict['labels'] = self.labels + + if self.groupId is not None: + thedict['groupId'] = self.groupId + + # TODO filters not implemented + ''' + if len(self.inputfilters): + thedict['inputfilters'] = {} + thedict['inputfilters']['type'] = 'list' + thedict['inputfilters']['specList'] = [f.to_dict() for f + in self.inputfilters] + ''' + + thedict = {k: v for k, v in thedict.items() if v is not None} + return thedict + + def from_dict(self, d): + """Method to load tilespec from json dictionary + + Paramters + --------- + d : dict + dictionary to use to set properties of this object + """ + self.tileId = d['tileId'] + self.z = d['z'] + self.width = d['width'] + self.height = d['height'] + self.minint = d.get('minIntensity') + self.maxint = d.get('maxIntensity') + self.frameId = d.get('frameId') + self.layout = Layout() + self.layout.from_dict(d.get('layout', None)) + self.minX = d.get('minX', None) + self.maxX = d.get('maxX', None) + self.maxY = d.get('maxY', None) + self.minY = d.get('minY', None) + mmld = d.get('mipmapLevels', {}) + self.ip = ImagePyramid({l: MipMap( + imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) + for l, v in mmld.items()}) + + tfl = TransformList(json=d['transforms']) + self.tforms = tfl.tforms + chd = d.get('channels', None) + if chd is None: + self.channels = None + else: + self.channels = [Channel(json=ch) for ch in chd] + + self.labels = d.get('labels', []) + self.groupId = d.get('groupId', None) + + # TODO filters not implemented -- should skip + ''' + self.inputfilters = [] + if d.get('inputfilters', None) is not None: + for f in d['inputfilters']['specList']: + f = Filter() + f.from_dict(f) + self.inputfilters.append(f) + ''' + + +@renderaccess +def get_tile_spec_renderparameters(stack, tile, host=None, port=None, + owner=None, project=None, + session=None, + render=None, **kwargs): + """renderapi call to get the render parameters of a specific tileId + + :func:`renderapi.render.renderaccess` decorated function + TODO provide example of return + + Parameters + ---------- + stack : str + name of render stack to retrieve + tile : str + tileId of tile to retrieve + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + dict + render-parameters json with the tilespec for that + tile and dereferenced transforms + + """ + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/render-parameters" % (tile) + return get_json(session, request_url) + + +@renderaccess +def get_tile_spec(stack, tile, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """renderapi call to get a specific tilespec by tileId + note that this will return a tilespec with resolved transform references + by accessing the render-parameters version of this tile + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + name of render stack to retrieve + tile : str + tileId of tile to retrieve + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + TileSpec + TileSpec with dereferenced transforms + """ + + try: + tilespec_json = get_tile_spec_renderparameters( + stack, tile, host, port, owner, project, session) + return TileSpec(json=tilespec_json['tileSpecs'][0]) + except Exception as e: + logger.error(e) + + +@renderaccess +def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """renderapi call to get a specific tilespec by tileId + note that this will return a tilespec without resolved transform references + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + name of render stack to retrieve + tile : str + tileId of tile to retrieve + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + TileSpec + TileSpec with referenced transforms intact + """ + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/" % (tile) + return TileSpec(json=get_json(session, request_url)) + + +@renderaccess +def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, + scale=1.0, host=None, + port=None, owner=None, project=None, + session=None, + render=None, **kwargs): + """renderapi call to get all tilespec that exist within a 2d bounding box + specified with min and max x,y values + note that this will return a tilespec with resolved transform references + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + name of render stack to retrieve + z : float + z value of bounding box + xmin : float + minimum x value + ymin : float + minimum y value + xmax : float + maximum x value + ymax : float + maximum y value + scale : float + scale to use when retrieving render parameters (not important) + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + :class:`TileSpec` objects with dereferenced tansforms + """ + x = xmin + y = ymin + width = xmax - xmin + height = ymax - ymin + return get_tile_specs_from_box(stack, z, x, y, width, height, + scale=scale, host=host, port=port, + owner=owner, project=project, + session=session) + + +@renderaccess +def get_tile_specs_from_box(stack, z, x, y, width, height, + scale=1.0, host=None, port=None, owner=None, + project=None, session=None, + render=None, **kwargs): + """renderapi call to get all tilespec that exist within a 2d bounding box + specified with min x,y values and width, height + note that this will return a tilespec with resolved transform references + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + name of render stack to retrieve + z : float + z value of bounding box + x : float + minimum x value + y : float + minimum y value + width : float + width of box (in scale=1.0 units) + height : float + height of box (in scale=1.0 units) + scale : float + scale to use when retrieving render parameters (not important) + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + TileSpec objects with dereferenced tansforms + """ + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( + z, x, y, width, height, scale) + logger.debug(request_url) + tilespecs_json = get_json(session, request_url) + return [TileSpec(json=tilespec_json) + for tilespec_json in tilespecs_json['tileSpecs']] + + +@renderaccess +def get_tile_specs_from_z(stack, z, host=None, port=None, + owner=None, project=None, session=None, + render=None, **kwargs): + """Get all TileSpecs in a specific z values. Returns referenced transforms. + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack + z : float + render z + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + list of TileSpec objects from that stack at that z + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) + logger.debug(request_url) + tilespecs_json = get_json(session, request_url) + + if len(tilespecs_json) == 0: + return None + else: + return [TileSpec(json=tilespec_json) + for tilespec_json in tilespecs_json] + + +@renderaccess +def get_tile_specs_from_stack(stack, host=None, port=None, + owner=None, project=None, + session=None, + render=None, **kwargs): + """get flat list of tilespecs for stack using i for sl in l for i in sl + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + list of TileSpec objects from that stack + """ + return [i for sl in [ + get_tile_specs_from_z(stack, z, host=host, port=port, + owner=owner, project=project, session=session) + for z in get_z_values_for_stack(stack, host=host, port=port, + owner=owner, project=project, + session=session)] for i in sl] + +# TODO: ADD FEATURES THAT REQUIRED THESE TO SUPPORT.. NOT YET FULLY IMPLEMENTED +# class ResolvedTileSpecMap: +# def __init__(self, tilespecs=[], transforms=[]): +# self.tilespecs = tilespecs +# self.transforms = transforms + +# def to_dict(self): +# d = {} +# d['tileIdToSpecMap'] = {} +# for ts in self.tilespecs: +# d['tileIdToSpecMap'][ts.tileId] = ts.to_dict() +# d['transformIdToSpecMap'] = {} +# for tf in self.transforms: +# d['transformIdToSpecMap'][tf.transformId] = tf.to_dict() +# return d + +# def from_dict(self, d): +# tsmap = d['tileIdToSpecMap'] +# tfmap = d['transformIdToSpecMap'] +# for tsd in tsmap.values(): +# ts = TileSpec() +# ts.from_dict(tsd) +# self.tilespecs.append(ts) +# for tfd in tfmap.values(): +# tf = load_transform_json(tfd) +# self.transforms.append(tf) + + +# class ResolvedTileSpecCollection: +# def __init__(self, tilespecs=[], transforms=[]): +# self.tilespecs = tilespecs +# self.transforms = transforms + +# def to_dict(self): +# d = {} +# d['tileCount'] = len(self.tilespecs) +# d['tileSpecs'] = [ts.to_dict() for ts in self.tilespecs] +# d['transformCount'] = len(self.transforms) +# d['transformSpecs'] = [tf.to_dict() for tf in self.transforms] +# return d + +# def from_dict(self, d): +# self.tilespecs = [] +# self.transforms = [] +# for i in range(d['tileCount']): +# ts = TileSpec() +# ts.from_dict(d['tileSpecs'][i]) +# self.tilespecs.append(ts) +# for i in range(d['tranformCount']): +# tfd = d['transformSpecs'][i] +# tf = load_transform_json(tfd) +# self.transforms.append(tf) + + +# class Filter: +# def __init__(self, classname, params={}): +# self.classname = classname +# self.params = params + +# def to_dict(self): +# d = {} +# d['className'] = self.classname +# d['params'] = self.params +# return d + +# def from_dict(self, d): +# self.classname = d['className'] +# self.params = d['params'] diff --git a/renderapi/transform/__init__.py b/renderapi/transform/__init__.py new file mode 100644 index 00000000..3a1277b9 --- /dev/null +++ b/renderapi/transform/__init__.py @@ -0,0 +1,3 @@ +from .transform import * +from .leaf import * +from .utils import * diff --git a/renderapi/transform/leaf/__init__.py b/renderapi/transform/leaf/__init__.py new file mode 100644 index 00000000..731d2eb1 --- /dev/null +++ b/renderapi/transform/leaf/__init__.py @@ -0,0 +1,5 @@ +from .transform import * +from .affine_models import * +from .polynomial_models import * +from .thin_plate_spline import * +from .utils import * diff --git a/renderapi/transform/leaf/affine_models.py b/renderapi/transform/leaf/affine_models.py new file mode 100644 index 00000000..11d67d43 --- /dev/null +++ b/renderapi/transform/leaf/affine_models.py @@ -0,0 +1,638 @@ +from .transform import Transform, logger +from .common import calc_first_order_properties +import numpy as np +from renderapi.errors import ConversionError, EstimationError + +try: + from scipy.linalg import svd, LinAlgError +except ImportError as e: + logger.info(e) + logger.info('scipy-based linalg may or may not lead ' + 'to better parameter fitting') + from numpy.linalg import svd + from numpy.linalg.linalg import LinAlgError +__all__ = [ + 'AffineModel', 'TranslationModel', + 'SimilarityModel', 'RigidModel'] + + +class AffineModel(Transform): + """Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + + className = 'mpicbg.trakem2.transform.AffineModel2D' + + def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, + transformId=None, labels=None, json=None, force_shear='x'): + """Initialize AffineModel, defaulting to identity + + Parameters + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str + unique transformId for this transform (optional) + labels : list of str + list of labels to give this transform + json : dict + json compatible representation of this transform + (will supersede all other parameters if not None) + + """ + self.force_shear = force_shear + if json is not None: + self.from_dict(json) + else: + self.M00 = M00 + self.M01 = M01 + self.M10 = M10 + self.M11 = M11 + self.B0 = B0 + self.B1 = B1 + self.className = 'mpicbg.trakem2.transform.AffineModel2D' + self.labels = labels + self.load_M() + self.transformId = transformId + + @property + def dataString(self): + """dataString string for this transform""" + return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( + self.M[0, 0], self.M[1, 0], self.M[0, 1], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + + def _process_dataString(self, datastring): + """generate datastring and param attributes from datastring""" + dsList = datastring.split() + self.M00 = float(dsList[0]) + self.M10 = float(dsList[1]) + self.M01 = float(dsList[2]) + self.M11 = float(dsList[3]) + self.B0 = float(dsList[4]) + self.B1 = float(dsList[5]) + self.load_M() + + def load_M(self): + """method to take the attribute of self and fill in self.M""" + self.M = np.identity(3, np.double) + self.M[0, 0] = self.M00 + self.M[0, 1] = self.M01 + self.M[1, 0] = self.M10 + self.M[1, 1] = self.M11 + self.M[0, 2] = self.B0 + self.M[1, 2] = self.B1 + + @staticmethod + def fit(A, B, return_all=False): + """function to fit this transform given the corresponding sets of points A & B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + N = A.shape[0] # total points + + M = np.zeros((2 * N, 6)) + Y = np.zeros((2 * N, 1)) + for i in range(N): + M[2 * i, :] = [A[i, 0], A[i, 1], 0, 0, 1, 0] + M[2 * i + 1, :] = [0, 0, A[i, 0], A[i, 1], 0, 1] + Y[2 * i] = B[i, 0] + Y[2 * i + 1] = B[i, 1] + + (Tvec, residuals, rank, s) = np.linalg.lstsq(M, Y) + if return_all: + return Tvec, residuals, rank, s + return Tvec + + def estimate(self, A, B, return_params=True, **kwargs): + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the parameter matrix + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + Tvec = self.fit(A, B, **kwargs) + self.M00 = Tvec[0, 0] + self.M10 = Tvec[2, 0] + self.M01 = Tvec[1, 0] + self.M11 = Tvec[3, 0] + self.B0 = Tvec[4, 0] + self.B1 = Tvec[5, 0] + self.load_M() + if return_params: + return self.M + + def concatenate(self, model): + """concatenate a model to this model -- ported from trakEM2 below: + :: + + final double a00 = m00 * model.m00 + m01 * model.m10; + final double a01 = m00 * model.m01 + m01 * model.m11; + final double a02 = m00 * model.m02 + m01 * model.m12 + m02; + + final double a10 = m10 * model.m00 + m11 * model.m10; + final double a11 = m10 * model.m01 + m11 * model.m11; + final double a12 = m10 * model.m02 + m11 * model.m12 + m12; + + Parameters + ---------- + model : AffineModel + model to concatenate to this one + + Returns + ------- + AffineModel + model after concatenating model with this model + """ + A = self.M.dot(model.M) + newmodel = AffineModel( + A[0, 0], A[0, 1], A[1, 0], + A[1, 1], A[0, 2], A[1, 2]) + return newmodel + + def invert(self): + """return an inverted version of this transformation + + Returns + ------- + AffineModel + an inverted version of this transformation + """ + inv_M = np.linalg.inv(self.M) + Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], + inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) + return Ai + + @staticmethod + def convert_to_point_vector(points): + """method to help reshape x,y points to x,y,1 vectors + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx3 array of x,y,1 points used for transformations + """ + Np = points.shape[0] + onevec = np.ones((Np, 1), np.double) + + if points.shape[1] != 2: + raise ConversionError('Points must be of shape (:, 2) ' + '-- got {}'.format(points.shape)) + Nd = 2 + points = np.concatenate((points, onevec), axis=1) + return points, Nd + + @staticmethod + def convert_points_vector_to_array(points, Nd=2): + """method for convertion x,y,K points to x,y vectors + + Parameters + ---------- + points : numpy.array + a Nx3 vector of points after transformation + Nd : int + the number of dimensions to cutoff (should be 2) + + Returns + ------- + numpy.array: a Nx2 array of x,y points + """ + points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T + return points + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(self.M, points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + def inverse_tform(self, points): + """transform a set of points through the inverse of this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after inverse transformation + """ + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(np.linalg.inv(self.M), points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + def calc_properties(self): + return calc_first_order_properties( + self.M[0:2, 0:2], + force_shear=self.force_shear) + + @property + def scale(self): + """tuple of scale for x, y""" + sx, sy, cx, cy, theta = self.calc_properties() + return (sx, sy) + + @property + def shear(self): + """shear""" + sx, sy, cx, cy, theta = self.calc_properties() + if self.force_shear == 'x': + return cx + else: + return cy + + @property + def translation(self): + """tuple of translation in x, y""" + return tuple(self.M[:2, 2]) + + @property + def rotation(self): + """counter-clockwise rotation""" + sx, sy, cx, cy, theta = self.calc_properties() + return theta + + def __str__(self): + return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( + self.M[0, 0], self.M[0, 1], self.M[1, 0], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + + +class TranslationModel(AffineModel): + """Translation fitting and estimation as an :class:`AffineModel` + Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + """ + + className = 'mpicbg.trakem2.transform.TranslationModel2D' + + def __init__(self, *args, **kwargs): + super(TranslationModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected dataString is 'tx ty'""" + tx, ty = map(float, dataString.split(' ')) + self.B0 = tx + self.B1 = ty + self.M00 = 1 + self.M10 = 0 + self.M01 = 0 + self.M11 = 1 + self.load_M() + + @staticmethod + def fit(src, dst): + """function to fit Translation transform given + the corresponding sets of points src & dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + t = dst.mean(axis=0) - src.mean(axis=0) + T = np.eye(3) + T[:2, 2] = t + return T + + def estimate(self, src, dst, return_params=True): + """method for setting this transformation with the best fit + given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + self.M = self.fit(src, dst) + if return_params: + return self.M + + +class RigidModel(AffineModel): + """model for fitting Rigid only transformations + (rotation+translation) + or + (determinate=1, orthonormal eigenvectors) + implemented as an :class:`AffineModel` + + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + className = 'mpicbg.trakem2.transform.RigidModel2D' + + def __init__(self, *args, **kwargs): + super(RigidModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected datastring is 'theta tx ty'""" + theta, tx, ty = map(float, dataString.split(' ')) + self.M00 = np.cos(theta) + self.M01 = -np.sin(theta) + self.M10 = np.sin(theta) + self.M11 = np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() + + @staticmethod + def fit(src, dst, rigid=True, **kwargs): + """function to fit this transform given the corresponding + sets of points src & dst + Umeyama estimation of similarity transformation + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + # TODO shape assertion + num, dim = src.shape + src_cld = src - src.mean(axis=0) + dst_cld = dst - dst.mean(axis=0) + A = np.dot(dst_cld.T, src_cld) / num + d = np.ones((dim, ), dtype=np.double) + if np.linalg.det(A) < 0: + d[dim - 1] = -1 + T = np.eye(dim + 1, dtype=np.double) + + rank = np.linalg.matrix_rank(A) + if rank == 0: + raise EstimationError('zero rank matrix A unacceptable -- ' + 'likely poorly conditioned') + + U, S, V = svd(A) + + if rank == dim - 1: + if np.linalg.det(U) * np.linalg.det(V) > 0: + T[:dim, :dim] = np.dot(U, V) + else: + s = d[dim - 1] + d[dim - 1] = -1 + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V)) + d[dim - 1] = s + else: + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V.T)) + + fit_scale = (1.0 if rigid else + 1.0 / src_cld.var(axis=0).sum() * np.dot(S, d)) + + T[:dim, dim] = dst.mean(axis=0) - fit_scale * np.dot( + T[:dim, :dim], src.mean(axis=0).T) + T[:dim, :dim] *= fit_scale + return T + + def estimate(self, A, B, return_params=True, **kwargs): + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + self.M = self.fit(A, B, **kwargs) + if return_params: + return self.M + + +class SimilarityModel(RigidModel): + """class for fitting Similarity transformations + (translation+rotation+scaling) + or + (orthogonal eigen vectors with equal eigenvalues) + + implemented as an :class:`AffineModel` + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + className = 'mpicbg.trakem2.transform.SimilarityModel2D' + + def __init__(self, *args, **kwargs): + super(SimilarityModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected datastring is 's theta tx ty'""" + s, theta, tx, ty = map(float, dataString.split(' ')) + self.M00 = s * np.cos(theta) + self.M01 = -s * np.sin(theta) + self.M10 = s * np.sin(theta) + self.M11 = s * np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() + + @staticmethod + def fit(src, dst, rigid=False, **kwargs): + """function to fit this transform given the corresponding + sets of points src & dst + Umeyama estimation of similarity transformation + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + return RigidModel.fit(src, dst, rigid=rigid) diff --git a/renderapi/transform/leaf/common.py b/renderapi/transform/leaf/common.py new file mode 100644 index 00000000..fa3fe4b8 --- /dev/null +++ b/renderapi/transform/leaf/common.py @@ -0,0 +1,57 @@ +import numpy as np + + +def calc_first_order_properties(M, force_shear='x'): + """calculate scale shear and rotation from a 2x2 matrix + could be M[0:2, 0:2] from an AffineModel or params[:, 1:3] + from a Polynomial2DTransform + + Parameters + ---------- + M : numpy array + 2x2 + force_shear : str + 'x' or 'y' + + Returns + ------- + sx : float + scale in x direction + sy : float + scale in y direction + cx : float + shear in x direction + cy : float + shear in y direction + theta : float + rotation angle in radians + """ + if force_shear == 'x': + sy = np.sqrt(M[1, 0] ** 2 + M[1, 1] ** 2) + theta = np.arctan2(M[1, 0], M[1, 1]) + rc = np.cos(theta) + rs = np.sin(theta) + sx = rc * M[0, 0] - rs * M[0, 1] + if rs != 0: + cx = (M[0, 0] - sx*rc) / (sx * rs) + else: + cx = (M[0, 1] - sx*rs) / (sx * rc) + cy = 0.0 + elif force_shear == 'y': + # shear in y direction + sx = np.sqrt(M[0, 0] ** 2 + M[0, 1] ** 2) + theta = np.arctan2(-M[0, 1], M[0, 0]) + rc = np.cos(theta) + rs = np.sin(theta) + sy = rs * M[1, 0] + rc * M[1, 1] + if rs != 0: + cy = (M[1, 1] - sy * rc) / (-sy * rs) + else: + cy = (M[1, 0] - sy * rs) / (sy * rc) + cx = 0.0 + else: + raise ValueError("%s not a valid option for force_shear." + " should be 'x' or 'y'") + # room for other cases, for example cx = cy + + return sx, sy, cx, cy, theta diff --git a/renderapi/transform/leaf/polynomial_models.py b/renderapi/transform/leaf/polynomial_models.py new file mode 100644 index 00000000..24080a95 --- /dev/null +++ b/renderapi/transform/leaf/polynomial_models.py @@ -0,0 +1,616 @@ +from .transform import Transform, logger +from .affine_models import AffineModel +import numpy as np +from .common import calc_first_order_properties +from renderapi.errors import ConversionError, EstimationError, RenderError + +try: + from scipy.linalg import svd, LinAlgError +except ImportError as e: + logger.info(e) + logger.info('scipy-based linalg may or may not lead ' + 'to better parameter fitting') + from numpy.linalg import svd, LinAlgError + +__all__ = [ + 'Polynomial2DTransform', 'NonLinearCoordinateTransform', + 'NonLinearTransform', 'LensCorrection'] + + +class Polynomial2DTransform(Transform): + """Polynomial2DTransform implemented as in skimage + + Attributes + ---------- + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + + """ + className = 'mpicbg.trakem2.transform.PolynomialTransform2D' + + def __init__(self, dataString=None, src=None, dst=None, order=2, + force_polynomial=True, params=None, identity=False, + labels=None, transformId=None, json=None, + force_shear='x', **kwargs): + """Initialize Polynomial2DTransform + This provides 5 different ways to initialize the transform which are + mutually exclusive and applied in the order specified here. + 1)json2)dataString,3)identity,4)params,5)(src,dst) + + Parameters + ---------- + json : dict + dictionary representation of the Polynomial2DTransform + generally used by TransformList + dataString : str + dataString representation of transform from mpicpg + identity : bool + whether to make this transform the identity + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + src : numpy.array + Nx2 array of source points to use for fitting (used with dst) + dst : numpy.array + Nx2 array of destination points to use for fitting (used with src) + order : int + degree of polynomial to store + force_polynomial : bool + whether to force this representation to return a Polynomial + regardless of degree (not implemented) + + + """ + self.force_shear = force_shear + if json is not None: + self.from_dict(json) + else: + self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' + if dataString is not None: + self._process_dataString(dataString) + elif identity: + self.params = np.array([[0, 1, 0], [0, 0, 1]]) + elif params is not None: + self.params = params + elif src is not None and dst is not None: + self.estimate(src, dst, order, return_params=False, **kwargs) + + if not force_polynomial and self.is_affine: + raise NotImplementedError('Falling back to Affine model is ' + 'not supported {}') + self.transformId = transformId + self.labels = labels + + @property + def is_affine(self): + """(boolean) TODO allow default to Affine""" + return False + # return self.order + + @property + def order(self): + """(int) order of polynomial""" + no_coeffs = len(self.params.ravel()) + return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) + + def calc_properties(self): + if self.order == 0: + return 1.0, 1.0, 0.0, 0.0, 0.0 + return calc_first_order_properties( + self.params[:, 1:3], + force_shear=self.force_shear) + + @property + def scale(self): + """tuple of scale for x, y""" + sx, sy, cx, cy, theta = self.calc_properties() + return (sx, sy) + + @property + def shear(self): + """shear""" + sx, sy, cx, cy, theta = self.calc_properties() + if self.force_shear == 'x': + return cx + else: + return cy + + @property + def translation(self): + """tuple of translation in x, y""" + return tuple(self.params[:, 0]) + + @property + def rotation(self): + """counter-clockwise rotation""" + sx, sy, cx, cy, theta = self.calc_properties() + return theta + + @property + def dataString(self): + """dataString of polynomial""" + return Polynomial2DTransform._dataStringfromParams(self.params) + + @staticmethod + def fit(src, dst, order=2): + """function to fit this transform given the corresponding sets + of points src & dst + polynomial fit + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : bool + order of polynomial to fit + + Returns + ------- + numpy.array + a [2,(order+1)*(order+2)/2] array with the best fit parameters + """ + xs = src[:, 0] + ys = src[:, 1] + xd = dst[:, 0] + yd = dst[:, 1] + rows = src.shape[0] + no_coeff = (order + 1) * (order + 2) + + if len(src) != len(dst): + raise EstimationError( + 'source has {} points, but dest has {}!'.format( + len(src), len(dst))) + if no_coeff > len(src): + raise EstimationError( + 'order {} is too large to fit {} points!'.format( + order, len(src))) + + A = np.zeros([rows * 2, no_coeff + 1]) + pidx = 0 + for j in range(order + 1): + for i in range(j + 1): + A[:rows, pidx] = xs ** (j - i) * ys ** i + A[rows:, pidx + no_coeff // 2] = xs ** (j - i) * ys ** i + pidx += 1 + + A[:rows, -1] = xd + A[rows:, -1] = yd + + # right singular vector corresponding to smallest singular value + _, s, V = svd(A) + Vsm = V[np.argmin(s), :] # never trust computers + return (-Vsm[:-1] / Vsm[-1]).reshape((2, no_coeff // 2)) + + def estimate(self, src, dst, order=2, + test_coords=True, max_tries=100, return_params=True, + **kwargs): + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : int + order of polynomial to fit + test_coords : bool + whether to test model after fitting to + make sure it is good (see fitgood) + max_tries : int + how many times to attempt to fit the model (see fitgood) + return_params : bool + whether to return the parameter matrix + **kwargs + dictionary of keyword arguments including those + that can be passed to fitgood + + Returns + ------- + numpy.array + a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix + (or None if return_params=False) + """ + def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): + """check if model produces a 'good' result + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + params : numpy.array + a Kx2 matrix of parameters + atol : float + absolute tolerance as in numpy.allclose for + transformed sample points + rtol : float + relative tolerance as in numpy.allclose for + transformed sample points + + Returns + ------- + bool + whether the goodness condition is met + """ + result = Polynomial2DTransform(params=params).tform(src) + t = np.allclose( + result, dst, + atol=atol, rtol=rtol) + return t + + estimated = False + tries = 0 + while (tries < max_tries and not estimated): + tries += 1 + try: + params = Polynomial2DTransform.fit(src, dst, order=order) + except (LinAlgError, ValueError) as e: + logger.debug('Encountered error {}'.format(e)) + continue + estimated = (fitgood(src, dst, params, **kwargs) if + test_coords else True) + + if tries == max_tries and not estimated: + raise EstimationError('Could not fit Polynomial ' + 'in {} attempts!'.format(tries)) + logger.debug('fit parameters in {} attempts'.format(tries)) + self.params = params + if return_params: + return self.params + + @staticmethod + def _dataStringfromParams(params=None): + """method for producing a dataString from the parameters""" + return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in params.flatten()]).replace('e', 'E') + + def _process_dataString(self, datastring): + """generate datastring and param attributes from datastring""" + dsList = datastring.split(' ') + self.params = Polynomial2DTransform._format_raveled_params(dsList) + + @staticmethod + def _format_raveled_params(raveled_params): + """method to reshape linear parameters into parameter matrix + + Parameters + ---------- + raveled_params : numpy.array + an K long vector of parameters + + Returns + ------- + numpy.array + a (2,K/2) matrix of parameters, with + first row for x and 2nd row for y + """ + halfway = int(len(raveled_params) / 2) + return np.array( + [[float(d) for d in raveled_params[:halfway]], + [float(d) for d in raveled_params[halfway:]]]) + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + dst = np.zeros(points.shape) + x = points[:, 0] + y = points[:, 1] + + o = int((-3 + np.sqrt(9 - 4 * (2 - len(self.params.ravel())))) / 2) + pidx = 0 + for j in range(o + 1): + for i in range(j + 1): + dst[:, 0] += self.params[0, pidx] * x ** (j - i) * y ** i + dst[:, 1] += self.params[1, pidx] * x ** (j - i) * y ** i + pidx += 1 + return dst + + def coefficients(self, order=None): + """determine number of coefficient terms in transform for a given order + + Parameters + ---------- + order : int, optional + order of polynomial, defaults to self.order + + Returns + ------- + int + number of coefficient terms expected in transform + + """ + if order is None: + order = self.order + return (order + 1) * (order + 2) + + def asorder(self, order): + '''return polynomial transform appoximation of this + transformation with a lower order + + Parameters + ---------- + order :int + desired order (must have order> current order) + + Returns + ------- + :class:`Polynomial2DTransform` + transform of lower order + + Raises + ------ + ConversionError + if target order < input order + ''' + if self.order > order: + raise ConversionError( + 'transformation {} is order {} -- conversion to ' + 'order {} not supported'.format( + self.dataString, self.order, order)) + new_params = np.zeros([2, self.coefficients(order) // 2]) + new_params[:self.params.shape[0], :self.params.shape[1]] = self.params + return Polynomial2DTransform(params=new_params) + + @staticmethod + def fromAffine(aff): + """return a polynomial transformation equavalent to a given Affine + + Parameters + ---------- + aff : AffineModel + transform to become equivalent to + + Returns + ------- + Polynomial2DTransform + Order 1 transform equal in effect to aff + + Raises + ------ + ConversionError + if input model is not AffineModel + """ + if not isinstance(aff, AffineModel): + raise ConversionError('attempting to convert a nonaffine model!') + return Polynomial2DTransform(order=1, params=np.array([ + [aff.M[0, 2], aff.M[0, 0], aff.M[0, 1]], + [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) + + +class NonLinearCoordinateTransform(Transform): + """ + render-python class that implements the + mpicbg.trakem2.transform.NonLinearCoordinateTransform class + + Parameters + ---------- + dataString: str or None + data string of transformation + labels : list of str + list of labels to give this transform + json: dict or None + json compatible dictionary representation of the transformation + + Returns + ------- + :class:`NonLinearTransform` + a transform instance + + + """ + + className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' + + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + if labels is not None: + self.labels = labels + self.transformId = transformId + self.className = ( + 'mpicbg.trakem2.transform.NonLinearCoordinateTransform') + + def _process_dataString(self, dataString): + + fields = dataString.split(" ") + + self.dimension = int(fields[0]) + self.length = int(fields[1]) + + # cutoff whitespace if there + fields = fields[0:2 + 4 * self.length + 2] + # last 2 fields are width and height + self.width = int(fields[-2]) + self.height = int(fields[-1]) + + data = np.array(fields[2:-2], dtype='float32') + try: + self.beta = data[0:2 * self.length].reshape(self.length, 2) + except ValueError as e: + raise RenderError( + 'Incorrect number of coefficients in ' + 'NonLinearCoordinateTransform. msg: {}'.format(e)) + if not (self.beta.shape[0] == self.length): + raise RenderError("not correct number of coefficents") + + # normMean and normVar follow + self.normMean = data[self.length * 2:self.length * 3] + self.normVar = data[self.length * 3:self.length * 4] + if not (self.normMean.shape[0] == self.length): + raise RenderError( + "incorrect number of normMean coefficents " + "{} != length {}".format(self.normMean.shape[0], self.length)) + if not (self.normVar.shape[0] == self.length): + raise RenderError( + "incorrect number of normVar coefficents " + "{} != {}".format(self.normVar.shape[0], self.length)) + + def kernelExpand(self, src, normMean=None, normVar=None): + """creates an expanded representation of the x,y + src points in a polynomial form + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a (N x self.length) array of coefficents + """ + x = src[:, 0] + y = src[:, 1] + + expanded = np.zeros([len(x), self.length]) + pidx = 0 + for i in range(1, self.dimension + 1): + for j in range(i, -1, -1): + expanded[:, pidx] = ( + np.power(x, j) * np.power(y, i - j)) + pidx += 1 + + if normMean is None: + normMean = self.normMean + if normVar is None: + normVar = self.normVar + + expanded[:, :-1] = ((expanded[:, :-1] - normMean[:-1]) / + normVar[:-1]) + expanded[:, -1] = 100.0 + return expanded + + def fit(self, A, B): + """function to fit this transform given the corresponding sets of points A & B + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + beta + a self.lengthx2 matrix with polynomial factors + normMean + a self.length vector of expanded means + normVar + a self.length vector of expanded standard deviations + """ + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + normMean = np.zeros(self.length).astype('float') + normVar = np.ones(self.length).astype('float') + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) + normMean = src_exp.mean(0) + normVar = src_exp.std(0) # poorly named variable + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) + + xcoeff, xresiduals, xrank, xs = np.linalg.lstsq(src_exp, B[:, 0]) + ycoeff, yresiduals, yrank, ys = np.linalg.lstsq(src_exp, B[:, 1]) + + beta = np.zeros((self.length, 2)) + beta[:, 0] = xcoeff + beta[:, 1] = ycoeff + + return beta, normMean, normVar + + def estimate(self, A, B, ndim=None, return_params=True, **kwargs): + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the dataString + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + dataString + """ + + beta, normMean, normVar = self.fit(A, B) + self.beta = beta + self.normMean = normMean + self.normVar = normVar + + if return_params: + return self.dataString + + def tform(self, src): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + + # final double[] featureVector = kernelExpand(position); + # return multiply(beta, featureVector); + nsrc = np.array(src, dtype=np.float64) + featureVector = self.kernelExpand(nsrc) + + dst = np.zeros(src.shape) + for i in range(0, featureVector.shape[1]): + dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i, 0]) + dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i, 1]) + return np.array(dst, dtype=src.dtype) + + @property + def dataString(self): + shapestring = '{} {}'.format(self.dimension, self.length) + betastring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.beta.ravel()]).replace('e', 'E') + meanstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normMean]).replace('e', 'E') + varstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normVar]).replace('e', 'E') + dimstring = '{} {}'.format(self.height, self.width) + return '{} {} {} {} {} '.format( + shapestring, betastring, meanstring, varstring, dimstring) + + +class NonLinearTransform(NonLinearCoordinateTransform): + className = 'mpicbg.trakem2.transform.nonLinearTransform' + + +class LensCorrection(NonLinearCoordinateTransform): + """ + a placeholder for the lenscorrection transform, same as NonLinearTransform + for now + """ + className = 'lenscorrection.NonLinearTransform' diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py new file mode 100644 index 00000000..793fc0f9 --- /dev/null +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -0,0 +1,490 @@ +import numpy as np +from renderapi.errors import RenderError, EstimationError +from renderapi.utils import encodeBase64, decodeBase64 +from .transform import Transform +import scipy.spatial +import logging +import sys +__all__ = ['ThinPlateSplineTransform'] + + +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler(sys.stdout)) + + +class ThinPlateSplineTransform(Transform): + """ + render-python class that can hold a dataString for + mpicbg.trakem2.transform.ThinPlateSplineTransform class. + Parameters + ---------- + dataString: str or None + data string of transformation + labels : list of str + list of labels to give this transform + json: dict or None + json compatible dictionary representation of the transformation + Returns + ------- + :class:`ThinPlateSplineTransform` + a transform instance + """ + + className = 'mpicbg.trakem2.transform.ThinPlateSplineTransform' + + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + self.labels = labels + self.transformId = transformId + self.className = ( + 'mpicbg.trakem2.transform.ThinPlateSplineTransform') + + def _process_dataString(self, dataString): + fields = dataString.split(" ") + + self.ndims = int(fields[1]) + self.nLm = int(fields[2]) + + if fields[3] != "null": + try: + values = decodeBase64(fields[3]) + self.aMtx = values[0:self.ndims*self.ndims].reshape( + self.ndims, self.ndims) + self.bVec = values[self.ndims*self.ndims:] + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + else: + self.aMtx = None + self.bVec = None + + try: + values = decodeBase64(fields[4]) + self.srcPts = values[0:self.ndims*self.nLm].reshape( + self.ndims, self.nLm, order='F') + self.dMtxDat = values[self.ndims*self.nLm:].reshape( + self.ndims, self.nLm, order='C') + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + return self.apply(points) + + def apply(self, points): + if not hasattr(self, 'dMtxDat'): + return points + + result = points + self.computeDeformationContribution(points) + if self.aMtx is not None: + result += self.aMtx.dot(points.transpose()).transpose() + if self.bVec is not None: + result += self.bVec + + return result + + def computeDeformationContribution(self, points): + disp = scipy.spatial.distance.cdist( + points, + self.srcPts.transpose(), + metric='sqeuclidean') + disp *= np.ma.log(np.sqrt(disp)).filled(0.0) + return disp.dot(self.dMtxDat.transpose()) + + def gradient_descent( + self, + pts, + gamma=1.0, + precision=0.0001, + max_iters=1000): + """based on https://en.wikipedia.org/wiki/Gradient_descent#Python + Parameters + ---------- + pts : numpy array + a Nx2 array of x,y points + gamma : float + step size is gamma fraction of current gradient + precision : float + criteria for stopping for differences between steps + max_iters : int + limit for iterations, error if reached + Returns + ------- + cur_pts : numpy array + a Nx2 array of x,y points, estimated inverse of pt + """ + cur_pts = np.copy(pts) + prev_pts = np.copy(pts) + step_size = 1 + iters = 0 + while (step_size > precision) & (iters < max_iters): + prev_pts[:, :] = cur_pts[:, :] + cur_pts -= gamma*(self.apply(prev_pts) - pts) + step_size = np.linalg.norm(cur_pts - prev_pts, axis=1).max() + iters += 1 + if iters == max_iters: + raise EstimationError( + 'gradient descent for inversion of ThinPlateSpline ' + 'reached maximum iterations: %d' % max_iters) + return cur_pts + + def inverse_tform( + self, + points, + gamma=1.0, + precision=0.0001, + max_iters=1000): + """transform a set of points through the inverse of this transformation + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + gamma : float + step size is gamma fraction of current gradient + precision : float + criteria for stopping for differences between steps + max_iters : int + limit for iterations, error if reached + Returns + ------- + numpy.array + a Nx2 array of x,y points after inverse transformation + """ + newpts = self.gradient_descent( + points, + gamma=gamma, + precision=precision, + max_iters=max_iters) + return newpts + + @staticmethod + def fit(A, B, computeAffine=True): + """function to fit this transform given the corresponding sets of points A & B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + dMatrix : numpy.array + ndims x nLm + aMatrix : numpy.array + ndims x ndims, affine matrix + bVector : numpy.array + ndims x 1, translation vector + """ + + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + # build displacements + ndims = B.shape[1] + nLm = B.shape[0] + y = (B - A).flatten() + + # compute K + # tempting to matricize this, but, nLm x nLm can get big + # settle for vectorize + kMatrix = np.zeros((ndims * nLm, ndims * nLm)) + for i in range(nLm): + r = np.linalg.norm(A[i, :] - A, axis=1) + nrm = np.zeros_like(r) + ind = np.argwhere(r > 1e-8) + nrm[ind] = r[ind] * r[ind] * np.log(r[ind]) + kMatrix[i * ndims, 0::2] = nrm + kMatrix[(i * ndims + 1)::2, 1::2] = nrm + + # compute L + lMatrix = kMatrix + if computeAffine: + pMatrix = np.tile(np.eye(ndims), (nLm, ndims + 1)) + for d in range(ndims): + pMatrix[0::2, d*ndims] = A[:, d] + pMatrix[1::2, d*ndims + 1] = A[:, d] + lMatrix = np.zeros( + (ndims * (nLm + ndims + 1), ndims * (nLm + ndims + 1))) + lMatrix[ + 0: pMatrix.shape[0], + kMatrix.shape[1]: kMatrix.shape[1] + pMatrix.shape[1]] = \ + pMatrix + pMatrix = np.transpose(pMatrix) + lMatrix[ + kMatrix.shape[0]: kMatrix.shape[0] + pMatrix.shape[0], + 0: pMatrix.shape[1]] = pMatrix + lMatrix[0: ndims * nLm, 0: ndims * nLm] = kMatrix + y = np.append(y, np.zeros(ndims * (ndims + 1))) + + wMatrix = np.linalg.solve(lMatrix, y) + + dMatrix = np.reshape(wMatrix[0: ndims * nLm], (ndims, nLm), order='F') + aMatrix = None + bVector = None + if computeAffine: + aMatrix = np.reshape( + wMatrix[ndims * nLm: ndims * nLm + ndims * ndims], + (ndims, ndims), + order='F') + bVector = wMatrix[ndims * nLm + ndims * ndims:] + + return dMatrix, aMatrix, bVector + + def estimate(self, A, B, computeAffine=True): + """method for setting this transformation with the best fit + given the corresponding points A,B + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + computeAffine: boolean + whether to include an affine computation + """ + + self.dMtxDat, self.aMtx, self.bVec = self.fit( + A, B, computeAffine=computeAffine) + (self.nLm, self.ndims) = B.shape + self.srcPts = np.transpose(A) + + @property + def dataString(self): + header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) + + if self.aMtx is not None: + blk1 = np.concatenate((self.aMtx.flatten(), self.bVec)) + b64_1 = encodeBase64(blk1) + else: + b64_1 = "null" + + blk2 = np.concatenate(( + self.srcPts.flatten(order='F'), + self.dMtxDat.flatten(order='C'))) + b64_2 = encodeBase64(blk2) + + return '{} {} {}'.format(header, b64_1, b64_2) + + @staticmethod + def mesh_refine( + new_src, + old_src, + old_dst, + old_tf=None, + computeAffine=True, + tol=1.0, + max_iter=50, + nworst=10, + niter=0): + """recursive kernel for adaptive_mesh_estimate() + Parameters + ---------- + new_src : numpy.array + Nx2 array of new control source points. Adapts during recursion. + Seeded by adaptive_mesh_estimate. + old_src : numpy.array + Nx2 array of orignal control source points. + old_dst : numpy.array + Nx2 array of orignal control destination points. + old_tf : ThinPlateSplineTransform + transform constructed from old_src and old_dst, passed through + recursion iterations. Created if None. + computeAffine : boolean + whether returned transform will have aMtx + tol : float + in units of pixels, how close should the points match + max_iter: int + some limit on how many recursive attempts + nworst : int + per iteration, the nworst matching srcPts will be added + niter : int + passed through the recursion for stopping criteria + + Returns + ------- + ThinPlateSplineTransform + """ + + if old_tf is None: + old_tf = ThinPlateSplineTransform() + old_tf.estimate(old_src, old_dst, computeAffine=computeAffine) + + new_tf = ThinPlateSplineTransform() + new_tf.estimate( + new_src, + old_tf.tform(new_src), + computeAffine=computeAffine) + new_dst = new_tf.tform(old_src) + + delta = np.linalg.norm(new_dst - old_dst, axis=1) + ind = np.argwhere(delta > tol).flatten() + + if ind.size == 0: + return new_tf + + if niter == max_iter: + raise EstimationError( + "Max number of iterations ({}) reached in" + " ThinPlateSplineTransform.mesh_refine()".format( + max_iter)) + + sortind = np.argsort(delta[ind]) + new_src = np.vstack((new_src, old_src[ind[sortind[0: nworst]]])) + + return ThinPlateSplineTransform.mesh_refine( + new_src, + old_src, + old_dst, + old_tf=old_tf, + computeAffine=computeAffine, + tol=tol, + max_iter=max_iter, + nworst=nworst, + niter=(niter + 1)) + + def adaptive_mesh_estimate( + self, + starting_grid=7, + computeAffine=True, + tol=1.0, + max_iter=50, + nworst=10): + """method for creating a transform with fewer control points + that matches the original transfom within some tolerance. + Parameters + ---------- + starting_grid : int + estimate will start with an n x n grid + computeAffine : boolean + whether returned transform will have aMtx + tol : float + in units of pixels, how close should the points match + max_iter: int + some limit on how many recursive attempts + nworst : int + per iteration, the nworst matching srcPts will be added + + Returns + ------- + ThinPlateSplineTransform + """ + + mn = self.srcPts.min(axis=1) + mx = self.srcPts.max(axis=1) + new_src = self.src_array( + mn[0], mn[1], mx[0], mx[1], starting_grid, starting_grid) + old_src = self.srcPts.transpose() + old_dst = self.tform(old_src) + + return ThinPlateSplineTransform.mesh_refine( + new_src, + old_src, + old_dst, + old_tf=self, + computeAffine=computeAffine, + tol=tol, + max_iter=max_iter, + nworst=nworst, + niter=0) + + @staticmethod + def src_array(xmin, ymin, xmax, ymax, nx, ny): + """create N x 2 array of regularly spaced points + + Parameters + ---------- + xmin : float + minimum of x grid + ymin : float + minimum of y grid + xmax : float + maximum of x grid + ymax : float + maximum of y grid + nx : int + number of points in x axis + ny : int + number of points in y axis + + Returns + ------- + src : :class:`numpy.ndarray` + (nx * ny) x 2 array of coordinated. + + """ + src = np.mgrid[xmin:xmax:nx*1j, ymin:ymax:ny*1j].reshape(2, -1).T + return src + + def scale_coordinates( + self, + factor, + ngrid=20, + preserve_srcPts=False): + """estimates a new ThinPlateSplineTransform from the current one + in a scaled transform space. + + Parameters + ---------- + factor : float + the factor by which to scale the space + ngrid : int + number of points per axis for the estimation grid + preserve_srcPts : bool + one might want to keep the original scaled srcPts + for example, if pts were made specially for a mask + or a crack or fold + + Returns + ------- + new_tform : :class:`renderapi.transform.ThinPlateSplineTransform` + the new transform in the scaled space + + """ + + new_tform = ThinPlateSplineTransform() + computeAffine = True + if self.aMtx is None: + computeAffine = False + + mn = self.srcPts.min(axis=1) + mx = self.srcPts.max(axis=1) + src = self.src_array(mn[0], mn[1], mx[0], mx[1], ngrid, ngrid) + + if preserve_srcPts: + # do not repeat close points + dist = scipy.spatial.distance.cdist( + src, + self.srcPts.transpose(), + metric='euclidean') + ind = np.invert(np.any(dist < 1e-3, axis=0)) + src = np.vstack((src, self.srcPts.transpose()[ind])) + + new_tform.estimate( + src * factor, + self.tform(src) * factor, + computeAffine=computeAffine) + + return new_tform diff --git a/renderapi/transform/leaf/transform.py b/renderapi/transform/leaf/transform.py new file mode 100644 index 00000000..ef7f6b34 --- /dev/null +++ b/renderapi/transform/leaf/transform.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python +"""handling mpicbg transforms in python + +Currently only implemented to facilitate Affine and Polynomial2D + used in Khaled Khairy's EM aligner workflow +""" +import logging + +from renderapi.utils import NullHandler + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) +__all__ = ['Transform'] + + +class Transform(object): + """Base transformation class + + Attributes + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced by + mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + """ + + def __init__(self, className=None, dataString=None, + transformId=None, labels=None, json=None): + """Initialize Transform + + Parameters + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced + by mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + labels : list of str + list of labels to give this transform + json : dict + json compatible representation of this transform + (supersedes className, dataString, and transformId if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.className = className + self.dataString = dataString + self.transformId = transformId + self.labels = labels + + def to_dict(self): + """serialization routine + + Returns + ------- + dict + json compatible representation of this transform + """ + d = {} + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = self.dataString + if self.transformId is not None: + d['id'] = self.transformId + if self.labels is not None: + d['metaData'] = {'labels': self.labels} + return d + + def from_dict(self, d): + """deserialization routine + + Parameters + ---------- + d : dict + json compatible representation of this transform + """ + self.className = d['className'] + self.transformId = d.get('id', None) + self._process_dataString(d['dataString']) + md = d.get('metaData', None) + if md is not None: + self.labels = md.get('labels', None) + else: + self.labels = None + + def _process_dataString(self, datastring): + """method meant to set state of transform from datastring + generic implementation only saves datastring at self.dataString. + should rewrite for all transform classes that want to + implement tform,fit,etc + + Parameters + ---------- + dataString : str + string which can be used to initialize mpicbg transforms in java + """ + self.dataString = datastring + + def __str__(self): + return 'className:%s\ndataString:%s' % ( + self.className, self.dataString) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return self.__str__() == other.__str__() + + def __hash__(self): + return hash((self.__str__())) diff --git a/renderapi/transform/leaf/utils.py b/renderapi/transform/leaf/utils.py new file mode 100644 index 00000000..c46116a1 --- /dev/null +++ b/renderapi/transform/leaf/utils.py @@ -0,0 +1,61 @@ +from .transform import Transform, logger +from renderapi.errors import RenderError +from .affine_models import ( + AffineModel, + TranslationModel, + RigidModel, + SimilarityModel) +from .polynomial_models import ( + Polynomial2DTransform, + NonLinearTransform, + NonLinearCoordinateTransform, + LensCorrection) +from .thin_plate_spline import ( + ThinPlateSplineTransform) +__all__ = ['load_leaf_json'] + + +def load_leaf_json(d): + """function to get the proper deserialization function for leaf transforms + + Parameters + ---------- + d : dict + json compatible representation of leaf transform to deserialize + + Returns + ------- + renderapi.transform.Transform + deserialized transformation + + Raises + ------ + RenderError + if d['type'] != leaf or is omitted + + """ + handle_load_leaf = { + AffineModel.className: lambda x: AffineModel(json=x), + Polynomial2DTransform.className: + lambda x: Polynomial2DTransform(json=x), + TranslationModel.className: lambda x: TranslationModel(json=x), + RigidModel.className: lambda x: RigidModel(json=x), + SimilarityModel.className: lambda x: SimilarityModel(json=x), + NonLinearTransform.className: lambda x: NonLinearTransform(json=x), + LensCorrection.className: lambda x: LensCorrection(json=x), + ThinPlateSplineTransform.className: + lambda x: ThinPlateSplineTransform(json=x), + NonLinearCoordinateTransform.className: + lambda x: NonLinearCoordinateTransform(json=x)} + + tform_type = d.get('type', 'leaf') + if tform_type != 'leaf': + raise RenderError( + 'Unexpected or unknown Transform Type {}'.format(tform_type)) + tform_class = d['className'] + try: + return handle_load_leaf[tform_class](d) + except KeyError as e: + logger.info('Leaf transform class {} not defined in ' + 'transform module, using generic'.format(e)) + return Transform(json=d) diff --git a/renderapi/transform/transform.py b/renderapi/transform/transform.py new file mode 100644 index 00000000..f0e14690 --- /dev/null +++ b/renderapi/transform/transform.py @@ -0,0 +1,242 @@ +import json +from renderapi.errors import RenderError +from renderapi.transform.leaf import load_leaf_json +__all__ = [ + 'TransformList', + 'ReferenceTransform', + 'InterpolatedTransform', + 'load_transform_json'] + + +class TransformList: + """A list of Transforms + + Attributes + ---------- + tforms : :obj:`list` of :class:`Transform` + transforms to apply + transformId : str, optional + uniqueId for this TransformList + """ + + def __init__(self, tforms=None, transformId=None, json=None): + """Initialize TransformList + + Parameters + ---------- + tforms : :obj:`list` of :class:`Transform` + transforms to apply + transformId : str, optional + uniqueId for this TransformList + json : dict, optional + json compatible dictionary to create + :class:`TransformList` via :method:`from_dict` + (will supersede tforms and transformId if not None) + """ + if json is not None: + self.from_dict(json) + else: + if tforms is None: + self.tforms = [] + else: + if not isinstance(tforms, list): + raise RenderError( + 'unexpected type {} for transforms!'.format( + type(tforms))) + self.tforms = tforms + self.transformId = transformId + + def to_dict(self): + """serialization function + + Returns + ------- + dict + json & render compatible representation of this TransformList + """ + d = {} + d['type'] = 'list' + d['specList'] = [tform.to_dict() for tform in self.tforms] + if self.transformId is not None: + d['id'] = self.transformId + return d + + def to_json(self): + """serialization function + + Returns + ------- + str + string representation of the json & render + representation of this TransformList + """ + return json.dumps(self.to_dict()) + + def from_dict(self, d): + """deserialization function + + Parameters + ---------- + d : dict + json compatible dictionary representation of this TransformList + """ + self.tforms = [] + if d is not None: + self.transformId = d.get('id') + for td in d['specList']: + self.tforms.append(load_transform_json(td)) + return self.tforms + + +class InterpolatedTransform: + """Transform spec defined by linear interpolation of + two other transform specs + + Attributes + ---------- + a : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + """ # noqa: E501 + + def __init__(self, a=None, b=None, lambda_=None, json=None): + """Initialize InterpolatedTransform + + Parameters + ---------- + a : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + json : dict + json compatible representation of this transform to + initialize via :method:`self.from_dict` + (will supersede a, b, and lambda_ if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.a = a + self.b = b + self.lambda_ = lambda_ + + def to_dict(self): + """serialization routine + Returns + ------- + dict + json compatible representation + """ + return dict(self) + + def from_dict(self, d): + """deserialization routine + Parameters + ---------- + d : dict + json compatible representation + """ + self.a = load_transform_json(d['a']) + self.b = load_transform_json(d['b']) + self.lambda_ = d['lambda'] + + def __iter__(self): + return iter([('type', 'interpolated'), + ('a', self.a.to_dict()), + ('b', self.b.to_dict()), + ('lambda', self.lambda_)]) + + +class ReferenceTransform: + """Transform which is simply a reference to a transform stored elsewhere + Attributes + ---------- + refId : str + transformId of the referenced transform + """ + + def __init__(self, refId=None, json=None): + """Initialize ReferenceTransform + Parameters + ---------- + refId : str + transformId of the referenced transform + json : dict + json compatible representation of this transform + (will supersede refId if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.refId = refId + + def to_dict(self): + """serialization routine + Returns + ------- + dict + json compatible representation of this transform + """ + d = {} + d['type'] = 'ref' + d['refId'] = self.refId + return d + + def from_dict(self, d): + """deserialization routine + Parameters + ---------- + d : dict + json compatible representation of this transform + """ + self.refId = d['refId'] + + def __str__(self): + return 'ReferenceTransform(%s)' % self.refId + + def __repr__(self): + return self.__str__() + + def __iter__(self): + return iter([('type', 'ref'), ('refId', self.refId)]) + + +def load_transform_json(d, default_type='leaf'): + """function to get the proper deserialization function + + Parameters + ---------- + d : dict + json compatible representation of Transform + default_type : str + what kind of transform should we assume this + if it is not specified in 'type' ('leaf','list','ref','interpolated') + + Returns + ------- + renderapi.transform.Transform + deserialized transformation using the most appropriate class + + Raises + ------ + RenderError + if d['type'] isn't one of ('leaf','list','ref','interpolated') + """ + handle_load_tform = {'leaf': load_leaf_json, + 'list': lambda x: TransformList(json=x), + 'ref': lambda x: ReferenceTransform(json=x), + 'interpolated': + lambda x: InterpolatedTransform(json=x)} + try: + return handle_load_tform[d.get('type', default_type)](d) + except KeyError as e: + raise RenderError('Unknown Transform Type {}'.format(e)) diff --git a/renderapi/transform/utils.py b/renderapi/transform/utils.py new file mode 100644 index 00000000..9c33b0f1 --- /dev/null +++ b/renderapi/transform/utils.py @@ -0,0 +1,92 @@ +from collections.abc import Iterable +from renderapi.errors import RenderError +from .leaf import AffineModel, Polynomial2DTransform +from .transform import TransformList, ReferenceTransform +__all__ = ['estimate_dstpts', + 'estimate_transformsum'] + + +def estimate_dstpts(transformlist, src=None, reference_tforms=None): + """estimate destination points for list of transforms. Recurses + through lists. + + Parameters + ---------- + transformlist : :obj:list of :obj:Transform + transforms that have a tform method implemented + src : numpy.array + a Nx2 array of source points + + Returns + ------- + numpy.array + Nx2 array of destination points + """ + dstpts = src + for tform in transformlist: + if isinstance(tform, list): + dstpts = estimate_dstpts(tform, dstpts, reference_tforms) + elif isinstance(tform, TransformList): + dstpts = estimate_dstpts(tform.tforms, dstpts, reference_tforms) + elif isinstance(tform, ReferenceTransform): + try: + tform_deref = next((tf for tf in reference_tforms + if tf.transformId == tform.refId)) + except TypeError: + raise RenderError( + "you supplied a set of tranforms that includes a " + "reference transform, but didn't supply a set of " + "reference transforms to enable dereferencing") + except StopIteration: + raise RenderError( + "the list of transforms you provided references " + "transorm {} but that transform could not be found " + "in the list of reference transforms".format(tform.refId)) + dstpts = estimate_dstpts([tform_deref], dstpts, reference_tforms) + else: + dstpts = tform.tform(dstpts) + return dstpts + + +def estimate_transformsum(transformlist, src=None, order=2): + """pseudo-composition of transforms in list of transforms + using source point transformation and a single estimation. + Will produce an Affine Model if all input transforms are Affine, + otherwise will produce a Polynomial of specified order + + Parameters + ---------- + transformlist : :obj:`list` of :obj:`Transform` + list of transform objects that implement tform + src : numpy.array + Nx2 array of source points for estimation + order : int + order of Polynomial output if transformlist + inputs are non-Affine + Returns + ------- + :class:`AffineModel` or :class:`Polynomial2DTransform` + best estimate of transformlist in a single transform of this order + """ + def flatten(l): + """generator-iterator to flatten deep lists of lists""" + for i in l: + if isinstance(i, Iterable): + try: + notstring = isinstance(i, basestring) + except NameError: + notstring = isinstance(i, str) + if notstring: + for sub in flatten(i): + yield sub + else: + yield i + + dstpts = estimate_dstpts(transformlist, src) + tforms = flatten(transformlist) + if all([(tform.className == AffineModel.className) + for tform in tforms]): + am = AffineModel() + am.estimate(A=src, B=dstpts, return_params=False) + return am + return Polynomial2DTransform(src=src, dst=dstpts, order=order) diff --git a/renderapi/utils.py b/renderapi/utils.py new file mode 100644 index 00000000..c46b2fb1 --- /dev/null +++ b/renderapi/utils.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python +''' +utilities to make render/java/web/life interfacing easier +''' +import tempfile +import logging +import copy +import json +import base64 +import zlib + +import numpy +import requests +try: + from inspect import getfullargspec +except ImportError: + from inspect import getargspec as getfullargspec + +from .errors import RenderError + +# use ujson if installed for faster json +try: + import ujson as requests_json +except ImportError: + import json as requests_json +requests.models.complexjson = requests_json + + +class NullHandler(logging.Handler): + """handler to avoid logging errors for, e.g., missing logger setup""" + def emit(self, record): + pass + + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class RenderEncoder(json.JSONEncoder): + """json Encoder in the following hierarchy for serialization: + obj.to_dict() + dict(obj) + JsonEncoder.default(obj) + obj.__dict__ + """ + def default(self, obj): + """default encoder for that handles Render objects + + Parameters + ---------- + obj : obj + any object that implements to_dict, dict(obj), + JsonEncoder.default(obj), or __dict__ (in order) + Returns + ------- + dict or list + json encodable datatype + + """ + if isinstance(obj, numpy.integer): + return int(obj) + to_dict = getattr(obj, "to_dict", None) + if callable(to_dict): + return obj.to_dict() + else: + try: + return dict(obj) + except TypeError as e: + logger.debug("{} object is not recognized dictionary".format( + type(obj))) + try: + return super(RenderEncoder, self).default(obj) + except TypeError as e: # pragma: no cover + logger.info(e) + logger.warning( + "cannot json serialize {}. " + "Defaulting to __dict__".format(type(obj))) + return obj.__dict__ + + +def post_json(session, request_url, d, params=None): + """POST requests with RenderError handling + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + d : dict + data payload (will be json dumps-ed) + params : dict + requests parameters + + Returns + ------- + requests.response: server response + + Raises + ------ + RenderError + if cannot post + """ + + headers = {"content-type": "application/json"} + if d is not None: + payload = json.dumps(d) + else: + payload = None + headers['Accept'] = "application/json" + r = session.post(request_url, data=payload, params=params, + headers=headers) + if r.status_code not in [200, 201, 204]: + raise RenderError( + 'cannot post {} to {} with params {} returned status_code ' + '{} with message {}'.format( + d, request_url, params, r.status_code, r.text)) + return r + + +def rest_delete(session, request_url, params=None): + """DELETE requests with RenderError handling + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + Returns + ------- + requests.response + server response + """ + r = session.delete(request_url) + if r.status_code not in [200, 202, 204]: + raise RenderError("delete of {} returned {} with message {}".format( + r.url, r.status_code, r.text)) + return r + + +def put_json(session, request_url, d, params=None): + """PUT requests with RenderError handling + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + d : dict + data payload (will be json dumps-ed) + params : dict + requests parameters + + Returns + ------- + requests.response + server response + + Raises + ------ + RenderError + if cannot post + """ + + headers = {"content-type": "application/json"} + if d is not None: + payload = renderdumps(d) + else: + payload = None + headers['Accept'] = "application/json" + r = session.put(request_url, data=payload, params=params, + headers=headers) + if r.status_code not in [200, 201, 204]: + raise RenderError( + 'put {} to {} returned status code {} with message {}'.format( + d, r.url, r.status_code, r.text)) + return r + + +def get_json(session, request_url, params=None, stream=False, **kwargs): + """get_json wrapper for requests to handle errors + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + params : dict + requests parameters + stream: bool + requests whether to stream + kwargs: dict + kwargs to shout into the dark + Returns + ------- + dict + json response from server + + Raises + ------ + RenderError + if cannot get json successfully + """ + + r = session.get(request_url, params=params, stream=stream) + if r.status_code != 200: + message = "request to {} returned error code {} with message {}" + raise RenderError(message.format(r.url, r.status_code, r.text)) + try: + return r.json() + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError(r.text) + + +def renderdumps(obj, *args, **kwargs): + """json.dumps using the RenderEncode + + Parameters + ---------- + obj : obj + object to dumps + *args + json.dumps args + **kwargs + json.dumps kwargs + + Returns + ------- + str + serialized object + """ + cls_ = kwargs.pop('cls', RenderEncoder) + return json.dumps(obj, *args, cls=cls_, **kwargs) + + +def renderdump(obj, *args, **kwargs): + """json.dump using the RenderEncoder + + Parameters + ---------- + obj : obj + object to dumps + *args + json.dump args + **kwargs + json.dump kwargs + """ + cls_ = kwargs.pop('cls', RenderEncoder) + return json.dump(obj, *args, cls=cls_, **kwargs) + + +def renderdump_temp(obj, *args, **kwargs): + """json.dump into a temporary file + renderdump_temp(obj), obj will be dumped through renderdump + into a temporary file + + Parameters + ---------- + obj : obj + object to dump + *args + json.dump args + **kwargs + json.dump kwargs + + Returns + ------- + str + path to location where temporary file was dumped + """ + + with tempfile.NamedTemporaryFile( + suffix=".json", mode='w', delete=False) as tf: + tempfilename = tf.name + renderdump(obj, tf, *args, **kwargs) + return tempfilename + + +def jbool(val): + """return string representing java string values of py booleans + + Parameters + ---------- + val : bool + boolean to encode + + Returns + ------- + str + 'true' or 'false' + + """ + if not isinstance(val, bool): + logger.warning('Evaluating javastring of non-boolean {} {}'.format( + type(val), val)) + return 'true' if val else 'false' + + +def stripLogger(logger_tostrip): # pragma: no cover + """remove all handlers from a logger -- useful for redefining + + Parameters + ---------- + logger_tostrip : :class:`logging.Logger` + logging logger to strip + """ + if logger_tostrip.handlers: + for handler in logger_tostrip.handlers: + logger_tostrip.removeHandler(handler) + + +def defaultifNone(val, default=None): + """simple default handler + + Parameters + ---------- + val : obj + value to fill in default + default : obj + default value + + Returns + ------- + obj + val if val is not None, else default + """ + return val if val is not None else default + + +def fitargspec(f, oldargs, oldkwargs): + """fit function argspec given input args tuple and kwargs dict + + Parameters + ---------- + f : func + function to inspect + oldargs : tuple + arguments passed to func + oldkwards : dict + keyword args passed to func + + Returns + ------- + new_args + args with values filled in according to f spec + new_kwargs + kwargs with values filled in according to f spec + """ + try: + arginfo = getfullargspec(f) + # args, varargs, keywords, defaults = inspect.getargspec(f) + num_expected_args = len(arginfo.args) - len(arginfo.defaults) + new_args = tuple(oldargs[:num_expected_args]) + new_kwargs = copy.copy(oldkwargs) + for i, arg in enumerate(oldargs[num_expected_args:]): + new_kwargs.update({arginfo.args[i + num_expected_args]: arg}) + return new_args, new_kwargs + except Exception as e: + logger.error('Cannot fit argspec for {}'.format(f)) + logger.error(e) + return oldargs, oldkwargs + + +def encodeBase64(src): + """encode an array or list of doubles + in Base64 binary-to-text encoding + same as in trakem2...ThinPlateSplineTransform.java + + Parameters + ---------- + src : 1D numpy array + floating point values to be encoded + + Returns + ------- + encoded: string + """ + return base64.b64encode( + zlib.compress( + src.byteswap().tobytes()) + ).decode('utf-8') + + +def decodeBase64(src): + """decode a string + encoded in base64 binary-to-text encoding + same as in trakem2...ThinPlateSplineTransform.java + + Parameters + ---------- + src : string + encoded string + + Returns + ------- + arr: length n numpy array of double-precision floats + """ + if src[0] == '@': + b = base64.b64decode(src[1:]) + else: + b = zlib.decompress(base64.b64decode(src)) + return numpy.frombuffer(b).byteswap() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..84f5af25 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +requests +scipy +numpy +pillow +sphinxcontrib-napoleon +decorator +six diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..4bba92fa --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[flake8] +ignore = E226,E126 +max-line-length = 79 diff --git a/setup.py b/setup.py deleted file mode 100644 index da0c5414..00000000 --- a/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup -setup(name='render-python', - version='1.0', - description=' a python API setup to interact via python with render databases see https://github.com/saalfeldlab/render', - author='Forrest Collman,Eric Perlman,Sharmi Seshamani', - author_email='forrest.collman@gmail.com', - url='https://github.com/fcollman/render-python', - packages = [''], - py_modules=['tilespec','renderapi']) diff --git a/test/rendersettings.py b/test/rendersettings.py new file mode 100644 index 00000000..2152fdae --- /dev/null +++ b/test/rendersettings.py @@ -0,0 +1,88 @@ +''' +default settings for render tests +''' +import os + +DEFAULT_RENDER = { + 'host': 'http://renderhost', + 'port': 8080, + 'owner': 'renderowner', + 'project': 'renderproject', + 'client_scripts': '/path/to/client_scripts' + } + +DEFAULT_RENDER_CLIENT = dict(DEFAULT_RENDER, **{ + 'client_script': '/path/to/client_scripts/run_ws_client.sh', + 'memGB': '2G'}) + +DEFAULT_RENDER_ENVIRONMENT_VARIABLES = { + 'RENDER_HOST': DEFAULT_RENDER['host'], + 'RENDER_PORT': DEFAULT_RENDER['port'], + 'RENDER_OWNER': DEFAULT_RENDER['owner'], + 'RENDER_PROJECT': DEFAULT_RENDER['project'], + 'RENDER_CLIENT_SCRIPTS': DEFAULT_RENDER['client_scripts'] + } + +DEFAULT_RENDER_CLIENT_ENVIRONMENT_VARIABLES = dict( + DEFAULT_RENDER_ENVIRONMENT_VARIABLES, **{ + 'RENDER_CLIENT_SCRIPT': DEFAULT_RENDER_CLIENT['client_script'], + 'RENDER_CLIENT_HEAP': DEFAULT_RENDER_CLIENT['memGB']}) + +TEST_FILES_DIR = os.path.join(os.path.dirname( + __file__), 'test_files') + +TEST_TILESPECS_FILE = os.path.join(TEST_FILES_DIR, 'tilespecs.json') + +TEST_THINPLATESPLINE_FILE = os.path.join( + TEST_FILES_DIR, + 'thin_plate_spline.json') +TEST_THINPLATEROUGH_FILE = os.path.join( + TEST_FILES_DIR, + 'thin_plate_from_rough.json') +TEST_THINPLATESPLINEAFFINE_FILE = os.path.join( + TEST_FILES_DIR, + 'thin_plate_spline_affine.json') + +INTERPOLATED_TRANSFORM_TILESPEC = os.path.join( + TEST_FILES_DIR, 'tilespec_interpolated.json') + +REFERENCE_TRANSFORM_TILESPEC = os.path.join( + TEST_FILES_DIR, 'tilespec_ref.json') + +REFERENCE_TRANSFORM_TILESPECS = os.path.join( + TEST_FILES_DIR, 'tilespec_references.json') + +REFERENCE_TRANSFORM_SPECS = os.path.join( + TEST_FILES_DIR, 'transform_references.json') + +NONLINEAR_TRANSFORM_KWARGS = { + 'className': "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + 'dataString': ( + "5 21 1103.117269905359 -5.606757285805906 " + "5.321142212984271 1041.3480932107918 -10.43426929509782 " + "7.428382306562735 -14.137269334106124 7.299563138046944 " + "-15.904206050362355 18.284978789358718 3.4106705605908587 " + "-23.24346650864392 -22.18066749731861 -3.583245180840507 " + "3.2706525226745384 27.117126387399466 20.96651792381158 " + "-1.2219610254066424 -10.531856407305256 0.3306037423046746 " + "2.054811912746283 28.849184526610635 34.799030853758985 " + "-7.4227129043141495 -8.211918715339516 -27.434469512200955 " + "-3.1066643785881 3.8503170392714026 1.7124046587349628 " + "1.8506627747180158 4.886666182237065 -2.949889582798017 " + "-4.5150121503501515 -11.363818954372583 -9.8507389567363 " + "8.672520277073502 5.89027776049669 4.924363838689751 " + "-1.4446630732848895 -2.3876262067533727 19.9189118558668 " + "21.72003155506043 1991.9406329917294 2171.99091870329 " + "5136325.014398676 4300827.048910938 5862798.046430213 " + "1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 " + "1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 " + "2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 " + "1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 " + "8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 " + "100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 " + "3334088.839525756 4464532.472685248 1.6068544862341637E10 " + "1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 " + "5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 " + "4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 " + "1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 " + "1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 ")} diff --git a/test/test_channel.py b/test/test_channel.py new file mode 100644 index 00000000..e83d222d --- /dev/null +++ b/test/test_channel.py @@ -0,0 +1,30 @@ +import renderapi +import json + +d = { + "name": "DAPI", + "maxIntensity": 255, + "minIntensity": 0, + "mipmapLevels": { + "0": { + "imageUrl": "file:///not/a/path", + "maskUrl": "file:///not/a/mask" + } + } +} + + +def test_channel(): + mml = renderapi.image_pyramid.MipMap( + imageUrl='file:///not/a/path', + maskUrl='file:///not/a/mask') + ip = renderapi.image_pyramid.ImagePyramid() + ip[0] = mml + channel = renderapi.channel.Channel(name='DAPI', + maxIntensity=255, + minIntensity=0, + ip=ip) + + a = json.loads(renderapi.utils.renderdumps(channel)) + b = json.loads(renderapi.utils.renderdumps(d)) + assert(a == b) diff --git a/test/test_client.py b/test/test_client.py new file mode 100644 index 00000000..39d9ebb6 --- /dev/null +++ b/test/test_client.py @@ -0,0 +1,169 @@ +import os +import pytest +import renderapi +import rendersettings +import requests + +args = { + 'host': 'renderhost', + 'port': 8080, + 'owner': 'renderowner', + 'project': 'renderproject', + 'client_scripts': '/path/to/client_scripts' + } + + +def test_render_client(): + r = renderapi.render.connect(**args) # noqa: F841 + + +def test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER, **kwargs): + rkwargs = dict(rkwargs, session=requests.Session()) + r = renderapi.connect(**dict(rkwargs, **kwargs)) + new_r = renderapi.connect(**dict(r.DEFAULT_KWARGS, **kwargs)) + assert(new_r.DEFAULT_KWARGS == r.DEFAULT_KWARGS == rkwargs) + + +def test_default_kwargs_client(): + test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER_CLIENT, + validate_client=False) + + +def test_environment_variables( + rkwargs=rendersettings.DEFAULT_RENDER, + renvkwargs=rendersettings.DEFAULT_RENDER_ENVIRONMENT_VARIABLES, + **kwargs): + def valstostring(d): + return {k: str(v) for k, v in d.items()} + old_env = os.environ.copy() + os.environ.update(valstostring(renvkwargs)) + + rkwargs = dict(rkwargs, session=requests.Session()) + env_render = renderapi.connect(session=rkwargs["session"], **kwargs) + + # restore environment + os.environ.clear() + os.environ.update(old_env) + + kwarg_render = renderapi.connect(**dict(valstostring(rkwargs), **kwargs)) + assert(valstostring(kwarg_render.DEFAULT_KWARGS) == + valstostring(env_render.DEFAULT_KWARGS) == + valstostring(rkwargs)) + + +def test_environment_variables_client(): + test_environment_variables( + rkwargs=rendersettings.DEFAULT_RENDER_CLIENT, + renvkwargs=rendersettings.DEFAULT_RENDER_CLIENT_ENVIRONMENT_VARIABLES, + validate_client=False) + + +@renderapi.render.renderaccess +def renderaccess_decorated(myparameter, owner=None, host=None, port=None, + project=None, client_scripts=None, **kwargs): + return (owner, host, port, project, client_scripts) + + +@renderapi.client.renderclientaccess +def renderclientaccess_decorated(myparameter, owner=None, host=None, + port=None, project=None, + client_scripts=None, client_script=None, + **kwargs): + return (owner, host, port, project, client_scripts, client_script) + + +def test_decorator(my_decorated=renderaccess_decorated): + r = renderapi.render.connect(**args) + (owner, host, port, project, client_scripts) = my_decorated(5, render=r) + assert(owner == args['owner']) + (owner, host, port, project, client_scripts) = my_decorated( + 5, owner='newowner', render=r) + assert(owner == 'newowner') + + +def test_renderaccess_decorator(tmpdir): + def checkexpected(expectation, values): + return all([i == j for i, j in zip(expectation, values)]) + + newargs = dict(args, **{'client_scripts': str(tmpdir)}) + + assert not os.path.isfile( + renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + expected = (newargs['owner'], newargs['host'], newargs['port'], + newargs['project'], newargs['client_scripts'], + renderapi.render.RenderClient.clientscript_from_clientscripts( + newargs['client_scripts'])) + + with open(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir)), 'w') as f: # noqa: F841 + # test that renderclientaccess decorated funtion works with Render + # objects missing client_script + assert checkexpected(expected, renderclientaccess_decorated( + 5, render=renderapi.render.Render(**newargs))) + # test that RenderClient objects continue to work + assert checkexpected(expected, renderclientaccess_decorated( + 5, render=renderapi.render.RenderClient(**newargs))) + # test with renderapi.connect set RenderObjects + assert checkexpected(expected, renderclientaccess_decorated( + 5, render=renderapi.connect(force_http=False, **newargs))) + + os.remove(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + +def test_renderclientaccess_decorator_fail(tmpdir): + # test that common methods of defining renderclient options fail quickly + newargs = dict(args, **{'client_scripts': str(tmpdir)}) + + assert not os.path.isfile( + renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + with pytest.raises(renderapi.errors.ClientScriptError): + _ = renderclientaccess_decorated( + 5, render=renderapi.render.Render(**newargs)) + + with pytest.raises(renderapi.errors.ClientScriptError): + _ = renderclientaccess_decorated( + 5, render=renderapi.render.RenderClient(**newargs)) # noqa: F841 + + with pytest.raises(renderapi.errors.ClientScriptError): + _ = renderclientaccess_decorated( # noqa: F841 + 5, render=renderapi.connect( + force_http=False, **newargs)) + + +def test_renderclientaccess_override(tmpdir): + def checkexpected(expectation, values): + return all([i == j for i, j in zip(expectation, values)]) + + newargs = dict(args, **{'client_scripts': str(tmpdir)}) + + assert not os.path.isfile( + renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + expected = ('newowner', newargs['host'], newargs['port'], + newargs['project'], newargs['client_scripts'], + renderapi.render.RenderClient.clientscript_from_clientscripts( + newargs['client_scripts'])) + + with open(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir)), 'w') as f: # noqa: F841 + # test that renderclientaccess decorated funtion works with Render + # objects missing client_script + assert checkexpected(expected, renderclientaccess_decorated( + 5, owner='newowner', render=renderapi.render.Render(**newargs))) + # test that RenderClient objects continue to work + assert checkexpected(expected, renderclientaccess_decorated( + 5, owner='newowner', + render=renderapi.render.RenderClient(**newargs))) + # test with renderapi.connect set RenderObjects + assert checkexpected(expected, renderclientaccess_decorated( + 5, owner='newowner', + render=renderapi.connect(force_http=False, **newargs))) + + os.remove(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) diff --git a/test/test_files/thin_plate_from_rough.json b/test/test_files/thin_plate_from_rough.json new file mode 100644 index 00000000..775c7b11 --- /dev/null +++ b/test/test_files/thin_plate_from_rough.json @@ -0,0 +1 @@ +{"type": "leaf", "className": "mpicbg.trakem2.transform.ThinPlateSplineTransform", "dataString": "ThinPlateSplineR2LogR 2 25 eJwBMADP/78yLRWQifN2vw/ErHm04Cs/IqdJx765j78ab8sN7GLJwCCjMIEXaI2/9qEQnRU3FUt4FyU= eJw9zFtMkmEYAOCJELrmIcvsoJ10Mipnx4VtfWLlRQ2PYFPCCNKVMv65iYeIiEw8QLFCy5b/nNp086KtRCNyvmOuzJjLuiBLRUGaW2ZYU1cbtnXzf9/lc/M4S2+NVnOM7c5LXS9/KrazhMYXIxPTn+aJHyuGOui6R8Tda+cXeywJxH2OV5z1qWVsJ/MJDe0Gu23Kij9i5iNmPmLmwyaf1UgnNy/tIx82/rDxh40/xuSjh2t+py0GyYeNP2z8YeOPMfmeavs8q6uL5MPGHzb+sPHHGPSsQ9JopwIKg2ZvIC8eien7u09ssgJ/JTaHX2RD1Mm43su5bqgKzeqXzFYAa+GLabRgL8oUNLDd3C0op9zMmz9gRjf+LU3T7gSQ58dy5+yfkVKUd+VJ2DdQJ3HLecUPkaQ47Nwc1YQaabbQwZVBibUSkk31KEMdDpPCCTBIHIG3+iY4Pfb6o89GobsCNeV5p4LrliMhH243QtWfo2lD72kweIOlG8pccCY1sXqN6kcmtsfrf84BnWU0qmOrCxrqJgfDHSEgVm3bGaX5jjSChYDaTqH03IEdIvlFkGplqjxxGdypOP5Vc+wUKvC0RW/Um5CupXe9qnkz4rsiWh1NflRb45CMZKSArLjNrA3UoqKrgsyDpgpU6ffFWFPjIXvlXpleIUf14z9ixuQdKH9Pl6dE3gri9F/jPfv/ogvPsqXU4UgoHBxWDuyKQEZWXOdyZAaSPOjU+brPQknDzMwELxRyNaGixBQdKG++uTablfQfS4+iWQ=="} diff --git a/test/test_files/thin_plate_spline.json b/test/test_files/thin_plate_spline.json new file mode 100644 index 00000000..591101be --- /dev/null +++ b/test/test_files/thin_plate_spline.json @@ -0,0 +1,6 @@ +{ + "type":"leaf", + "id":"", + "className":"mpicbg.trakem2.transform.ThinPlateSplineTransform", + "dataString":"ThinPlateSplineR2LogR 2 995 null eJx8u3k4VV//Pk6SWYbmUpIGpaIkVNurJJREMkSJDJGxjMk8z8Mxz2dyznHOUZQ5bcmQZI6ERKYMRSEZwve89ea5nufzu37+sK591t5rr+E13Pe91mZi+v/+gyf//+Xafen/Xf6f32P+NG9OVXGG6KbKUBE2DET8T31AXuMZsuMCRA0UiHYf3wlRSWX0IXcXSOG5/0BX8gcEeP29Lxn+lt7FmB3HPG0hKXZfNz9qA1H4Q7fu9fRDwoN2QqKwEESldNPHe6chyXz5GbUvAYLT9Rb5ZU5D9L1Wh47qHAj4ddS209cWMNjKxstakRB0LlzOUG03xNfL3m38+BMiymWWFxouQMzU0S0nlcXXxhOxVmpwCnrKQpJgu+sMmECY8N/fUyT+lmFbfzxk198GUY/Tl/sIjZB829UPhQvg/vZByeseHvDD/r0vwe5vGTKlsTR2lBPiHuowhVg8hhCTJ/c0lH4CZkg36KHYCPgo8PeTOzMhzoxXSWv5K8RsM9CkCJVDSgjPw0fJwhBc8+/8rP9benl0qpwNngasuOrl6vqnYKf3mKhOL4ekDaXnu3eIQDr6my395hTgLp/M7ts/D+6Izczsj/MQk0o18fx0E2LtdkU6alpAQtuNPSYq2hBf4DFUJGEA8Z58cxyWJRBRy9/mOZcPqWrXb3lz6kCssOkeJDgL4uZE8vE8nyCgao/uq1AniC0Q3uhspQwJnHwGg+aZkHhGdX1NaRqk7hT+Kr0+AxwP7xcrGbCGuO+3RMbPYyCmuLN6vf5TwCApnOJZdEhSsDq2qBD+f+zKm3mrS7eRHSTs+ZW4tVgPYl92LzmWKgK2sldkmT0YEuvXBRy/GQKJu4Tcy5S9wOd2JW2IIANp62cLBz6fhTDuDz5zF0Uh4eYC8cOAB4Q3bp+UMr0LGFGP3UeHH4DrCZ3WtMckSKSe6eHYaA8Bs3/fm5j63/2I/9cu43DS6WfjgyHpSuA4F+cMRLvt/XKlpwXi9M9U+OAkwBc/fv1z7X6I2c8WFhCKBQyO2XdutwkkqFaRI/fjIeZq/kuNY28gzTRc2CpQAJJcKxc6269D1EXXOdNn3BD7vKNkwGoR0vGdW3Zfc4K4fJwo2yVFiAyRUHe1bIeIagWRpzoPIG7sblua1leIPif/YC/tCkRWtpoHUj5AzPjNRsGGUUga5/NNJGlB0M+vopKncyDCZEtX7MbvEJ354FFZjyfE7LyzoMypD8k31Hp2s4tBzNydN4ZuF8DXQF3WvF0PUs5W7xe+PAdxKaJ143qlEKeu6pO62AKJwpxfrx0XgJR9HId633Yy7EDUyL7oDkS/vPxT8foOiBFSkZp41waxOmWB2excEG0rH6kkAZBm0T9gXlsFiRKnNGLsQgFT08G2ES2BhIBfJJ+CQxAVeK8m4hENElnPYs9e4YTguBujbHanIVEpVv1AMDOEO8DZ5vNkSCkmHXg2KQEhkTnCiPEIJJ8oXgrwZazzjyKf7MpaSDuUf2CjwQWIenb28bq2NEgxTKCaTxdA5O/XRW9uXWH4kbXOcoo2YA59dCMp1AHF8L/XOc1AYdsfdWWIzZmW22u8BCkLPvPqMfEQU7nHst99BGLvkKe1yC0Qc5s1gFnaCGJcDh27WDACkQqOi/UziZAkVpqIF2kDv+UGl3TnaxC/sSqPmrANAgveaD16uh1iafjLBy09Iax6f3/n8TCID0xdOMmfCaFTPe+411kDQYIp9ERDMDzeJZ/nlZkNGLnXRzUe9EO0qUOWUlggRFUI3vriYwYZv380qGVzA6bydffL/A7IMB+89H5uGDIShr5tHXsK6UJH5uQnDeAx90/Sh0TGOHzcNnO4eYO917vgD9m/Ia5VrJe44wAkjmbS6zVZIa7e2FOmSgESTPRvyZaWQGwDLnz/9yZIMWUVES6VgfjzXX0PHr2CpDeXZ6pT9ACzLv/51okvgO397/lLXH95aufm4+DdFhFrzvwAkmVO+JuxjoOP2qZ+6ZQ08DVnuMnxKAg/apMjGRoDMb9qKqwprhCkLBJlnmgH7np/ulBmXQjh/WbGKbwJIvad+u17sxRCYWZa5gqOEWe2SFu76oCf+lvL4jAPSBHah+NoMgZ/WT/pP5ujIPhks+FGKXXAHRbTu9woCZmtFLVq6VzA1SgctWnYBAlnM047FglCWi4z96/cGogzcNXY3sAGqbIeB9Pev4D4CaGy0BQqpB4M4C47IgpeDbmHY5eZAN/FtKejZCdgyeTWMo54wGYFEWYTj4ITL9MbudJYSBa/6po0LwBWXk/MbxdaQpq8dV7dfra1eVnNZw7wqNBtXQ6kSk7w+F1lBmd9z8aKWQFIHwymP7I7Bu5Gog/lni5DMsce/H2RPHC9F9oZeyob4ovmcmjHtoLDxfzAD3NOkIC5gfvhJQGP/OuE2+6mQxyHVLOxlC/YmcqONI1gABPou+miHgHS/rBjaOaNkOCR/F0wVnI1jq31K2lu46fUq1kQFHF1NlfnwGreW6sPC6l0r3BFACv9gC59Qx2SCRuwWo7ukPZAbCJOLRcwabx41WofINQ2DfAVSIKn4I7NNxa6IP0+ro6P13KtHdK/8TO9v3Jr7d69EHDVZ0KNeTOkLn753X4WA/HINd/Hl/cBhtvChJGIIGT3/vjzM4qrz621Exm8aZfPlmrA1oT43Yx3gMRlUcddfZGQwWv5lG+fCWDO96TZ8ksAjml0+JUyw0739PVmDJsD/jtzoPz8awgIGUv0qyZChmuCDCNjgC+TwzLT/leQse7FPU/LHZDYx9a464QZpLZtYGlzjodMpmS/La7xgEuoQjDcbICpyGIZtVMFYrrjzhgW97V+4V/9LTFyZ6tOEBMhXIk/aPr4MAQLqyRe5j0PWDVVrdFxYQiRfdP0pCMN0p9gLt8PvA2BZ7qsa9lmIBXd4OI2YgihW/xX4mHqN3ne9x/1INMZb02nVQG2qEp76sgncFywzcGwEAC3rWoouVUcUge/DYw2VULgadGA7KPvIO3uIY7oR3LgczVk+sdHRbDZsS9m3GAcMgxE8GeWQ9f6+6/fwh29hYkmtk2Q7jyktf00BzjY/NJJ/hkEWEStx0iUDcxUNP8xaMBOJ2s61nSuPZ/x7/Npew/yigs8AXeOXU7hC+2ArWt3dPi9D9K+eA+F8JyCjOkbwS1usuB5x0Y8cVAebKR468NLvwP+DYuocVgC3MDib4c/agZc09/2LO7taBG8TQPcF+fQz1XTYCAXcsNH5gbg80P/Wbe19xOxf0t1O39PCaMaIOT8a1//1q/iV7PvmbtP7DgGxLqlNwrCfHDr29hS+ZkMwIphC+wrpyFj7/k+3Wuq4NPgMtSs1wxYDy8JbG4p+FTf69xd+RVw5ywM7T/vBvfNihg+mVuQnu1belXkLdwrK8Wxpmeurvtav3Bb/Y/3T+lB8Bd9RvrRBqwtz6On2wLBZaPD6/ydHyC9Vu/e8CUSuHC1yn5W3gN4ZvmIV7O+YH3giDVTnj6kH0utSeEdhIec1Wn2F8YhQf/ngRGXNvA/KWSWnccCOMMf/SEeDuDInOkoh7wC/DFZ5cSZDeDKVJ748z1jXm2N9cibfcHP/zH3u1e3Ac++8X7MZDR4jSAlP9dJA86206njsST4iXL67BGLBuyDJxs0mXQhoG538QCmFLCe7gsfMmPBauhBlbxsC3h3iqPEsW6I6pfyrV04A/GDQ2ZZPorwcJPG1HBy8Kod/MeuzP+W+myXOEmqDD90tV5Ho43BLWNGKydVV9f5P37jLFD/JaMCDK78WcEX/67rWj3BrLCn//AGsJ0hav++nby6zmv1XpW3JbIvzQDG/IBespcKuFSd2F+o8gSiKqPEfloy8JXH7aLFvgmw7wChW5I3AY/7vrI+9uUzO3PuPAWKZ4DDlSPOcO/AritRe68A8dDfdo3uPlzxe4JqY/FWHmO4q9mB/ZH5AzKj/tYb1mzokNm8A4gdFarY176gUypF8ZAZA+cwl+d26x0Z8ZekVJKVDxajMryNpFSIqJiaOSvZAx6BxxfWj7tBhDaXwFWOdHBM7Uyy7/gG4TJ6zOPfg9fG5cH03+Nc5UXWR5Lvfj8lAiHSWoNPJ3HgeK06z6/dEoI/TK/EQcdmWfVirVkIcN5zavtLgFuKNgHVuvIQ2l8tqv3bYK29EPhb3onRVOcQb4TAitnP1j3Ma/U+f+0akp/lCiyeqAbnl+cbXVyKIPEaJ37HgxR4vPXR8SNmBRCXxGx4Xf04uC+ksDrksUE8HiVSyrXg7p5S1KhxGYgvo05WMx0G5+utt1xIXUC8bdgt/yUB7GSKZcnlj1fnc+29q3676scP4ryRSsNxiCof9Tz7mxkCXD6r+zRqgtvpoOzxF1Jw/yvr4BSvF3j9tUPQbSw3jXKPXrt2sfvmNzubCdaXVb493yUCxmT7KxqnD4PfAd2VdbHrviF+Nv07+P5SCrGm0//PvLv8uHV43QkO8HnosqDwMQ2sHlKbm87XQcBkHVx+OAYOuXwtXvenwb2zUFf/CwV8KhvzzgU3gK+iZFdtPAasnG9/JSuQwDWgX9l2fQrY/E/7Dq++aEp/LgbHet9WywOvwDNo0erhdhEG72VLWDdkAj6Pjb61UvnB3WlqyDurGDyWSxodLs2D/3dkj5zOGPhasOdu4sCDjXHzm8/qDL9ne/tpNmYBgsReLDAIDcTnNXSHGk6CS9sW+W5TBn7aODUwsc4HHKTXL/D7FkD8mTsxy/sHwcdgQaRp0BASDfZ72ruTwLrrZs3bE7yruOU//thBXMEbTnL0G6U/CoEU5HX1ZKQr3NkX7W5pSgWS7AMP5zZRsDMJMbfxMIbMK7yD+4MPwgOFT4VRC6bgVp3w4/u5AHjsFj3mv7sIfBfPy19OpIFT0rcLI8GpEMJ++MXFQW6wG2P9i5ut+fWaN4yDY0CD6AvcEQijqSx8tQoFN5ErNa8aMGv9+ndewUrt5sn+W/pgH/47123wGOg4KfIuvq0CZ5FvBuKfmMG7JT9J9FArhNjamDyTBAi2r34zOvgS/A6OubBcYvDZ6raM6UVrCOUO6GUfyYPAp1Lv0htNINDWrSNTxARCHD8N5onegVD28AMO7HwQFvSm+RS/HiP/etyR+LoM4cuCcedGL4FPVmzhs2AtiFLlsrPYTYHAH/PklJJPgDmlImj0KRb8nrb/uMf6FpKbNr7isB0Dk6U/Dbt090Nq/kAzSd0ILDzCVvJ6qlIg+e1nVrge6BLdt/kkJBr+Hae6yZXUAr8wiIPbol8MO8AqXPdCoVQ+xAZey5L7fRW08Sw6BK31gHn13+sX9W17UlJSMjjMm6zgEQyvJFuhnxmYxo/6Hknigpii/IN41kdwT2Az9XfkBOiJOemTJirAzpkiyuXCAlGPkm7sfXUR/NxavBdT8BC18GaBnicGnpt2X/hyQBiSR4a2lr/MBLv+RmOvddshSS6BtpDyHYy5Hix+fhkFmIcq+cb7SuFBwl0Jp1caENF8Avnycg+Y3/bHohocEPWv36olNl+3Dt0NkfV87TKGyeC613Ql32EwpAhk0g8e9bC8HX+3AFEfMKdmOaLB2oERdh06V+PW2njDs8MHryi3gn3aEMYuxwpicI6VIiyG4EXwe1bo3Azh+a0vncYd4KaQ0T9C0GqcXXt+Vb+5rd4x/d5NEDAH+tTzReXAO879UNO6PvCsTa3RmdoApnZx/qf7qiD4+M4n9Wz+cPvxvYFNXydW4+xae6GkWwrPuk+D2Qevzh0xYhAw0mSl8GsO7p+c2RbKfx4y2oreW6p6AiZS+L7NiQEwMfDHtx9fhPgp840Cr13h9tSlgrSCO5DEty7zq38BmG//XfjGhxNSOMSLzdDHa+9ZtRPtZDr33YEWSLU6B6LPEUgOjqpgX6JDQl306NLQAljciJS2UE6CuAtKE3rx19ae/9dugHJMo75SIRrsFRS6uwQY74nitpWe8IKks4Ki55cokGybJbj7VT6kSJ4N7DoiBUnqB5fYnyxAUtOB/gc3/sGHYtpyGTKQYveQAbNsIOW5ru+iugikSmfArmuvIS36ZN6z+VhIctm636uCA7DlQA1Q/8rg7d9GdkXQgfyvbnZVmIFuN5kBXpCNaGRfBHGnPr4tUtSG9MDAFTydaqL08BolH7Bzr4NCb9yCjEymb/MWKPgFdWYb7lkAbLvphk1bTkOqxBJhy47dDB63V+KmyR7ABkSpNgzSIRHKGO6rC1i5oNtx3aoQvyMnZ/Lxc4hPNT3DLvgOMvquBKs7yUE6qhfPZYuFuKlrpolfeACH4/HfcuMDROVgvkzYlwP+akJ++OaDkHDNQN7lRStkwEbnpyYbIPHd7osSuY+AqHLgtmz8AiTrI2dCttdCfHMcf8+fG4CTaZU6YzcBxP3dlJf7uCH17hsxW1NBILIw0IsaMyQa+TDC8wXIPKukw62tDrFz8ZIPzd8zeK3qNazGRUgqGRi85ooBgo7XieKoHIjvctNDZH8DURQdZFk4DdEFTnc26vEB/rbLzQvRDHxSZivgaeAPBLbJTy2tapA4wLxk8usq4C6FHGw6hIf4JxFE00wnwN2M0OtOPQSJmmd2HL6aC4RdL1b0p9jao1NzRl6QGTvcs5mYDQm+hTvXmysCsUhpS2SEB8QvmT+RbBcEQpcbOUbHEGIiNx/2k+EA8tvoXp+w03D3arr3u0svIFk7p8pkzhOw+rErfkS+845+9EcrWJWlnhesDmDg1a3XT7S8AJxAmDnzg+1A/R9/oqjlf7ekVEPiHa88vSEmyJLjpOydTIJEVnut9YgmhFyl2Evr+gD+RelWhRETSCOIRnYsnoaMyT/ApsMN0TWtEkdgBvCc7GY7vnJDlNbR2ohj4oBvwqSWFFyGjBS2lfiVUbAjaGLpMJAejK7E8QSm0YZ77YzxMwkWNyazQWL/kxt54WeAtO/le+cvxRCTEmf7h98AMoUuruhZmGqL5meDKUBAqlbiKubw4FYjxTgg2VVLHzZwgeT9jqczsjyAvMsl5aPMGYhzGXewKr4HGdeWuZ27AyG5//cDJqFbgO2xTxIPY6y/8tyR138sIONIoTd0A+May6CF+8DF1yHwT7Eo4NXrTAQa+SBrnOsV1gYLaVfxzEoHu4G4h66H76+CuPslJWj2NcDqELKaZ5gh6au0bNsAEfzT/XpvWPACfu/tv/PqWC/5p0UNsCl9XV8nK4Bo28eAQww+LSe+7srJLsAGqXN+WY8B/D0XHoNwhh3n5ml0ogchPUpp372BzUCQMtshVOQCYaJe+rJ9rmv4blVPJTg5ahyrfglRuw6zdkfGAGnbtDID2EGcoUTVmORZyLzLoXR/NhVS+Hzb30dzAz5WYKuqai9ENUancD77ApmBt2TY/Tkh9JflOm6VkrX2V/V78oF1RedFwwF7oKXMCf+LwdsxQRbV1hDJ7V4SFXIASKIba03OvoKoLL71hb8ygez2/DSFex4SXFp4tMU+AsnnvLDciSOQpi/RX598HTLb3VfmAyMiGcAgzED5eefojjEEkspuznqukweyRZVql+sgJPmeYsAAPSAdDkqS3RsCKW+4HnVwTAJ2opZk/dIeMLKChhua2oBUzNpnseUiJMYkymkWDQKluDwqQlcI0rbpmU8XCEPYE++sEzEMfqxqb/qtoh+y4vv6keXt8LCIs2hPfDpQlNkrcipvQvLS+ePvxe8C+YKW5cWdxZBSqdiv+bsRsj5WT0X5/4JknWg3vAAvkAmvFgKmH0Pa4rrSuR0ngXa4+EeF6k7wU86zaVF8DrT15xZ4NGYh/VbG9xF3NsjayRT2lSUHUoLZRL7QdYHi37XCU+OIv8ru87wArOiQ71lDFkilTymGtXAB9vi8LeFPLUTMMheZDZwBHCYVryW2BDHakrM7dB8Atj5tXDtEHaLi5wel1XcDTpGfOlTzG8JfdJ8ac9AFvKiXtOmWLgjdVlw3P8HwywPON67kPoWIuId2h6UCIV3txkq+Dn2qFcZeeAdwrfov0bN8EBje5av86yzgv3wY9b/oAkFn5K5OeyYBdudEE5vgDwgJr31auqMBiBIfKew1B8DXanNKhTwj/ulVPH5Szw7+7J909+uzAIHl1uaWpl3gbbw5bqnTDIg6t/XNWQkQNHlbYcNgPSTuD5JSPf0V8EwhU57oIqTqyrn1/jkOaeXDMZLYMMCTBGNY3I0h/Vkx5ymeQ4Cvyw1qwc1BSmaX4qFD5yDTcSihyRKFyOuVBmoWDD55/kPLPfCGtKhryY3znkDNsLiqPkYDQp0NsfbaRaCQ6yaCil0go12PxTv2FVB7ilZwQerxUS12O35ILXvBX938BcIxBKMkF26g79h+64KwNKTORhWpBHFAVs69s9fOkMEoW3fxVmE0pMcr7HtRPQaRsoz/XG6QJUSA6FeBELSt/mabgRVkFuxLMZs7Cf7h8+sj1ssBsTB7rIM5AkLfH9h6xn8YwkN0JHy1QgDvtc+MV6MFgtzfCAywazLslp+f9Us5RPrv4hK8+QhwzcVn+o4ZADngm0XsNnXAuyhfuV1nArjKJdzwrjxIJbeXEat3Q3JMoZi4ihTg4t5tVq8VAnyAhnx4XjCkiUhwuulVAp5np8v4oXuQciI1mm0eCwTLhBVdPNlOR0mjQQwyZ91YHklehIyY3R6Zs1eBuKlvAbt7AjLUDYf9I39D1h+D2v3XsQxc5b50AmsGWYqnyqvZwxm4Yb3e4DIRKALZHGWpxoA7gK7Xf6oJ5E/eX+PdnkJGSOfQhbJEoP/S1jZ/bQTpR+QmNcpjgS708dBO8h9IJOCOR/H5AeXbtSfdDe6g1dLbwBV9bC3+eKzmp1ltzYDIFvAK4h2zO8ACmWfuGrxwloOgWC2TgrtLQLhW0zDzsBhwvg1nWPp4wHMqNkffbQlwzkb9ChLtkHzOWmpz7gXA+/e/OlPZAjHvW0SiFwAIsq1XOEYigFR42aq8loFLvjumqjhfAKITcoU5QADSvuAu7bS+CJlsfCv7bxlmRcFn3Bn2LvFo882jDL+4Lv1W7sFmSN1/4bH4G3bAB1t03z73FrKu38zd2D4AWNWLQV+Tn6yNZ1XfpPI7CGop6QG+9eeWsdamtfpVfTdrsMLQ0hUPuJKlOx+z6ZAltc9I+9ZRwAv3MhLQIaBVRUranN4MyXXj2/8cDgJqa9iTmrY3kHz6s1HjUs5ae6t6ebaphLRC0DdIFf57TY9Jz8UTuyDFky9NRcMXqL0jxzKPzUGsY/p1XYw5UN19/hG4wPo1uMQLvge68P/ghbESq37dw+AyJN7s9/ohUDfMb/pxnQfcoqT5pdaPQmLlt+uGvwaAsJedr/o3DoieD7OF1gsBdrjUgEtNAGJunNh45Ns4YG3PrcQPD1PmhEETe8gcHkuWWaiFrLc9OGzjNoh7tXfPNXUsxJqmxh5VYeRT11q3rZQYoO6Mt5Nb8ATfP9bZRVKRQLKYtCneKQ9Yu7d/47cwm/FO3SuA244mX532Avp9/0GxkgHwCB8u3ZIqCnHNDa8bLl0GvNDblbhCnfd5bNv3FXDUT1EImRVo2qJD2xqOgt3Qh9aKMzcYcWqDbuVLhn+VHfYZdheHFJL84FuhzZCh9Ej95G4Gj3nK9rCt9yEQhhMsdxxWgyykchqeCkG09dQPqZw+oE590K63Og4JklMbw0yWgKpyYr2YnhUkpnzcK6boADRlJUO95zQIFkxmYyRCyHrJFP8isQa8T6mm+GmZAvWARCou9ANgZzP07+APgn9eubFzehoQAjpW9lUyCsRXeC6WfKzYQG8Z8FvqA2qD/wD28W8OByF28ON4cp/BjAHfVhzgRGbE7ZNHvn1v9WPEC9JsaIILZH3QOho4Gwgx1m9HwuufAyVfNuGOeAHE3A/zeiP3Emhy7MeyDAshyuZd1Gi6LZC79tO2C32GWKVj32QfpQMZd2d73qdvEK2oPZu9n2E/5Vzz1hM1gNGpXlA9ux0o1vyz/XQGTnxfrxlSHgUUz1eHKqwpEF5gU+d9lWE/ki850nntISopXsRbSnnNflf5U5apy5Hh0q0Qyf/+8+6jw0BNNEGH2UohVOfvuQQaTunzrLIbxJx6zql2OgXoipM39V8MQ9ybGcNBoQNAzdp0d6CZBBGRJBVGQgFaaYfSk02PISzBggHbHgF9/5GICEIq+OCPy1//cwDoQfgxr2/zEGVQXLG+bhyoj0VPhfxRhsAAopXT5V6gXSas7PM5/zoreq5uFKiR/IMZR6gQ1002XQzpApqiyleH8TKGH6Su6DHU3Kvsar47wWtpL07rGgpEwVaeqD5GXg/rkGdzq4NUty8CTgscgEusbJtLlYXw4hpFweNBQPANOHNpcQxoNWJHFy2uQsLuNJuQ2S1AG+geeSWSDNFhS/sOzU0z8tXvT65lJRBYpEjoz3AGenMo+USPOsTw+BoUKrpBFBsnTmM3E2TaBNHlAnjBTeKpYe+YOWArBY2kGhCImrW99F1UHwgjd6altvdD9Ga9lMbWMiBWHOy3FrkASbxqfQEPHRk8hif3gvg7wGMUBI48fgq4g/zP2Q6NgE91z4r+QpwcO/qLtRlCNZ6DGjELiDeiw7Z7aK+t56r+SouRM5JZ/gOeJ/XXm18MX6tf1Z+olm+XKPzi4CRAOCaz9ylkq9fNyDCJQLRd8g51njuQ7c+1iRmphFjsv/Hsgs57zlZbiJXgKeG8bgFBvea7lQQGAPfn7YcNkg/BJVD7qidvKxBbePdqSvdATPOQ97dWKuBK9xwLWx8A9GHjDCkeNQi7gtMQWq5a68+qvkufp+8rjDsK/sq4qT2XvkO2EP2N2K808D5C27eJWR7oPz4uJ4g6QFAt695urN7a86v6L533w/Ge8lSwIDPg/KZh8JQxtmIAJCAmNey7HDgA9rLiQbuVBAH34fSXhDtHIVj5VPM5hw9ASCioynHpAOf1XIeuH8MBIdopwbL8B7gnOv2qnfwIBJ+7bWj/B/B/HLqy30x61OUVpJ4KD7o1nDbd1ASSuFQdDfkMD7myRfXFdgHxqo2rg5wJ+LRMFvxMOgWZXfeuxZRygjPpt2RkFStkPg0J7I/IgQfvo58NJs9D5nH3K41clqCZMK9q9PHumm5/r4vV+PtGxvVTGtNyJ0DEImcdZtN9iHiTPdjsFQ03XG65fUx8Dv8q5bCmmK/age7HetJXQ/D+nnQ0scwTguOiMi1yHcFrz5U5X95zkFY11xEl9gQw1Ar+PdynIL10U1643FeIVsGF9TVPAlFMSTZHdAY8THpu1s0z8J9nn4TWCSPwOlow++P8Jcj8nCqPz5YHF1Ois0u3EZC3hwo7jEaA8/y+k07XBIA8SzdnEFVw17fuP2HRC+S+HU6aCg7gRxu98KRZFkh1E1l9pXPgLTp6zo7QDhSZxbH5SA/wRlOVBlKEIZNPavHwXSr4njKtb+MMBpJ3np/GiVlwu7azZZ7HBUgNbU7BtdvBP/BpiuCW20COO7lix8GdfWHSnUFAUkBTZD5wQjAP6w9bi4tA2ojXd0juBp9Y5dGyP11AufDHeXnzbgh8NfDPAQMgmyVfCzzQBBGnTW8UoXNA+kEVWogJgShbgRtDfSJAst1zuNdDD8I9nux7KpkLBI/3bSXPtkHaO+kVnS97vRMvb4EbJOFfuruJ4yCbzQ6HHlGBBGen9rvXWdfsdVXvyuZNW8kHcYPqlws2FEHWwbfY0D8i4KcUIdkugIMshZKNtux1gJNOXhkPnTnztSow8um6TZMqBtvW2lvdH6WdElXy/Hkd8C1bD2yYrwZKRMTKvhZR5/rKuTTatZy+jNQRwFo3ZfiiZpAdLr9je0DJ2v7oanur+5WUpUwX7527gXAxaxkzWQD0PMLJPwGNgB1j6h27VwE00qnJrA+HgFBGvX359iTQ2UXVQ080MvIfU9CsEQnoy7V2OVYXAW/cvMlGlReyUtk+HrTvA+L+bdnmvs5A/rHLkv77IoNXOxTvw6cCxfJTu8rBaEbc+6XnLroNsk9GeAcd/sTInwl6fFsa1vq3uv+a+SVWVtqHgWeiC54d+pQIJF4aySI0HwiU9j8Jcc+BviB7NT/uERAqpV0ldwcA6fe3QltrJyAoq8vPRG0G8obX900bzYFYcC63xoRhLxJPrFqiTwA+Z/IwZ5w/ZB4qiX9+ShUIXCw5lbohjP4nsH1xbAbypr/xhVwSOyyZvReIF61X/JTCnv1U7Go7ZAbUbDPa8wGITY+uhepGANFn3RUtc0Y+zT4SJOLSDJmp91l1dD5ApqJ5lN3CNyC0ZybYTj0BqkGi4adiKqM/YYnnHjH4rN6fgNCMT5AZSVbJrvkD1Fdtj4UlGyGj/k2FuqoK0Ne9W9mHyThVFFFaXQX0nceqdRF/wJoKCrW9716br9X9ZlL5yVyxdbqAm5o6/FPxOpBmfKml7WGQ2bJt/iW2F7DralIHyPlAVTQPz9G5AuSnt9guKd4Bkp7aXSmKB1AKHVueRAgBKaiJU6+uDXAmt39uPfYTqDK/7vZmDAPpUvqKfkK6PqSVYVMNxOdpvxjEF6iNKiYwKcvA9wnyG3uVgeTxckXfpV8Jq6RbdTHmweHuTg8uoN3uEczWHwdiD8mu/ORZyFpsdqisXwDSmR/c14U6gT4ZPdD2kXdNZ6VWjVirnswH8ta+uJEvuUB72dHiJnsaSPpcWvXRE5CVzvsu3pIx77yRMoR8N6AHnFv2OVIGWZ7DeWWR9Ws8YHWeKHkF8skqFZApdN/wWY08ZE1wm9dW5kMW857yl5kCQFjPdSMjlmEX+YgFq/s2IPZxPYobF4RMb4caadOtQGT+lCx4ljE/orsGFLi2AaGfZftXyfeMOFb5l4coJj7WLrIHwtvnxWEeDL/JRhlhYwsQaMqP8L5XAf+1gs2KlRuImcEKW+lVQFTtO/x+ZC8QBO+s7O8TlGJn27e8Bzx28vhBqVnAsQgPWeUyeJqR2hddR27A/4zlGLzFBwS9Z49eZJcDfmtJUWaDPuCrhutONWwDXFePa3R8PBDLLetf9G4G3I70kLx5ZSC4hLi/6y4BnOH57h17RwF/ndVy7Isp4ObYhISUGOv2ru7khzcigP11WbPzIQWIlrkr53ZSF9/YGG2gQSaneV4682dI/7Gl2HN4HAh5xZejlLggA1OhlJmzDMTfbwe3TX1n8DjXz0zNhUD4cIHL4CNjHBmTu/Y3OQBZdaPRu7opwIdr21dOMfKr7esTqr5mgFO/6nIn/BiQ5h5ptf26D3Q1vHxdOsM/DI67UR1DIHuHtrVuzQQQWyNiaULqa+u4mj+zOUdy54ttgcT+P+v8b/ylziiyWqmHAklbgnJlYD1k/KlGXecfAk3k6cEXx78BdaTH5VRKBVCfniMqqyhCukz1PsYCAdEZc7XlJxVSRrYcetcsCoSPIn6PRU0g2WYXnUHUgGAsvbIPlXShR/Tj4m3IFMudqa3fBknceiM0sUEginYK3qZ+gOQi3OON99uBiNFYqPqcCvEcVMXWPWRG3PhtIzRaDwkVimMOOuVALB6wDzmjCwlHeSVqCQeBtFVMWUpVBgjusRX92+WB9N23k2fxB6R7ZZ/zTw0H0i9aOeOGVf35/5yTXv2d+sSX3ElchKwas+un1z+CNM3ZQ+dOjQFB/sWeEA4WSD/64Km7RDbgrzwaTT53DDKyZ/4RJgA3883BcVc/xG4JIhutewSk9fzfy0JTASMwWp/uOwCZz4R3bL7xB+Ke7/JnJCwgp/UUy6RcX9M513hzRjVt0zdhIM37hOmaO0O6kZmX3lFGPO/dtYdsKgVYlQf/+rOvzQdnO0gNfykndAKA5OKtq+khDsmoAN5zpwVkjj3h+DQbC0nss4+8fhkA6d4VHUPZAxCDV/1az6IJZG+1wnfPayDh8p4DlsRYIDsajTzVJwC58KMSp1c4ZB2/EC35jQvofp0FT9y1gfT8Nf/Fd31r/VyNN9g+GxsFqitkr0M93/8qgKwwAep2hXSgbLosz1hooDSx3N3z9C1Qvi8Inh4TA2z5PI9UyghQ3yAlG4RpQAt1WNlnzRK5p8a86T1QAoR97lPGgHr4hbqq8hOg5Xgo3LvdDdQjrmUnAxn8o8PHie9gHNBCflwM+/0O4mj2rxPKMiEzi3haiOAJES53h0b9GbhGhGmF/4fw5fH2WsdCZog1ii3ZAwFy/XZKUUWQaXLtSfKzdPCfsfox18LgG47Jz+9h5yDqYcc7c+UyIC0XSN1IMQasgniw6kU3yIzt0nC9tBVSPF7+kwYgE5H7zVEoC0lYyX8OjADFyFAo46oMYLwXFDaqqDDyyoTLJp5WiIvczmv2jg9IjWIxXL09gKvVn7QVqgLypwrz0M5YoE+YHYg2OQ8Un3eaDxJ2QXbsrmdxI32Qpf4vfsm6v2I/lG221x3Hk4HSOLGiV9PueSkYzBtB2E9p/0S9Nsjc9I4vsdUFInqTjAWmG4Go62WrVOACQcPez5+MxzHshb9FVkgXotqFibfPzwPptEYci+QABNVrD4upZDL860hSq8YfCGw5/jW99zCQAxOCEkbiIbpj2PpYux+QazVtZlkNIb0paRe7gwLQ2sDuxv5JoOGXh+u6GfYjMzfDCARr9vGvbgMUUb0Dg0pbgRK47ceeInGgJkcWiR/5CJRjNyW2KJ8H8u+mosayaiBX/Nn0dTwaKLUne3WT64BstRF1qsmD2BIOoc+ndYGi8AG3++128PsuzRPkwchXlw59xx+/D9TY+x8HTlwEWjjat7eGAiGVVec/DpswcEdHrNv8QXBz8U39KPAYKI882AbJkRBxePocS3w4UK5pFxXEaYLP3adR8S+SgJwo02sqsQH8z2xipDMDoMhlcJ6zI4L3SYI/XDoPpLz9hZZnisCn/FD0tvJEhj3GiOGK5yFMcWPPGRM9IB85cZuRICD0ctiWm2wdQO1oIF7cGgherf1vzWMRyLp4NNr8YQ4E3swu2JljyIg386LUTqk1Xro6b6Eek2cvkGwgi0z+Rydd4y2r8WlVl1y933/dZmDZrwb0kMcr5/BD84veRbBbAaW9DT/78htgdIto14i+QNU7kWJ5+Cf4Vd2x09XJAdrRBrVQDzUIok+yqVE6gdZUYEmMeQuRpXqYXfXxQFXNuLDkowGu3MFc593wQPcsXDnPGulnYaNLOA50RaPfGuvbwM2c55iG6j6gXn6XnJaZCF5VvhcrPBi4mvu2QG13D4TdZ0G/yboCDX1R9WhSCiLHqPYP+AaAphKwSzrBCsLYelZ0K3pRPv8WSjFghqgzYJwGtAn9FZwQ/z/zEy2Tem9LZAQjDjBYj2szRMaSVe5nC0H2sTM129mH1nj86v1evS0YRN8faIt/7Ot1pMEVe6dBUr0LsjSv182gMuCYXNxruKQMtCh+JkMXvjXevPZ87bYzQ4YMv+QqOv/9yw5wFN+5ci4nq++e4EFRSbAbFR4Si9sPlBBpk19X94CTb9ABakQeUIsEGbDm0dr6rfqDTXmqgnimK9Ap2tWlw5oQfSpqw+XUT4z4p2B2vOc1RGOSCmWGmhk8wa7+wjnM2nmK/9UFsrHeVlULswx7OjMd+KcYso8cTD4k1QKODvLvMHKHIVtfLiCWRRfMlHsVS35JARXDHlhrcg6iRef3lc1mQdaAgPosbIOIBYee9d0MPHf/skXq4zqIej5w/kXYSaDc+jNeZvoJMANLG88+HwQKdv3K/nZsRcXhDcZvIItC8D147yrEdH1SWXTdBNlMW1fy2Srvzr7i7eblt2dt/Nl8f8s76VOvO7t8GThXSihn03G48WrhNgufNtDUuPSEu6sBsyz1bD7mKWT5lfnnF7JBwu4tn/pTFIEyuX0ljsUXamw99oORF4wFLVU9T0Gcf5bZ4zu7gDLdp/K6PBcS5ka9P/E8A4qs4/JQWzjEratIM7gsA3QWzQTtt3IQu53r4LsZhv2xFL8PojDu15VuPD8pCPR8W4LWegTil3hvTpakMfz77tlRqx1rOuDq/KcealJ633oIaHZi2zzc2NZ07tX6uKemTJi630AtkMdqKJRBkpzVMnnMisGTZspMuxYhySrXW4/tOdAqTLa4xtX8h/cmBCYE794LyXvbVTkqGoDe+YInTnAOUthPy5XG2jF4DJtcygZvSEut1nGU7AJ6RvwFWUMHSOX8qxtnGx1DmmYOQhr5QB2+6y5kad/1llLeAAmf2joGuP2ANvl9+0swh0QnQ8ejWQFA/RyfppG3BEQeh5VzYFkP2lfO66bcOrlvqa+Wwbv5/vCIy0BKT11V0SctIC9ms0j2UyD16HLADsNWoF44LWOYlA6p48d37p+RAvK1uGfPLwlCOk2t723EDaCcDE9QMy2DdNUPK+coqTd/3L3aMgnJXu9V+z8bAbVdfwUXYMPVChXduIA8ncKtGIQFLERNBYk8giy7/kP2pVFACC5iwMnLQDnHVl9H5oG0p3teZjtuB9pujWJ7zXxI26ey7jNTC8P/jOualkIhg5b/cNOP60AOvbpOaZ0QZOzjn7PoZ+CPP+03Du97BTjkynhFAiOPfpZqsSUsAv7xi8lnskSgDGw525EpAkSPyvKxmjNADjue9kv5OxDdhkZrDHWAElp3rm5UDEiHrrf1PqgCysadUkTVx5Ap/31B3bodKDzmlZ/LX0PmC5l72REKkJXHZtVwxwGI474r58my1LsuZz3wYODA55PizF+B0izDCLQNa/sya/swxdxMjnaM9sfcP1XLbwLyFqMM3I4EoPC5y8bP+wIJm9xvGM7ByLfiShU8EoA3Zj/xh86Ih8v3/+5D9CQYD9C+Ap3VY4s68Tbgnkcf0sMIAe30oZXvNPBdASllPG+Aes3nyRXmAMDHXNV7Jf8baKJ7jp+sPQEEkugpeshpoHGOqN+gH2bw5KG/+zxS3MLcG9cBmXtYW0iBwcNzBZJ3ZiQB8a59a1jeU6D1yIbT9zNwtgvzYd43DHwwarLV1FlzTTdZ4yGWQRxqt/WBfmpoJT9mPnPsZhBooB3n7WYVOwWkE569IRPPgT5zZfFBjhJkmv+gYV7dBaotioR3qjDe/+oGXawMqIEFr3hwWWu8ZW3+PC+unDui7Rq28bj7icE7lre9XTBi+L887y8zFSCmCRPaOc8B/Ztjg1vTqTVe9L+4dm3frGdCPPa5N9B/uh0NO6y4pgtle/Ka908w+CoqKRUqwg70pt/3H5rggP4+LcC09zhQX5zN3NyQzsjbcz779K8BPc25O/A4A7/7Lx9rFDYBsrDdzgCbi4AvIWy7EZIKlF3cMaNHJYBwzWDinJwQZHFblSr/5AJcp8jgI0sGH3GLYtobPgHkB3cn7a/5Almk2Xl0l8Wa7pOtFxAZ4d69pnutjWfi9cr3O9TpV3skp22BPHT5H8GXEU+SNoxLvIOsQ7Gd7/fUA71ulxYTdxyQ91au6MF0s/iV73/IFxu9BHragebTtnJemaIsIhXw4zLQv9pSfnfZAOWJkJaPAAPHHj0jrHkrHTLNroQTXgtA9vGpmEtJF4C8jvfkgbQ5yIZWbr+H5YDlPv3Ojn4G6KHBxNM+V9f6Sfv3e0z6N/kRBwIArV2roMKJe61+NX+s5tG152R2LmzOdwA6Pye7G+ni2n3ZQX/LLKqgbbnuAtA6HQhfm09BVkLRz6v5cYz5aPU107ABGvKwtXXdFcg+87lF8RkGshq0fLRypCFbzTLRWCYPsk87zRvpRQCd+n0ad/Ao0GWH6AGs5gzcXvT+ibkBkLdWljYeqgHM7LkiZ9oJyMLvXfHXUJ29XYGuDDs2CtkaaY6Co/em/CSqEWQzn7VK+NgINvFpOLMt+av9/U+eDL7Wf3iDCTh5/fc4VvPqv/O0dj+N8re8KcC9kXmBkVcTv/grZ70EPcmvNEcTCYi/lqpkYGwMBO1SXr1THyB24tYfn1RG3Gc9lntugQfu/RTI9s5m8Nvsik2mwxiwpVxI5zyPB+OsUml+Kda1fmhcOmHRk0AD7RcO9ZJOKWu4xNC12iBYRBMwbbUxYuf4QC0YQcQkHq/tj5D4ec09FfWAmBHLcS7eBDJCfgR9eF8I+M97qfIsVyBpt/sBiW+SQPbaFrDl+BaIS7o+EpwyA/SPuG8zMkxr7ayut5NKPJnu3QpkY4ErXdhMeHxEpVijqgpIReIfE5+NglMMy+CJ3YYMfsFvIPvgDLj70U8p9VoAOeYdAz44gGGKrN8OOiM/5ekwd0y/B+vKTFG3Qi4G/ln6Jw+tzev/flf4Lx9bu16NE2Y3S3ZuN2TwKUJ0j9Oul2B96Up6o7EXkD3LPBnAFm4ffPj2igwPIx8cidzUdRNua/4DbxyBbHnfMqauGe4PY2vqjeuAFnag3KPkJ2TS/btVnZqBfKLsBntc85pOuabzEk7KfOA0Brprp31byTOg0ruORt1igVued1fiuaejfpvANyuwt145Ry/PnHtpIwiQkXydSVZZB0/kzWjlJLemShmvNp/olVtl6FvOCqv3b7egj4uXJmpI+1Aq16TxLss6ZGBYXiuE7zw6se/43sQccWTAFjE9fv8wErJltjG63hqZbhfAb5guR78UTs+I1rSgbWITZkyFgPwc41Dtq+JGJouG9j68ZIRMXW/R5arlQ7qcAmsxy6/lN/mIpmjNx6IT+7EBPU6GyKjjbMefg23ojMZNWw0cBkFtso85dfMjbbI5L5w7lpCOukd9Ga+V0L5KcaxcwDDyKWGgWOlIOVLF+Sj8m0Q3+nPvYSWxUwPoe2pUkc96STT3ZI5g2sNFpC7B+pt+M3sZ0yJnzzHhXCSJh8v11zZdZIgpFK/G14D2Ti6wtVzEI2MyblWt9Rh05Pf9qTbeCHRIsfjmuvOLaPQxthrTX7JIPz45Z6NSLzLQ/DLv6sQO5FOp6EWN1iX0C63czj29D33fv+S42FyHfI+ntBV+HkUn+Z+pm93qQD/+OsqrJaeLvpQG8cahBfRrflpfzMbfyPMEt6gG4Qyk7VTKRiXndYj3NkPKNh41dK7+zgOMFAv6q+jZlbp7Gmi7GbW22FUUbZdgCpL74oGOa4dJxxfkoU1sGrsFbOzQQdESDkelc8ikQoS+8QcvtJmvRfjalBji+WxB/paUEvI6TC43W/4t0h82Y/z6y2a0d6ZWZlfuE/TT4bCT0UcT0SEf1f1pW6qQFvPWbptjZHTogUTryyZBpH66yeBhmjTSNskxfJqJG3npjPaFE2eR8Mmcqyc3tCEDcNQpiCcWwU4EdeCn3ZHWUe2SLNMy9IOS6Uf/4BG0Yi4r8tJyODLs2agf4YogXQIzkj4R95HvSq/85nK0kTfLCSOJznno190bnAWSCejcSBJr1HAOMilu73lE8yMy2lXVctvxBFqBP7D8kJSK9A86DKd92YyUQQTH7mA2dFBVHRSa/ZDh9NbP+Wki6GuDFwJvs62RVltd97xFA6SCyr2ep5uKzIgdHpZSsUda7xo9Hpa/i9Qqnd0YvM0XacHwjAb7jyBf2lk13l/LR8a17m9MUnogzyRau/n3hh5kKHrfzVMJquiXeD6K275ltGORw/r10T6ksJ0vhCm4DZ3mlAlZ+APIEBp1PVbJAE3aokBXw+Wi9d0LNidVriIjl5TViqIM0Bb7C+9vnShFYi1lkV2Gvmhm46Jra1Qhgv1pOnak2QyZNMjH+LzZjv6gPz+sa1mJzkYKbubvd0V9AlMdngvuRD6/mG3ybRFHJiUdlWae70bG4kN3nHtligxNnmH6WX8H7c18WK+VUoPWJPE73NoviAzHkdSjHG4gy8sLhjHTCWjnxWN64750pDDa15Hp9Sa0+4TvniJfH3m2o3FHsvQ7kDqW577WpdpI8z6vZ+eW96OVdjy40Rg99Edu6PVd46eRlgRvtPI6J9I85R3ru74H6WdJ6kqdE0AnDW3Ks7WK0EmLJz+4Pj5GB10uq2m4yiK9pQZn0vV7EZLCiNOcoiXSfHtzQWLwC7ThF/z65XUb6XM21PN+F4728n4kCPQmoW1Zpy46vjNBuvqDbfDtcfLMG6rurrO7gQ4eKxbbyuqKfHB2s3h+4Q3yyajyScM1Ejqsv2Oc3pSPvrbX2m0dZYF2YB5lVrYeQD8Znv+EsVlGUbDleZ55Hak7xzJdc6AEWdyQ+fjb0DlkULntLe9OE3QIA3biegR0Rt3x8LGJRGRgTHPxKZGG9vqdP5t8mRUtlxy8z935C+3tKz7wZPoW0mkbtIWAbkXfMD2DtnsjyMDhu7vTpBfQkryARo202rJ117J/eaLMZUzmF7Y7sGii0y+55nM7iehI/LPznjqh8izZS0FWNg+QNm0ZgcbaNrTmZUniS+VDCG6rz4LGwwmkTS7iefb0BNL1fKemgm420tcWKC5GPoV+KRvLf80ljVaWW7RkZfKiExW3km59uom2Brzli3DbjYxrN8ZIB9egXZEBVCn2U2hVv0A0q4cGMsRRbsJVa4p0auRdnqTcROsxeo0tDqfRNKenYvpW9sgTr8ST0QZU9Mtyzpafh2zRpo37tqcpv0F/tFsqsHC/QqfCMY0sP/EoXTSw641kE9q8Iaw0MpkRP7PcAvbIJCCTGUctXm6OLGMWDpVMW7RFfk7ZfXwavA/p5ItCYl7rIzOv9puKJG5BZicHn7Xh7qIjR0yZsYl1KGW2bqdvmzUyqoAtzwucRltzNU9v1GpGp0b5Q9VfN6Gzz8YFbS2W5FmJaUbflZWQLpAouVd/EF2spXuNvOREfp7uOlCmE41OiX8XTDEqRweDuLkqDo2jSxZIbHXkM3S67IGXTEYs2npo+f2u49xlHL1aNAv+AWQ64QDt8SQFXfhcqTBex4L8/LrpNKZ8Vp45CFeW8JOCtIvdeMc9PIIm8L+7pallhLayqk69HuxDPxC7XIrnd6Ij0zTaR10fdEThvvqo8hFk5lz81ymbOOST+D6z2HINdCy/htxc9ofRD4mcgCRp9OuL8pCu+m1It3tm7bv6WHmmzdl9lj6zZTxsWBKfcI88y3ez/EkDHLLU3z9+i0VFnl/jSWTa5WF5gTIhg2CBkTI2k4lxyec66Njn8TF/r5fy7NmHzxTG26C9LHyd25Nd0F9boqtLte+gM4O39zv2oejMQ+xxtQvcZWzzScq8ZkXIb1ypYlu3FFo1VGxOUuxG+u2UT+CPxqNfku6dvLXzPjqo3NfSxYoiC23x1DHRm8hHylRzmO5PZMYCynsqC5AvhicUcNt3IS2Cr+IaH+5Ffp5T8HCTnJRnyddi73t1XX59/abt1aVT6KLDoLFmpHoZq3Ji/+mpKKSFZWJHdN1d5Jms/f6F7zbI+9pt68uYSWUsTBtjc0aK5Nmy1hlM3PNEBml50zeNf6FNF1Ed23dh8szN3tMcu2+iP06WOLrLmaDzH+gfYzaWIb+d5KfFXjAhk1GnLTcI/UL6dZJ4g/T2oO89E991qjDicqDjHR1tJbRgs9BXhQRupAlPwlRXT6GfZjIbL7yqRd5et8FdS6tD+4NM5NVJM0hjgqTUtmYdpG/p3BNF7rfITC4mR2eChsy2KSl/3XAJnbtXf3XTYAo6mGQPu1LKkeZg0SK7bCO0zwATvQfLggwmPiPb879Cm/aQTBcGTcvY43a3G3WgyNStI4WB7VnoABfHTOiCN9I//ziicYs9+idkotH1XiHS7p0gmNN+BfmqMeW8pOxQtl5597Ee5zq0U87AeCJcGh0IFLj70NwLfX9aM2fJLBddvuR5Nl78HVol5HO89wMTMleXXLjx5CjSiTFf0FWsk1/ffWww5qwA0r/3yxObSU5kqudbwWTgLNIZxPHq6fPL6Gjw9r2FphNo4xzz6B4vJ/RjatIZu13X0e+IcG6j7T1k4A4Hn6ojHZ122WPMecQA6bS6H7EJyUXa3EKfcP/sRT8fDSglHh9DnocWCOxwOIL0lGWzBBe6IkVH4Bn/gw601/Qc/f23i2iqBrt47QsiMnN6w3Yzn7dof97mjnbWa2jzXSVt0RxZZLg7GtM/ZYWU3Ph5o35+I9LwuaHk4Pa3aMu6P/WzShZILzaas/oMDQky2L7LQH0a7f143jD4+XWkXOzTuJhGB/LJQuXsgmAT8lEH/SqWV4c2jjlbCd2noa0Rb0U+l5xDc7STeLX3N6CT9/bHL17vRTuUkCf+BWeQ9xxzzYeSHyPjDdajTZ7daP98p/mgrifS41/P3Smji2ZdYLXzIuWhr288endjXhNx85iJfxnOjL6f4mvIsGNG27Hvds6dzEFahS/9MvihibxvzcmkvrVAl+p/JyUIPES/CL6v7vP5fw2Xhz8VbBiGSXYhFVIiadkZlfUoKk1RiYRkJEl2ShlZURriE5khM3tHj7333uNwrGMee/v8Ce/vee/7vi6DHY56tcYYwwSFYRKdg9u1MJdpeTBZ+SI0FiCt0NfDMGg2cFLmrTKkBJVLvkZFTHebfDBm8RrHY14ds9a5jIOlI68gNh5eEbj2KnOZYqXQl8jQ9G0oy645KbJkD4000zNdS0kw6Hn27l0PVxzWCY/Kb92FFX5nfy7eOI6kmhfgEHAOUv/w8d4leGAlm46bxvVg+C3QT/JV+AbEhdbUY6uM0P7tjlWA31Mo75NzMqCcQgsGzm7NFTZonEqleM7bDoS3uyap+Ddwxnuv2q9MRVh08XdXDNyH/xGPhYh+yYSqc2Lbh0q0cO3q55QB70mI4ShcajN9Bt2Xs6a1CuqxjjZP5fyQHHR8ebUan/gQ60WfWLx9VwXe2Zxmr224cdqDTUHXMBgm+P/RMpjtxg4Jm/KhV/ehcOSZzZ7Dh3BoD8UapwsjEi/rSh244wLt584XPw2RgbIHvoNtPZkweKry9tNQMex92S5/ogOx5eyavgzXOUh+9N6XINWDfXs32Q58UoU65tG49xWiUH/yyMS20THon/nz31xmJLalWQmJnT0FLS3n1KbWabDw89rFLyaVSByuLRWjU0TCNPduURdfLPdZeNLQmwGVFK2sA4JuGH9dKChJQheaPM4Zni/cwvdnKZyTZn9Awf1vMRZadyG1gL6rui0eW428/vJzVmDXRON5Vg4zTBzN+bwiKY0FwmetqR8ToFN4r8Qrzx9Ql6UxTeZ9idOyj33+ZehA+yGjpPclgzA1QaXAPhANo8xd9RuzbVg55EPUk6jHwSxVId9jM9CiUUqILd2EHqHjZ+d8XYBYcJfD2sMO+sLon4Rf84dyA79eZXYdbLU8eJVxXQnLi4S4rx76D4vGh85eTefCpuDxm/MRnkA4uPtwilE6FMySj7oLseMApXmRUbcozp2i4dUcXcemOQtmatF0mI+0KLk5qwCzmrkfvkTXQVv10e5eBwro5yp4MVnRiH50JjmO+R+wYvxrjUw/4rTaIjk0/jn0drhysm/uhz+OxeeySU3Y3fzm5ONAPxxLspg1LVZBwhElNVLLDJQmHbjRuL2MVUzKvxNHz0DfWrv098FCIBUEShmfPooTz4l+loZKGOaiES9xcg5mhYOm3u5WwivWuj3Mft9x8LVuTlurD0zajFPePfoG/rNSFZfp9oCm9OTERCMmqBqo5GkZJuGQ94EzNx6ZwrQid2L+zRCYsn3rt6r/CxoYk50rpT5Ck4z1oP2DBhxJI93o4PyBaRcFpp/kjcCCdceGy6tdWOB2KVLUyQtq8ltEvOJVseFv8jxXwEUI6X/NqjZyBzcuaZ8Q6MlDokS8ZyJDEQ5YWuR0ATesBi39e733ABYmrsU6v4/EbdmfpPWMJeyrTqhw7lmA5bZHZfBAGJpUzErmYAHybz2uMVsrwjIZ1eIaQSkYNGQ5+NpKFWY043pOnddAwvOzBZWug5Bx4kJSA0MnBi6x30t0NYbGN0Xm382IMPu+XT/nVA42vLL8umLFBYUPtOj7BMnQ21M7Phv+B+Zl7irV3fUD1x+d87dHOnHy2rcEI9oG6Lu860BxWPwO31yKePiSD/K+lybVqtJiz2jE+fi6bVzXcM2s/BEC7fXOG1v2p6CZ0e5IiWkl9Dd2dejV3ccBbOt+VMWD43nDrmIfOqGpv7pOUNMf5xeXPUzUb+C0/a48/40uqNyvLrOWcx5ixP86P//vGU66+f9xYSuECrMj64ESQkD2u2fSdWMdevbW+d3ycMRJlhLnlQuDmP+1W9fmgzK0ZL17/PNiJYxEO1IQzB7g2Glvrk9y+jgq4HJK2s4Ppwz2nlPk1kSyfENIkZogBmRvbTTP58Ewje1Y28QLIOwl8lTbhEG78J5Uj3pFrPYfvG7PFQejGqceGvyqhMUTP79Z/sqF8Q2NfUiwA9+jKVV3DLdhfFe2W5vxDRwS4Y3X//0cRh/oqwu07sXJ8F3IuP85dCSOmXiadkFb2BEhc/NJnHd17ly71QCkdy33KNOu42JNXt5kohH2Pjie5SRxHuroAideKtRDF1V0EW8jG84ez6DiniqRp3h4wZPopZhP+XP6NVuXHXY6aU3/XovBWdfNPI9ORxy5w+QmSt2A4ZYl3yYZOGEwtLRqSmcLh+LmC5rddvhKU7D/QC8HTJ+m65FQkYIt8Z+CCRd8cHOqtXwoWRyGFFveWWdkw6yIm3rnzTWoNMuR3ce9O39X+OPqYRMu7H+4kLOn7BwM0+oVsFX64pKUA4xeiMRCdddXq7L7Yd1us36GKhMH85tqBnT9oYD5cquC9Cb2EYdcVK6fg2XVqKdxH7SRTMtcx6U5B4s0PcYRT5dwkSdifT+TBzSfrq5TNFyEQmFvhmd9b4A0uM+K16sin3o49KDxuhMuj/AJ5zC7QNf+96Nn01ugp+viZQNJJczZ716a/vIEjhU+Za56z4jj3Dnz3P90YcSS+bhLpSEuBrSYzmj6YHmzVvls6V8YZCusOE2rD6Ox4QPntWswvMbnlmSCJOZlMK05n3GGCtfjYhIbTPDhadF/+TOm2FNyO5VCUBY6+mmGRc5LQWFRd+dDws7eqT5i8jnQhL3pjHwSZks4QVupmqM+Dn13BqTUlL5AJ1+Nm51uCHQ+flT/u/8qEBopRw9FiUFD3daFq+f55RkF0pb1Nl/gWPkF59OljDgSx0PYyrDF6j1UFz/fqoAPbdKOrL8DkMzHn97r0IQ9J99oXaTygtmP1rFvqc/j0Kx9v2mLL4TfnFpJPFGM/ZbFAovRFyFpz/0VNaZS8H/w7xud7xIsx+EVDt05JIy4ND/82wetdrIdzflJSErf7nOhv4aEk3YaNplD8M2qMKo5ORVbSr7F656bgdW8Ju9vR25BC2/yBv+4Mow7xPyJdq8GQkXqsYKSTuyMVpuxuJeMBBoJH3HyXSi/BxcaVdSxITRvhcqQC5dOd0Q/i3BEbzuuADZXaajgT0xbUg7C4gLFDePJKpx/w7geZiMPhIAg/lO/+GE5h2dU+lgOdrfJ+dEYysOS9CnLecE4KNnLE6CUIgCNDefSTvsP4NRvJ38OmauY4JVSeeTnJyBa2r6/+OczLMu1mIiX0WKLlZ6I0YYYzCQLXpk59geG5rle9LQFY4PxQ/+H0sq4Xv825MetAmy2arm/7pWP09MWd/iodvYpaFPT5x43zD+/Oe2bdRYWh02oJLpu5NMYM724ceNV/q5/wj9kFcSB+Ot+bjh5FKu4fzE5DlHCMqXURtrVh9guO5nUw1IKixNDX98828TOtNySPpIGlO4S194c44ORDi3SmMBx7KOmjZk5vYHNTFJavl8LcOT7HuP4/npo8lNS+DmhAmMlCmtr2ZpY0zmuryvSAUX6zS8ar7+GUb23agXPnyMx/mOH2CdBaCnSr3qu+QS6FQ+l+UrE4OQvi+z3u4pxUJxl2OufEXbqBfLU9IYgoSA0f6liEeok4t57y9dgt5vT0RXFdsicKczVfpuG7aP3Jh3rvsKYHE+AR9RtDEl2lNp70waJrnNNIYwbSKILT9bq7pLfFdgu/rw1DXNHFt6orvVD+YWOz8W4FxZjXOLzjEohzonisPDQSRwjcam8V9sNBOlYwnfxnf3QJZx/lvMcpyJYwt5tGODwaLX/kals6FSLbtT55wZDA0JXzN5UYlnfH9HLQcXw5+5XDv6mdIgRi1n1V+CGoiuGdCcLTLHRfVQ+S1oKf5yJ3ZTiPQ4LcTduxRyjhQ79FfMJVnvsCr3KPxVcAxXTR7/Y2uTg7L2lCtdDgtjU8NbvzB0VIKWfl2N+fhYnVfea6b5ogYkjbz3jM6awtNv6TviKJlRusBweVKKDwUaPh1wXz8DYkyfl5+/X7Hia23P6BC5sO5qRr2w3j2Fzoa/9it2wYEQ3V8ChG0fUpTejNA3lKd6dOUgIYMqn4TtHmPf/i2Pu2gSf6A+4uM/89H+qejDLP9WsdnoXjv/9+qsl5T8YNPXw6OP/CduxUOwaV4ub8oytNDE0ME/UqXggpQ7Nh//aFn7pwqEoZcrRd8+wtKru7ZPrv7Akzd9768sXSGIzNrV6WwpDF/he6LGtY0n+ETtxkgeOmbP/8B1gxOkA9UWvmnb8nWeWt+zAj3VsiZ8TdvYhOfk6LwttI5LKbuwWLYjEpXqTynMZbTt8aXG6785D2PKiGLBWVIBGb6c/d1kcsK743eBj3XH8Q1J+f42/CEpsZ0t+cUYCaV5Q3eK2KA4wdl1W1zDGjunQqQOlh5BAmLOqm++HdSrK8DkpdihJkaPRDDLOp3/0ok8i7QZsaLUfUPX5CFv+IxfWR4SwNlL2aniHJDQluXdUpnJA96dNymVvehitj3C/WOECUxsagcp6bTjSRbwY8TcbRo7iogQDM7bJC+orx9bi2CO2r3bF2TDRrnznSmAXEFiO7j8g9wua3ML22rzvxEGL5Fwhozs4cs72o/QDIrbu19Ud2c7CycfEe/xakUD+hprPPxTBmP2YcDmfHgze5Xw/lMsFBOqAr3tfa+FCHpHzbfVLnFIdmDuyZoNDHtals47GOKjM8UJb5TouNUn8nI7a8YSWcTlHnlMYQVx4M5fJgpGMZdMJ6Zyw+ZsjeFLYAKaMp+aDP63jKKt7Q+eZqzvvgSBKOTskvQs7X4AO2Ch246vwi8cwr6DzcFK3BR6I9m4P9B2EdoUmq7mkZFx2vLWr+j0FzDzbW/9EdRnWrZfU6jtlgFjHtiEgZI8tFuq8Ed2vcPXjAdYHxuxYu0f072SCD46+G7dlCHWGUdNv6n9HtSHiwLTGJ0k7JDccoVE364a+sazy/NAI0DancS4q28IxgfM3h717YIKdXJNhloPk4DaL6VAtaDY8OaHiSpCnAt1ueseDOFh/z2vm91Esl4v56STYjD29EaNbScVQK0lJbl6bhayQv00nOkax4o7hhTtzr7CZleH4uGMNjImYP7FbeozL/bEN6Xb6WOqet07x3AWHx/bMt701xYZ83eRmhwEYiq3Sr24zAFIKG8VhBcAq9rt/2MIPwtpd3tpQM3Yctz7P5uc7AKOkj4WKFVRYa6cV01zth51Nb+/+/krEYmrFFz+b1nCB8LGJ5+IFqAl88NIR5KG30bek8NRzmHVjPCT42Asnf/fSeRiWYuFlkS3Rw38hBc8Oh2kFw0xHxxJ/iweMu8l7nw+lg8nVL/FHU79jcUvzUKJ5BJLTtJcu3neGUAfeS4zxL6DUMDTp7lF1MGlje9enooOEsILrVNMpONxy8mZnZAcuLZv/N89Xh017j8ktU2fDaNYTTwbewzgjNe95tTcLSXXCMUoy0UB+w1flmuOF351H5WRqHXdy9t7C4E4uvhbqcmmuPInJShRN2w+ssPV7MnM93WHs65Muav1wHIqc1mS9icEwULHAs+aXBJ3+5qaX1S7gRhlJi71MHaY8RGsVNT5DgcRNniOKk1gjUn6tjWoMUnPfWIZO9kKCfs+LF247d730WU/bNgjCx/338fG/hAyNlYjEZAkgHZEyWVHwwqIO1aTNTwE4zqdN2U4kYG543SW1xRBY9am0Gz7ogIVrq98NL/Fi94ZNqcvRCZg7YlfQVqKJCezxsVsrb2C68n6e6r0kUJqRwghtH+y7dKL9CNLALE9Yphm3DbTx7TNhcfwNw26HOD/0fcH57t1EzUPzEHliTqOOJw/ahNLjevlE83f5LAX0UDZin/jp5GuEdGwPo+ExcvWBTvI94c+1NFh+JFS64Yst9nyvaG5buoLxRR8TtkaZcO7PR8l3IUkwq+Qm+SaQBJNCC3pxVxlw4FNT6PaAAmT4OGI5dw8M6o7XyVmcgd+FXrmdKiewTdqJpF/JBGOOR/aNetfB8sEGCWLaVewoS5GzDw6GpT+7lwbPGuG8082Us+8wnzIpyNdD0XmHY94w6LO8QnKeSzb3rp/QJls29niAhM0lVx1j3Tmhb9+Z/aE+etDfQzWe4N8BxJJkmpLvNjgrZ0hIm92L5BQjG46wHJy7S8dZf8gR1/RYc1i1iqFg31zvkRuf4W8Pw+zW4H7svMTDECY9AYVSlQ+EGD1hcM76Iq+9M3jWOR5t//4Xu239f9NVX8QpPDAgvXxR/mDi8n5PCSKMB3/2cadnkN8nJPO40+I1zvLd+Jj5akOeqlrQXjxgIZ9FxY4yUDgIX7ze1C7TOIMNJD/jl6z8OMDFoGRSQIC8AnNz3+yj2N4+Qplssy9/D/WJA3PT+6BZI7lP4ugWLkSyjkYH6OOIghW9dNM/7B08JuKRdg8WDxtsG5P4QE/+svHAkiqWrtImkas/QYfGs7ros2q4zKxWhVGRWDCwFR0poAZTFotBmUYi2NKRpnTUywjWr1Hbd72LxPF0G1/IrcRiU86hmS8jsBF/+v5/Z6WAQKPHHnd4DuKK/kpVpe9wzFPp5m0GVpz1N2CmLq2Afx+DP2mKOOL31R+GvAJvgCyn1Sf8KBLblt9YHZK7jUNnth28Am5g2xXjDostTRh7sBJr9PYqkMtFFN/TKmNrSE0z746Pkpgt33bwDcFYQWhFpMkA9MbnjNNeOA5Ze++3T2fw4bz2eNHKMj+2WrOvPGYKxyojASnp3puQuf7zhI00FwwS5rjv3+eF9u5kdoGpUSw/Xd4nnBaIVVHyOdcidWEgRcJiSSoUJsxzKk90EGDyVqLJlQ+a8hRefumcIpIYY2puJdXWB6tHCdVxt2qgtfJXpd3DdFg029dI9U8IUyvCbwarfkBMfy2Y/UEfO+4cXWeipcKh27P665R3sPsgDXuBuiSuZSgwNQ6EQFZuo8QWFxmXWSYvN9PIY+YzQUZdvUu4Jk7V+Op7J9bX2l1uTEvFqaHuOIJKPPQajfzm8veE5R8X9H76UULVaOD72OwSWFyyY3mYmwIFFwnEJIdz2PZC5v7bcFdoklsrSWyRhDRmAcXbjxihg526lFJAEPqtL4srP9wHVd76Ueu+OxzJHqc/80wCh+uc3gT63sHG2trDhfmUOOQwNpqx5yd0lCpEfNN9DGP6PYf7zfiQnFVCTRkAOHeK0VPaRwlHWGBDJrkB63WFSPn0/EAonbXwCODD5WtcldtDz7Df5IBrqFIqjqZaPk5/NAnpB32lb9WEYEPPVAdViz0QEy7xGzxNgibJlWXRDRNssVcfVSyggTKmdYXHUnLYlHncYGzTHGeYcijzOouxlhAo/CWKH7r7khimHFfRKL5I6cOlVEwKuKVVpXoCF4LFg8fff8DXvvcbO3lHYfab9NEqGiJMvXa9fc/sFI4mdvzHHN2L87UlPU8tq3GyU/LWWxExWCpeuJdfVg4NnLyuURpVaF//4N+lJAkY0B+bWt0liO3Bu6PuUBzBqVRnzUNOdkDQvXpYPuoeTE5S5vz0sMD2P4kyFARmmBXdeiPy4gFO/Cf74+h7X5z1TmXd5jgHZEWiW0j+HPSXuoYrbMdhVj8Xzyphpyd7VGfeBX2HuumAl9y7VHFx4+m11wP+2LCQWna2jEKe4pwK5Y8TY5A4Ys4YzXt2Z1+aBRzOK+CCgJTuncYnSHbR6P0cpQxN84XXU9qeYCQtQYzFtB8Ttl5afWfd2Q+GNzZ0L0nYeIklzOuTEKbxB1He0qWHYtHTf1YekaB/8obSsy17yO3Juvr1BSsWVUd+E8qNwOwXhJGyzKswshIDlAcYsCOmurNN8Ao2BGt57GtehLkf0y/jA29A18L09dxaPiS5+aRt8U1jTyN7WObSAIzBtar02m/Ym2T59Uv7eyzqPMnY4BQF82W/uO9trcHcvUfnR9S/wKyy1oFiP34sUBb2ZNcQgHnvzmbxvFRsMqxQ/a2pKc9ybf7zVa0I7LWfNBy7GorEVqppCe0KIDIwVt0P+Y3E6CgJ95+0MLHQXCwenIfzJczHBC43IZmL7/P2p1VMkPm2+NvTHtuJKZ5+7QiZqXnfq7cJ6Kbd9IiPgQzDd0gUplfXYPBSvrzPsgrOcJ5LvvqGChaeVdt1R9+DVnvG/9xSFOFP+KVHU09dceLESotQ4TSuhWZKk2OP4vDUP2Mni+/Q46Fpv9f8NUyZt8oFFVHCiK3w5yvenUhwWDSIW2XCDF61NL1pgKlk+421TD0k5L835PUOBcJbX/Mr1v2YrPZu8OIAFw4rrXnnOrJht1zjwIyTDS5/D6kqtQeoLTA+TrJ2hT634xHx5fdg1J8+219oDAtulh4/d2gAcppt6JaYuXf4aLGINssKv1n3fE6YuIbDsUHjNLfiYU5dzvxT7BmYDeVW7GZ4CXNvJOSWtwyxOeJ8eHpjIXbPK8+PvPyEOfu+WnERH+NQ53lN6uESHAleqLKLfoXTveQLx+xcIXXWyOHvQTsYP2Tj2xL9H1TK+nB4Tmng+G0WuTnnC7BoeJLzSEcGVuZ2OOkq7vS82FCC6Y5fTj37rUaRNAfRrvyHmFB2h/MTbzZXFMGC5vLaQUUf+LdIty1brYOhKaLJPfx/sErG6ctVdlp5ihedy1Ovb+OEO4lK1ogbp703iyZf6kDvgRvVrom8uLZ5lirlex+szHBfX2V3gg1SmrxlQgEu5CYH+qeHyO+1eRBDXtTG9fJptitVJNwCMzVdgVrYXKa9eLTrMpDt2T6N/HwHXYczQnIia2HgyIu8dHVG6Psm25ybyIyLZ06bXNBgh9Vyf7HV92swdvdZT9Grctz8tovV6u4rrFS/MxcTlYSzf79R7xOJBqKKoXvKU28c/iuX7VNaCEu7fw+FW0dAAuezMzUW9TgVWh7W1tuCQ1JFTvYRXbiod7niLrrAgpYAbWTWO5yq1hynmaLCYgdyAk/ZGk5xSgia3nbErf1iQXNXu3FUmmrGO0sXicc/KSlo0uG0Ce0wTdg6rBwf4wgR2YNh3nQempb78ymrM+KaCzKw+epc5y3+zxjhq621q2Uyn6KGpGNgQIvla9k8S0bHoUGEWOh/QQmWeZmkNpltcHNJ1muD5h1syTNJLvgEylOK/BThv0KEaRF68mUKR3naMdexP6f7cOZLUOKNj4Y4qcxpcM1iD0yY3lGhzX+VT3lk9q8e119cE7XqMs29iqP2XDPCF87g5viVe/JPzuDabp2PURkMMB/1VpMcsx/FP07Y+DzShaXmiAe1VGxIeDZk8dSIBQfkyve6prPDJsXsylTOcxiJGpZqNDTGRg4uvh8X6qCQwT85r8wQFglGfV1zFzDucY126JK8PANzq6wmg6A8BYnFbterx/lMC5EVNA3q+VQ3T4wecDgN7Y8UfNhZjmFdbO/r548cYONjZu2gEAtuP+4t76xMhjXFP92H3tDCwuE6nQ9C8VhEmp4TDgyHxQWzKxM8dDgmOXSNQU0Bp0TuhtfJmEABpfv33doWMHDTRspwUB4wV+9vqFYZzE3rfX6tn4+EFy2zu/KZcHw2/55grjp08D1p6tbWRKKDILtJZSzMhzFNGd3izqcR9VQFeWUY7KY8Rx1oCRPcufN8HbdhddJ8X/1BSuh7b8aRlcqKA+Zud+cOiKG/ClXgq+EveJfOYmCBHJ9PRRkzv/pLH9ai9xWzvDMCso/xfwYhXvm7veioe8sYYe1aW1AMEx2uMsvJ94pkYOf1pUrTEpF86vsxHKSYs1DG/1CpPsQWZgP2Z8t7/MrfP+UxeXaDDRbpfmdQipjj6h6xFjkZKiQOf/0cMuSJK23XOem97sGYJU297O8SmLMbN3HS1cUCfv7lrhPv5fefXK8cqTwAo2RChpFtLfwdCD3S7OQEg1836KRLmjGq4MZZrT1rUL8Y9JKSRRX+PaGTd7vFD/1ctDP0TB9hbO/XUWErSmxaPZZCnhXF8jqDr10ct8HJ7+Ef1bkFqEumfXRSbBZJFKeK9CcP4QRpMEKefye/hUXmHZ+HIePqiGa14hp2DEqLLDDbY6aVtskeCwpI0fi0jztQBKaD5P7srmOF1rUNDp6iaiizNvbspxSGzOuBzG2PGHDyBlOkKGkNq1ftUymiFTF3//jU+q5cmKN414x18VDJ2JQ84ysK5SsnDjwXaYceBaxy6rGEbplJ85YnrFgtZsH1J60Se4ki679SuHByWGBBkliCDceN4nbsBouCD7/4qHILxiezTU7xT2LT7ft7lTl8oSLsx5mfOXm4YN/QK/tEFqZv/4iOueME+TMBKq7fuLCT2ND+eI8QFpwAHoabj2DO9HgU8aksEE4JDHzfzoZq//JGV+l4yHIYrlzO1IBsF+naIjUxrH+Zm2fk8QiLeEafhflKYIeS74IBxzkstm/s/S5FBwOGzvzuuIIEod7eKNorOCN4Lj74EwvOtgyobJTHQdNi3n4heVcY8V/s+MyRjJ5RQjXHC+/C0oBdRjVFI1SRDSeNmwWw37Q6rihWCZvqhkYf11Xv+HaTqrgMCw4+3VcQ/XQnB41Jm/qC/3Du3ddoe8FrWOaoZzvmeQ+Kwp5mUNBO4Uj5Qa3VA25A/P7UQ1MsBNtS+YcnStNhvIf5ZI34Uj51vbfiuIA3zI9nmWwrHYSmd5x7/nLkYjNroJVlQSXO0C5MpIuegYFr1pkap6VxILvwvPXFE0i4YPWlaTUSO6MWy/gVt2BqNiiNdRcdDFdApUOfLJZfoPhBF7aNuRtjONgTAmksaRzAqwzEMB3dDzlCOPzI16dNQgjJ9Bm3S/IEYemMvlZ8Tz0WlixRLknKQc+GeJ7PqUNQpXGqoY33EpJ6OT5cXXoLI4HtxSBMj3M8EfQHaiSx5Q177tr8MZx98k7j5NgnIMcmH1KqG9/Zw2GCLTcz+q74mTiNncEBCvHCHDc1aFY6IuLs3IZltitnnnTEox23sIh9syGUhgUyKmiIQ/Ww5sSTRF/ImN9n1R+pg11TXtnv3L/gcNSHQyrdeZh9s9Y97+8QEM+q/eXdImLh7cTsEwefwqxU1vyhNUccxRMXHThqYLZvNEjqxWeoss1sobkwjPMO84cefu6EJYmQHIEH8TBxZX96r7IrkLfJtxQCqVFfc5jgssCG8cwXDOvXjmLT69JXVBn/oPv3gwz/bH0YrDE7IfxoL4z1CV2ocrHE4vOv0l73xUNKxm2OwH1rUHTPOeTE+ePYJqW5LHlJENqUZPccfYIwosJvdFeJEyK7jW0tYwug2V/yQeWOfxJvE2PH7urh4IbJuwcSJByX1fPid+6HxTBH3VWZWQyOv3SalvkUtPoplw+RZWDiLvfPdaUcLPP59cWmix46tcKiGYfTYUmxJWZ2ShcIZ7ztNTh2eH/yV0Wk5wiOnOptdyLvwZ7rz5U0ry9DXMZ/J+89VMSirHM33h7txyrXW1nEWUNs2Hq2GC1JgOKlQLrQ8RZYCVk4pdyWim0bLGphLzmRfCUukwgGOCvo3Xdw+Ad2UqnJaKREYdE267HQTUYs8ayI/cHshf6FYyY1xQ7YlUrF97HIDxpDt6+cK+uBFkWrbTlvTpyN2tp/rVAP5qpf2BxYzMcaeiNvNpoZ7NS/ZUbp8B4mxWXucl+3gL49OtUJdFHQa7DOIXqyBXOvPWfL5n4Dww+nkiKknuCQ+6VBh8690HWxhGdX1AlMIeteV6Bag/LMk7vIqXRYJ58tobY+B3Oyqilv5VShteMNFRO1Tj7lLaGyKmpFrFHafVzyXBO2/kj8Bjt+0F8lYv/Pyh+j/xm7609vYp2lIuupt69geMbiyw36GKgP9oEfEVdwiGfZ7pZPIYwUJRz7WbLTy0l/JF8OtmPqsmvvWM1tzDTNNdK8FIR5Bz21izlNYUQ/50V20W1Md8+stszyw681F5JXjFextdo23Po5K+QMnCKshsRCxxkijZ+iJ0xL2srK7N3xCPecxcnlN9B3dubI8JIV5pX+Lm5bGQAUJtskGp3Chley3/p99kMR01qh/ktBLJ7S+itT7QblTSq0l9RvIqHmaX6Q9wB0MCjukdk1CvOTvHtU57qgun9P4pfw69jGRq99VzAS6hfOM/d8egAkb5oeMwt/mDE79TCnyh3nvnsIFrXcweSwhGKHSRkYcKnQeRbbiTazE5wV/MewWto3xeke5Q7HWTU8sKrGV/vorgOLIxQpZzF2Orbj+HO+fJYcd+z7UnQ04/JzLKtdtDH4JoVd2VUVdN/Nd/47JVXUR1Ug0ujrCBVel6c84Wk75CWI5Z1Fxy8N2kF/l/GP9XZeGLnkMCgodAhiTzGfMnTZBuKRN25Di3FQM6C6Yv6sFUgdKfe43YShwZ3S/7tiMHR4aPeeM/iFpX4J+XmRCbgCCSSDPd7Q8UlwTGP0BHaKTUQk9nZiU0pLU4/XC1h0v/ttEr9iA5fqHUtxP8ittmQ/R50Giw7s+UMXB4As0KDHX6kC4+FRn9hKriDJV0g5e/QwRka+iT0y4YrZxUcZllOGcVD3Dr+BxRy2UT4UZ/73C4mWo48VhtVxKMSZZ920C4uDGnWuOg8iaVvx1pM1N1x+98Qs+9EIzLIgnxP/CxhfdtnM7zgKE/20POd3ei8zc/HZzf0WOBAWd45SrA/St3dHDk9PQsuUVQifYwikd2hrRIk9wHG1TF6yTjy27W2/ZEVyg7TdMiOfnCeBcOVGgQXbFVxqWSZGuM5ApkYdVSsPK6Qcl322P/kyLnNMKnCcPQHzQckpdk/eAbGog9z9334Y3T9XevBUJPRMf38QWDqJGHylVkXkMZAyFWd3mwN0Zk29jDDxwQkDl+6SbWFIVS/uELOuhNA+3VbeFicsPyTY1u60hNhLPnA8IROHAq7GnLkcgG3e7kv9dKcwqftardlLhOGAV0c2J3yh6fPxVYOJGUyN6uJNW3DAIb/AoMtGclDOxKggX/8Cl3uW2X8YPof57RcRDyLP4LCnaNCNrDM4qu9+ufemBwzG+APXZyacDPv5W0VcAmYTKFXhyTjMIUU/s3ArYN8Tl6DhK7hgyu596LwTxJiFNK1dEsNOQeKMSKIa9l8OYohzd8SSRRE1jevxMEgy+BR5yB9GhbW8mut7sY+fKZH4XR0rA/oSvForYDjqKgbEuEFLFj3Hy198WJ51SoNP+CoMF8yfSz/yZsezjtND8G6odudamgppg6gPDolJOcdw0J3tRZg6PaTS5entX2SAQVvfSBqKW5BxQOOlYn8LNkqOXLkWsoRtjz7/reyRASNbte39U9YwsJoRcqZqETP3ZaeedaDDbW1/+kvVjFAibi3pb+4JHftTv11LCcGyXXHknmkTmAtO5ru8JAb1Zw8eL0qZR/Lj7OeGLRHYJLxg2C9tij08j6kmVL9AD7/Xrrldf5E8ST6vOS8Paaw0nyzJTjjsETHScf8+1t2MnDQnPgHSZZfLnrXGOLhbPjXvmxKMaRg8dqLXxIYF3dOVFEPYFZ8Sfj+7GueGxfz2GbyB/OJ+ljRfIk5f5Ru9RM0O37rN5B+2v4e5R/iXtXARlsuPpark5uDcCxFHXSlRJH+YLxf8Vg6TAwaDQa84kUBKXuHwssbZw4cPdsVkY8bnHIssO0bo2SPh+arqKxTJWk8ktN/BVmvfXg9RJiTQ+L30BRUYGLyccEVnZMen46PUl8xxsfeEU+ZDOsh4JLEl0+0Am+w4lCpshHXzY+EKzgnQ875OdVc4B5DIP77+sOyB5Hk1asUFKozuzeAJGvoCAy+pwwwsuHfyN0+/KXIABoVPco0y/Yd1Ly92nnmnDYMXegIHNnY40efg29j2Qhw0/GZT7N6D1S4JDjZx7PjlzqUKncBuJP6yP/ZqZARbnz2zcQv1hyFOwdBnpmo4qiUu9P7bJExl0aSoTStDwaPrHhqnQ6D2PenLWStdXEylWFppeYM91UUne9p5cUEzgPl15ztMvE7xPqTsDqyt0BUyfz6Co6NBv/Z+PI0RywWblXnTWOSt/SxX0gDL1BgVLdS0cf7aKVrXYkqYLpjQOCHoDaOUqiIPtfqgU3HF0bZ4HSqLHtk30CrC2K3YPZEHymG4m+Ir6x1qWLgnevOU0QQSb3qqP+0+CQsCbyWN2F2QRHD7p3u2Ftr2DzfZpv3A3x8XXYPJxrBu+u+CSdQaLoW+ZPv0KhZLq7xSJty6Ed9Yj3nfc4dRqhmdacN8XLF4O9yXzgFdVcI82hXmQG7qMF4Jn4RiIauiZ4KaMJYrqm2sZY2Lf5jmL/YNwqQghTnNhBUu82RzlInH4Gzaod0CLldwUjB6ovesOYzfJeUw0tZC+/Z56kyLUzATbP2BYjZDnurzr/jQ9Yp8Sq3m0ZowSpj//u6ehvYEDs2WTaklCUNhe+qdhldPMWHw/h93PxJ07ybpXTA3hs4Y9SN/Ri/h0KaN6LWIL0h25y17cVIHE3WlYLWSgE2/A5X8497h5Ou4uKKl/5AQdlm/ycgCyL/+Lr/avoPkX59PzmcO4fT7wLw4oiSQZZK+z7fshUVMMaxrVsfhzfVC+r2qUBPc1NfO4wbxctf+GNhpYB5BNVH26jlsEL3OmsY1BkTnUUduYQVo2DP0x4huDWeIjidCHFtgVumuljcvL8zy/Sj1p9fIp6Q+0SK3jx7mE4RMuSMtYOQjHVzXyoH2tPuvYiwlcDwhOC6o6S3mUi2d/XklCKsoFFboLwniJhXbGq+MJU4y/lHSequAxTrjdWWdKtjiftUxV+AhTJM/eBwIzMYmffxHf0UWFyz5Au0WS4FgPfy2zysK+gZ1eRKfHYOJ3/HnRbxskGhU1virMQJmCNbheR/Nsa1u7aZ9hzg28CrSPbhni4tfLOLEbN7h7OqCeQ7cgDG1GeLRnxpALL79mSr3A5Dd1MrjT76HxV3ORR8+6EJrwsmSb8VnYKJT8bio1DGs+2vYp0bJgisV7P2bB7NxQtP2tluuAxKr8zWHBC7BIuM/EZrKTGxOKuSrvFaKhTom1coqT3FVMvfymsYlnDas+ptKLQIdlJHd/qbtOLWwzubByAgrKizW/3mLw6Ji4sjb01z5dKyh2d6dJTg32vk6QOwidObVsKn0foOeuqlL53fPYX9gcaORrjas/Awvzw94jVOP9m79F/cPt53d3l7PqoDNqt1BZlU9mCcw+LLJ2Q/JHdODVXkl8rv36j+2JbAhSUDXY9KTIE9jPao+wXceh2IDvaU2DLEx610wNWs7RAjmUX1/voBzkTcON7h/gI3qh3vvzWfD2At7K2LNAk6cY0haldglTyHVZdj9/B1OJYbIqLjV4Ij7Ypdk7X1Y/uMu95GpEdeqG6KvC5Tj9kGvelsrTRjMDVimuU8NvaayPUXrO3cc/+QpWfEP/iQr2qZXdiIp6+VDfWVRJPBF9nGUnMR6bt+jHYxnoV/HXM/P+y42XGzqMhFbRLKjq9vrsP4dDyTKhl9pgZbtyIAo0yQcNj8gbUXJK7+b6tfhKp0qnI6a372vXgImdXweinUNQGWqVy3ZuBMmdpP4RXMCgVzwkZvbxx17YvQ2DuMFGDpSp3Ty2mXoYN01rNaaCz3XLzNp1O3BBXOmzRPxEbgkIRfVn38PZuvkJr5Ge+KA5lS6tTklTB1w2MMvUA6k08nTLy20YZlJ6Hfo4QBcYpGxp+RaweXi8kc+It74e9Yk1PWaNgxZe57fSrZA17HTMQPXDuJgDu8thdc12LDn9U3KRQeY/fn3x4j4UxhTVhzbdOLGGs5fX5hXqrHf6Vxk7M6eTrzTDg0wU8HNj6tBCw7NEPzSVz/DN0R+tybHOjQnwPK00lDndyGc2BZ3HqYIhpbV02F/OpOwTnNydzq9FdTp50dJUw1Aa8ve59defYd51aKfdRl7YWD49EB/QSnkPLQ9qXBBFTezt+eMFpNhaf/5lKeUGtB59tobgzs6OCtiExtjyyFPUTooGWEYmk8zl3atT5AIi5NlhzZt6IHw89NXbbFoqC7+uvLq4H2cPXb2seFzP6xVElFfsPSFIV42qw+nHXBwkkP0l8gPXLwSN/tLLFueIWc53zc7EOfm/+RonLkHi4a2euWCJkiidSFLy6ohsU1YN4hbGdo4RQ8x3/SEEU77tlN8DrBgnEc9N2yJ01krpZmnhrB+kF9YUp4EBCujq3R6nDix5mxBQZMI49h+jOLYJViQNnucwemAoS2h2XQLb6HtmpmzcSAjjDffNEqK9pCnOiOZw8p9AbM2XofWfnsI8zQ9bBcX9uRTW2sQ/lz5nU8ZCGUPP5yECe9GB/N1kXyKpt6kMq44WMp5qEhPYsVGrXukvsVtXHXNEohS/IX9c2zzx9nqYUnwzDWqVnUovF3mwNiegUvUsuTI5XFcrTL5WkoziwtFd/TENl12uKQx2Xt8GOeJpUxdpvGwvqbLXvlbE9MjDoRrKTxF8ooL9aPiThxx2rddHFoIbu4OarZ1opDx+FHfM60+XP7i2bhEjsMp+vJfwyxbOMl+if+5mSaMOmyuSztrYz+dj+XrczyYR156PmY9gpOfRmKTuZOAMM3UdyWTGbKDXameBIriglZZ+QavCRADFFN4Ltrl72Kmt2T4qg8jdXIdx8EdF8WuT8kX52AdqU3a8ZItkqV1ki4fC8d5VpOjyqKSsMD70ukrUwpMD63eYhpqxCy3Z8yOxnZYyvshKWvIHXtpuPZHuwzD1M/Yk+8O+2CXJiVn5hg7NvfHRByV94PaY0k6xtPuUH5Z2my4cQRnl3Ss+c/J4Ig4w979lesw5ixR9M/sNoykfLNUusYD4/orHL38/wFRJ2xleIsVCK8bOk0WumG5rux6FL9D/i45ucVuSTGcrLQ7nSd/F9s/VHcqPtv+Hx0ll90=" +} diff --git a/test/test_files/thin_plate_spline_affine.json b/test/test_files/thin_plate_spline_affine.json new file mode 100644 index 00000000..3a6a6fcf --- /dev/null +++ b/test/test_files/thin_plate_spline_affine.json @@ -0,0 +1,5 @@ +{ + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.ThinPlateSplineTransform", + "dataString" : "ThinPlateSplineR2LogR 2 100 @P4R64SLPrGq8kudjBEERFDykx/MHaYpjP2iTdKla/W/AWp5BXNVtaMBXQhhumRdE eJx9VH1UjHkULps6ks+S1IqOWkQpW9k4Xu3aPUeFxIxEUdmtlTgVq0+bmSaqpVCRYWoxanvLUFrK8qYimzmkqCWl8bErsaeRb0527/x+ve85c+3Z+ec59/bc5zy/e583AwP9n1ciy5YaOUwV6kyWZbf+HSXUuYgvh7/XDvER6p+hLkxaJdRKxC8Wi8UhsmNCzSK+yuA/f7yvj3DAn1DnIv6AP6Ee8CfUSsQf8CfULOKr/n9PH/nCmIv4vD++5v3xtRLxeX98zSK+Sp+P7/bRnrAvjLw/vub98bUS8Xl/fM0ivkqfL/jDd8J7wT74GiPvj6+ViM/742sW8VX6fMEfzg2+E94L9sHXGJWIz/vjaxbxVfp8wR/OMc4NvhPeC/ahRDweeX98zSK+Sp8v+MPfFc4xzg2+E94L9sHXGFnEV+nzBX/4O8ffFc4xzg2+E94L9sEiHo8qfb7gD/3fwd85/q5wjnFu8J3wXrAPoUbInDRbmS9yDOZKG/peG/YXcNK6ZwFmtm7c7rqQ9L7wOdyu+mh3qXk3YKjMdZsX9IMdzarKOGm9lXGo7C3MPWp0C6oGnRUvG236mZxV0fcmKGO5/Nm/jOc8zJmcs/3q8zfdmO2SpwVlRWrmpwW3mGcWpoCaMvtQF+i/+GrFowfA6321cpotzMktPH3DQWed9rvTrcz+COtI81RLTpFaVd8U/RsTnXvpfYN/IRd7LPvJ1e9TuIhp2Qb3ZH2A6+6rk5yhL9k7UbsdeDvT14+8DXMlnhXLI0HH4sozwwgm762ka8LcGk7+IO2a7/FgJlnatqSjOJTzLhiUvTU2kfGLE7VHdY4D7J4bJcrhvBVZ1W3nKoGnGO1zRg5zW47WvksGneQJ85vvMHkFL5r67vlx8vphnTuHHGYSwpTDPRPVXKBoQ3fTLAfGbprGasySU4yds/Pwyph50HcbX/bSGniRd2c4tXDyug8nJy8TgU6vw9rRIVSvPZjqbe4nel+0hxO93xM8iJ5vxWCil6UOJXolylSi5zKpiOr5Xqd6Trb0vYP30/cGBJL3dpYGkPfuan1D3rtBWkbeu/7QbvLejsqZ5L2+K03Ie+tS/iDv/drImN6jfAS9x2Jnco+L5iJyD7V9MrnHg/mL6D2sbOk9ao3pPdKayT3Kn04n99Aeiqd5SUqhedl4muSFW9NJ8lL6IZPmZfQkmpfKQJqXGzdIXgJbltG8SCJJXsJnLaN5dmqlee7R0Dwfy6B5XhBE8/xlB82zVwPNs+YHkueQC440z5vH0jybx+j0TFQ/toD+xsv3vRLBt1VVS10x7GPbJCNtOez5uU37egVF732kP9jahPIaUumciyHVkY4B/V7toahR4FvZZf3wLOyj8ta56puwZ+mI8MO6u5vaqFoZijtCaL8mk/LudNO5G3KqE+8A73utmHG3CPbxbsPjmHjYc3XFUI9CuF/j9fx5NZCL/QVrehIJrm4bR/oHZuwhPNP0q3RunTXVOVWm28dn5Seu6fY8e+2sBN39ikdG+EIusl2VDrm6vIm0cwwJ9sRYkn7FRcobnuNP5yyXU52E97BniWLq3G/gfur3l+3EkIv6vVODNjN+W2SyKxWvGLspigsLJxYSdP02lvRbaw8Q3vSwE3TO3onqaP4CXPJppuM26D/tXfSwCXju/ilB8YxffFWOYXczY+caHZ61NIGg7LAd6ZsF5BGepNKNzC3es4noZHyw0uVi5yKfJ7q8FR3J6NLluK3LyVj3fYwtHTZZ992V7ziYSlCS5kn658WTCe+uhQedc59OdfwzIEejLjDiX2Gvz0XO8XGw5/wWhwYx7L1kb9ZjJ7hD3GKTkmaK6W9If7f/Lco7uJDOxZlSnejVcOeepgN/7oO7K5Z2yM5ADo7bX3LfBLlIDCsqGAo5GRRrVD2e4tI+0i8OHUp4FzX/0LlNNlSn6xPIYeD2jXVvIJdRQ45OuQQ5tUye+XkN5HbrneC8PMix9nZSXhjFgym0/0pDeK79OWTuyHMXqtNo8S+ZhThG" +} \ No newline at end of file diff --git a/test/test_files/tilespec_interpolated.json b/test/test_files/tilespec_interpolated.json new file mode 100644 index 00000000..8109351b --- /dev/null +++ b/test/test_files/tilespec_interpolated.json @@ -0,0 +1,70 @@ +{ + "maxX": 7275, + "maxY": -27014, + "layout": { + "sectionId": "2266.0" + }, + "width": 3840, + "height": 3840, + "minX": 1857, + "minY": -32318, + "minIntensity": 0, + "tileId": "20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.2266.0", + "meshCellSize": 64, + "maxIntensity": 65535, + "z": 2266, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + } + }, + "transforms": { + "specList": [ + { + "a": { + "specList": [ + { + "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 ", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000", + "type": "leaf" + } + ], + "type": "list" + }, + "b": { + "specList": [ + { + "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 ", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000", + "type": "leaf" + } + ], + "type": "list" + }, + "type": "interpolated", + "lambda": 0.2 + } + ], + "type": "list" + } +} diff --git a/test/test_files/tilespec_ref.json b/test/test_files/tilespec_ref.json new file mode 100644 index 00000000..bc4e1278 --- /dev/null +++ b/test/test_files/tilespec_ref.json @@ -0,0 +1,29 @@ +{ + "maxX": 7275, + "maxY": -27014, + "layout": { + "sectionId": "2266.0" + }, + "width": 3840, + "height": 3840, + "minX": 1857, + "minY": -32318, + "minIntensity": 0, + "tileId": "20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.2266.0", + "meshCellSize": 64, + "maxIntensity": 65535, + "z": 2266, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + } + }, + "transforms": { + "specList": [ + { "type": "ref", + "refId": "lc_em_test_2266.2266.0" + } + ], + "type": "list" + } +} diff --git a/test/test_files/tilespec_references.json b/test/test_files/tilespec_references.json new file mode 100644 index 00000000..18d5a50b --- /dev/null +++ b/test/test_files/tilespec_references.json @@ -0,0 +1,283 @@ +[ + { + "tileId": "150304141121052104.3407.0", + "z": 3407, + "width": 2560, + "height": 2160, + "layout": { + "sectionId": "3407.0", + "temca": "7", + "camera": "0", + "imageRow": 104, + "imageCol": 52, + "stageX": 97500, + "stageY": 189800, + "rotation": 180 + }, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3407/col0052_row0104_cam0.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera0" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.988634000000 0.036349000000 -0.034204000000 0.948211000000 222.545418368929 100.738559261896" + } + ] + } + }, + { + "tileId": "150304141121052105.3407.0", + "z": 3407, + "width": 2560, + "height": 2160, + "layout": { + "sectionId": "3407.0", + "temca": "7", + "camera": "0", + "imageRow": 105, + "imageCol": 52, + "stageX": 97500, + "stageY": 191650, + "rotation": 180 + }, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3407/col0052_row0105_cam0.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera0" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.988756000000 0.038429000000 -0.033347000000 0.946228000000 115.792562368937 1965.888095261878" + } + ] + } + }, + { + "tileId" : "150304141121053104.3407.0", + "z" : 3407.0, + "width" : 2560.0, + "height" : 2160.0, + "layout" : { + "sectionId" : "3407.0", + "temca" : "7", + "camera" : "0", + "imageRow" : 104, + "imageCol" : 53, + "stageX" : 99450.0, + "stageY" : 189800.0, + "rotation" : 180.0 + }, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3407/col0053_row0104_cam0.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera0" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.988139000000 0.037567000000 -0.031542000000 0.948947000000 2195.971461368928 368.161869261879" + } + ] + } + }, + { + "tileId" : "150304141121053105.3407.0", + "layout" : { + "sectionId" : "3407.0", + "temca" : "7", + "camera" : "0", + "imageRow" : 105, + "imageCol" : 53, + "stageX" : 99450.0, + "stageY" : 191650.0, + "rotation" : 180.0 + }, + "z" : 3407.0, + "width" : 2560.0, + "height" : 2160.0, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3407/col0053_row0105_cam0.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera0" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.987798000000 0.038469000000 -0.035973000000 0.947612000000 2098.872003368931 2199.856228261895" + } + ] + } + }, + { + "tileId": "150304141121055030.3408.1", + "layout": { + "sectionId": "3408.1", + "temca": "7", + "camera": "3", + "imageRow": 30, + "imageCol": 55, + "stageX": 103350, + "stageY": 54800, + "rotation": 180 + }, + "z": 3408, + "width": 2560, + "height": 2160, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3408/col0055_row0030_cam3.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam3.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera3" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.982457000000 0.035841000000 -0.042949000000 0.944933000000 702.755960368930 1277.236107261851" + } + ] + } + }, + { + "tileId": "150304141121055031.3408.1", + "layout": { + "sectionId": "3408.1", + "temca": "7", + "camera": "3", + "imageRow": 31, + "imageCol": 55, + "stageX": 103350, + "stageY": 56650, + "rotation": 180 + }, + "z": 3408, + "width": 2560, + "height": 2160, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3408/col0055_row0031_cam3.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam3.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera3" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.982235000000 0.035777000000 -0.042126000000 0.943634000000 590.676579368926 3122.673506261839" + } + ] + } + }, + { + "tileId" : "150304141121056030.3408.1", + "z" : 3408.0, + "width" : 2560.0, + "height" : 2160.0, + "layout" : { + "sectionId" : "3408.1", + "temca" : "7", + "camera" : "2", + "imageRow" : 30, + "imageCol" : 56, + "stageX" : 105000.0, + "stageY" : 54800.0, + "rotation" : 180.0 + }, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3408/col0056_row0030_cam2.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam2.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera2" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.982463000000 0.042319000000 -0.038484000000 0.945792000000 2502.850133368935 1376.114864261879" + } + ] + } + }, + { + "tileId" : "150304141121056031.3408.1", + "z" : 3408.0, + "width" : 2560.0, + "height" : 2160.0, + "layout" : { + "sectionId" : "3408.1", + "temca" : "7", + "camera" : "2", + "imageRow" : 31, + "imageCol" : 56, + "stageX" : 105000.0, + "stageY" : 56650.0, + "rotation" : 180.0 + }, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3408/col0056_row0031_cam2.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam2.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera2" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.982244000000 0.041618000000 -0.037589000000 0.944900000000 2387.030142368923 3217.084657261847" + } + ] + } + } + ] + \ No newline at end of file diff --git a/test/test_files/tilespecs.json b/test/test_files/tilespecs.json new file mode 100644 index 00000000..f990337a --- /dev/null +++ b/test/test_files/tilespecs.json @@ -0,0 +1,132 @@ +[ + { + "tileId": "20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.2266.0", + "layout": { + "sectionId": "2266.0" + }, + "z": 2266, + "minX": 1857, + "minY": -32318, + "maxX": 7275, + "maxY": -27014, + "width": 3840, + "height": 3840, + "minIntensity": 0, + "maxIntensity": 65535, + "mipmapLevels": { + "0": { + "imageUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + }, + "1": { + "imageUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8_mmL1.tif.tif", + "maskUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8_mmL1_mask.tif.tif" + }, + "2": { + "imageUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8_mmL2.tif.tif" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000" + } + ] + }, + "meshCellSize": 64 + }, + { + "tileId": "20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8.2266.0", + "layout": { + "sectionId": "2266.0" + }, + "z": 2266, + "minX": 3915, + "minY": -34958, + "maxX": 9339, + "maxY": -29519, + "width": 3840, + "height": 3840, + "minIntensity": 0, + "maxIntensity": 65535, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8.tif" + }, + "1": { + "imageUrl": "file:/data/20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8_mmL1.tif.tif" + }, + "2": { + "imageUrl": "file:/data/20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8_mmL2.tif.tif" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000" + } + ] + }, + "meshCellSize": 64 + }, + { + "tileId": "20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8.2266.0", + "layout": { + "sectionId": "2266.0" + }, + "z": 2266, + "minX": 5957, + "minY": -37482, + "maxX": 11373, + "maxY": -32062, + "width": 3840, + "height": 3840, + "minIntensity": 0, + "maxIntensity": 255, + "mipmapLevels": { + "0": { + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8.tif" + }, + "1": { + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8_mmL1.tif.tif" + }, + "2": { + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8_mmL2.tif.tif" + }, + "5": { + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8_mmL5.tif.tif" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961993" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000" + } + ] + }, + "meshCellSize": 64 + } +] diff --git a/test/test_files/transform_references.json b/test/test_files/transform_references.json new file mode 100644 index 00000000..493c9afb --- /dev/null +++ b/test/test_files/transform_references.json @@ -0,0 +1,51 @@ +[ + { + "type" : "list", + "id" : "LENS_CORRECTION_temca7_camera0", + "specList" : [ + { + "type" : "leaf", + "className" : "lenscorrection.NonLinearTransform", + "dataString" : "5 21 700.707955266298 -2.2001080559868793 13.873040384630464 627.0743514702998 -13.529483853406793 7.228735711439736 -10.839669176402502 -6.139021966264066 -11.72071825781854 -10.33928052302308 4.278978527660939 -3.4594411956970808 -2.5020177814252778 0.8912979016273739 3.35982841409578 -8.64411101181073 0.09093625547696149 8.332230284245867 7.479082855684247 -1.284703107571685 1.8807143771547103 5.941259214658612 4.209581504855379 5.257758911157933 2.3978334920077202 3.754133480575426 -0.5929877413140048 -0.7779841026367293 -2.944571086212573 1.1933155763290975 0.4661295617851611 -2.412931393336346 -2.6955605399934246 -0.22775323647894358 0.7122079198088755 -2.648021400179547 -1.6624574262499792 -0.02708163303879585 1.1475940290759183 0.23485421235502013 11.471758011138729 11.048994469455984 1147.1734708175197 1104.8993194529953 1794114.7301473576 1265879.0975329224 1604717.7004822914 3.2387227602706795E9 1.9734583414478405E9 1.8483776775925295E9 2.6122108229352436E9 6.344304427022081E12 3.546610513583933E12 2.87805285556924E12 3.0253326094556655E12 4.532508846629051E12 1.3094445313371616E16 6.91662907507397E15 5.157572930400959E15 4.70862651090562E15 5.276554689026626E15 8.193085649459779E15 100.0 691.4672425752566 619.6213386843245 1767931.6182683974 1129448.610690605 1399095.6047029237 4.185162060049768E9 2.476443778259326E9 2.1902776293974566E9 2.899873921130305E9 9.847375063509006E12 5.576800271136418E12 4.489070210105462E12 4.3313999043551543E12 5.925923866832138E12 2.3253941132175516E16 1.2802457301674196E16 9.817920026167838E15 8.617375639277423E15 8.677623871959977E15 1.2087680046683428E16 0.0 2560 2160 " + }, + { + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "1.0225774 0.004573957 -0.008069136 1.0538727 130.0 30.0" + } + ] + }, + { + "id" : "LENS_CORRECTION_temca7_camera2", + "type" : "list", + "specList" : [ + { + "type" : "leaf", + "className" : "lenscorrection.NonLinearTransform", + "dataString" : "5 21 711.4721917133976 -5.589914769754155 -1.1212879916039302 611.9788313567354 0.9060213772013888 17.225578046304864 -3.375415049826092 -13.228357077450797 -4.0000251672817235 -9.45662859209138 -1.634495499587608 -10.304113753791569 -1.213393557789903 3.280829150417878 3.806378900638064 -3.504968777660551 7.201798010176688 4.68489990495069 6.5940053140106905 5.003072665329844 2.207293734167834 2.8728531024175634 -0.05073765949401299 3.623864247802219 2.95885160398298 0.8934478162984338 -4.8060384709988995 -0.13469418887316031 -2.0248910123223816 -1.208647910444716 -0.6461420209825857 -1.1353187792931874 -0.33149925100723077 -1.1122145006790851 -0.5921228750450704 -1.0352160750358692 -0.9975754041991061 0.5723447761009257 1.4718233160631549 -0.4660362839593981 13.271729787926228 11.36085817904796 1327.1706044725697 1136.0903783862736 2272546.952672654 1506473.4529777775 1650604.9178121507 4.337314511226062E9 2.5744385721076655E9 2.194297853083332E9 2.663116597892426E9 8.809638594278018E12 4.903814127489643E12 3.748933696078432E12 3.551527713373412E12 4.561781316801743E12 1.864076900958326E16 9.938324654099888E15 7.134639652469553E15 6.067832412917443E15 6.103406845480805E15 8.129116449151267E15 100.0 714.9731259974685 599.9322125157199 1909272.5769361807 1216360.74472006 1355493.1773583877 4.666723318413752E9 2.795306737181569E9 2.369412143315229E9 2.7942938019494762E9 1.1235656814375736E13 6.477705273329607E12 5.098545422604573E12 4.656013779749117E12 5.679481768816695E12 2.6992898463066668E16 1.51679732641352E16 1.1460688242542052E16 9.727988954220464E15 9.253755629882418E15 1.1537151295307158E16 0.0 2560 2160 " + }, + { + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.98033196 -0.011563861 -0.007845614 1.0093758 130.0 30.0" + } + ] + }, + { + "id" : "LENS_CORRECTION_temca7_camera3", + "type" : "list", + "specList" : [ + { + "type" : "leaf", + "className" : "lenscorrection.NonLinearTransform", + "dataString" : "5 21 716.6873371364047 6.053095456716258 -7.33011813717707 615.9404061487677 -10.10392129154934 -3.178743380467174 -0.9080793854570661 -7.555073092930774 6.836976455645333 -10.919574061891632 6.010802574254718 3.4312960837342352 1.9319897093811633 2.278773522479664 4.261230047320696 -3.3766138130897687 -10.037252509049736 8.879376829985034 0.2511892646304901 -7.474464984035155 -0.12651470010799137 3.018706620210251 -0.6984470933503166 1.997670980472821 -0.33445113401323834 1.8050086636786895 10.515312559392513 -6.9198312805652265 -0.03371009293810445 2.8556473768134767 -0.6863440894714427 -0.8946743620783042 0.6330640226897923 0.1726669199954567 0.6353096977003143 -0.8782956379063966 0.3367041976112679 -0.6744341635057034 -3.6657658385583005 2.8912392853921967 12.576628285752045 9.997393590654209 1257.663252302061 999.7429242926293 2093303.624799589 1245004.0742310893 1366319.5248503662 3.9345304836190944E9 2.0606718153677976E9 1.6892732155723114E9 2.1396317407128017E9 7.919490965364902E12 3.8544763741048306E12 2.7808203804989346E12 2.632847197055338E12 3.6107639405689927E12 1.6662293914167926E16 7.721975010374804E15 5.17652292334484E15 4.3144423730759265E15 4.428686756822473E15 6.384364622813321E15 100.0 715.2689070153682 605.6814246284501 1880885.1217286212 1109432.3712253342 1320610.7518000966 4.546331733157403E9 2.4653417254374003E9 2.0918157620442765E9 2.6594039686218805E9 1.0867868593386514E13 5.60885189992376E12 4.3212574170000156E12 4.01855921622566E12 5.314489633143806E12 2.5987814164131128E16 1.297613219424449E16 9.483497223595874E15 8.031119671564667E15 7.84530320604815E15 1.0658513461698616E16 0.0 2560 2160 " + }, + { + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "1.0059261 0.0090332 0.002712006 0.99720263 130.0 30.0" + } + ] + } + ] + \ No newline at end of file diff --git a/test/test_imagepyramid.py b/test/test_imagepyramid.py new file mode 100644 index 00000000..683948e1 --- /dev/null +++ b/test/test_imagepyramid.py @@ -0,0 +1,100 @@ +from renderapi import image_pyramid +from renderapi.errors import RenderError +import pytest + +image_filename = "not_a_file.jpg" +alt_image_filename = "not_a_file2.jpg" +mask_filename = "not_a_mask.png" +alt_mask_filename = "not_a_mask2.png" + + +def test_basic_pyramid(): + + ip = image_pyramid.ImagePyramid() + ip[0] = image_pyramid.MipMap(imageUrl=image_filename, + maskUrl=mask_filename) + + assert(ip[0].imageUrl == image_filename) + assert(ip[0].maskUrl == mask_filename) + assert(len(ip.levels) == 1) + with pytest.raises(RenderError): + ip['not_a_level'] + + with pytest.raises(RenderError): + ip[-1] + + assert(ip.to_dict()['0']['imageUrl'] == image_filename) + + ip[1] = image_pyramid.MipMap(imageUrl=image_filename, + maskUrl=mask_filename) + assert(ip[1].imageUrl == image_filename) + + # test __setitem__ interface + ip[1]['imageUrl'] = alt_image_filename + ip[1]['maskUrl'] = alt_mask_filename + assert(ip[1].imageUrl == alt_image_filename) + assert(ip[1].maskUrl == alt_mask_filename) + + with pytest.raises(KeyError): + ip[1]['notvalid'] = 'test' + + assert(len(ip) == 2) + ip.pop(1) + assert(len(ip) == 1) + with pytest.raises(KeyError): + ip[1] + + +def test_pyramid_deserialize(): + d = { + "0": { + "imageUrl": image_filename, + "maskUrl": mask_filename + } + } + ip = image_pyramid.ImagePyramid.from_dict(d) + assert(ip[0].imageUrl == image_filename) + assert(ip[0].maskUrl == mask_filename) + + mm2 = image_pyramid.MipMap(imageUrl=image_filename, + maskUrl=mask_filename) + ip2 = image_pyramid.ImagePyramid({0: mm2}) + assert(ip == ip2) + + assert(ip[0] == d["0"]) + mm3 = image_pyramid.MipMap(imageUrl=image_filename) + ip3 = image_pyramid.ImagePyramid({0: mm3}) + assert(ip != ip3) + + assert(mm3['imageUrl'] == image_filename) + assert(mm3['maskUrl'] is None) + + with pytest.raises(KeyError): + mm3['not_a_key'] + + +def test_mipmaplevel_deprecated(): + with pytest.deprecated_call(): + mml = image_pyramid.MipMapLevel( + 0, + imageUrl=image_filename, + maskUrl=mask_filename) + assert(mml['imageUrl'] == image_filename) + assert(mml['maskUrl'] == mask_filename) + with pytest.raises(KeyError): + mml['not_a_key'] + + assert(mml == mml) + with pytest.deprecated_call(): + mml2 = image_pyramid.MipMapLevel(0, + imageUrl=image_filename) + assert(mml != mml2) + assert(len([k for k, v in mml]) == 2) + + +def test_transformed_mapping(): + d = image_pyramid.TransformedDict() + d[1] = 'a' + d[2] = 'b' + mylist = [k for k in d] + assert(len(mylist) == 2) diff --git a/test/test_pointmatch.py b/test/test_pointmatch.py new file mode 100644 index 00000000..3239e2a7 --- /dev/null +++ b/test/test_pointmatch.py @@ -0,0 +1,114 @@ +import copy + +import pytest +import six + +import renderapi + + +@pytest.fixture(scope="module") +def matches(): + return [{ + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "1", + "qId": "1-1", + "matches": { + "p": [[0, 0], + [100, 100], + [0, 100], + [100, 0]], + "q": [[0, 0], + [100, 100], + [0, 100], + [100, 0]], + "w": [1, 1, 1, 1]}, + "matchCount": 4 + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "2", + "qId": "2-1", + "matches": { + "p": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "q": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "w": [1, 1, 1, 1], + }, + "matchCount": 4 + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "0", + "qId": "0-2", + "matches": { + "p": [ + [100, 100], + [100, 0] + ], + "q": [ + [0, 100], + [0, 0] + ], + "w": [1, 1], + }, + "matchCount": 2 + } + ] + + +@pytest.fixture(scope="module") +def match(matches): + return matches[1] + + +def set_tuple_subscript(obj, tuple_subscript, value): + o = obj + for s in tuple_subscript[:-1]: + o = o.__getitem__(s) + return o.__setitem__(tuple_subscript[-1], value) + + +def copymatch_and_compare(input_match, tuple_subscript_to_change, + target_value=None): + orig_match = copy.deepcopy(input_match) + copied_match = renderapi.pointmatch.copy_match_explicit(orig_match) + set_tuple_subscript(orig_match, tuple_subscript_to_change, target_value) + return orig_match == copied_match + + +def test_copy_match(match): + # test for top level keys + for k in (six.viewkeys(match) - {"matches"}): + assert not copymatch_and_compare(match, (k,)) + # test for nested values in matches + for subtup in (("matches", "p", 0, 0), + ("matches", "q", 0, 0), + ("matches", "w", 0)): + assert not copymatch_and_compare(match, subtup) + + +def test_copy_matches(matches): + assert all([i == j for i, j in zip( + matches, renderapi.pointmatch.copy_matches_explicit(matches))]) + + +@pytest.mark.parametrize("do_copy", [True, False]) +def test_swap_matchpair(matches, do_copy): + for match, swapped_match in zip( + matches, (renderapi.pointmatch.swap_matchpair( + match, copy=do_copy) for match in matches)): + assert match != swapped_match + assert match == renderapi.pointmatch.swap_matchpair( + swapped_match, do_copy) diff --git a/test/test_resolvedtiles.py b/test/test_resolvedtiles.py new file mode 100644 index 00000000..484aa63f --- /dev/null +++ b/test/test_resolvedtiles.py @@ -0,0 +1,48 @@ +import renderapi +import pytest +import json +import rendersettings + + +@pytest.fixture(scope='module') +def referenced_tilespecs_and_transforms(): + with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS, 'r') as fp: + ds = json.load(fp) + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ds] + + with open(rendersettings.REFERENCE_TRANSFORM_SPECS, 'r') as fp: + ds = json.load(fp) + transforms = [renderapi.transform.load_transform_json(tf) for tf in ds] + return tilespecs, transforms + + +@pytest.fixture(scope='module') +def resolvedtiles_object(referenced_tilespecs_and_transforms): + tilespecs, transforms = referenced_tilespecs_and_transforms + resolved_tiles = renderapi.resolvedtiles.ResolvedTiles( + tilespecs=tilespecs, transformList=transforms) + return resolved_tiles + + +@pytest.fixture(scope="module") +def resolvedtiles_objects(resolvedtiles_object): + # one resolvedtiles object per tile in the resolvedtiles_object + rts_l = [renderapi.resolvedtiles.ResolvedTiles( + tilespecs=[ts], transformList=resolvedtiles_object.transforms) + for ts in resolvedtiles_object.tilespecs] + return rts_l + + +def test_resolvedtiles_from_dict(resolvedtiles_object, + referenced_tilespecs_and_transforms): + tilespecs, transforms = referenced_tilespecs_and_transforms + d = resolvedtiles_object.to_dict() + resolved_tiles = renderapi.resolvedtiles.ResolvedTiles(json=d) + assert(len(tilespecs) == len(resolved_tiles.tilespecs)) + assert(len(transforms) == len(resolved_tiles.transforms)) + + +def test_combine_resolvedtiles(resolvedtiles_object, resolvedtiles_objects): + assert(resolvedtiles_object.to_dict() == + renderapi.resolvedtiles.combine_resolvedtiles( + resolvedtiles_objects).to_dict()) diff --git a/test/test_stack.py b/test/test_stack.py new file mode 100644 index 00000000..a95a5bb2 --- /dev/null +++ b/test/test_stack.py @@ -0,0 +1,9 @@ +import renderapi + + +def test_blank_stackversion(): + sv = renderapi.stack.StackVersion() + der_sv = renderapi.stack.StackVersion(**sv.to_dict()) + fd_sv = renderapi.stack.StackVersion() + fd_sv.from_dict(sv.to_dict()) + assert(sv.to_dict() == der_sv.to_dict() == fd_sv.to_dict()) diff --git a/test/test_tilespec.py b/test/test_tilespec.py new file mode 100644 index 00000000..9602c29f --- /dev/null +++ b/test/test_tilespec.py @@ -0,0 +1,57 @@ +import json +from operator import eq +import renderapi +import rendersettings +from renderapi.image_pyramid import ImagePyramid + + +def test_load_tilespecs_json(): + with open(rendersettings.TEST_TILESPECS_FILE, 'r') as f: + ts_json = json.load(f) + # current TileSpec objects do not put, e.g. min/max X/Y values in in dict + ts_json_expected_fields = ['layout', 'width', 'height', 'tileId', + 'minIntensity', 'maxIntensity', 'mipmapLevels', + 'z', 'transforms'] + ts_json_expected = [{k: v for k, v in ts.items() + if k in ts_json_expected_fields} + for ts in ts_json] + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ts_json] + assert(all(map( + lambda x: eq(*x), + zip(json.loads( + renderapi.utils.renderdumps(tilespecs)), ts_json_expected)))) + + +def test_load_tilespecs_args(): + with open(rendersettings.TEST_TILESPECS_FILE, 'r') as f: + ts_json = json.load(f) + # current TileSpec objects do not put, e.g. min/max X/Y values in in dict + ts_json_expected_fields = ['layout', 'width', 'height', 'tileId', + 'minIntensity', 'maxIntensity', 'mipmapLevels', + 'z', 'transforms'] + ts_json_expected = [{k: v for k, v in ts.items() + if k in ts_json_expected_fields} + for ts in ts_json] + + tilespecs = [renderapi.tilespec.TileSpec( + tileId=ts['tileId'], + z=ts['z'], + width=ts['width'], + height=ts['height'], + imagePyramid=ImagePyramid.from_dict(ts['mipmapLevels']), + layout=renderapi.tilespec.Layout( + force_pixelsize=False, **ts['layout']), + minint=ts['minIntensity'], maxint=ts['maxIntensity'], + tforms=renderapi.transform.TransformList( + json=ts['transforms']).tforms) + for ts in ts_json_expected] + assert(all(map( + lambda x: eq(*x), + zip([ts.to_dict() for ts in tilespecs], ts_json_expected)))) + + +def test_bbox_shape(): + with open(rendersettings.TEST_TILESPECS_FILE, 'r') as f: + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in json.load(f)] + + assert(all([len(ts.bbox) == 4 for ts in tilespecs])) diff --git a/test/test_transform.py b/test/test_transform.py new file mode 100644 index 00000000..e977c03f --- /dev/null +++ b/test/test_transform.py @@ -0,0 +1,1085 @@ +import json +import renderapi +import numpy as np +import scipy.linalg +import rendersettings +import importlib +import pytest +from scipy.spatial.distance import cdist + + +EPSILON = 0.0000000001 +EPSILON2 = 0.000000001 + + +def cross_py23_reload(module): + try: + reload(module) + except NameError: + importlib.reload(module) + + +def test_TransformList_init(): + tlist = renderapi.transform.TransformList() # noqa: F841 + + +def test_simple_TransformList_init(): + aff = renderapi.transform.AffineModel() + tlist = renderapi.transform.TransformList(tforms=[aff]) # noqa: F841 + js = tlist.to_json() + assert(isinstance(js, str)) + j = json.loads(js) + assert(isinstance(j, dict)) + + +def test_fail_TransformList_init(): + with pytest.raises(renderapi.errors.RenderError): + aff = renderapi.transform.AffineModel() + tlist = renderapi.transform.TransformList(tforms=aff) # noqa: F841 + + +def test_fail_loadtransform_json(): + d = {'type': 'not_a_type', + 'dataString': 'junkstring', + 'transformId': 'badtform'} + with pytest.raises(renderapi.errors.RenderError): + renderapi.transform.load_transform_json(d) + with pytest.raises(renderapi.errors.RenderError): + renderapi.transform.load_leaf_json(d) + + +def test_load_unknown_tform(): + d = {'className': 'mpicpg.not_supported_transform', + 'dataString': 'crazy_parameters'} + tform = renderapi.transform.load_transform_json(d) + assert(isinstance(tform, renderapi.transform.Transform)) + + +def test_fail_bad_tform(): + d = {'type': 'leaf', + 'className': 'mpicpg.not_supported_transform'} + with pytest.raises(renderapi.errors.RenderError): + tform = renderapi.transform.load_transform_json(d) # noqa: F841 + + +def test_similarity_rot_90(): + am = renderapi.transform.SimilarityModel() + # setup a 90 degree clockwise rotation + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float64) + points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float64) + am.estimate(points_in, points_out) + + assert(np.abs(am.scale[0] - 1.0) < .00001) + assert(np.abs(am.scale[1] - 1.0) < .00001) + assert(np.abs(am.rotation + np.pi / 2) < .000001) + assert(np.abs(am.translation[0]) < .000001) + assert(np.abs(am.translation[1]) < .000001) + assert(np.all(np.abs(am.shear) < .000001)) + + points = np.array([[20, 30], [1, 2], [10, -5], [-4, 3], [5.6, 2.3]]) + new_points = am.tform(points) + + old_points = am.inverse_tform(new_points) + assert(np.sum(np.abs(points - old_points)) < (.0001 * len(points.ravel()))) + + am_inverse = renderapi.transform.AffineModel() + am_inverse.estimate(points_out, points_in) + + identity = am.concatenate(am_inverse) + assert(np.abs(identity.scale[0] - 1.0) < .00001) + assert(np.abs(identity.scale[1] - 1.0) < .00001) + assert(np.abs(identity.rotation) < .000001) + assert(np.abs(identity.translation[0]) < .000001) + assert(np.abs(identity.translation[1]) < .000001) + assert(np.all(np.abs(identity.shear) < .000001)) + print(str(am)) + + +def test_affine_fail(): + am = renderapi.transform.AffineModel() + # setup a 90 degree clockwise rotation + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float64) + points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float64) + # catch error + with pytest.raises(renderapi.errors.EstimationError): + am.estimate(points_in, points_out[0:-2, :]) + + +def test_json_affine(): + M00 = .9 + M10 = -0.2 + M01 = 0.3 + M11 = .85 + B0 = 245.3 + B1 = -234.1 + am = renderapi.transform.AffineModel(M00=M00, + M10=M10, + M01=M01, + M11=M11, + B0=B0, + B1=B1) + + props = np.array(am.calc_properties()) + jam = am.to_dict() + am2 = renderapi.transform.AffineModel(json=jam) + jprops = np.array(am2.calc_properties()) + + assert(np.all(np.abs(props - jprops) < EPSILON)) + + +def test_affine_random(): + M00 = .9 + M10 = -0.2 + M01 = 0.3 + M11 = .85 + B0 = 245.3 + B1 = -234.1 + am = renderapi.transform.AffineModel(M00=M00, + M10=M10, + M01=M01, + M11=M11, + B0=B0, + B1=B1) + + points_in = np.random.rand(10, 2) + points_out = am.tform(points_in) + + am_fit = renderapi.transform.AffineModel() + am_fit.estimate(points_in, points_out) + + assert(np.sum(np.abs(am.M.ravel() - am_fit.M.ravel())) < (.001 * 6)) + + # return_all test + tvec, res, rank, s = am.fit(points_in, points_out, return_all=True) + compare_tvec = np.array([M00, M01, M10, M11, B0, B1]) + tvec = tvec.flatten() + assert(np.linalg.norm(tvec - compare_tvec) < EPSILON) + assert(res < EPSILON) + + +def test_invert_Affine(): + am = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + Iam = am.invert() + assert(np.allclose(Iam.concatenate(am).M, np.eye(3))) + assert(np.allclose(am.concatenate(Iam).M, np.eye(3))) + + +def test_polynomial_scale(): + p = np.transpose(np.array([[0, 0]])) + t = renderapi.transform.Polynomial2DTransform(params=p) + assert(t.order == 0) + assert(t.scale == (1.0, 1.0)) + + p = np.transpose(np.array([[0, 0], [1.1, 0], [0, 0.9]])) + t = renderapi.transform.Polynomial2DTransform(params=p) + assert(t.order == 1) + assert(t.scale == (1.1, 0.9)) + + +def test_Polynomial_estimation(use_numpy=False): + if use_numpy: + try: + import builtins + except ImportError: + import __builtin__ as builtins + realimport = builtins.__import__ + + def noscipy_import(name, globals=None, locals=None, + fromlist=(), level=0): + if 'scipy' in name: + raise ImportError + return realimport(name, globals, locals, fromlist, level) + builtins.__import__ = noscipy_import + + cross_py23_reload(renderapi.transform.polynomial_models) + + assert(renderapi.transform.polynomial_models.svd is np.linalg.svd + if use_numpy else + renderapi.transform.polynomial_models.svd is scipy.linalg.svd) + + datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') + default_pt = renderapi.transform.Polynomial2DTransform( + dataString=datastring) + srcpts = np.random.rand(30, 2) + dstpts = default_pt.tform(srcpts) + derived_pt = renderapi.transform.Polynomial2DTransform( + src=srcpts, dst=dstpts) + assert(np.allclose(derived_pt.params, default_pt.params)) + assert(not default_pt.is_affine) + + with pytest.raises(renderapi.errors.EstimationError): + derived_pt.estimate(srcpts, dstpts[0:-2, :]) + + with pytest.raises(renderapi.errors.EstimationError): + derived_pt.estimate(srcpts, dstpts[0:-2, :], order=500) + + if use_numpy: + builtins.__import__ = realimport + cross_py23_reload(renderapi.transform.polynomial_models) + assert(renderapi.transform.polynomial_models.svd is scipy.linalg.svd) + cross_py23_reload(renderapi.transform) + + +def test_Polynomial_estimation_numpy(): + test_Polynomial_estimation(use_numpy=True) + + +def notatest_transformsum_polynomial_identity(): + # test not used currently in favor of more reproducible affine + srcpts = np.random.rand(50, 2) + am = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + invam = am.invert() + + datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') + pt = renderapi.transform.Polynomial2DTransform( + dataString=datastring) + ptest_dstpts = pt.tform(srcpts) + invpt = renderapi.transform.Polynomial2DTransform( + src=ptest_dstpts, dst=srcpts) + + tformlist = [am, [pt, invpt], invam] + new_tform = renderapi.transform.estimate_transformsum( + tformlist, src=srcpts) + + poly_identity = renderapi.transform.Polynomial2DTransform( + identity=True).asorder(new_tform.order) + assert all([i < 1e-3 for i in + (new_tform.params[:, 0] - poly_identity.params[:, 0]).ravel()]) + + assert np.allclose( + new_tform.params[:, 1:-1], + poly_identity.params[:, 1:-1], atol=1e-5) + + +def test_transformsum_affine_concatenate(): + srcpts = np.random.rand(50, 2) + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + am2 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=-100, + B1=3) + am3 = renderapi.transform.AffineModel(M00=1.9, + M10=-0.2, + M01=0.3, + M11=1.85, + B0=-25.3, + B1=60.1) + am4 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=2.3, + B1=100.1) + + tformlist = [am1, [[am2, am3], am4]] + new_tform = renderapi.transform.estimate_transformsum( + tformlist, src=srcpts) + concat_tform = am4.concatenate(am3.concatenate(am2)).concatenate(am1) + assert np.allclose(new_tform.M, concat_tform.M) + + +def test_load_polynomial(): + datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') + pt = renderapi.transform.Polynomial2DTransform( + dataString=datastring) + pt_dict = renderapi.transform.Polynomial2DTransform(json=pt.to_dict()) + pt_dataString = renderapi.transform.Polynomial2DTransform( + dataString=pt.dataString) + pt_params = renderapi.transform.Polynomial2DTransform(params=pt.params) + assert (pt_dict.to_dict() == pt_dataString.to_dict() == + pt_params.to_dict() == pt.to_dict()) + + +def test_Polynomial_from_affine(): + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + pt = renderapi.transform.Polynomial2DTransform.fromAffine(am1) + pt_params_raveled = pt.params.ravel() + assert pt.order == 1 + assert pt_params_raveled[0] == am1.B0 + assert pt_params_raveled[1] == am1.M00 + assert pt_params_raveled[2] == am1.M01 + assert pt_params_raveled[3] == am1.B1 + assert pt_params_raveled[4] == am1.M10 + assert pt_params_raveled[5] == am1.M11 + + +def test_interpolated_transform(): + with open(rendersettings.INTERPOLATED_TRANSFORM_TILESPEC, 'r') as f: + j = json.load(f) + ts = renderapi.tilespec.TileSpec(json=j) + it_ts = [tform for tform in ts.tforms + if isinstance( + tform, renderapi.transform.InterpolatedTransform)][0] + it_args = renderapi.transform.InterpolatedTransform( + it_ts.a, it_ts.b, it_ts.lambda_) + it_dd = renderapi.transform.InterpolatedTransform( + json=it_ts.to_dict()) + + assert (dict(it_args) == it_args.to_dict() == dict(it_ts) == + it_ts.to_dict() == dict(it_dd) == it_dd.to_dict()) + + +def test_reference_transform(): + with open(rendersettings.REFERENCE_TRANSFORM_TILESPEC, 'r') as f: + j = json.load(f) + ts = renderapi.tilespec.TileSpec(json=j) + ref_ts = [tform for tform in ts.tforms + if isinstance( + tform, renderapi.transform.ReferenceTransform)][0] + ref_args = renderapi.transform.ReferenceTransform(ref_ts.refId) + ref_dd = renderapi.transform.ReferenceTransform(json=ref_ts.to_dict()) + + assert (dict(ref_args) == ref_args.to_dict() == dict(ref_ts) == + ref_ts.to_dict() == dict(ref_dd) == ref_dd.to_dict()) + print(ref_dd) + + +def test_transform_hash_eq(): + t1 = renderapi.transform.Transform( + **rendersettings.NONLINEAR_TRANSFORM_KWARGS) + t2 = renderapi.transform.Transform( + **rendersettings.NONLINEAR_TRANSFORM_KWARGS) + + assert t1 is not t2 + assert t1 == t2 + assert len({t1, t2}) == 1 + + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + am2 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + assert am1 is not am2 + assert am1 == am2 + assert len({am1, am2}) == 1 + + +def test_polynomial_transform_asorder_identity(): + srcpts = np.random.rand(50, 2) + pt1 = renderapi.transform.Polynomial2DTransform(identity=True) + pt2 = renderapi.transform.Polynomial2DTransform(identity=True).asorder(2) + + dstpts1 = pt1.tform(srcpts) + dstpts2 = pt2.tform(srcpts) + + assert pt1.order != pt2.order + assert np.allclose(dstpts1, dstpts2) + + +def test_transformsum_identity_polynomial(): + srcpts = np.random.rand(50, 2) + pt = renderapi.transform.Polynomial2DTransform(identity=True) + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + am2 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=-100, + B1=3) + tformlist = [pt, am1, am2] + new_tform = renderapi.transform.estimate_transformsum( + tformlist, src=srcpts, order=1) + + new_srcpts = np.random.rand(50, 2) + new_dstpts_comp = new_tform.tform(new_srcpts) + new_dstpts = am2.concatenate(am1).tform(new_srcpts) + assert np.allclose(new_dstpts_comp, new_dstpts) + + +def estimate_homography_transform( + do_scale=True, do_translate=True, do_rotate=True, transformclass=None): + scale = np.random.rand() + random_scale = (renderapi.transform.AffineModel( + M00=scale, M11=scale) + if do_scale else renderapi.transform.AffineModel()) + + random_translate = (renderapi.transform.AffineModel( + B0=np.random.rand(), B1=np.random.rand()) + if do_translate else renderapi.transform.AffineModel()) + + theta = np.random.rand() * 2 * np.pi + random_rotate = (renderapi.transform.AffineModel( + M00=np.cos(theta), M01=-np.sin(theta), + M10=np.sin(theta), M11=np.cos(theta)) + if do_rotate else renderapi.transform.AffineModel()) + + target_tform = random_translate.concatenate( + random_rotate.concatenate(random_scale)) + + src_pts = np.random.rand(50, 2) + dst_pts = target_tform.tform(src_pts) + tform = transformclass() + tform.estimate(src_pts, dst_pts, return_params=False) + M = tform.estimate(src_pts, dst_pts, return_params=True) + assert(M.shape == (3, 3)) + + assert np.allclose(target_tform.M, tform.M) + if do_scale: + assert np.isclose(tform.scale[0], scale) + if do_translate: + assert np.allclose(tform.translation, random_translate.translation) + if do_rotate: + assert np.isclose(tform.rotation, random_rotate.rotation) + # currently forces as affines + am = renderapi.transform.AffineModel(json=tform.to_dict()) + assert am.to_dict() == tform.to_dict() + + +def test_estimate_similarity_transform(): + estimate_homography_transform( + transformclass=renderapi.transform.SimilarityModel) + + +def test_estimate_rigid_transform(): + estimate_homography_transform( + do_scale=False, transformclass=renderapi.transform.RigidModel) + + +def test_estimate_translation_transform(): + estimate_homography_transform( + do_scale=False, do_rotate=False, + transformclass=renderapi.transform.TranslationModel) + + +def test_non_linear_transform(): + lens_tform = renderapi.transform.NonLinearTransform(dataString=( + "6 28 535.9261337198133 -0.07156495284083819 " + "1.5429761616475304 482.5391851962856 -1.3964665434772456 " + "-6.442573400985017 -2.9852709783360574 4.123111145658342 " + "-11.540309826482599 5.014980246618293 13.616409494909078 " + "6.8938038217739255 -2.580163142352532 -5.848702515778433 " + "11.520166623181623 -0.26414636893418164 17.725654626324285 " + "-3.6910106603259862 -5.747999298096374 6.850067634843171 " + "16.792611164648633 7.81339608279572 -1.6748576247746882 " + "7.294185157592906 -8.213479728551924 -5.181177837004834 " + "-2.363460931085683 2.9307897710181123 -19.60977878575318 " + "-5.626357722731385 -3.8857454141075323 -15.21120604177655 " + "-21.23967741658828 -7.433492911261084 5.986054967656173 " + "-0.2472470831304605 -4.252500955671167 6.285397956497022 " + "-10.087631670227893 -2.936382384586807 13.850132289055523 " + "-1.6230114467674603 -6.137660325940608 12.358797331540874 " + "15.361852639613971 -7.395509537102164 -3.4680250777444144 " + "11.543534617091353 2.1153010560743724 -6.1108581863128535 " + "2.4532196427825284 -1.6047644982741716 5.190795110418094 " + "0.40139303958456196 11.96302944378589 12.660577579061767 " + "1196.3041637890071 1266.0596612286872 1719846.5445062846 " + "1485783.6202875 1838252.7229766874 2.714668570418218E9 " + "2.106666890772152E9 2.1421214796751451E9 2.8349699962925887E9 " + "4.528468416208078E12 3.2949497103892886E12 3.0161488655484473E12 " + "3.2930355498080786E12 4.550261299826646E12 7.824367950003595E15 " + "5.464560367850104E15 4.686502506566912E15 4.616510470870998E15 " + "5.276444951800235E15 7.530553575860649E15 1.3844135530148071E19 " + "9.408014352057864E18 7.7275759384866017E18 7.1369460697416591E18 " + "7.373674200122966E18 8.7221483757320745E18 1.277368891891578E19 " + "100.0 537.4423453941299 485.2431449404924 1253541.1543424227 " + "899439.7598253534 1082433.2436210443 2.5451681382231455E9 " + "1.814150348079708E9 1.6692374367919033E9 2.1769137889095693E9 " + "4.992425732284977E12 3.5258872445513765E12 3.074020072484778E12 " + "3.1163128442286265E12 4.266956980967905E12 9.700243843041522E15 " + "6.781968704541316E15 5.745611297869626E15 5.443791159762913E15 " + "5.856309886603154E15 8.280136088463922E15 1.8821205899653665E19 " + "1.3042150458228838E19 1.0829624014456685E19 9.899910593314652E18 " + "9.885988268659214E18 1.1051253229808925E19 1.6002173610912907E19 " + "0.0 2048 2048 "), transformId="testing") + + ticks = np.arange(0, 2048, 64, np.float64) + xx, yy = np.meshgrid(ticks, ticks) + x = np.ravel(xx).T + y = np.ravel(yy).T + xy = np.vstack((x, y)).T + + xyp = lens_tform.tform(xy) + dv = xyp-xy + mean_disp = np.mean(np.sqrt(np.sum(dv**2, axis=1))) + assert((mean_disp-0.7570507) < .01) + + +def test_estimate_non_linear_transform(): + dataString = ( + "5 21 1134.6315 27.041916 49.84044 1069.9268 8.754167 " + "-8.966886 -67.81902 -26.337778 -42.44015 47.329388 " + "-57.457996 -19.826283 12.059551 16.236008 25.214308 " + "-6.14843 13.517356 -69.09145 59.978294 9.0963745 2.8096974 " + "17.50649 1.8950082 -16.095743 -6.6861115 5.4056945 1.330886 " + "67.93751 -21.105675 -1.9647278 -2.1779869 -8.596665 " + "0.25304613 0.07506427 -2.4152899 10.851463 5.036614 " + "-8.608638 -2.1753247 -22.031227 18.718458 20.106474 " + "1871.7964 2010.6288 4716919.0 3762185.0 5205234.5 " + "13463128000.0 950763500.0 9750733000.0 14992724000.0 " + "41204630000000.0 27199392000000.0 24678496000000.0 " + "28117224000000.0 45919358000000.0 1.3180739E+17 " + "8.3419896E+16 7.0674915E+16 7.122505E+16 8.621747E+16 " + "1.4640968E+17 100.0 1101.5017 1078.2463 4353785.0 3244154.8 " + "4338782.5 15909609000.0 11250277000.0 11111349000.0 " + "15978368000.0 57421630000000.0 39513703000000.0 " + "36268440000000.0 38755805000000.0 57893900000000.0 " + "2.0705793E+17 1.4025713E+17 1.2428827E+17 1.23013825E+17 " + "1.3686534E+17 2.0921157E+17 0.0 3840 3840 ") + + n = 500 + x = np.linspace(0, 3840, n) + xp, yp = np.meshgrid(x, x) + src = np.transpose(np.vstack((xp.flatten(), yp.flatten()))) + + tf = renderapi.transform.NonLinearCoordinateTransform( + dataString=dataString) + dst = tf.tform(src) + tf.estimate(src, dst) + fdst = tf.tform(src) + + assert(np.abs(fdst-dst).max() < 1e-5) + + +@pytest.mark.parametrize("transform_class,transform_json", [ + (renderapi.transform.Transform, { + 'className': 'some_class', + 'dataString': '1234_some_data', + 'id': 'myTransform', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}), + (renderapi.transform.AffineModel, { + 'className': 'mpicbg.trakem2.transform.AffineModel2D', + 'dataString': ( + "-0.6433267575 0.7655917209 -0.7655917209 " + "-0.6433267575 115071.0014383696 23073.7167961992"), + 'id': 'myAffineModel', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}), + (renderapi.transform.Polynomial2DTransform, { + 'className': 'mpicbg.trakem2.transform.PolynomialTransform2D', + 'dataString': ( + '67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06'), + 'id': 'myPolynomialModel', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}), + (renderapi.transform.NonLinearCoordinateTransform, + dict(rendersettings.NONLINEAR_TRANSFORM_KWARGS, **{ + 'id': 'myNLCTransform', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}))]) +def test_load_json_transforms(transform_class, transform_json): + tform = transform_class(json=transform_json) + tform_d = tform.to_dict() + + # avoid comparing datastrings as strings because of rounding + assert (tform.transformId == tform_d['id'] == transform_json['id']) + assert (tform.labels == tform_d['metaData']['labels'] == + transform_json['metaData']['labels']) + assert (tform.className == tform_d['className'] == + transform_json['className']) + + +@pytest.fixture(scope='module') +def referenced_tilespecs_and_transforms(): + with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS, 'r') as fp: + ds = json.load(fp) + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ds] + + with open(rendersettings.REFERENCE_TRANSFORM_SPECS, 'r') as fp: + ds = json.load(fp) + transforms = [renderapi.transform.load_transform_json(tf) for tf in ds] + return tilespecs, transforms + + +def test_estimate_dstpoints_reference(referenced_tilespecs_and_transforms): + tilespecs, transforms = referenced_tilespecs_and_transforms + + ticks = np.arange(0, 2048, 64, np.float64) + xx, yy = np.meshgrid(ticks, ticks) + x = np.ravel(xx).T + y = np.ravel(yy).T + xy = np.vstack((x, y)).T + + xyt = renderapi.transform.estimate_dstpts( + tilespecs[0].tforms, xy, transforms) + assert(xy.shape == xyt.shape) + with pytest.raises(renderapi.errors.RenderError): + renderapi.transform.estimate_dstpts(tilespecs[0].tforms, xy) + with pytest.raises(renderapi.errors.RenderError): + renderapi.transform.estimate_dstpts( + tilespecs[0].tforms, xy, [transforms[2]]) + + +def test_fail_convert_points(): + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float64) + with pytest.raises(renderapi.errors.ConversionError): + renderapi.transform.AffineModel.convert_to_point_vector(points_in.T) + + +def test_translation_transform_init(): + d = {'type': 'leaf', + 'className': 'mpicbg.trakem2.transform.TranslationModel2D', + 'dataString': '10 0'} + tform = renderapi.transform.load_transform_json(d) + assert(isinstance(tform, renderapi.transform.TranslationModel)) + assert(tform.M[0, 2] == 10) + + +def test_rigid_init(): + d = {'type': 'leaf', + 'className': 'mpicbg.trakem2.transform.RigidModel2D', + 'dataString': '{} 10 5'.format(np.pi/2)} + tform = renderapi.transform.load_transform_json(d) + assert(isinstance(tform, renderapi.transform.RigidModel)) + assert(tform.M[0, 2] == 10) + assert(tform.M[0, 0] < EPSILON) + + +def test_similarity_init(): + d = {'type': 'leaf', + 'className': 'mpicbg.trakem2.transform.SimilarityModel2D', + 'dataString': '2.0 0.0 10 5'} + tform = renderapi.transform.load_transform_json(d) + assert(isinstance(tform, renderapi.transform.SimilarityModel)) + assert(np.abs(tform.M[0, 0]-2) < EPSILON) + assert(tform.M[0, 2] == 10) + + +def test_affine_properties(): + # scale + sx = 1.5 + sy = 1.1 + # shear + cx = 0.0 + cy = 0.0 + # angle + theta = 0.0 + pt = np.array( + [1.234, 4.567, 1]) + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + rnd_tf.force_shear = 'y' + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + theta = 0.1234 + pt = np.array( + [1.234, 4.567, 1]) + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + rnd_tf.force_shear = 'y' + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + cx = 0.2 + cy = 0.0 + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + rnd_tf.force_shear = 'y' + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + + cx = 0.0 + cy = 0.1 + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + + rnd_tf.force_shear = 'y' + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cy) < 1e-8) + + cx = 0.3 + cy = 0.1 + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + rnd_tf.force_shear = 'y' + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.shear - cy) < 1e-8) + + +def affine_property_function(sx, sy, cx, cy, theta, pt): + scale = np.array([ + [sx, 0, 0], + [0, sy, 0], + [0, 0, 1]]) + shear = np.array([ + [1, cx, 0], + [cy, 1, 0], + [0, 0, 1]]) + rotation = np.array([ + [np.cos(theta), -np.sin(theta), 0], + [np.sin(theta), np.cos(theta), 0], + [0, 0, 1]]) + M = scale.dot(shear.dot(rotation)) + + testpt = M.dot(pt)[0:2] + + rnd_tf = renderapi.transform.SimilarityModel( + M00=M[0, 0], + M01=M[0, 1], + M10=M[1, 0], + M11=M[1, 1]) + return M, testpt, rnd_tf + + +def test_thinplatespline_inverse(): + j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + assert (j['dataString'].split(' ')[-1] == t.dataString.split(' ')[-1]) + x = np.linspace(0, 3840, 10) + xt, yt = np.meshgrid(x, x) + src_pts = np.transpose( + np.vstack((xt.flatten(), yt.flatten()))) + src_inv_est = t.inverse_tform(t.tform(src_pts)) + assert (src_inv_est.shape == src_pts.shape) + for i in range(src_pts.shape[0]): + assert(np.linalg.norm(src_pts[i, :] - src_inv_est[i, :]) < 0.1) + with pytest.raises(renderapi.errors.EstimationError): + src_inv_est = t.inverse_tform( + t.tform(src_pts), + max_iters=5) + + +def test_thinplatespline(): + j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + assert (j['dataString'].split(' ')[-1] == t.dataString.split(' ')[-1]) + t.aMtx = np.zeros(4) + t.bVec = np.zeros(2) + t2 = renderapi.transform.ThinPlateSplineTransform( + dataString=t.dataString) + assert (t == t2) + # test incorrect lengths + with pytest.raises(renderapi.errors.RenderError): + s = t2.dataString.split(' ') + s[1] = str(int(s[1])+1) + renderapi.transform.ThinPlateSplineTransform( + dataString=" ".join(s)) + with pytest.raises(renderapi.errors.RenderError): + s = t2.dataString.split(' ') + s[2] = str(int(s[2])-4) + renderapi.transform.ThinPlateSplineTransform( + dataString=" ".join(s)) + + x = np.linspace(0, 3840, 10) + xt, yt = np.meshgrid(x, x) + src_pts = np.transpose( + np.vstack((xt.flatten(), yt.flatten()))) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + dst_pts = t.tform(src_pts) + assert(dst_pts.shape == src_pts.shape) + + # check load from json + jt = t.to_dict() + nt = renderapi.transform.ThinPlateSplineTransform( + json=jt) + assert nt == t + + +def test_thinplatespline_apply(): + # tests some copied behavior from trakem2 + j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + del t.dMtxDat + pt = np.array([1.234, 45.678]) + npt = t.apply(pt) + assert np.all(pt == npt) + + +def estimate_test(jpath, computeAffine=True): + # test that the estimate method can produce same results + with open(jpath, 'r') as f: + j = json.load(f) + + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + # exact points + src1 = np.transpose(t.srcPts) + # some in-between points + x = np.linspace( + src1[:, 0].min(), + src1[:, 0].max(), + int(np.sqrt(t.nLm)) * 3) + y = np.linspace( + src1[:, 1].min(), + src1[:, 1].max(), + int(np.sqrt(t.nLm)) * 3) + xt, yt = np.meshgrid(x, y) + src2 = np.transpose(np.vstack((xt.flatten(), yt.flatten()))) + dst1_a = t.tform(src1) + dst2_a = t.tform(src2) + + # estimate + t.estimate(src1, dst1_a, computeAffine=computeAffine) + dst1_b = t.tform(src1) + dst2_b = t.tform(src2) + delta1 = np.linalg.norm(dst1_b - dst1_a, axis=1) + delta2 = np.linalg.norm(dst2_b - dst2_a, axis=1) + + assert delta1.max() < EPSILON2 + assert delta2.max() < EPSILON2 + + with pytest.raises(renderapi.errors.EstimationError): + t.estimate(src1, dst1_a[1:, :]) + + +def test_thinplatespline_estimate(): + estimate_test( + rendersettings.TEST_THINPLATESPLINE_FILE, + computeAffine=False) + estimate_test( + rendersettings.TEST_THINPLATESPLINEAFFINE_FILE, + computeAffine=True) + thinplate_estimate_nojson(computeAffine=True) + thinplate_estimate_nojson(computeAffine=False) + + +def thinplate_estimate_nojson(computeAffine=True): + # an estimate test that does not depend on pre-computed json + x = np.linspace(0, 1000, 40) + xt, yt = np.meshgrid(x, x) + src = np.transpose(np.vstack((xt.flatten(), yt.flatten()))) + + def poly2d(src): + dst = np.zeros_like(src) + dx = (src[:, 0] - src[:, 0].mean()) / (np.ptp(src[:, 0])) + dy = (src[:, 1] - src[:, 1].mean()) / (np.ptp(src[:, 1])) + dst[:, 0] = src[:, 0] + 0.5 * dx * dy + 7.0 * dx * dy * dy + dst[:, 1] = src[:, 1] + 0.7 * dy * dy - 4.0 * dx * dx * dy + a = np.array([[1.1, -0.04], [-0.02, 0.95]]) + b = np.array([3.0, 4.0]) + dst = np.transpose(a.dot(np.transpose(dst))) + b + return dst + + dst = poly2d(src) + t = renderapi.transform.ThinPlateSplineTransform() + t.estimate(src, dst, computeAffine=computeAffine) + + x = np.linspace(0, 1000, 120) + xt, yt = np.meshgrid(x, x) + test_src = np.transpose(np.vstack((xt.flatten(), yt.flatten()))) + p_dst = poly2d(test_src) + t_dst = t.tform(test_src) + + delta = np.linalg.norm(p_dst - t_dst, axis=1) + # can it match the polynomial within a pixel? + # for low-N, it won't + assert delta.max() < 1.0 + + +def test_encode64(): + # case for Stephan's '@' character + s = '@QAkh+fAbhm6/8AAAAAAAAA==' + x = renderapi.utils.decodeBase64(s) + assert(x.size == 2) + assert(x[0] == 3.14159) + assert(x[1] == -1.0) + # all other + x = np.array([1.0, 3.0, 3.14159, -4.0, 10.0]) + s = renderapi.utils.encodeBase64(x) + y = renderapi.utils.decodeBase64(s) + assert(np.all(x == y)) + + +def test_adaptive_estimate(): + with open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r') as f: + j = json.load(f) + + tf = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + + tol = 1.0 + ntf = tf.adaptive_mesh_estimate(tol=1.0) + + src = ntf.srcPts.transpose() + dsta = tf.tform(src) + dstb = ntf.tform(src) + assert(np.linalg.norm(dsta - dstb, axis=1).max() <= tol) + + src = tf.srcPts.transpose() + dsta = tf.tform(src) + dstb = ntf.tform(src) + nover = np.argwhere(np.linalg.norm(dsta - dstb, axis=1) >= tol).size + assert(nover == 0) + + with pytest.raises(renderapi.errors.EstimationError): + tf.adaptive_mesh_estimate(max_iter=1) + + # invoke the recursion directly, without passing self + mn = tf.srcPts.min(axis=1) + mx = tf.srcPts.max(axis=1) + xt, yt = np.meshgrid( + np.linspace(mn[0], mx[0], 5), + np.linspace(mn[1], mx[1], 5)) + new_src = np.vstack((xt.flatten(), yt.flatten())).transpose() + old_src = tf.srcPts.transpose() + old_dst = tf.tform(old_src) + ntf = tf.mesh_refine( + new_src, + old_src, + old_dst, + old_tf=None, + computeAffine=True, + tol=1.0, + max_iter=50, + nworst=10, + niter=0) + dsta = tf.tform(src) + dstb = ntf.tform(src) + assert(np.linalg.norm(dsta - dstb, axis=1).max() <= tol) + + +def test_polynomial_shear(): + # make sure it gives the same answer as affine + M = np.array([ + [1.1, -0.2, 7.0], + [0.3, 0.8, -9.0], + [0.0, 0.0, 1.0]]) + + atf = renderapi.transform.AffineModel() + atf.M = M + + params = np.zeros((2, 3)) + params[:, 1:] = M[0:2, 0:2] + params[:, 0] = M[0:2, 2] + + ptf = renderapi.transform.Polynomial2DTransform(params=params) + + for shear in ['x', 'y']: + atf.force_shear = shear + ptf.force_shear = shear + + assert atf.scale == ptf.scale + assert atf.shear == ptf.shear + assert atf.rotation == ptf.rotation + assert atf.translation == ptf.translation + + +@pytest.mark.parametrize('ngrid', [5, 20]) +@pytest.mark.parametrize('preserve_srcPts', [True, False]) +@pytest.mark.parametrize('computeAffine', [True, False]) +@pytest.mark.parametrize('factor', [1e-3, 1e3, 1e5]) +def test_scale_thinplate(factor, computeAffine, preserve_srcPts, ngrid): + # use case is to scale a rough alignment thinplate spline + with open(rendersettings.TEST_THINPLATEROUGH_FILE, 'r') as f: + tform = renderapi.transform.load_transform_json( + json.load(f)) + + if not computeAffine: + tform.aMtx = None + tform.bVec = None + + mn = tform.srcPts.min(axis=0) + mx = tform.srcPts.max(axis=0) + + npts = 1000 + src = np.random.rand(npts, 2) + src[:, 0] = src[:, 0] * (mx[0] - mn[0]) + mn[0] + src[:, 1] = src[:, 1] * (mx[1] - mn[1]) + mn[1] + dst = tform.tform(src) + + scaled_tform = tform.scale_coordinates( + factor, ngrid=ngrid, preserve_srcPts=preserve_srcPts) + dst_scaled = scaled_tform.tform(src * factor) + + delta = np.linalg.norm(dst_scaled - dst * factor, axis=1) + scale = np.ptp(dst_scaled, axis=0).mean() + tol = 1e-4 + if ngrid == 5: + tol = 1e-3 + assert (delta.max() / scale) < tol + + if preserve_srcPts: + dist = cdist( + tform.srcPts.transpose() * factor, + scaled_tform.srcPts.transpose(), + metric='euclidean') + # check that the original srcPts have a close neighbor still + # in the scaled srcPts + assert np.all(np.any(dist < 1e-3, axis=1)) diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 00000000..a2500e62 --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,56 @@ +import importlib +import json +import renderapi +import pytest +import numpy as np +import ujson + + +def cross_py23_reload(module): + try: + reload(module) + except NameError: + importlib.reload(module) + + +@pytest.mark.parametrize("use_ujson", [True, False]) +def test_json_load(use_ujson): + if not use_ujson: + try: + import builtins + except ImportError: + import __builtin__ as builtins + realimport = builtins.__import__ + + def noujson_import(name, globals=None, locals=None, + fromlist=(), level=0): + if 'ujson' in name: + raise ImportError + return realimport(name, globals, locals, fromlist, level) + builtins.__import__ = noujson_import + cross_py23_reload(renderapi.utils) + assert (renderapi.utils.requests_json is ujson + if use_ujson else renderapi.utils.requests_json is json) + assert ( + renderapi.utils.requests.models.complexjson is ujson + if use_ujson else renderapi.utils.requests.models.complexjson is json) + + +def test_jbool(): + assert(renderapi.utils.jbool(True) == 'true') + assert(renderapi.utils.jbool(False) == 'false') + assert(renderapi.utils.jbool(0) == 'false') + assert(renderapi.utils.jbool(1) == 'true') + + +def test_renderdumps_simple(): + s = renderapi.utils.renderdumps({'a': 1}) + assert(s == '{"a": 1}') + + s = renderapi.utils.renderdumps(5) + assert(s == '5') + + +def test_renderdumps_fails(): + with pytest.raises(AttributeError): + renderapi.utils.renderdumps(np.zeros(3)) diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..74e88cc7 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,11 @@ +coverage>=4.1 +mock>=2.0.0 +pep8>=1.7.0 +pytest>4.6,<5.0 +pytest-cov>=2.2.1 +pytest-pep8>=1.0.6 +pytest-xdist>=1.14 +flake8>=3.0.4 +pylint>=1.5.4 +ujson +jinja2 diff --git a/tilespec.py b/tilespec.py deleted file mode 100644 index 3ae5967a..00000000 --- a/tilespec.py +++ /dev/null @@ -1,410 +0,0 @@ -import numpy as np - -class ResolvedTileSpecMap(): - def __init__(self,tilespecs=[],transforms=[]): - - self.tilespecs = tilespecs - self.transforms = transforms - - def to_dict(self): - d={} - d['tileIdToSpecMap']={} - for ts in self.tilespecs: - d['tileIdToSpecMap'][ts.tileId]=ts.to_dict() - d['transformIdToSpecMap']={} - for tf in self.transforms: - d['transformIdToSpecMap'][tf.transformId]=tf.to_dict() - return d - - def from_dict(self,d): - tsmap = d['tileIdToSpecMap'] - tfmap = d['transformIdToSpecMap'] - for tsd in tsmap.values(): - ts = TileSpec() - ts.from_dict(tsd) - self.tilespecs.append(ts) - for tfd in tfmap.values(): - tf.Transform() - tf.from_dict(tfd) - self.transforms.append(tf) - - -class ResolvedTileSpecCollection(): - def __init__(self,tilespecs=[],transforms=[]): - - self.tilespecs = tilespecs - self.transforms = transforms - def to_dict(self): - d ={} - d['tileCount']=len(self.tilespecs) - d['tileSpecs']=[ts.to_dict() for ts in self.tilespecs] - d['transformCount']=len(self.transforms) - d['transformSpecs']=[tf.to_dict() for tf in self.transforms] - return d - - def from_dict(self,d): - self.tilespecs = [] - self.transforms = [] - for i in range(d['tileCount']): - ts = TileSpec() - ts.from_dict(d['tileSpecs'][i]) - self.tilespecs.append(ts) - for i in range(d['tranformCount']): - tfd = d['transformSpecs'][i] - if tfd['className'] is 'mpicbg.trakem2.transform.AffineModel2D': - tf = AffineModel() - else: - tf = Transform() - tf.from_dict(tfd) - self.transforms.append(tf) - -class StackVersion(): - def __init__(self,cycleNumber=1,cycleStepNumber=1,stackResolutionX=1, - stackResolutionY=1,stackResolutionZ=1,materializedBoxRootPath=None, - versionNotes="",createTimestamp=None): - import time - - self.cycleNumber=cycleNumber - self.cycleStepNumber=cycleStepNumber - self.stackResolutionX=stackResolutionX - self.stackResolutionY=stackResolutionY - self.stackResolutionZ=stackResolutionZ - self.materializedBoxRootPath=materializedBoxRootPath - if createTimestamp is None: - createTimestamp=time.strftime('%Y-%M-%dT%H:%M:%S.00Z') - self.createTimestamp = createTimestamp - self.versionNotes = versionNotes - def to_dict(self): - d = {} - d['cycleNumber']=self.cycleNumber - d['cycleStepNumber']=self.cycleStepNumber - d['stackResolutionX']=self.stackResolutionX - d['stackResolutionY']=self.stackResolutionY - d['stackResolutionZ']=self.stackResolutionZ - d['createTimestamp']=self.createTimestamp - d["materializedBoxRootPath"]= "string" - d['mipmapPathBuilder']={'numberOfLevels':0} - d['versionNotes']=self.versionNotes - return d - def from_dict(self,d): - for key in d.keys(): - eval('self.%s=d[%s]'%(key,key)) - -class Filter(): - def __init__(self,classname,params={}): - self.classname = classname - self.params = params - - def to_dict(self): - d = {} - d['className']=self.classname - d['params'] = self.params - return d - - def from_dict(self,d): - self.classname = d['className'] - self.params = d['params'] - -class ReferenceTransform(): - def __init__(self,refId=None,json=None): - if json is not None: - self.from_dict(json) - else: - self.refId = refId - def to_dict(self): - d={} - d['type']='ref' - d['refId']=self.refId - return d - def from_dict(self,d): - self.refId = d['refId'] - - def __str__(self): - return 'ReferenceTransform(%s)'%self.refId - def __repr__(self): - return self.__str__() - -class Transform(): - def __init__(self,className=None,dataString=None,transformId=None,json=None): - - if json is not None: - self.from_dict(json) - else: - self.className = className - self.dataString = dataString - self.transformId = transformId - def to_dict(self): - d ={} - d['type']='leaf' - d['className']=self.className - d['dataString']=self.dataString - if self.transformId is not None: - d['transformId']=self.transformId - return d - def from_dict(self,d): - self.dataString = d['dataString'] - self.className = d['className'] - self.transformId = d.get('transformId',None) - - def __str__(self): - return 'className:%s\ndataString:%s'%(self.className,self.dataString) - def __repr__(self): - return self.__str__() - - -class AffineModel(Transform): - className='mpicbg.trakem2.transform.AffineModel2D' - def __init__(self,M00=1.0,M01=0.0,M10=0.0,M11=1.0,B0=0.0,B1=0.0): - self.M00= M00 - self.M01= M01 - self.M10 = M10 - self.M11 = M11 - self.B0 = B0 - self.B1 = B1 - self.className = 'mpicbg.trakem2.transform.AffineModel2D' - self.load_M() - - def load_M(self): - self.M = np.identity(3,np.double) - self.M[0,0]=self.M00 - self.M[0,1]=self.M01 - self.M[1,0]=self.M10 - self.M[1,1]=self.M11 - self.M[0,2]=self.B0 - self.M[1,2]=self.B1 - - def to_dict(self): - d = {} - d['type']='leaf' - d['className']=self.className - d['dataString']="%.10f %.10f %.10f %.10f %.10f %.10f"%(self.M[0,0],self.M[1,0],self.M[0,1],self.M[1,1],self.M[0,2],self.M[1,2]) - return d - - def from_dict(self,d): - ds = d['dataString'].split() - (self.M00,self.M10,self.M01,self.M11,self.B0,self.B1)=map(float,ds) - self.load_M() - - def invert(self): - Ai = AffineModel() - Ai.M=np.linalg.inv(self.M) - return Ai - - def convert_to_point_vector(self,points): - Np = points.shape[0] - - zerovec = np.zeros((Np,1),np.double) - onevec = np.ones((Np,1),np.double) - assert(points.shape[1]==2) - Nd = 2 - points=np.concatenate((points,onevec),axis=1) - return points,Nd - - def convert_points_vector_to_array(self,points,Nd): - points=points[:,0:Nd]/np.tile(points[:,2],(Nd,1)).T - return points - - def tform(self,points): - points,Nd = self.convert_to_point_vector(points) - pt=np.dot(self.M,points.T).T - return self.convert_points_vector_to_array(pt,Nd) - - def concatenate(self,model): - #final double a00 = m00 * model.m00 + m01 * model.m10; - #final double a01 = m00 * model.m01 + m01 * model.m11; - #final double a02 = m00 * model.m02 + m01 * model.m12 + m02; - - #final double a10 = m10 * model.m00 + m11 * model.m10; - #final double a11 = m10 * model.m01 + m11 * model.m11; - #final double a12 = m10 * model.m02 + m11 * model.m12 + m12; - a00 = self.M[0,0] * model.M[0,0] + self.M[0,1] * model.M[1,0] - a01 = self.M[0,0] * model.M[0,1] + self.M[0,1] * model.M[1,1] - a02 = self.M[0,0] * model.M[0,2] + self.M[0,1] * model.M[1,2] + self.M[0,2]; - - a10 = self.M[1,0] * model.M[0,0] + self.M[1,1] * model.M[1,0] - a11 = self.M[1,0] * model.M[0,1] + self.M[1,1] * model.M[1,1] - a12 = self.M[1,0] * model.M[0,2] + self.M[1,1] * model.M[1,2] + self.M[1,2]; - - newmodel = AffineModel(a00,a01,a10,a11,a02,a12) - return newmodel - - - - def inverse_tform(self,points): - points,Nd = self.convert_to_point_vector(points) - pt = np.dot(np.linalg.inv(self.M),points.T).T - return self.convert_points_vector_to_array(pt,Nd) - - def __str__(self): - return "M=[[%f,%f],[%f,%f]] B=[%f,%f]"%(self.M[0,0],self.M[0,1],self.M[1,0],self.M[1,1],self.M[0,2],self.M[1,2]) - - - -class Layout(): - def __init__(self,sectionId=None,scopeId=None,cameraId=None,imageRow=None, - imageCol=None,stageX=None,stageY=None,rotation=None,pixelsize=0.100): - self.sectionId = str(sectionId) - self.scopeId = str(scopeId) - self.cameraId = str(cameraId) - self.imageRow = imageRow - self.imageCol = imageCol - self.stageX = stageX - self.stageY = stageY - self.rotation = rotation - self.pixelsize = pixelsize - - def to_dict(self): - d = {} - d['sectionId'] = self.sectionId - d['temca'] = self.scopeId - d['camera'] = self.cameraId - d['imageRow'] = self.imageRow - d['imageCol'] = self.imageCol - d['stageX'] = self.stageX - d['stageY'] = self.stageY - d['rotation'] = self.rotation - d['pixelsize'] = self.pixelsize - return d - def from_dict(self,d): - if d is None: - d={} - - self.sectionId = d.get('sectionId',None) - self.cameraId = d.get('camera',None) - self.scopeId = d.get('temca',None) - self.imageRow = d.get('imageRow',None) - self.imageCol = d.get('imageCol',None) - self.stageX = d.get('stageX',None) - self.stageY = d.get('stageY',None) - self.rotation = d.get('rotation',None) - self.pixelsize = d.get('pixelsize',None) - - -class TileSpec(): - - def __init__(self, - tileId=None,z=None,width=None,height=None,imageUrl=None,frameId=None, - maskUrl=None,minint=0,maxint=65000,layout=Layout(), - tforms = [],inputfilters=[],scale3Url=None,scale2Url=None,scale1Url=None,json=None): - - if json is not None: - self.from_dict(json) - else: - self.tileId = tileId - self.z = z - self.width = width - self.height = height - self.layout = layout - self.imageUrl = imageUrl - self.maskUrl = maskUrl - self.minint= minint - self.maxint = maxint - self.tforms = tforms - self.frameId = frameId - self.layout=layout - self.inputfilters = inputfilters - self.scale3Url = scale3Url - self.scale2Url = scale2Url - self.scale1Url = scale1Url - - - def to_dict(self): - thedict={} - thedict['tileId']=self.tileId - thedict['z']=self.z - thedict['width']=self.width - thedict['height']=self.height - thedict['minIntensity']=self.minint - thedict['maxIntensity']=self.maxint - thedict['frameId']=self.frameId - if self.layout is not None: - thedict['layout']=self.layout.to_dict() - mipmapdict={} - mipmapdict['0']={} - mipmapdict['0']['imageUrl']=self.imageUrl - if self.scale1Url is not None: - mipmapdict['1']={} - mipmapdict['1']['imageUrl']=self.scale1Url - if self.scale3Url is not None: - mipmapdict['3']={} - mipmapdict['3']['imageUrl']=self.scale3Url - if self.scale2Url is not None: - mipmapdict['2']={} - mipmapdict['2']['imageUrl']=self.scale2Url - if self.maskUrl is not None: - mipmapdict['0']['maskUrl']=self.maskUrl - thedict['mipmapLevels']=mipmapdict - thedict['transforms']={} - thedict['transforms']['type']='list' - #thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] - thedict['transforms']['specList'] = [] - for t in self.tforms: - strlist = {} - #added by sharmi - if your speclist contains a speclist (can happen if you run the optimization more than once) - if isinstance(t,list): - strlist['type'] = 'list' - strlist['specList'] = [tt.to_dict() for tt in t] - thedict['transforms']['specList'].append(strlist) - else: - thedict['transforms']['specList'].append(t.to_dict()) - ############################################################################################################ - - - thedict['inputfilters']={} - thedict['inputfilters']['type']='list' - thedict['inputfilters']['specList']=[f.to_dict() for f in self.inputfilters] - return thedict - - def from_dict(self,d): - '''Method to load tilespec from json dictionary''' - self.tileId = d['tileId'] - self.z = d['z'] - self.width = d['width'] - self.height = d['height'] - self.minint = d['minIntensity'] - self.maxint = d['maxIntensity'] - self.frameId = d.get('frameId',None) - self.layout = Layout() - self.layout.from_dict(d.get('layout',None)) - self.minX = d.get('minX',None) - self.maxX = d.get('maxX',None) - self.maxY = d.get('maxY',None) - self.minY = d.get('minY',None) - self.imageUrl = d['mipmapLevels']['0']['imageUrl'] - self.maskUrl = d['mipmapLevels']['0'].get('maskUrl',None) - if d['mipmapLevels'].get('2',None) is not None: - self.scale2Url = d['mipmapLevels']['2'].get('imageUrl',None) - else: - self.scale2Url = None - if d['mipmapLevels'].get('1',None) is not None: - self.scale1Url = d['mipmapLevels']['1'].get('imageUrl',None) - else: - self.scale1Url = None - if d['mipmapLevels'].get('3',None) is not None: - self.scale3Url = d['mipmapLevels']['3'].get('imageUrl',None) - else: - self.scale3Url = None - - self.tforms = [] - for t in d['transforms']['specList']: - if t['type']=='ref': - tf = ReferenceTransform(refId=t['refId']) - elif t['type']=='leaf': - if t['className']==AffineModel.className: - tf = AffineModel() - tf.from_dict(t) - else: - tf = Transform(json=t) - self.tforms.append(tf) - self.inputfilters = [] - if d.get('inputfilters',None) is not None: - for f in d['inputfilters']['specList']: - f['type'] - f = Filter() - f.from_dict(f) - self.inputfilters.append(f) - - - - - diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..143a2850 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +# Tox (https://tox.readthedocs.io/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + + +[tox] +envlist = py27, py36 + +[testenv] +commands = pytest +deps = -rrequirements.txt + -rtest_requirements.txt +passenv = RENDER_HOST RENDER_PORT RENDER_EXAMPLE_DATA RENDER_CLIENT_SCRIPTS RENDER_PYTHON_TEST_POOL_SIZE RENDER_JAVA_HOME