From e80e84076b3997ad264355fc899738c8203ed5da Mon Sep 17 00:00:00 2001 From: Russel Date: Thu, 12 Jan 2017 17:15:54 -0800 Subject: [PATCH 001/766] renderapi: remove pathos requirement, enable env variable defaults --- renderapi.py | 168 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 60 deletions(-) diff --git a/renderapi.py b/renderapi.py index 8a361f0e..4d83e421 100755 --- a/renderapi.py +++ b/renderapi.py @@ -1,26 +1,26 @@ -import tempfile +import logging import os import json import subprocess import sys -import requests -import numpy as np -from tilespec import TileSpec,StackVersion -this = sys.modules[__name__] +from functools import partial +import tempfile from io import BytesIO +import time +import requests import numpy as np from PIL import Image -import pathos.multiprocessing as mp -import time -from functools import partial +from tilespec import TileSpec,StackVersion +# import pathos.multiprocessing as mp +try: + from pathos.multiprocessing import ProcessingPool as Pool + has_pathos = True +except ImportError as e: + logging.warning(e) + has_pathos = False + from multiprocessing import Pool -# DEFAULT_HOST = "renderer.int.janelia.org" -this.DEFAULT_HOST = "ibs-forrestc-ux1.corp.alleninstitute.org" -this.DEFAULT_PORT = 8080 -this.DEFAULT_OWNER = "Forrest" -this.DEFAULT_PROJECT = "M246930_Scnn1a_4" -this.DEFAULT_CLIENT_SCRIPTS = "/pipeline/render/render-ws-java-client/src/main/scripts" # GET http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/world-to-local-coordinates/{x},{y} # curl "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates/40000,40000" @@ -38,18 +38,71 @@ # ] class Render(object): - DEFAULT_HOST = "ibs-forrestc-ux1.corp.alleninstitute.org" - DEFAULT_PORT = 8080 - DEFAULT_OWNER = "Forrest" - DEFAULT_PROJECT = "M246930_Scnn1a_4" - DEFAULT_CLIENT_SCRIPTS = "/pipeline/render/render-ws-java-client/src/main/scripts" - - def __init__(self,default_host,default_port,default_owner,default_project,default_client_scripts=this.DEFAULT_CLIENT_SCRIPTS): - self.DEFAULT_HOST = default_host - self.DEFAULT_PORT = default_port - self.DEFAULT_PROJECT = default_project - self.DEFAULT_OWNER = default_owner - self.DEFAULT_CLIENT_SCRIPTS = default_client_scripts + def __init__(self, host=None, port=None, owner=None, project=None, + client_scripts=None): + + # FIXME: combine port and host into server + # FIXME port handling might have weird casting requirements + # TODO maybe pull this out to a separate renderapi.connect() function? + if host is None: + if 'RENDER_HOST' not in os.environ: + host = str(raw_input("Enter Render Host: ")) + if host == '': + logging.critical('Render Host must not be empty!') + raise ValueError('Render Host must not be empty!') + host = (host if host.startswith('http') + else 'http://{}'.format(host)) + else: + host = os.environ['RENDER_HOST'] + self.DEFAULT_HOST = host + + if port is None: + if 'RENDER_PORT' not in os.environ: + port = str(int(raw_input("Enter Render Port: "))) + if port == '': + # TODO better (no) port handling + logging.critical('Render Port must not be empty!') + raise ValueError('Render Port must not be empty!') + else: + port = str(int(os.environ['RENDER_PORT'])) + self.DEFAULT_PORT = port + + if project is None: + if 'RENDER_PROJECT' not in os.environ: + project = str(raw_input("Enter Render Project: ")) + if project == '': + logging.critical('Render Project must not be empty!') + raise ValueError('Render Project must not be empty!') + else: + project = str(os.environ['RENDER_PROJECT']) + self.DEFAULT_PROJECT = project + + if owner is None: + if 'RENDER_OWNER' not in os.environ: + owner = str(raw_input("Enter Render Owner: ")) + if owner == '': + logging.critical('Render Owner must not be empty!') + raise ValueError('Render Owner must not be empty!') + else: + owner = str(os.environ['RENDER_OWNER']) + self.DEFAULT_OWNER = owner + + if client_scripts is None: + if 'RENDER_CLIENT_SCRIPTS' not in os.environ: + client_scripts = str(raw_input( + "Enter Render Client Scripts location: ")) + if client_scripts == '': + logging.critical('Render Client Scripts must not be empty!') + raise ValueError('Render Client Scripts must not be empty!') + else: + client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) + self.DEFAULT_CLIENT_SCRIPTS = client_scripts + + logging.debug('Render object created with ' + 'host={h}, port={p}, project={pr}, ' + 'owner={o}, scripts={s}'.format( + h=self.DEFAULT_HOST, p=self.DEFAULT_PORT, pr=self.DEFAULT_PROJECT, + o=self.DEFAULT_OWNER, s=self.DEFAULT_CLIENT_SCRIPTS)) def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): #def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): @@ -82,7 +135,7 @@ def delete_stack(self,stack,host=None,port=None,owner=None,project=None,session= (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) request_url = self.format_preamble(host,port,owner,project,stack) r=session.delete(request_url) - print r.text + print r.text return r def create_stack(self,stack,cycleNumber=1,cycleStepNumber=1, @@ -115,10 +168,10 @@ def create_stack(self,stack,cycleNumber=1,cycleStepNumber=1, # proc.wait() # if verbose: # print proc.stdout.read() - + def import_single_json_file(self,stack,jsonfile,transformFile=None, client_scripts=DEFAULT_CLIENT_SCRIPTS,host=None,port=None,owner=None,project=None,verbose=False): - + (host,port,owner,project,client_scripts)=\ self.process_defaults(host,port,owner,project,client_scripts) @@ -147,12 +200,11 @@ def import_jsonfiles_and_transforms_parallel_by_z(self,stack,jsonfiles,transform self.process_defaults(host,port,owner,project,client_scripts) self.set_stack_state(stack,'LOADING',host,port,owner,project) + pool = Pool(poolsize) - pool = mp.ProcessingPool(poolsize) - partial_import = partial(self.import_single_json_file,stack, client_scripts=client_scripts,host=host,port=port,owner=owner,project=project,verbose=verbose) - + rs = pool.amap(partial_import, jsonfiles,transformfiles) rs.wait() @@ -161,14 +213,13 @@ def import_jsonfiles_and_transforms_parallel_by_z(self,stack,jsonfiles,transform def import_jsonfiles_parallel(self,stack,jsonfiles,poolsize=20,transformFile=None,client_scripts=DEFAULT_CLIENT_SCRIPTS, host=None,port=None,owner=None,project=None,close_stack=True,verbose=False): - + (host,port,owner,project,client_scripts)=\ self.process_defaults(host,port,owner,project,client_scripts) self.set_stack_state(stack,'LOADING',host,port,owner,project) + pool = Pool(poolsize) - pool = mp.ProcessingPool(poolsize) - partial_import = partial(self.import_single_json_file,stack,transformFile=transformFile, client_scripts=client_scripts,host=host,port=port,owner=owner,project=project,verbose=verbose) partial_import(jsonfiles[0]) @@ -186,7 +237,7 @@ def import_jsonfiles(self,stack,jsonfiles,transformFile = None, self.process_defaults(host,port,owner,project,client_scripts) self.set_stack_state(stack,'LOADING',host,port,owner,project) - + if transformFile is None: transform_params = [] else: @@ -206,8 +257,8 @@ def import_jsonfiles(self,stack,jsonfiles,transformFile = None, print proc.stdout.read() if close_stack: self.set_stack_state(stack,'COMPLETE',host,port,owner,project) - - + + def world_to_local_coordinates(self,stack, z, x, y, host = None, port = None, owner = None, project = None, session=requests.session()): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) @@ -271,7 +322,7 @@ def get_stack_metadata_by_owner(self,owner=None,host=None,port=None,session=requ # PUT http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/local-to-world-coordinates # with request body containing JSON array of local coordinate elements - # curl -H "Content-Type: application/json" -X PUT --data @coordinate-local.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/local-to-world-coordinates" + # curl -H "Content-Type: application/json" -X PUT --data @coordinate-local.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/local-to-world-coordinates" # [ # { # "tileId": "140422184419063136", @@ -291,7 +342,7 @@ def world_to_local_coordinates_batch(self,stack, z, data, host = None, port = No #print r.text return r.json() - # curl -H "Content-Type: application/json" -X PUT --data @coordinate-world.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates" + # curl -H "Content-Type: application/json" -X PUT --data @coordinate-world.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates" # [ # [ # { @@ -317,7 +368,7 @@ def get_z_values_for_stack(self,stack,project = None, except: print(r.text) return None - + def get_z_value_for_section(self,stack,sectionId,project = None, host = None,port = None,owner = None,session=requests.session()): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) @@ -328,7 +379,7 @@ def get_z_value_for_section(self,stack,sectionId,project = None, return r.json() except: print(r.text) - return None + return None def get_tile_image_data(self,stack,tileId,host=None,port=None,owner=None,project=None,session=requests.session(),verbose=False): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/png-image" % (tileId) @@ -340,7 +391,7 @@ def get_tile_image_data(self,stack,tileId,host=None,port=None,owner=None,project array = np.asarray(img) return array except: - print (r.text) + print (r.text) return None def world_to_local_coordinates_array(self,stack, dataarray, tileId, z=0,host = None, port = None, owner = None, project = None, session=requests.session()): @@ -353,7 +404,7 @@ def world_to_local_coordinates_array(self,stack, dataarray, tileId, z=0,host = N d['world']=[dataarray[i,0],dataarray[i,1]] dlist.append(d) jsondata=json.dumps(dlist) - + r = session.put(request_url, data=jsondata, headers={"content-type":"application/json"}) json_answer = r.json() @@ -371,7 +422,7 @@ def world_to_local_coordinates_array(self,stack, dataarray, tileId, z=0,host = N except: print json_answer return None - + def local_to_world_coordinates_array(self,stack, dataarray, tileId, z=0,host = None, port = None, owner = None, project = None, session=requests.session()): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/local-to-world-coordinates" % (z) @@ -382,7 +433,7 @@ def local_to_world_coordinates_array(self,stack, dataarray, tileId, z=0,host = N d['local']=[dataarray[i,0],dataarray[i,1]] dlist.append(d) jsondata=json.dumps(dlist) - + r = session.put(request_url, data=jsondata, headers={"content-type":"application/json"}) json_answer = r.json() @@ -401,12 +452,12 @@ def local_to_world_coordinates_array(self,stack, dataarray, tileId, z=0,host = N print json_answer return None - + def local_to_world_coordinates_batch(self,stack, data, z, host = None, port = None, owner = None, project = None, session=requests.session()): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/local-to-world-coordinates" % (z) - - + + r = session.put(request_url, data=data, headers={"content-type":"application/json"}) #print r.text() @@ -434,18 +485,18 @@ def put_resolved_tilespecs(self,stack,data,host = None,port = None,owner = None, request_url = self.format_preamble(host,port,owner,project,stack)+"/resolvedTiles" if verbose: print request_url - + r = session.put(request_url, data=data, headers={"content-type":"application/json","Accept":"text/plain"}) return r - + # http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/tile/140422184419060139 def get_tile_spec(self,stack, tile, host = None, port = None, owner = None, project = None, session=requests.session()): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/render-parameters"%(tile) tilespec_json= self.process_simple_url_request(request_url,session) - + return TileSpec(json=tilespec_json['tileSpecs'][0]) @@ -486,7 +537,7 @@ def get_bounds_from_z(self,stack,z,host = None,port = None,owner = None,project (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) request_url = self.format_preamble(host,port,owner,project,stack)+'/z/%f/bounds'%(z) return self.process_simple_url_request(request_url,session) - + # # API for doing the bulk requests locally (i.e., to be run on the cluster) # Full documentation here: http://wiki.int.janelia.org/wiki/display/flyTEM/Coordinate+Mapping+Tools @@ -502,8 +553,8 @@ def set_stack_state(self,stack,state='LOADING',host = None,port = None,owner = N if verbose: request_url r=session.put(request_url,data=None,headers={"content-type":"application/json"}) - return r - + return r + def batch_local_work(self,stack, z, data, host = None, port = None, owner = None, project = None, localToWorld=False, deleteTemp=True, threads=16): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) fromJson = tempfile.NamedTemporaryFile(suffix=".json", mode='w', delete=False) @@ -550,7 +601,7 @@ def get_png_tile(self,stack,z,x,y,width,height,scale=1.0,host=None,port=None,own except: print r.text return image - + def world_to_local_coordinates_batch_local(self,stack, z, data, host = None, port = None, owner = None, project = None): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) return batch_local_work(stack, z, data, host, port, owner, project, localToWorld=False) @@ -608,7 +659,7 @@ def get_matches_with_group(self,matchCollection,pgroup,owner=None,host=None,port request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/pGroup/%s/matches/"%\ (owner,matchCollection,pgroup) return self.process_simple_url_request(request_url, session) - + def get_match_groupIds_from_only(self,matchCollection,owner=None,host=None,port=None,verbose=False,session=requests.session()): (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/pGroupIds"%\ @@ -620,6 +671,3 @@ def get_match_groupIds_to_only(self,matchCollection,owner=None,host=None,port=No request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/qGroupIds"%\ (owner,matchCollection) return self.process_simple_url_request(request_url, session) - - - From e7ef7e47b7d839f761a594af220edbe37c2eb244 Mon Sep 17 00:00:00 2001 From: Russel Date: Fri, 13 Jan 2017 10:35:37 -0800 Subject: [PATCH 002/766] renderapi: clean up imports, fix client_script bug, PEP8 --- renderapi.py | 658 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 414 insertions(+), 244 deletions(-) diff --git a/renderapi.py b/renderapi.py index 4d83e421..36a584d8 100755 --- a/renderapi.py +++ b/renderapi.py @@ -5,12 +5,12 @@ import sys from functools import partial import tempfile -from io import BytesIO +import io import time import requests import numpy as np from PIL import Image -from tilespec import TileSpec,StackVersion +from tilespec import TileSpec, StackVersion # import pathos.multiprocessing as mp try: @@ -92,8 +92,10 @@ def __init__(self, host=None, port=None, owner=None, project=None, client_scripts = str(raw_input( "Enter Render Client Scripts location: ")) if client_scripts == '': - logging.critical('Render Client Scripts must not be empty!') - raise ValueError('Render Client Scripts must not be empty!') + logging.critical('Render Client Scripts must ' + 'not be empty!') + raise ValueError('Render Client Scripts must ' + 'not be empty!') else: client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) self.DEFAULT_CLIENT_SCRIPTS = client_scripts @@ -101,17 +103,19 @@ def __init__(self, host=None, port=None, owner=None, project=None, logging.debug('Render object created with ' 'host={h}, port={p}, project={pr}, ' 'owner={o}, scripts={s}'.format( - h=self.DEFAULT_HOST, p=self.DEFAULT_PORT, pr=self.DEFAULT_PROJECT, - o=self.DEFAULT_OWNER, s=self.DEFAULT_CLIENT_SCRIPTS)) + h=self.DEFAULT_HOST, p=self.DEFAULT_PORT, + pr=self.DEFAULT_PROJECT, o=self.DEFAULT_OWNER, + s=self.DEFAULT_CLIENT_SCRIPTS)) - def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): + def process_defaults(self, host, port, owner, project, + client_scripts=None): #def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): #utility function which will convert arguments to default arguments if they are None #allows Render object to be used with defaults if lazy, but allows projects/hosts/owners to be changed #from call to call if desired. #used by many functions convert default None arguments to default values. if host is None: - host=self.DEFAULT_HOST + host = self.DEFAULT_HOST if port is None: port = self.DEFAULT_PORT if owner is None: @@ -120,36 +124,42 @@ def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_ project = self.DEFAULT_PROJECT if client_scripts is None: client_scripts = self.DEFAULT_CLIENT_SCRIPTS - return (host,port,owner,project,client_scripts) + return (host, port, owner, project, client_scripts) - def make_stack_params(self,host,port,owner,project,stack): + def make_stack_params(self, host, port, owner, project, stack): #utility function to turn host,port,owner,project,stack combinations #to java CLI based argument list for subprocess calling #returns [--baseDataUrl,self.format_baseurl(host,port),--owner - baseurl = self.format_baseurl(host,port) - project_params = ['--baseDataUrl', baseurl, '--owner', owner, '--project', project] - stack_params= project_params + ['--stack', stack] + baseurl = self.format_baseurl(host, port) + project_params = ['--baseDataUrl', baseurl, + '--owner', owner, '--project', project] + stack_params = project_params + ['--stack', stack] return stack_params - def delete_stack(self,stack,host=None,port=None,owner=None,project=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack) - r=session.delete(request_url) + def delete_stack(self, stack, host=None, port=None, owner=None, + project=None, session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble(host, port, owner, project, stack) + r = session.delete(request_url) print r.text return r - def create_stack(self,stack,cycleNumber=1,cycleStepNumber=1, - client_scripts = None,host = None,port = None,owner = None, - project = None,verbose=False,session=requests.session()): + def create_stack(self, stack, cycleNumber=1, cycleStepNumber=1, + client_scripts=None, host=None, port=None, owner=None, + project=None, verbose=False, session=requests.session()): - - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project,client_scripts) - sv = StackVersion(cycleNumber=cycleNumber,cycleStepNumber=cycleStepNumber) - request_url = self.format_preamble(host,port,owner,project,stack) + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project, client_scripts) + sv = StackVersion( + cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber) + request_url = self.format_preamble(host, port, owner, project, stack) if verbose: - print "stack version2",request_url,sv.to_dict() + print "stack version2", request_url, sv.to_dict() payload = json.dumps(sv.to_dict()) - r = session.post(request_url,data=payload,headers={"content-type":"application/json","Accept":"application/json"}) + r = session.post(request_url, data=payload, + headers={"content-type": "application/json", + "Accept": "application/json"}) try: return r except: @@ -169,23 +179,24 @@ def create_stack(self,stack,cycleNumber=1,cycleStepNumber=1, # if verbose: # print proc.stdout.read() - def import_single_json_file(self,stack,jsonfile,transformFile=None, - client_scripts=DEFAULT_CLIENT_SCRIPTS,host=None,port=None,owner=None,project=None,verbose=False): + def import_single_json_file(self, stack, jsonfile, transformFile=None, + client_scripts=None, host=None, port=None, + owner=None, project=None, verbose=False): - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project, client_scripts) if transformFile is None: - transform_params =[] + transform_params = [] else: - transform_params = ['--transformFile',transformFile] - import subprocess - my_env= os.environ.copy() - stack_params = self.make_stack_params(host,port,owner,project,stack) + transform_params = ['--transformFile', transformFile] + my_env = os.environ.copy() + stack_params = self.make_stack_params( + host, port, owner, project, stack) cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - [jsonfile] + stack_params + \ + transform_params + \ + [jsonfile] if verbose: print cmd proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) @@ -193,62 +204,73 @@ def import_single_json_file(self,stack,jsonfile,transformFile=None, if verbose: print proc.stdout.read() - def import_jsonfiles_and_transforms_parallel_by_z(self,stack,jsonfiles,transformfiles,poolsize=20,client_scripts=DEFAULT_CLIENT_SCRIPTS, - host=None,port=None,owner=None,project=None,close_stack=True,verbose=False): + def import_jsonfiles_and_transforms_parallel_by_z( + self, stack, jsonfiles, transformfiles, poolsize=20, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, verbose=False): - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) - self.set_stack_state(stack,'LOADING',host,port,owner,project) + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project, client_scripts) + self.set_stack_state(stack, 'LOADING', host, port, owner, project) pool = Pool(poolsize) - partial_import = partial(self.import_single_json_file,stack, - client_scripts=client_scripts,host=host,port=port,owner=owner,project=project,verbose=verbose) + partial_import = partial(self.import_single_json_file, stack, + client_scripts=client_scripts, host=host, + port=port, owner=owner, project=project, + verbose=verbose) - rs = pool.amap(partial_import, jsonfiles,transformfiles) + rs = pool.amap(partial_import, jsonfiles, transformfiles) rs.wait() if close_stack: - self.set_stack_state(stack,'COMPLETE',host,port,owner,project) + self.set_stack_state(stack, 'COMPLETE', host, port, owner, project) - def import_jsonfiles_parallel(self,stack,jsonfiles,poolsize=20,transformFile=None,client_scripts=DEFAULT_CLIENT_SCRIPTS, - host=None,port=None,owner=None,project=None,close_stack=True,verbose=False): + def import_jsonfiles_parallel( + self, stack, jsonfiles, poolsize=20, transformFile=None, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, verbose=False): - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) - self.set_stack_state(stack,'LOADING',host,port,owner,project) + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project, client_scripts) + self.set_stack_state(stack, 'LOADING', host, port, owner, project) pool = Pool(poolsize) - partial_import = partial(self.import_single_json_file,stack,transformFile=transformFile, - client_scripts=client_scripts,host=host,port=port,owner=owner,project=project,verbose=verbose) + partial_import = partial(self.import_single_json_file, stack, + transformFile=transformFile, + client_scripts=client_scripts, + host=host, port=port, owner=owner, + project=project, verbose=verbose) partial_import(jsonfiles[0]) rs = pool.amap(partial_import, jsonfiles) rs.wait() if close_stack: - self.set_stack_state(stack,'COMPLETE',host,port,owner,project) + self.set_stack_state(stack, 'COMPLETE', host, port, owner, project) + def import_jsonfiles(self, stack, jsonfiles, transformFile=None, + client_scripts=None, host=None, port=None, + owner=None, project=None, close_stack=True, + verbose=False): - def import_jsonfiles(self,stack,jsonfiles,transformFile = None, - client_scripts=DEFAULT_CLIENT_SCRIPTS,host = None,port = None, - owner = None,project = None,close_stack=True,verbose=False): - (host,port,owner,project,client_scripts)=\ - self.process_defaults(host,port,owner,project,client_scripts) + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project, client_scripts) - self.set_stack_state(stack,'LOADING',host,port,owner,project) + self.set_stack_state(stack, 'LOADING', host, port, owner, project) if transformFile is None: transform_params = [] else: - transform_params = ['--transformFile',transformFile] - import subprocess + transform_params = ['--transformFile', transformFile] + my_env = os.environ.copy() - stack_params = self.make_stack_params(host,port,owner,project,stack) + stack_params = self.make_stack_params( + host, port, owner, project, stack) cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - jsonfiles + stack_params + \ + transform_params + \ + jsonfiles if verbose: print cmd proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) @@ -256,13 +278,17 @@ def import_jsonfiles(self,stack,jsonfiles,transformFile = None, if verbose: print proc.stdout.read() if close_stack: - self.set_stack_state(stack,'COMPLETE',host,port,owner,project) - + self.set_stack_state(stack, 'COMPLETE', host, port, owner, project) - def world_to_local_coordinates(self,stack, z, x, y, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) + def world_to_local_coordinates(self, stack, z, x, y, host=None, port=None, + owner=None, project=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) r = session.get(request_url) try: @@ -283,10 +309,15 @@ def world_to_local_coordinates(self,stack, z, x, y, host = None, port = None, ow # 2239.0 # ] # } - def local_to_world_coordinates(self,stack, tileId, x, y, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) + def local_to_world_coordinates(self, stack, tileId, x, y, host=None, + port=None, owner=None, project=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) r = session.get(request_url) try: @@ -296,29 +327,38 @@ def local_to_world_coordinates(self,stack, tileId, x, y, host = None, port = Non print(r.text) return None - def get_owners(self,host=None,port=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,None,None) - request_url = "%s/owners/"%self.format_baseurl(host,port) - return self.process_simple_url_request(request_url,session) + def get_owners(self, host=None, port=None, session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, None, None) + request_url = "%s/owners/" % self.format_baseurl(host, port) + return self.process_simple_url_request(request_url, session) - def get_projects_by_owner(self,owner=None,host=None,port=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) + def get_projects_by_owner(self, owner=None, host=None, port=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) metadata = self.get_stack_metadata_by_owner(owner) projects = list(set([m['stackId']['project'] for m in metadata])) return projects - def get_stacks_by_owner_project(self,owner=None,project=None,host=None,port=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) + def get_stacks_by_owner_project(self, owner=None, project=None, host=None, + port=None, session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) metadata = self.get_stack_metadata_by_owner(owner) - stacks =([m['stackId']['stack'] for m in metadata if m['stackId']['project']==project]) + stacks = ([m['stackId']['stack'] for m in metadata + if m['stackId']['project'] == project]) return stacks - def get_stack_metadata_by_owner(self,owner=None,host=None,port=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = "%s/owner/%s/stacks/"%(self.format_baseurl(host,port),owner) + def get_stack_metadata_by_owner(self, owner=None, host=None, port=None, + session=requests.session(), verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = "%s/owner/%s/stacks/" % ( + self.format_baseurl(host, port), owner) if verbose: request_url - return self.process_simple_url_request(request_url,session) + return self.process_simple_url_request(request_url, session) # PUT http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/local-to-world-coordinates # with request body containing JSON array of local coordinate elements @@ -333,12 +373,16 @@ def get_stack_metadata_by_owner(self,owner=None,host=None,port=None,session=requ # ] # } # ] - def world_to_local_coordinates_batch(self,stack, z, data, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/world-to-local-coordinates" % (z) - r = session.put(request_url, data=data, headers={"content-type":"application/json"}) - - + def world_to_local_coordinates_batch(self, stack, z, data, host=None, + port=None, owner=None, project=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/world-to-local-coordinates" % (z) + r = session.put(request_url, data=data, + headers={"content-type": "application/json"}) #print r.text return r.json() @@ -356,10 +400,13 @@ def world_to_local_coordinates_batch(self,stack, z, data, host = None, port = No # } # ] # ] - def get_z_values_for_stack(self,stack,project = None, - host = None,port = None,owner = None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/zValues/" + def get_z_values_for_stack(self, stack, project=None, host=None, port=None, + owner=None, session=requests.session(), + verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + "/zValues/" if verbose: print request_url r = session.get(request_url) @@ -369,10 +416,13 @@ def get_z_values_for_stack(self,stack,project = None, print(r.text) return None - def get_z_value_for_section(self,stack,sectionId,project = None, - host = None,port = None,owner = None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/section/%s/z"%(sectionId) + def get_z_value_for_section(self, stack, sectionId, project=None, + host=None, port=None, owner=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + "/section/%s/z" % (sectionId) r = session.get(request_url) try: #print(r.json()) @@ -380,97 +430,122 @@ def get_z_value_for_section(self,stack,sectionId,project = None, except: print(r.text) return None - def get_tile_image_data(self,stack,tileId,host=None,port=None,owner=None,project=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/png-image" % (tileId) + + def get_tile_image_data(self, stack, tileId, host=None, port=None, + owner=None, project=None, + session=requests.session(), verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/png-image" % (tileId) if verbose: print request_url r = session.get(request_url) try: - img = Image.open(BytesIO(r.content)) + img = Image.open(io.BytesIO(r.content)) array = np.asarray(img) return array except: print (r.text) return None - def world_to_local_coordinates_array(self,stack, dataarray, tileId, z=0,host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/world-to-local-coordinates" % (z) - dlist =[] + def world_to_local_coordinates_array(self, stack, dataarray, tileId, z=0, + host=None, port=None, owner=None, + project=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/world-to-local-coordinates" % (z) + dlist = [] for i in range(dataarray.shape[0]): - d ={} - d['tileId']=tileId - d['world']=[dataarray[i,0],dataarray[i,1]] + d = {} + d['tileId'] = tileId + d['world'] = [dataarray[i, 0], dataarray[i, 1]] dlist.append(d) - jsondata=json.dumps(dlist) + jsondata = json.dumps(dlist) - r = session.put(request_url, data=jsondata, headers={"content-type":"application/json"}) + r = session.put(request_url, data=jsondata, + headers={"content-type": "application/json"}) json_answer = r.json() #print json_answer try: answer = np.zeros(dataarray.shape) - for i,coord in enumerate(json_answer): + for i, coord in enumerate(json_answer): c = coord['local'] - answer[i,0]=c[0] - answer[i,1]=c[1] + answer[i, 0] = c[0] + answer[i, 1] = c[1] return answer except: print json_answer return None - def local_to_world_coordinates_array(self,stack, dataarray, tileId, z=0,host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/local-to-world-coordinates" % (z) - dlist =[] + def local_to_world_coordinates_array(self, stack, dataarray, tileId, z=0, + host=None, port=None, owner=None, + project=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/local-to-world-coordinates" % (z) + dlist = [] for i in range(dataarray.shape[0]): - d ={} - d['tileId']=tileId - d['local']=[dataarray[i,0],dataarray[i,1]] + d = {} + d['tileId'] = tileId + d['local'] = [dataarray[i, 0], dataarray[i, 1]] dlist.append(d) - jsondata=json.dumps(dlist) + jsondata = json.dumps(dlist) - r = session.put(request_url, data=jsondata, headers={"content-type":"application/json"}) + r = session.put(request_url, data=jsondata, + headers={"content-type": "application/json"}) json_answer = r.json() #print json_answer try: answer = np.zeros(dataarray.shape) - for i,coord in enumerate(json_answer): + for i, coord in enumerate(json_answer): c = coord['world'] - answer[i,0]=c[0] - answer[i,1]=c[1] + answer[i, 0] = c[0] + answer[i, 1] = c[1] return answer except: print json_answer return None + def local_to_world_coordinates_batch(self, stack, data, z, host=None, + port=None, owner=None, project=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/local-to-world-coordinates" % (z) - def local_to_world_coordinates_batch(self,stack, data, z, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/local-to-world-coordinates" % (z) - - - r = session.put(request_url, data=data, headers={"content-type":"application/json"}) + r = session.put(request_url, data=data, + headers={"content-type": "application/json"}) #print r.text() return r.json() - def format_baseurl(self,host,port): - return 'http://%s:%d/render-ws/v1'%(host,port) + def format_baseurl(self, host, port): + return 'http://%s:%d/render-ws/v1' % (host, port) - def format_preamble(self,host,port,owner,project,stack): - preamble = "%s/owner/%s/project/%s/stack/%s"%(self.format_baseurl(host,port),owner,project,stack) + def format_preamble(self, host, port, owner, project, stack): + preamble = "%s/owner/%s/project/%s/stack/%s" % ( + self.format_baseurl(host, port), owner, project, stack) return preamble - def process_simple_url_request(self,request_url,session): + def process_simple_url_request(self, request_url, session): r = session.get(request_url) try: #print(r.json()) @@ -479,64 +554,91 @@ def process_simple_url_request(self,request_url,session): #print e print(r.text) return None - def put_resolved_tilespecs(self,stack,data,host = None,port = None,owner = None,project = None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/resolvedTiles" + + def put_resolved_tilespecs(self, stack, data, host=None, port=None, + owner=None, project=None, + session=requests.session(), verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + "/resolvedTiles" if verbose: print request_url - r = session.put(request_url, data=data, headers={"content-type":"application/json","Accept":"text/plain"}) + r = session.put(request_url, data=data, + headers={"content-type":"application/json", + "Accept":"text/plain"}) return r # http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/tile/140422184419060139 - def get_tile_spec(self,stack, tile, host = None, port = None, owner = None, project = None, session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/tile/%s/render-parameters"%(tile) + def get_tile_spec(self, stack, tile, host=None, port=None, owner=None, + project=None, session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/render-parameters" % (tile) - tilespec_json= self.process_simple_url_request(request_url,session) + tilespec_json = self.process_simple_url_request(request_url, session) return TileSpec(json=tilespec_json['tileSpecs'][0]) - - def get_tile_specs_from_minmax_box(self,stack,z,xmin,xmax,ymin,ymax,scale=1.0,host=None,port=None,owner=None, - project=None,session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) + def get_tile_specs_from_minmax_box(self, stack, z, xmin, xmax, ymin, ymax, + scale=1.0, host=None, port=None, + owner=None, project=None, + session=requests.session(), + verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) x = xmin y = ymin - width = xmax-xmin - height = ymax -ymin - return self.get_tile_specs_from_box(stack, z, x, y, width, height,scale,host,port,owner,project,session,verbose) - - def get_tile_specs_from_box(self,stack,z,x,y,width,height,scale=1.0,host=None, port=None,owner=None,project=None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters"%(z,x,y,width,height,scale) + width = xmax - xmin + height = ymax - ymin + return self.get_tile_specs_from_box(stack, z, x, y, width, height, + scale, host, port, owner, project, + session, verbose) + + def get_tile_specs_from_box(self, stack, z, x, y, width, height, scale=1.0, + host=None, port=None, owner=None, project=None, + session=requests.session(), verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( + z, x, y, width, height, scale) if verbose: print request_url - tilespecs_json=self.process_simple_url_request(request_url,session)['tileSpecs'] + tilespecs_json = self.process_simple_url_request( + request_url, session)['tileSpecs'] #return tilespecs_json - return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json] - - - def get_tile_specs_from_z(self,stack,z,host = None,port = None,owner=None,project=None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+'/z/%f/tile-specs'%(z) + return [TileSpec(json=tilespec_json) + for tilespec_json in tilespecs_json] + + def get_tile_specs_from_z(self, stack, z, host=None, port=None, owner=None, + project=None, session=requests.session(), + verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) if verbose: print request_url - tilespecs_json=self.process_simple_url_request(request_url,session) - if len(tilespecs_json)==0: + tilespecs_json = self.process_simple_url_request(request_url, session) + if len(tilespecs_json) == 0: return None else: - return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json] - - def get_bounds_from_z(self,stack,z,host = None,port = None,owner = None,project = None, - session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+'/z/%f/bounds'%(z) - return self.process_simple_url_request(request_url,session) + return [TileSpec(json=tilespec_json) + for tilespec_json in tilespecs_json] + + def get_bounds_from_z(self, stack, z, host=None, port=None, owner=None, + project=None, session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + '/z/%f/bounds' % (z) + return self.process_simple_url_request(request_url, session) # # API for doing the bulk requests locally (i.e., to be run on the cluster) @@ -545,28 +647,42 @@ def get_bounds_from_z(self,stack,z,host = None,port = None,owner = None,project MAP_COORD_SCRIPT = "/groups/flyTEM/flyTEM/render/bin/map-coord.sh" - def set_stack_state(self,stack,state='LOADING',host = None,port = None,owner = None,project = None, - session=requests.session(),verbose=False): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - assert state in ['LOADING','COMPLETE','OFFLINE'] - request_url = self.format_preamble(host,port,owner,project,stack)+"/state/%s"%state + def set_stack_state(self, stack, state='LOADING', host=None, port=None, + owner=None, project=None, session=requests.session(), + verbose=False): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] + request_url = self.format_preamble( + host, port, owner, project, stack) + "/state/%s" % state if verbose: request_url - r=session.put(request_url,data=None,headers={"content-type":"application/json"}) + r = session.put(request_url, data=None, + headers={"content-type": "application/json"}) return r - def batch_local_work(self,stack, z, data, host = None, port = None, owner = None, project = None, localToWorld=False, deleteTemp=True, threads=16): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - fromJson = tempfile.NamedTemporaryFile(suffix=".json", mode='w', delete=False) + def batch_local_work(self, stack, z, data, host=None, port=None, + owner=None, project=None, localToWorld=False, + deleteTemp=True, threads=16): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + fromJson = tempfile.NamedTemporaryFile( + suffix=".json", mode='w', delete=False) fromJson.write(data) fromJson.flush() fromJson.close() - toJson = tempfile.NamedTemporaryFile(suffix=".json", mode='r', delete=False) + toJson = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) toJson.close() #cmd = "%s --owner %s --project %s --stack %s --z %d --fromJson %s --toJson %s --baseDataUrl http://tem-services.int.janelia.org:8080/render-ws/v1 --numberOfThreads %d" % (MAP_COORD_SCRIPT, owner, project, stack, z, fromJson.name, toJson.name, threads) - cmd = "%s --owner %s --project %s --stack %s --z %d --fromJson %s --toJson %s --baseDataUrl http://10.40.3.162:8080/render-ws/v1 --numberOfThreads %d" % (MAP_COORD_SCRIPT, owner, project, stack, z, fromJson.name, toJson.name, threads) + cmd = ("%s --owner %s --project %s --stack %s --z %d --fromJson %s " + "--toJson %s --baseDataUrl http://10.40.3.162:8080/render-ws/v1 " + "--numberOfThreads %d") % ( + MAP_COORD_SCRIPT, owner, project, stack, z, + fromJson.name, toJson.name, threads) + if localToWorld: cmd = cmd + " --localToWorld" #print(cmd) @@ -588,86 +704,140 @@ def batch_local_work(self,stack, z, data, host = None, port = None, owner = None return outdata - def get_png_tile(self,stack,z,x,y,width,height,scale=1.0,host=None,port=None,owner=None,project=None,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host,port,owner,project,stack)+"/z/%d/box/%d,%d,%d,%d,%3.2f/png-image"%(z,x,y,width,height,scale) + def get_png_tile(self, stack, z, x, y, width, height, scale=1.0, host=None, + port=None, owner=None, project=None, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%3.2f/png-image" % ( + z, x, y, width, height, scale) #print request_url r = session.get(request_url) - import io - from PIL import Image - from array import array try: image = np.asarray(Image.open(io.BytesIO(r.content))) except: print r.text return image - def world_to_local_coordinates_batch_local(self,stack, z, data, host = None, port = None, owner = None, project = None): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - return batch_local_work(stack, z, data, host, port, owner, project, localToWorld=False) - - def local_to_world_coordinates_batch_local(self,stack, z, data, host = None, port = None, owner = None, project = None): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - return batch_local_work(stack, z, data, host, port, owner, project, localToWorld=True) - - def get_matchcollection_owners(self,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,None,None) - request_url=self.format_baseurl(host,port)+"/matchCollectionOwners" + def world_to_local_coordinates_batch_local(self, stack, z, data, host=None, + port=None, owner=None, + project=None): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + return batch_local_work(stack, z, data, host, port, owner, project, + localToWorld=False) + + def local_to_world_coordinates_batch_local(self, stack, z, data, host=None, + port=None, owner=None, + project=None): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + return batch_local_work(stack, z, data, host, port, owner, project, + localToWorld=True) + + def get_matchcollection_owners(self, host=None, port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, None, None) + request_url = self.format_baseurl(host, port) + \ + "/matchCollectionOwners" return self.process_simple_url_request(request_url, session) - def get_matchcollections(self,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollections"%owner + def get_matchcollections(self, owner=None, host=None, port=None, + verbose=False, session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollections" % owner return self.process_simple_url_request(request_url, session) - def get_match_groupIds(self,matchCollection,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/groupIds"%(owner,matchCollection) + def get_match_groupIds(self, matchCollection, owner=None, host=None, + port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) return self.process_simple_url_request(request_url, session) - def get_section_z_value(self,stack,sectionId,host=None,port=None,owner=None,project=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,project) - request_url = self.format_preamble(host, port, owner, project, stack)+"/section/%s/z"%sectionId + def get_section_z_value(self, stack, sectionId, host=None, port=None, + owner=None, project=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, project) + request_url = self.format_preamble( + host, port, owner, project, stack) + "/section/%s/z" % sectionId return float(self.process_simple_url_request(request_url, session)) - def get_matches_outside_group(self,matchCollection,groupId,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup"%\ - (owner,matchCollection,groupId) + def get_matches_outside_group(self, matchCollection, groupId, owner=None, + host=None, port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( + owner, matchCollection, groupId) return self.process_simple_url_request(request_url, session) - def get_matches_within_group(self,matchCollection,groupId,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup"%\ - (owner,matchCollection,groupId) + def get_matches_within_group(self, matchCollection, groupId, owner=None, + host=None, port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( + owner, matchCollection, groupId) return self.process_simple_url_request(request_url, session) - def get_matches_from_group_to_group(self,matchCollection,pgroup,qgroup,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/matchesWith/%s"%\ - (owner,matchCollection,pgroup,qgroup) + def get_matches_from_group_to_group(self, matchCollection, pgroup, qgroup, + owner=None, host=None, port=None, + verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( + owner, matchCollection, pgroup, qgroup) return self.process_simple_url_request(request_url, session) - def get_matches_from_tile_to_tile(self,matchCollection,pgroup,pid,qgroup,qid,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/group/%s/id/%s/matchesWith/%s/id/%s"%\ - (owner,matchCollection,pgroup,pid,qgroup,qid) + def get_matches_from_tile_to_tile(self, matchCollection, pgroup, pid, + qgroup, qid, owner=None, host=None, + port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + ("/owner/%s/matchCollection/%s/group/%s/id/%s/" + "matchesWith/%s/id/%s" % ( + owner, matchCollection, pgroup, pid, qgroup, qid)) return self.process_simple_url_request(request_url, session) - def get_matches_with_group(self,matchCollection,pgroup,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/pGroup/%s/matches/"%\ - (owner,matchCollection,pgroup) + def get_matches_with_group(self, matchCollection, pgroup, owner=None, + host=None, port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( + owner, matchCollection, pgroup) return self.process_simple_url_request(request_url, session) - def get_match_groupIds_from_only(self,matchCollection,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/pGroupIds"%\ - (owner,matchCollection) + def get_match_groupIds_from_only(self, matchCollection, owner=None, + host=None, port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) return self.process_simple_url_request(request_url, session) - def get_match_groupIds_to_only(self,matchCollection,owner=None,host=None,port=None,verbose=False,session=requests.session()): - (host,port,owner,project,client_scripts)=self.process_defaults(host,port,owner,None) - request_url = self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/qGroupIds"%\ - (owner,matchCollection) + def get_match_groupIds_to_only(self, matchCollection, owner=None, + host=None, port=None, verbose=False, + session=requests.session()): + (host, port, owner, project, client_scripts) = self.process_defaults( + host, port, owner, None) + request_url = self.format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) return self.process_simple_url_request(request_url, session) From 0ac6444742b4394d4e314136c534265cf7228118 Mon Sep 17 00:00:00 2001 From: Russel Date: Fri, 13 Jan 2017 10:42:10 -0800 Subject: [PATCH 003/766] renderapi: add Forrest's normalizeForMatching Flag --- renderapi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/renderapi.py b/renderapi.py index 36a584d8..901686fb 100755 --- a/renderapi.py +++ b/renderapi.py @@ -431,14 +431,16 @@ def get_z_value_for_section(self, stack, sectionId, project=None, print(r.text) return None - def get_tile_image_data(self, stack, tileId, host=None, port=None, - owner=None, project=None, + def get_tile_image_data(self, stack, tileId, normalizeForMatching=True, + host=None, port=None, owner=None, project=None, session=requests.session(), verbose=False): (host, port, owner, project, client_scripts) = self.process_defaults( host, port, owner, project) request_url = self.format_preamble( host, port, owner, project, stack) + \ "/tile/%s/png-image" % (tileId) + if normalizeForMatching: + request_url += "?normalizeForMatching=true" if verbose: print request_url r = session.get(request_url) From 9d90564e036f46cd2f087a0048fc0e771fe28c97 Mon Sep 17 00:00:00 2001 From: Russel Date: Sun, 15 Jan 2017 16:47:09 -0800 Subject: [PATCH 004/766] module: moving tilespec and renderapi to module --- renderapi.py => renderapi/renderapi.py | 0 tilespec.py => renderapi/tilespec.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename renderapi.py => renderapi/renderapi.py (100%) rename tilespec.py => renderapi/tilespec.py (100%) diff --git a/renderapi.py b/renderapi/renderapi.py similarity index 100% rename from renderapi.py rename to renderapi/renderapi.py diff --git a/tilespec.py b/renderapi/tilespec.py similarity index 100% rename from tilespec.py rename to renderapi/tilespec.py From 5abfa3c7beb870cbbb4a91a0385bae6a43bd662a Mon Sep 17 00:00:00 2001 From: Russel Date: Sun, 15 Jan 2017 16:50:08 -0800 Subject: [PATCH 005/766] render: remove commented prints --- renderapi/renderapi.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/renderapi/renderapi.py b/renderapi/renderapi.py index 901686fb..270362b7 100755 --- a/renderapi/renderapi.py +++ b/renderapi/renderapi.py @@ -292,7 +292,6 @@ def world_to_local_coordinates(self, stack, z, x, y, host=None, port=None, r = session.get(request_url) try: - #print(r.json()) return r.json() except: print(r.text) @@ -321,7 +320,6 @@ def local_to_world_coordinates(self, stack, tileId, x, y, host=None, r = session.get(request_url) try: - #print(r.json()) return r.json() except: print(r.text) @@ -383,7 +381,6 @@ def world_to_local_coordinates_batch(self, stack, z, data, host=None, "/z/%d/world-to-local-coordinates" % (z) r = session.put(request_url, data=data, headers={"content-type": "application/json"}) - #print r.text return r.json() # curl -H "Content-Type: application/json" -X PUT --data @coordinate-world.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates" @@ -425,7 +422,6 @@ def get_z_value_for_section(self, stack, sectionId, project=None, host, port, owner, project, stack) + "/section/%s/z" % (sectionId) r = session.get(request_url) try: - #print(r.json()) return r.json() except: print(r.text) @@ -473,7 +469,6 @@ def world_to_local_coordinates_array(self, stack, dataarray, tileId, z=0, headers={"content-type": "application/json"}) json_answer = r.json() - #print json_answer try: answer = np.zeros(dataarray.shape) @@ -509,7 +504,6 @@ def local_to_world_coordinates_array(self, stack, dataarray, tileId, z=0, headers={"content-type": "application/json"}) json_answer = r.json() - #print json_answer try: answer = np.zeros(dataarray.shape) @@ -536,7 +530,6 @@ def local_to_world_coordinates_batch(self, stack, data, z, host=None, r = session.put(request_url, data=data, headers={"content-type": "application/json"}) - #print r.text() return r.json() def format_baseurl(self, host, port): @@ -614,7 +607,6 @@ def get_tile_specs_from_box(self, stack, z, x, y, width, height, scale=1.0, print request_url tilespecs_json = self.process_simple_url_request( request_url, session)['tileSpecs'] - #return tilespecs_json return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json] @@ -687,8 +679,6 @@ def batch_local_work(self, stack, z, data, host=None, port=None, if localToWorld: cmd = cmd + " --localToWorld" - #print(cmd) - try: rc = subprocess.call(cmd, shell="True") if rc != 0: @@ -715,7 +705,6 @@ def get_png_tile(self, stack, z, x, y, width, height, scale=1.0, host=None, host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/png-image" % ( z, x, y, width, height, scale) - #print request_url r = session.get(request_url) try: image = np.asarray(Image.open(io.BytesIO(r.content))) From c163f4e35251995b19319e9c93da287a7cf47f19 Mon Sep 17 00:00:00 2001 From: Russel Date: Sun, 15 Jan 2017 23:59:19 -0800 Subject: [PATCH 006/766] module: add client and stack modules as example for module, add init --- renderapi/__init__.py | 13 ++++ renderapi/client.py | 138 ++++++++++++++++++++++++++++++++++++++++ renderapi/renderapi.py | 139 +++++------------------------------------ renderapi/stack.py | 47 ++++++++++++++ 4 files changed, 213 insertions(+), 124 deletions(-) create mode 100644 renderapi/__init__.py create mode 100644 renderapi/client.py create mode 100644 renderapi/stack.py diff --git a/renderapi/__init__.py b/renderapi/__init__.py new file mode 100644 index 00000000..267bb6bf --- /dev/null +++ b/renderapi/__init__.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +from . import render +from . import tilespec +from . import client +from . import errors +from . import stack +from .render import connect + +from .render import Render + +__all__ = ['render', 'client', 'tilespec', 'errors', + 'stack', 'connect', 'Render'] diff --git a/renderapi/client.py b/renderapi/client.py new file mode 100644 index 00000000..d6482ba5 --- /dev/null +++ b/renderapi/client.py @@ -0,0 +1,138 @@ +import json +from functools import partial +import logging +import subprocess +from renderapi.render import Render # TODO make relative for final +from renderapi.stack import set_stack_state, make_stack_params + +try: + from pathos.multiprocessing import ProcessingPool as Pool + has_pathos = True +except ImportError as e: + logging.warning(e) + has_pathos = False + from multiprocessing import Pool + + +def import_single_json_file(stack, jsonfile, render=None, transformFile=None, + client_scripts=None, host=None, port=None, + owner=None, project=None, verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + import_single_json_file(stack, jsonfile, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_scripts=client_scripts, **{'verbose': verbose, + 'transformFile': None})) + return + if transformFile is None: + transform_params = [] + else: + transform_params = ['--transformFile', transformFile] + my_env = os.environ.copy() + stack_params = make_stack_params( + host, port, owner, project, stack) + cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ + stack_params + \ + transform_params + \ + [jsonfile] + if verbose: + print cmd + proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) + proc.wait() + if verbose: + print proc.stdout.read() + + +def import_jsonfiles_and_transforms_parallel_by_z( + stack, jsonfiles, transformfiles, render=None, poolsize=20, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + import_jsonfiles_and_transforms_parallel_by_z( + stack, jsonfile, transformfiles, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_scripts=client_scripts, **{'verbose': verbose, + 'close_stack': close_stack, + 'poolsize': poolsize})) + return + set_stack_state(stack, 'LOADING', host, port, owner, project) + pool = Pool(poolsize) + partial_import = partial(import_single_json_file, stack, render=render, + client_scripts=client_scripts, host=host, + port=port, owner=owner, project=project, + verbose=verbose) + rs = pool.amap(partial_import, jsonfiles, transformfiles) + rs.wait() + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +def import_jsonfiles_parallel( + stack, jsonfiles, render=None, poolsize=20, transformFile=None, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + import_jsonfiles_parallel( + stack, jsonfiles, **render.make_kwargs( + host=host, port=port, owner=owner, + project=project, client_scripts=client_scripts, + **{'verbose': verbose, + 'close_stack': close_stack, + 'poolsize': poolsize, + 'transformFile': transformFile})) + return + set_stack_state(stack, 'LOADING', host, port, owner, project) + pool = Pool(poolsize) + partial_import = partial(import_single_json_file, stack, render=render, + transformFile=transformFile, + client_scripts=client_scripts, + host=host, port=port, owner=owner, + project=project, verbose=verbose) + partial_import(jsonfiles[0]) + rs = pool.amap(partial_import, jsonfiles) + rs.wait() + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, + client_scripts=None, host=None, port=None, + owner=None, project=None, close_stack=True, + verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + import_jsonfiles( + stack, jsonfiles, **render.make_kwargs( + host=host, port=port, owner=owner, + project=project, client_scripts=client_scripts, + **{'verbose': verbose, + 'close_stack': close_stack, + 'transformFile': transformFile})) + return + + set_stack_state(stack, 'LOADING', host, port, owner, project) + if transformFile is None: + transform_params = [] + else: + transform_params = ['--transformFile', transformFile] + my_env = os.environ.copy() + stack_params = make_stack_params( + host, port, owner, project, stack) + cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ + stack_params + \ + transform_params + \ + jsonfiles + if verbose: + print cmd + proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) + proc.wait() + if verbose: + print proc.stdout.read() + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) diff --git a/renderapi/renderapi.py b/renderapi/renderapi.py index 270362b7..29bf94d5 100755 --- a/renderapi/renderapi.py +++ b/renderapi/renderapi.py @@ -107,6 +107,21 @@ def __init__(self, host=None, port=None, owner=None, project=None, pr=self.DEFAULT_PROJECT, o=self.DEFAULT_OWNER, s=self.DEFAULT_CLIENT_SCRIPTS)) + @property + def DEFAULT_KWARGS(self): + return self.make_kwargs() + + def make_kwargs(self, host=None, port=None, owner=None, project=None, + client_scripts=None, **kwargs): + processed_kwargs = { + 'host': self.DEFAULT_HOST if host is None else host, + 'port': self.DEFAULT_PORT if port is None else port, + 'owner': self.DEFAULT_OWNER if owner is None else owner, + 'client_scripts': (self.DEFAULT_CLIENT_SCRIPTS if client_scripts + is None else client_scripts)} + processed_kwargs.update(kwargs) + return processed_kwargs + def process_defaults(self, host, port, owner, project, client_scripts=None): #def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): @@ -126,16 +141,6 @@ def process_defaults(self, host, port, owner, project, client_scripts = self.DEFAULT_CLIENT_SCRIPTS return (host, port, owner, project, client_scripts) - def make_stack_params(self, host, port, owner, project, stack): - #utility function to turn host,port,owner,project,stack combinations - #to java CLI based argument list for subprocess calling - #returns [--baseDataUrl,self.format_baseurl(host,port),--owner - baseurl = self.format_baseurl(host, port) - project_params = ['--baseDataUrl', baseurl, - '--owner', owner, '--project', project] - stack_params = project_params + ['--stack', stack] - return stack_params - def delete_stack(self, stack, host=None, port=None, owner=None, project=None, session=requests.session()): (host, port, owner, project, client_scripts) = self.process_defaults( @@ -166,120 +171,6 @@ def create_stack(self, stack, cycleNumber=1, cycleStepNumber=1, print r.text return None - # import subprocess - # my_env = os.environ.copy() - # stack_params = self.make_stack_params(host,port,owner,project,stack) - # cmd = [os.path.join(client_scripts, 'manage_stacks.sh')] + \ - # stack_params + \ - # ['--action', 'CREATE', '--cycleNumber', '%d'%cycleNumber, '--cycleStepNumber', '%d'%cycleStepNumber] - # if verbose: - # print cmd - # proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - # proc.wait() - # if verbose: - # print proc.stdout.read() - - def import_single_json_file(self, stack, jsonfile, transformFile=None, - client_scripts=None, host=None, port=None, - owner=None, project=None, verbose=False): - - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project, client_scripts) - - if transformFile is None: - transform_params = [] - else: - transform_params = ['--transformFile', transformFile] - my_env = os.environ.copy() - stack_params = self.make_stack_params( - host, port, owner, project, stack) - cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - [jsonfile] - if verbose: - print cmd - proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - proc.wait() - if verbose: - print proc.stdout.read() - - def import_jsonfiles_and_transforms_parallel_by_z( - self, stack, jsonfiles, transformfiles, poolsize=20, - client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, verbose=False): - - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project, client_scripts) - self.set_stack_state(stack, 'LOADING', host, port, owner, project) - - pool = Pool(poolsize) - - partial_import = partial(self.import_single_json_file, stack, - client_scripts=client_scripts, host=host, - port=port, owner=owner, project=project, - verbose=verbose) - - rs = pool.amap(partial_import, jsonfiles, transformfiles) - rs.wait() - - if close_stack: - self.set_stack_state(stack, 'COMPLETE', host, port, owner, project) - - def import_jsonfiles_parallel( - self, stack, jsonfiles, poolsize=20, transformFile=None, - client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, verbose=False): - - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project, client_scripts) - self.set_stack_state(stack, 'LOADING', host, port, owner, project) - - pool = Pool(poolsize) - - partial_import = partial(self.import_single_json_file, stack, - transformFile=transformFile, - client_scripts=client_scripts, - host=host, port=port, owner=owner, - project=project, verbose=verbose) - partial_import(jsonfiles[0]) - rs = pool.amap(partial_import, jsonfiles) - rs.wait() - - if close_stack: - self.set_stack_state(stack, 'COMPLETE', host, port, owner, project) - - def import_jsonfiles(self, stack, jsonfiles, transformFile=None, - client_scripts=None, host=None, port=None, - owner=None, project=None, close_stack=True, - verbose=False): - - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project, client_scripts) - - self.set_stack_state(stack, 'LOADING', host, port, owner, project) - - if transformFile is None: - transform_params = [] - else: - transform_params = ['--transformFile', transformFile] - - my_env = os.environ.copy() - stack_params = self.make_stack_params( - host, port, owner, project, stack) - cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - jsonfiles - if verbose: - print cmd - proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - proc.wait() - if verbose: - print proc.stdout.read() - if close_stack: - self.set_stack_state(stack, 'COMPLETE', host, port, owner, project) - def world_to_local_coordinates(self, stack, z, x, y, host=None, port=None, owner=None, project=None, session=requests.session()): diff --git a/renderapi/stack.py b/renderapi/stack.py new file mode 100644 index 00000000..1d0dda78 --- /dev/null +++ b/renderapi/stack.py @@ -0,0 +1,47 @@ +import json +import logging +import requests + + +def format_baseurl(host, port): + return 'http://%s:%d/render-ws/v1' % (host, port) + + +def format_preamble(host, port, owner, project, stack): + preamble = "%s/owner/%s/project/%s/stack/%s" % ( + format_baseurl(host, port), owner, project, stack) + return preamble + + +def set_stack_state(stack, render=None, state='LOADING', host=None, port=None, + owner=None, project=None, session=requests.session(), + verbose=False, **kwargs): + + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return set_stack_state( + stack, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'verbose': verbose, 'state': state, 'session': session})) + + assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] + request_url = format_preamble( + host, port, owner, project, stack) + "/state/%s" % state + if verbose: + request_url + r = session.put(request_url, data=None, + headers={"content-type": "application/json"}) + return r + + +def make_stack_params(host, port, owner, project, stack): + '''utility function to turn host,port,owner,project,stack combinations + to java CLI based argument list for subprocess calling + returns [--baseDataUrl,self.format_baseurl(host,port),--owner + ''' + baseurl = format_baseurl(host, port) + project_params = ['--baseDataUrl', baseurl, + '--owner', owner, '--project', project] + stack_params = project_params + ['--stack', stack] + return stack_params From 8784dbc84d2dfb8c7b1b4da79e20a29171146871 Mon Sep 17 00:00:00 2001 From: Russel Date: Mon, 16 Jan 2017 10:12:18 -0800 Subject: [PATCH 007/766] fix: testing draft module -- default project bug, requirements, init --- renderapi/__init__.py | 10 ++++++---- renderapi/renderapi.py | 29 +++++++++++++++-------------- renderapi/stack.py | 1 + requirements.txt | 2 ++ 4 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 requirements.txt diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 267bb6bf..b825cadb 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -1,13 +1,15 @@ #!/usr/bin/env python -from . import render +#from . import render from . import tilespec from . import client from . import errors from . import stack -from .render import connect +from . import renderapi +from . import renderapi as render +#from .render import connect -from .render import Render +#from .render import Render __all__ = ['render', 'client', 'tilespec', 'errors', - 'stack', 'connect', 'Render'] + 'stack', 'render']#'connect', 'Render'] diff --git a/renderapi/renderapi.py b/renderapi/renderapi.py index 29bf94d5..5eeec319 100755 --- a/renderapi/renderapi.py +++ b/renderapi/renderapi.py @@ -50,8 +50,8 @@ def __init__(self, host=None, port=None, owner=None, project=None, if host == '': logging.critical('Render Host must not be empty!') raise ValueError('Render Host must not be empty!') - host = (host if host.startswith('http') - else 'http://{}'.format(host)) + # host = (host if host.startswith('http') + # else 'http://{}'.format(host)) else: host = os.environ['RENDER_HOST'] self.DEFAULT_HOST = host @@ -64,40 +64,40 @@ def __init__(self, host=None, port=None, owner=None, project=None, logging.critical('Render Port must not be empty!') raise ValueError('Render Port must not be empty!') else: - port = str(int(os.environ['RENDER_PORT'])) + port = int(os.environ['RENDER_PORT']) self.DEFAULT_PORT = port if project is None: if 'RENDER_PROJECT' not in os.environ: project = str(raw_input("Enter Render Project: ")) - if project == '': - logging.critical('Render Project must not be empty!') - raise ValueError('Render Project must not be empty!') else: project = str(os.environ['RENDER_PROJECT']) + if project == '': + logging.critical('Render Project must not be empty!') + raise ValueError('Render Project must not be empty!') self.DEFAULT_PROJECT = project if owner is None: if 'RENDER_OWNER' not in os.environ: owner = str(raw_input("Enter Render Owner: ")) - if owner == '': - logging.critical('Render Owner must not be empty!') - raise ValueError('Render Owner must not be empty!') else: owner = str(os.environ['RENDER_OWNER']) + if owner == '': + logging.critical('Render Owner must not be empty!') + raise ValueError('Render Owner must not be empty!') self.DEFAULT_OWNER = owner if client_scripts is None: if 'RENDER_CLIENT_SCRIPTS' not in os.environ: client_scripts = str(raw_input( "Enter Render Client Scripts location: ")) - if client_scripts == '': - logging.critical('Render Client Scripts must ' - 'not be empty!') - raise ValueError('Render Client Scripts must ' - 'not be empty!') else: client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) + if client_scripts == '': + logging.critical('Render Client Scripts must ' + 'not be empty!') + raise ValueError('Render Client Scripts must ' + 'not be empty!') self.DEFAULT_CLIENT_SCRIPTS = client_scripts logging.debug('Render object created with ' @@ -117,6 +117,7 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, 'host': self.DEFAULT_HOST if host is None else host, 'port': self.DEFAULT_PORT if port is None else port, 'owner': self.DEFAULT_OWNER if owner is None else owner, + 'project': self.DEFAULT_PROJECT if project is None else project, 'client_scripts': (self.DEFAULT_CLIENT_SCRIPTS if client_scripts is None else client_scripts)} processed_kwargs.update(kwargs) diff --git a/renderapi/stack.py b/renderapi/stack.py index 1d0dda78..5dcabd3a 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -1,6 +1,7 @@ import json import logging import requests +from renderapi import Render def format_baseurl(host, port): diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..83c48de8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +numpy From c284b5cd8ccf75570eb49661c32f592379ed30fa Mon Sep 17 00:00:00 2001 From: Russel Date: Mon, 16 Jan 2017 10:16:51 -0800 Subject: [PATCH 008/766] errors: adding error module --- renderapi/errors.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 renderapi/errors.py diff --git a/renderapi/errors.py b/renderapi/errors.py new file mode 100644 index 00000000..94b5e6d8 --- /dev/null +++ b/renderapi/errors.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + + +class ClientScriptError(Exception): + pass From 4e1b28551a38e3ca3c2f848df14676f0c5b34e2e Mon Sep 17 00:00:00 2001 From: Russel Date: Mon, 16 Jan 2017 10:18:48 -0800 Subject: [PATCH 009/766] client: trial-run dependencies and gitignore --- .gitignore | 2 ++ renderapi/client.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..850de96e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +*source_me.sh diff --git a/renderapi/client.py b/renderapi/client.py index d6482ba5..00425cda 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -2,8 +2,8 @@ from functools import partial import logging import subprocess -from renderapi.render import Render # TODO make relative for final -from renderapi.stack import set_stack_state, make_stack_params +from renderapi import Render +from stack import set_stack_state, make_stack_params try: from pathos.multiprocessing import ProcessingPool as Pool From e17fd0df531b41dedee95cef1d6ae26968149d8c Mon Sep 17 00:00:00 2001 From: Russel Date: Mon, 16 Jan 2017 10:28:55 -0800 Subject: [PATCH 010/766] requirements: adding PIL --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 83c48de8..b88e5f3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests numpy +pillow From b6f0ddd912dad23f53124839880fcf2f3080022d Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 12:52:37 -0800 Subject: [PATCH 011/766] client: add exmple docstrings, shebang --- renderapi/client.py | 51 ++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 00425cda..8a32be5f 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import json from functools import partial import logging @@ -17,14 +18,19 @@ def import_single_json_file(stack, jsonfile, render=None, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, verbose=False, **kwargs): + ''' + calls client script to import given jsonfile: + transformFile: ? + ''' + # process render-based default configuration if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - import_single_json_file(stack, jsonfile, **render.make_kwargs( + return import_single_json_file(stack, jsonfile, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, **{'verbose': verbose, 'transformFile': None})) - return + if transformFile is None: transform_params = [] else: @@ -48,16 +54,24 @@ def import_jsonfiles_and_transforms_parallel_by_z( stack, jsonfiles, transformfiles, render=None, poolsize=20, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, verbose=False, **kwargs): + ''' + imports json files and transform files in parallel: + jsonfiles: "list of tilespec" jsons to import + transformfiles: ? + poolsize: number of processes for multiprocessing pool + close_stack: mark render stack as COMPLETE after successful import + ''' + # process render-based default configuration if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - import_jsonfiles_and_transforms_parallel_by_z( + return import_jsonfiles_and_transforms_parallel_by_z( stack, jsonfile, transformfiles, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, **{'verbose': verbose, 'close_stack': close_stack, 'poolsize': poolsize})) - return + set_stack_state(stack, 'LOADING', host, port, owner, project) pool = Pool(poolsize) partial_import = partial(import_single_json_file, stack, render=render, @@ -74,18 +88,23 @@ def import_jsonfiles_parallel( stack, jsonfiles, render=None, poolsize=20, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, verbose=False, **kwargs): + ''' + import jsons using client script in parallel + jsonfiles: list of jsonfiles to upload + poolsize: number of upload processes spawned by multiprocessing pool + transformFile: ? + ''' + # process render-based default configuration if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - import_jsonfiles_parallel( + return import_jsonfiles_parallel( stack, jsonfiles, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, - **{'verbose': verbose, - 'close_stack': close_stack, - 'poolsize': poolsize, - 'transformFile': transformFile})) - return + **{'verbose': verbose, 'close_stack': close_stack, + 'poolsize': poolsize, 'transformFile': transformFile})) + set_stack_state(stack, 'LOADING', host, port, owner, project) pool = Pool(poolsize) partial_import = partial(import_single_json_file, stack, render=render, @@ -104,17 +123,21 @@ def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, verbose=False, **kwargs): + ''' + import jsons using client script serially + jsonfiles: iterator of filenames to be uploaded + transformFile: ? + close_stack: ? + ''' if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - import_jsonfiles( + return import_jsonfiles( stack, jsonfiles, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, - **{'verbose': verbose, - 'close_stack': close_stack, + **{'verbose': verbose, 'close_stack': close_stack, 'transformFile': transformFile})) - return set_stack_state(stack, 'LOADING', host, port, owner, project) if transformFile is None: From 3607b700b6ee287f240c910a53c91f76c6a14d49 Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 12:53:23 -0800 Subject: [PATCH 012/766] render: add connect(), some docstring --- renderapi/renderapi.py | 69 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/renderapi/renderapi.py b/renderapi/renderapi.py index 5eeec319..0b4bff1e 100755 --- a/renderapi/renderapi.py +++ b/renderapi/renderapi.py @@ -37,6 +37,7 @@ # } # ] + class Render(object): def __init__(self, host=None, port=None, owner=None, project=None, client_scripts=None): @@ -109,10 +110,19 @@ def __init__(self, host=None, port=None, owner=None, project=None, @property def DEFAULT_KWARGS(self): + ''' + kwargs to which the render object falls back. Depends on: + self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, + self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS + ''' return self.make_kwargs() def make_kwargs(self, host=None, port=None, owner=None, project=None, client_scripts=None, **kwargs): + ''' + make kwargs using this render object's defaults and any + designated kwargs passed in + ''' processed_kwargs = { 'host': self.DEFAULT_HOST if host is None else host, 'port': self.DEFAULT_PORT if port is None else port, @@ -724,3 +734,62 @@ def get_match_groupIds_to_only(self, matchCollection, owner=None, request_url = self.format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) return self.process_simple_url_request(request_url, session) + + +def connect(host=None, port=None, owner=None, project=None, + client_scripts=None): + '''helper function to connect to a render instance''' + if host is None: + if 'RENDER_HOST' not in os.environ: + host = str(raw_input("Enter Render Host: ")) + if host == '': + logging.critical('Render Host must not be empty!') + raise ValueError('Render Host must not be empty!') + # host = (host if host.startswith('http') + # else 'http://{}'.format(host)) + else: + host = os.environ['RENDER_HOST'] + + if port is None: + if 'RENDER_PORT' not in os.environ: + port = str(int(raw_input("Enter Render Port: "))) + if port == '': + # TODO better (no) port handling + logging.critical('Render Port must not be empty!') + raise ValueError('Render Port must not be empty!') + else: + port = int(os.environ['RENDER_PORT']) + + if project is None: + if 'RENDER_PROJECT' not in os.environ: + project = str(raw_input("Enter Render Project: ")) + else: + project = str(os.environ['RENDER_PROJECT']) + if project == '': + logging.critical('Render Project must not be empty!') + raise ValueError('Render Project must not be empty!') + + if owner is None: + if 'RENDER_OWNER' not in os.environ: + owner = str(raw_input("Enter Render Owner: ")) + else: + owner = str(os.environ['RENDER_OWNER']) + if owner == '': + logging.critical('Render Owner must not be empty!') + raise ValueError('Render Owner must not be empty!') + + # TODO should client_scripts be required? + if client_scripts is None: + if 'RENDER_CLIENT_SCRIPTS' not in os.environ: + client_scripts = str(raw_input( + "Enter Render Client Scripts location: ")) + else: + client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) + if client_scripts == '': + logging.critical('Render Client Scripts must ' + 'not be empty!') + raise ValueError('Render Client Scripts must ' + 'not be empty!') + + return Render(host=host, port=port, owner=owner, project=project, + client_scripts=client_scripts) From f088c22fd56e06b450215ed4eb0c52c63a0770ec Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 13:37:40 -0800 Subject: [PATCH 013/766] docs: moving example scripts to examples --- .../examples/add_downsample_to_render_project.py | 0 .../examples/apply_TEM2_to_render_stack.py | 0 .../examples/apply_alignment_from_render_stack.py | 0 .../examples/apply_alignment_transforms_to_otherStacks2.ipynb | 0 .../examples/apply_ztransforms_to_render_stack.py | 0 create_mipmaps.py => docs/examples/create_mipmaps.py | 0 .../examples/detect_and_drop_stitching_mistakes.py | 0 remove_outer_tiles.py => docs/examples/remove_outer_tiles.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename add_downsample_to_render_project.py => docs/examples/add_downsample_to_render_project.py (100%) rename apply_TEM2_to_render_stack.py => docs/examples/apply_TEM2_to_render_stack.py (100%) rename apply_alignment_from_render_stack.py => docs/examples/apply_alignment_from_render_stack.py (100%) rename apply_alignment_transforms_to_otherStacks2.ipynb => docs/examples/apply_alignment_transforms_to_otherStacks2.ipynb (100%) rename apply_ztransforms_to_render_stack.py => docs/examples/apply_ztransforms_to_render_stack.py (100%) rename create_mipmaps.py => docs/examples/create_mipmaps.py (100%) rename detect_and_drop_stitching_mistakes.py => docs/examples/detect_and_drop_stitching_mistakes.py (100%) rename remove_outer_tiles.py => docs/examples/remove_outer_tiles.py (100%) diff --git a/add_downsample_to_render_project.py b/docs/examples/add_downsample_to_render_project.py similarity index 100% rename from add_downsample_to_render_project.py rename to docs/examples/add_downsample_to_render_project.py diff --git a/apply_TEM2_to_render_stack.py b/docs/examples/apply_TEM2_to_render_stack.py similarity index 100% rename from apply_TEM2_to_render_stack.py rename to docs/examples/apply_TEM2_to_render_stack.py diff --git a/apply_alignment_from_render_stack.py b/docs/examples/apply_alignment_from_render_stack.py similarity index 100% rename from apply_alignment_from_render_stack.py rename to docs/examples/apply_alignment_from_render_stack.py diff --git a/apply_alignment_transforms_to_otherStacks2.ipynb b/docs/examples/apply_alignment_transforms_to_otherStacks2.ipynb similarity index 100% rename from apply_alignment_transforms_to_otherStacks2.ipynb rename to docs/examples/apply_alignment_transforms_to_otherStacks2.ipynb diff --git a/apply_ztransforms_to_render_stack.py b/docs/examples/apply_ztransforms_to_render_stack.py similarity index 100% rename from apply_ztransforms_to_render_stack.py rename to docs/examples/apply_ztransforms_to_render_stack.py diff --git a/create_mipmaps.py b/docs/examples/create_mipmaps.py similarity index 100% rename from create_mipmaps.py rename to docs/examples/create_mipmaps.py diff --git a/detect_and_drop_stitching_mistakes.py b/docs/examples/detect_and_drop_stitching_mistakes.py similarity index 100% rename from detect_and_drop_stitching_mistakes.py rename to docs/examples/detect_and_drop_stitching_mistakes.py diff --git a/remove_outer_tiles.py b/docs/examples/remove_outer_tiles.py similarity index 100% rename from remove_outer_tiles.py rename to docs/examples/remove_outer_tiles.py From f273d7b631cf857fa2411c94cab2ac83de06a461 Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 13:50:53 -0800 Subject: [PATCH 014/766] README: add document links to readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9966b2c5..ecf50132 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@ it presently interacts with render via a web-api, though a couple functions are tilespec.py is a set of python objects that facilitate serializing and deserializing objects to/from json. -add_downsample_to_render_project.py is an example script that makes use of these files to read tilespecs from a render stack, +[add_downsample_to_render_project.py](docs/examples/add_downsample_to_render_project.py) is an example script that makes use of these files to read tilespecs from a render stack, modify the tilespecs to include paths to downsampled images, and then write those tilespecs to disk, upload them back to render, and launch all the jobs to create those downsampled images in a parallel fashion. This program assumes a filestructure path that is unique to the Synapse Biology group at the Allen Institute, but nonetheless demonstrate the general utility of the above files. -create_mipmaps.py is a simple python program for using Pillow to create downsampled images from a single image that is included simply for reference. +[create_mipmaps.py](docs/examples/create_mipmaps.py) is a simple python program for using Pillow to create downsampled images from a single image that is included simply for reference. From 1912e229e837f2e4e8afa7bb62f10656cceed29c Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 16:09:56 -0800 Subject: [PATCH 015/766] render: renderapi->render, add likelyUniqueid --- renderapi/__init__.py | 11 ++++------- renderapi/client.py | 2 +- renderapi/{renderapi.py => render.py} | 0 renderapi/stack.py | 15 +++++++++++++++ 4 files changed, 20 insertions(+), 8 deletions(-) rename renderapi/{renderapi.py => render.py} (100%) diff --git a/renderapi/__init__.py b/renderapi/__init__.py index b825cadb..a82b00c1 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -1,15 +1,12 @@ #!/usr/bin/env python -#from . import render +from . import render from . import tilespec from . import client from . import errors from . import stack -from . import renderapi -from . import renderapi as render -#from .render import connect - -#from .render import Render +from .render import connect +from .render import Render __all__ = ['render', 'client', 'tilespec', 'errors', - 'stack', 'render']#'connect', 'Render'] + 'stack', 'connect', 'Render'] diff --git a/renderapi/client.py b/renderapi/client.py index 8a32be5f..15e47804 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -3,7 +3,7 @@ from functools import partial import logging import subprocess -from renderapi import Render +from render import Render from stack import set_stack_state, make_stack_params try: diff --git a/renderapi/renderapi.py b/renderapi/render.py similarity index 100% rename from renderapi/renderapi.py rename to renderapi/render.py diff --git a/renderapi/stack.py b/renderapi/stack.py index 5dcabd3a..bd4a5569 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import json import logging import requests @@ -36,6 +37,20 @@ def set_stack_state(stack, render=None, state='LOADING', host=None, port=None, return r +def likelyUniqueId(render=None, host=None, port=None, + session=requests.session(), **kwargs): + '''return hex-code nearly-unique id from render server''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return likelyUniqueId(**render.make_kwargs(host=host, port=port, + **{'session': session})) + + request_url = '{}/likelyUniqueId'.format(format_baseurl(host, port)) + return session.get(request_url, data=None, + headers={"content-type": "application/json"}) + + def make_stack_params(host, port, owner, project, stack): '''utility function to turn host,port,owner,project,stack combinations to java CLI based argument list for subprocess calling From 57ec1758414b89587294311517cf08f1235b528a Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 16:25:10 -0800 Subject: [PATCH 016/766] stack: move create and delete stack to stack.py, print to logging --- renderapi/render.py | 30 ------------------------------ renderapi/stack.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 0b4bff1e..c043ac38 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -152,36 +152,6 @@ def process_defaults(self, host, port, owner, project, client_scripts = self.DEFAULT_CLIENT_SCRIPTS return (host, port, owner, project, client_scripts) - def delete_stack(self, stack, host=None, port=None, owner=None, - project=None, session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble(host, port, owner, project, stack) - r = session.delete(request_url) - print r.text - return r - - def create_stack(self, stack, cycleNumber=1, cycleStepNumber=1, - client_scripts=None, host=None, port=None, owner=None, - project=None, verbose=False, session=requests.session()): - - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project, client_scripts) - sv = StackVersion( - cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber) - request_url = self.format_preamble(host, port, owner, project, stack) - if verbose: - print "stack version2", request_url, sv.to_dict() - payload = json.dumps(sv.to_dict()) - r = session.post(request_url, data=payload, - headers={"content-type": "application/json", - "Accept": "application/json"}) - try: - return r - except: - print r.text - return None - def world_to_local_coordinates(self, stack, z, x, y, host=None, port=None, owner=None, project=None, session=requests.session()): diff --git a/renderapi/stack.py b/renderapi/stack.py index bd4a5569..27ee1132 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -2,7 +2,8 @@ import json import logging import requests -from renderapi import Render +from render import Render +from tilespec import StackVersion def format_baseurl(host, port): @@ -61,3 +62,45 @@ def make_stack_params(host, port, owner, project, stack): '--owner', owner, '--project', project] stack_params = project_params + ['--stack', stack] return stack_params + + +def delete_stack(stack, render=None, host=None, port=None, owner=None, + project=None, session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return delete_stack(stack, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session})) + + request_url = format_preamble(host, port, owner, project, stack) + r = session.delete(request_url) + logging.debug(r.text) + return r + + +def create_stack(self, stack, cycleNumber=1, cycleStepNumber=1, + host=None, port=None, owner=None, project=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return create_stack(stack, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session, 'cycleNumber': cycleNumber, + 'cycleStepNumber': cycleStepNumber, 'verbose': verbose})) + + sv = StackVersion( + cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber) + request_url = format_preamble(host, port, owner, project, stack) + if verbose: + print "stack version2", request_url, sv.to_dict() + payload = json.dumps(sv.to_dict()) + r = session.post(request_url, data=payload, + headers={"content-type": "application/json", + "Accept": "application/json"}) + try: + return r + except: + logging.error(r.text) + return None From 1f12d69b355ff4cf455b45c72f1d4983c14d8c35 Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 17:16:32 -0800 Subject: [PATCH 017/766] image: make image-retrieval module --- renderapi/__init__.py | 3 +- renderapi/image.py | 99 +++++++++++++++++++++++++++++++++++++++++++ renderapi/render.py | 2 +- 3 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 renderapi/image.py diff --git a/renderapi/__init__.py b/renderapi/__init__.py index a82b00c1..611365b4 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -5,8 +5,9 @@ from . import client from . import errors from . import stack +from . import image from .render import connect from .render import Render __all__ = ['render', 'client', 'tilespec', 'errors', - 'stack', 'connect', 'Render'] + 'stack', 'image', 'connect', 'Render'] diff --git a/renderapi/image.py b/renderapi/image.py new file mode 100644 index 00000000..0824b285 --- /dev/null +++ b/renderapi/image.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +import io +from PIL import Image +import numpy as np + +# define acceptable image formats -- currently render generates png, jpeg, tiff +IMAGE_FORMATS = {'png': 'png-image', + '.png': 'png-image', + 'jpg': 'jpeg-image', + 'jpeg': 'jpeg-image', + '.jpg': 'jpeg-image', + 'tif': 'tiff-image', + '.tif': 'tiff-image' + 'tiff': 'tiff-image'} + + +def format_baseurl(host, port): + return 'http://%s:%d/render-ws/v1' % (host, port) + + +def format_preamble(host, port, owner, project, stack): + preamble = "%s/owner/%s/project/%s/stack/%s" % ( + format_baseurl(host, port), owner, project, stack) + return preamble + + +def get_bb_image(stack, z, x, y, width, height, render=None, scale=1.0, + host=None, port=None, owner=None, project=None, + img_format=None, session=requests.session(), **kwargs): + ''' + render image from a bounding box defined in xy and return numpy array: + z: layer + x: leftmost point of bounding rectangle + y: topmost pont of bounding rectangle + width: extent to right in x + height: extent down in y + ''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_bb_image( + stack, z, x, y, width, height, + **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'scale': scale, 'img_format': img_format, + 'session': session})) + + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%3.2f/%s" % ( + z, x, y, width, height, scale, image_ext) + r = session.get(request_url) + try: + image = np.asarray(Image.open(io.BytesIO(r.content))) + return image + except: + logging.error(r.text) + + +def get_tile_image_data(stack, tileId, render=None, + normalizeForMatching=True, host=None, port=None, + owner=None, project=None, img_format=None, + session=requests.session(), verbose=False, **kwargs): + ''' + render image from a tile with all transforms and return numpy array + ''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_tile_image_data(stack, tileId, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{img_format': img_format, 'session': session})) + + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = self.format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/png-image" % (tileId) + if normalizeForMatching: + request_url += "?normalizeForMatching=true" + if verbose: + print request_url + r = session.get(request_url) + try: + img = Image.open(io.BytesIO(r.content)) + array = np.asarray(img) + return array + except: + logging.error(r.text) + return None diff --git a/renderapi/render.py b/renderapi/render.py index c043ac38..c37a27f8 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -317,7 +317,7 @@ def get_tile_image_data(self, stack, tileId, normalizeForMatching=True, array = np.asarray(img) return array except: - print (r.text) + logging.error(r.text) return None def world_to_local_coordinates_array(self, stack, dataarray, tileId, z=0, From 2b8f6c8b3ecbd432d64b6a7984d7d4dd11a96a1d Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 17:26:32 -0800 Subject: [PATCH 018/766] image: fix errors, default to png --- renderapi/image.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index 0824b285..99020710 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -11,8 +11,9 @@ 'jpeg': 'jpeg-image', '.jpg': 'jpeg-image', 'tif': 'tiff-image', - '.tif': 'tiff-image' - 'tiff': 'tiff-image'} + '.tif': 'tiff-image', + 'tiff': 'tiff-image', + None: 'png-image'} # Default to png def format_baseurl(host, port): @@ -75,7 +76,7 @@ def get_tile_image_data(stack, tileId, render=None, raise ValueError('invalid Render object specified!') return get_tile_image_data(stack, tileId, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - **{img_format': img_format, 'session': session})) + **{'img_format': img_format, 'session': session})) try: image_ext = IMAGE_FORMATS[img_format] From 50e493146dc20111623689f7611a188ec8f89e8e Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 17:31:16 -0800 Subject: [PATCH 019/766] image: more cleanup, import error --- renderapi/image.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/image.py b/renderapi/image.py index 99020710..b4dab550 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import io +import requests from PIL import Image import numpy as np @@ -83,7 +84,7 @@ def get_tile_image_data(stack, tileId, render=None, except KeyError as e: raise ValueError('{} is not a valid render image format!'.format(e)) - request_url = self.format_preamble( + request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/png-image" % (tileId) if normalizeForMatching: From e920deb3a4db46349bde78643108b4fed9db0e4e Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 18:02:19 -0800 Subject: [PATCH 020/766] pointmatch: added pointmatch calls; render: separated connect() functionality --- renderapi/client.py | 3 + renderapi/pointmatch.py | 214 ++++++++++++++++++++++++++++++++++++++++ renderapi/render.py | 187 ----------------------------------- 3 files changed, 217 insertions(+), 187 deletions(-) create mode 100644 renderapi/pointmatch.py diff --git a/renderapi/client.py b/renderapi/client.py index 15e47804..f43cf333 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +''' +render functions relying on render-ws client scripts +''' import json from functools import partial import logging diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py new file mode 100644 index 00000000..2bf14da3 --- /dev/null +++ b/renderapi/pointmatch.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +''' +Point Match APIs +''' +import requests +from render import Render + + +def format_baseurl(host, port): + return 'http://%s:%d/render-ws/v1' % (host, port) + + +def get_matchcollection_owners(render=None, host=None, port=None, + verbose=False, session=requests.session(), + **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_matchcollection_owners(**render.make_kwargs( + host=host, port=port, **{'verbose': verbose, + 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/matchCollectionOwners" + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_matchcollections(render=None, owner=None, host=None, port=None, + verbose=False, session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_matchcollections(**render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollections" % owner + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_match_groupIds(matchCollection, render=None, owner=None, host=None, + port=None, verbose=False, + session=requests.session()): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_match_groupIds(matchCollection, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_matches_outside_group(matchCollection, groupId, render=None, + owner=None, host=None, port=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_matches_outside_group( + matchCollection, groupId, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( + owner, matchCollection, groupId) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_matches_within_group(matchCollection, groupId, owner=None, + host=None, port=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_matches_within_group( + matchCollection, groupId, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( + owner, matchCollection, groupId) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, + render=None, owner=None, host=None, + port=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_matches_from_group_to_group( + matchCollection, pgroup, qgroup, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( + owner, matchCollection, pgroup, qgroup) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, + qgroup, qid, render=None, owner=None, + host=None, port=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_matches_from_tile_to_tile( + matchCollection, pgroup, pid, qgroup, qid, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + ("/owner/%s/matchCollection/%s/group/%s/id/%s/" + "matchesWith/%s/id/%s" % ( + owner, matchCollection, pgroup, pid, qgroup, qid)) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, + host=None, port=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_matches_with_group( + matchCollection, pgroup, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( + owner, matchCollection, pgroup) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_match_groupIds_from_only(matchCollection, render=None, owner=None, + host=None, port=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_match_groupIds_from_only( + matchCollection, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_match_groupIds_to_only(matchCollection, render=None, owner=None, + host=None, port=None, verbose=False, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_match_groupIds_to_only( + matchCollection, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'verbose': verbose, 'session': session})) + + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) diff --git a/renderapi/render.py b/renderapi/render.py index c37a27f8..3b7b6492 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -41,64 +41,10 @@ class Render(object): def __init__(self, host=None, port=None, owner=None, project=None, client_scripts=None): - - # FIXME: combine port and host into server - # FIXME port handling might have weird casting requirements - # TODO maybe pull this out to a separate renderapi.connect() function? - if host is None: - if 'RENDER_HOST' not in os.environ: - host = str(raw_input("Enter Render Host: ")) - if host == '': - logging.critical('Render Host must not be empty!') - raise ValueError('Render Host must not be empty!') - # host = (host if host.startswith('http') - # else 'http://{}'.format(host)) - else: - host = os.environ['RENDER_HOST'] self.DEFAULT_HOST = host - - if port is None: - if 'RENDER_PORT' not in os.environ: - port = str(int(raw_input("Enter Render Port: "))) - if port == '': - # TODO better (no) port handling - logging.critical('Render Port must not be empty!') - raise ValueError('Render Port must not be empty!') - else: - port = int(os.environ['RENDER_PORT']) self.DEFAULT_PORT = port - - if project is None: - if 'RENDER_PROJECT' not in os.environ: - project = str(raw_input("Enter Render Project: ")) - else: - project = str(os.environ['RENDER_PROJECT']) - if project == '': - logging.critical('Render Project must not be empty!') - raise ValueError('Render Project must not be empty!') self.DEFAULT_PROJECT = project - - if owner is None: - if 'RENDER_OWNER' not in os.environ: - owner = str(raw_input("Enter Render Owner: ")) - else: - owner = str(os.environ['RENDER_OWNER']) - if owner == '': - logging.critical('Render Owner must not be empty!') - raise ValueError('Render Owner must not be empty!') self.DEFAULT_OWNER = owner - - if client_scripts is None: - if 'RENDER_CLIENT_SCRIPTS' not in os.environ: - client_scripts = str(raw_input( - "Enter Render Client Scripts location: ")) - else: - client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) - if client_scripts == '': - logging.critical('Render Client Scripts must ' - 'not be empty!') - raise ValueError('Render Client Scripts must ' - 'not be empty!') self.DEFAULT_CLIENT_SCRIPTS = client_scripts logging.debug('Render object created with ' @@ -299,27 +245,6 @@ def get_z_value_for_section(self, stack, sectionId, project=None, print(r.text) return None - def get_tile_image_data(self, stack, tileId, normalizeForMatching=True, - host=None, port=None, owner=None, project=None, - session=requests.session(), verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/tile/%s/png-image" % (tileId) - if normalizeForMatching: - request_url += "?normalizeForMatching=true" - if verbose: - print request_url - r = session.get(request_url) - try: - img = Image.open(io.BytesIO(r.content)) - array = np.asarray(img) - return array - except: - logging.error(r.text) - return None - def world_to_local_coordinates_array(self, stack, dataarray, tileId, z=0, host=None, port=None, owner=None, project=None, @@ -568,22 +493,6 @@ def batch_local_work(self, stack, z, data, host=None, port=None, return outdata - def get_png_tile(self, stack, z, x, y, width, height, scale=1.0, host=None, - port=None, owner=None, project=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/box/%d,%d,%d,%d,%3.2f/png-image" % ( - z, x, y, width, height, scale) - r = session.get(request_url) - try: - image = np.asarray(Image.open(io.BytesIO(r.content))) - except: - print r.text - return image - def world_to_local_coordinates_batch_local(self, stack, z, data, host=None, port=None, owner=None, project=None): @@ -600,31 +509,6 @@ def local_to_world_coordinates_batch_local(self, stack, z, data, host=None, return batch_local_work(stack, z, data, host, port, owner, project, localToWorld=True) - def get_matchcollection_owners(self, host=None, port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, None, None) - request_url = self.format_baseurl(host, port) + \ - "/matchCollectionOwners" - return self.process_simple_url_request(request_url, session) - - def get_matchcollections(self, owner=None, host=None, port=None, - verbose=False, session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollections" % owner - return self.process_simple_url_request(request_url, session) - - def get_match_groupIds(self, matchCollection, owner=None, host=None, - port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) - return self.process_simple_url_request(request_url, session) - def get_section_z_value(self, stack, sectionId, host=None, port=None, owner=None, project=None, verbose=False, session=requests.session()): @@ -634,77 +518,6 @@ def get_section_z_value(self, stack, sectionId, host=None, port=None, host, port, owner, project, stack) + "/section/%s/z" % sectionId return float(self.process_simple_url_request(request_url, session)) - def get_matches_outside_group(self, matchCollection, groupId, owner=None, - host=None, port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( - owner, matchCollection, groupId) - return self.process_simple_url_request(request_url, session) - - def get_matches_within_group(self, matchCollection, groupId, owner=None, - host=None, port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( - owner, matchCollection, groupId) - return self.process_simple_url_request(request_url, session) - - def get_matches_from_group_to_group(self, matchCollection, pgroup, qgroup, - owner=None, host=None, port=None, - verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( - owner, matchCollection, pgroup, qgroup) - return self.process_simple_url_request(request_url, session) - - def get_matches_from_tile_to_tile(self, matchCollection, pgroup, pid, - qgroup, qid, owner=None, host=None, - port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - ("/owner/%s/matchCollection/%s/group/%s/id/%s/" - "matchesWith/%s/id/%s" % ( - owner, matchCollection, pgroup, pid, qgroup, qid)) - return self.process_simple_url_request(request_url, session) - - def get_matches_with_group(self, matchCollection, pgroup, owner=None, - host=None, port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( - owner, matchCollection, pgroup) - return self.process_simple_url_request(request_url, session) - - def get_match_groupIds_from_only(self, matchCollection, owner=None, - host=None, port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) - return self.process_simple_url_request(request_url, session) - - def get_match_groupIds_to_only(self, matchCollection, owner=None, - host=None, port=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = self.format_baseurl(host, port) + \ - "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) - return self.process_simple_url_request(request_url, session) - def connect(host=None, port=None, owner=None, project=None, client_scripts=None): From 13d74398764a9e4c0167694fa041fe5d0f2710ac Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 18:05:23 -0800 Subject: [PATCH 021/766] render: cleanup imports --- renderapi/render.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 3b7b6492..2386cb8c 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -3,14 +3,10 @@ import json import subprocess import sys -from functools import partial import tempfile -import io -import time import requests import numpy as np -from PIL import Image -from tilespec import TileSpec, StackVersion +from tilespec import TileSpec # import pathos.multiprocessing as mp try: From 6da790ea7bc08cfcc729db21d1c309ca34344f5b Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 17 Jan 2017 18:09:06 -0800 Subject: [PATCH 022/766] more import cleanup --- renderapi/image.py | 1 + renderapi/render.py | 11 +---------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index b4dab550..4358c80e 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -4,6 +4,7 @@ import requests from PIL import Image import numpy as np +from render import Render # define acceptable image formats -- currently render generates png, jpeg, tiff IMAGE_FORMATS = {'png': 'png-image', diff --git a/renderapi/render.py b/renderapi/render.py index 2386cb8c..ec00af90 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python import logging import os import json @@ -8,16 +9,6 @@ import numpy as np from tilespec import TileSpec -# import pathos.multiprocessing as mp -try: - from pathos.multiprocessing import ProcessingPool as Pool - has_pathos = True -except ImportError as e: - logging.warning(e) - has_pathos = False - from multiprocessing import Pool - - # GET http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/world-to-local-coordinates/{x},{y} # curl "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates/40000,40000" # returns: From a7c673ba1bde77db180daa7c9a686f43185df99d Mon Sep 17 00:00:00 2001 From: Russel Date: Wed, 18 Jan 2017 06:38:08 -0800 Subject: [PATCH 023/766] tilespec: PEP8 --- renderapi/tilespec.py | 447 ++++++++++++++++++++++-------------------- 1 file changed, 230 insertions(+), 217 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index a07226b7..96899b56 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,22 +1,23 @@ +import time import numpy as np -class ResolvedTileSpecMap(): - def __init__(self,tilespecs=[],transforms=[]): - + +class ResolvedTileSpecMap: + def __init__(self, tilespecs=[], transforms=[]): self.tilespecs = tilespecs self.transforms = transforms - + def to_dict(self): - d={} - d['tileIdToSpecMap']={} + d = {} + d['tileIdToSpecMap'] = {} for ts in self.tilespecs: - d['tileIdToSpecMap'][ts.tileId]=ts.to_dict() - d['transformIdToSpecMap']={} + d['tileIdToSpecMap'][ts.tileId] = ts.to_dict() + d['transformIdToSpecMap'] = {} for tf in self.transforms: - d['transformIdToSpecMap'][tf.transformId]=tf.to_dict() + d['transformIdToSpecMap'][tf.transformId] = tf.to_dict() return d - - def from_dict(self,d): + + def from_dict(self, d): tsmap = d['tileIdToSpecMap'] tfmap = d['transformIdToSpecMap'] for tsd in tsmap.values(): @@ -27,22 +28,22 @@ def from_dict(self,d): tf.Transform() tf.from_dict(tfd) self.transforms.append(tf) - - -class ResolvedTileSpecCollection(): - def __init__(self,tilespecs=[],transforms=[]): - + + +class ResolvedTileSpecCollection: + def __init__(self, tilespecs=[], transforms=[]): self.tilespecs = tilespecs self.transforms = transforms + def to_dict(self): - d ={} - d['tileCount']=len(self.tilespecs) - d['tileSpecs']=[ts.to_dict() for ts in self.tilespecs] - d['transformCount']=len(self.transforms) - d['transformSpecs']=[tf.to_dict() for tf in self.transforms] + d = {} + d['tileCount'] = len(self.tilespecs) + d['tileSpecs'] = [ts.to_dict() for ts in self.tilespecs] + d['transformCount'] = len(self.transforms) + d['transformSpecs'] = [tf.to_dict() for tf in self.transforms] return d - - def from_dict(self,d): + + def from_dict(self, d): self.tilespecs = [] self.transforms = [] for i in range(d['tileCount']): @@ -56,178 +57,196 @@ def from_dict(self,d): else: tf = Transform() tf.from_dict(tfd) - self.transforms.append(tf) - -class StackVersion(): - def __init__(self,cycleNumber=1,cycleStepNumber=1,stackResolutionX=1, - stackResolutionY=1,stackResolutionZ=1,materializedBoxRootPath=None, - versionNotes="",createTimestamp=None): - import time - - self.cycleNumber=cycleNumber - self.cycleStepNumber=cycleStepNumber - self.stackResolutionX=stackResolutionX - self.stackResolutionY=stackResolutionY - self.stackResolutionZ=stackResolutionZ - self.materializedBoxRootPath=materializedBoxRootPath + self.transforms.append(tf) + + +class StackVersion: + def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=1, + stackResolutionY=1, stackResolutionZ=1, + materializedBoxRootPath=None, versionNotes="", + createTimestamp=None): + self.cycleNumber = cycleNumber + self.cycleStepNumber = cycleStepNumber + self.stackResolutionX = stackResolutionX + self.stackResolutionY = stackResolutionY + self.stackResolutionZ = stackResolutionZ + self.materializedBoxRootPath = materializedBoxRootPath if createTimestamp is None: - createTimestamp=time.strftime('%Y-%M-%dT%H:%M:%S.00Z') + createTimestamp = time.strftime('%Y-%M-%dT%H:%M:%S.00Z') self.createTimestamp = createTimestamp self.versionNotes = versionNotes + def to_dict(self): d = {} - d['cycleNumber']=self.cycleNumber - d['cycleStepNumber']=self.cycleStepNumber - d['stackResolutionX']=self.stackResolutionX - d['stackResolutionY']=self.stackResolutionY - d['stackResolutionZ']=self.stackResolutionZ - d['createTimestamp']=self.createTimestamp - d["materializedBoxRootPath"]= "string" - d['mipmapPathBuilder']={'numberOfLevels':0} - d['versionNotes']=self.versionNotes + d['cycleNumber'] = self.cycleNumber + d['cycleStepNumber'] = self.cycleStepNumber + d['stackResolutionX'] = self.stackResolutionX + d['stackResolutionY'] = self.stackResolutionY + d['stackResolutionZ'] = self.stackResolutionZ + d['createTimestamp'] = self.createTimestamp + d["materializedBoxRootPath"] = "string" + d['mipmapPathBuilder'] = {'numberOfLevels': 0} + d['versionNotes'] = self.versionNotes return d - def from_dict(self,d): + + def from_dict(self, d): for key in d.keys(): - eval('self.%s=d[%s]'%(key,key)) - -class Filter(): - def __init__(self,classname,params={}): + eval('self.%s=d[%s]' % (key, key)) + + +class Filter: + def __init__(self, classname, params={}): self.classname = classname self.params = params - + def to_dict(self): d = {} - d['className']=self.classname + d['className'] = self.classname d['params'] = self.params return d - - def from_dict(self,d): + + def from_dict(self, d): self.classname = d['className'] self.params = d['params'] -class ReferenceTransform(): - def __init__(self,refId=None,json=None): + +class ReferenceTransform: + def __init__(self, refId=None, json=None): if json is not None: self.from_dict(json) else: self.refId = refId + def to_dict(self): - d={} - d['type']='ref' - d['refId']=self.refId + d = {} + d['type'] = 'ref' + d['refId'] = self.refId return d - def from_dict(self,d): + + def from_dict(self, d): self.refId = d['refId'] - + def __str__(self): - return 'ReferenceTransform(%s)'%self.refId + return 'ReferenceTransform(%s)' % self.refId + def __repr__(self): return self.__str__() - -class Transform(): - def __init__(self,className=None,dataString=None,transformId=None,json=None): - + + +class Transform: + def __init__(self, className=None, dataString=None, + transformId=None, json=None): if json is not None: self.from_dict(json) else: self.className = className self.dataString = dataString self.transformId = transformId + def to_dict(self): - d ={} - d['type']='leaf' - d['className']=self.className - d['dataString']=self.dataString + d = {} + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = self.dataString if self.transformId is not None: - d['transformId']=self.transformId + d['transformId'] = self.transformId return d - def from_dict(self,d): + + def from_dict(self, d): self.dataString = d['dataString'] - self.className = d['className'] - self.transformId = d.get('transformId',None) - + self.className = d['className'] + self.transformId = d.get('transformId', None) + def __str__(self): - return 'className:%s\ndataString:%s'%(self.className,self.dataString) + return 'className:%s\ndataString:%s' % ( + self.className, self.dataString) + def __repr__(self): return self.__str__() - - + + class AffineModel(Transform): - className='mpicbg.trakem2.transform.AffineModel2D' - def __init__(self,M00=1.0,M01=0.0,M10=0.0,M11=1.0,B0=0.0,B1=0.0): - self.M00= M00 - self.M01= M01 + className = 'mpicbg.trakem2.transform.AffineModel2D' + + def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0): + self.M00 = M00 + self.M01 = M01 self.M10 = M10 self.M11 = M11 self.B0 = B0 self.B1 = B1 self.className = 'mpicbg.trakem2.transform.AffineModel2D' self.load_M() - + def load_M(self): - self.M = np.identity(4,np.double) - self.M[0,0]=self.M00 - self.M[0,1]=self.M01 - self.M[1,0]=self.M10 - self.M[1,1]=self.M11 - self.M[0,3]=self.B0 - self.M[1,3]=self.B1 - + self.M = np.identity(4, np.double) + self.M[0, 0] = self.M00 + self.M[0, 1] = self.M01 + self.M[1, 0] = self.M10 + self.M[1, 1] = self.M11 + self.M[0, 3] = self.B0 + self.M[1, 3] = self.B1 + def to_dict(self): d = {} - d['type']='leaf' - d['className']=self.className - d['dataString']="%f %f %f %f %f %f"%(self.M[0,0],self.M[0,1],self.M[1,0],self.M[1,1],self.M[0,3],self.M[1,3]) + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = "%f %f %f %f %f %f" % ( + self.M[0, 0], self.M[0, 1], self.M[1, 0], + self.M[1, 1], self.M[0, 3], self.M[1, 3]) return d - - def from_dict(self,d): + + def from_dict(self, d): ds = d['dataString'].split() - (self.M00,self.M01,self.M10,self.M11,self.B0,self.B1)=map(float,ds) + (self.M00, self.M01, self.M10, self.M11, self.B0, self.B1) = map( + float, ds) self.load_M() - + def invert(self): Ai = AffineModel() - Ai.M=np.linalg.inv(self.M) + Ai.M = np.linalg.inv(self.M) return Ai - - def convert_to_point_vector(self,points): + + def convert_to_point_vector(self, points): Np = points.shape[0] - zerovec = np.zeros((Np,1),np.double) - onevec = np.ones((Np,1),np.double) - - if points.shape[1]==2: + zerovec = np.zeros((Np, 1), np.double) + onevec = np.ones((Np, 1), np.double) + + if points.shape[1] == 2: Nd = 2 - points=np.concatenate((points,zerovec),axis=1) - points=np.concatenate((points,onevec),axis=1) - elif points.shape[1]==3: - points=np.concatenate((points,onevec),axis=1) + points = np.concatenate((points, zerovec), axis=1) + points = np.concatenate((points, onevec), axis=1) + elif points.shape[1] == 3: + points = np.concatenate((points, onevec), axis=1) Nd = 3 - assert(points.shape[1]==4) - return points,Nd - - def convert_points_vector_to_array(self,points,Nd): - points=points[:,0:Nd]/np.tile(points[:,3],(Nd,1)).T + assert(points.shape[1] == 4) + return points, Nd + + def convert_points_vector_to_array(self, points, Nd): + points = points[:, 0:Nd] / np.tile(points[:, 3], (Nd, 1)).T return points - - def tform(self,points): - points,Nd = self.convert_to_point_vector(points) - pt=np.dot(self.M,points.T).T - return self.convert_points_vector_to_array(pt,Nd) - - def inverse_tform(self,points): - points,Nd = self.convert_to_point_vector(points) - pt = np.dot(np.linalg.inv(self.M),points.T).T - return self.convert_points_vector_to_array(pt,Nd) - + + def tform(self, points): + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(self.M, points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + def inverse_tform(self, points): + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(np.linalg.inv(self.M), points.T).T + return self.convert_points_vector_to_array(pt, Nd) + def __str__(self): - return "M=[[%f,%f],[%f,%f]] B=[%f,%f]"%(self.M[0,0],self.M[0,1],self.M[1,0],self.M[1,1],self.M[0,3],self.M[1,3]) + return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( + self.M[0, 0], self.M[0, 1], self.M[1, 0], + self.M[1, 1], self.M[0, 3], self.M[1, 3]) - - -class Layout(): - def __init__(self,sectionId=None,scopeId=None,cameraId=None,imageRow=None, - imageCol=None,stageX=None,stageY=None,rotation=None,pixelsize=0.100): + +class Layout: + def __init__(self, sectionId=None, scopeId=None, cameraId=None, + imageRow=None, imageCol=None, stageX=None, stageY=None, + rotation=None, pixelsize=0.100): self.sectionId = str(sectionId) self.scopeId = str(scopeId) self.cameraId = str(cameraId) @@ -250,28 +269,28 @@ def to_dict(self): d['rotation'] = self.rotation d['pixelsize'] = self.pixelsize return d - def from_dict(self,d): - self.sectionId = d.get('sectionId',None) - self.cameraId = d.get('camera',None) - self.scopeId = d.get('temca',None) - self.imageRow = d.get('imageRow',None) - self.imageCol = d.get('imageCol',None) - self.stageX = d.get('stageX',None) - self.stageY = d.get('stageY',None) - self.rotation = d.get('rotation',None) - self.pixelsize = d.get('pixelsize',None) - - -class TileSpec(): - - def __init__(self, - tileId=None,z=None,width=None,height=None,imageUrl=None,frameId=None, - maskUrl=None,minint=0,maxint=65000,layout=Layout(), - tforms = [],inputfilters=[],scale3Url=None,scale2Url=None,scale1Url=None,json=None): + def from_dict(self, d): + self.sectionId = d.get('sectionId', None) + self.cameraId = d.get('camera', None) + self.scopeId = d.get('temca', None) + self.imageRow = d.get('imageRow', None) + self.imageCol = d.get('imageCol', None) + self.stageX = d.get('stageX', None) + self.stageY = d.get('stageY', None) + self.rotation = d.get('rotation', None) + self.pixelsize = d.get('pixelsize', None) + + +class TileSpec: + def __init__(self, tileId=None, z=None, width=None, height=None, + imageUrl=None, frameId=None, maskUrl=None, + minint=0, maxint=65000, layout=Layout(), tforms=[], + inputfilters=[], scale3Url=None, scale2Url=None, + scale1Url=None, json=None): if json is not None: self.from_dict(json) - else: + else: self.tileId = tileId self.z = z self.width = width @@ -279,65 +298,64 @@ def __init__(self, self.layout = layout self.imageUrl = imageUrl self.maskUrl = maskUrl - self.minint= minint + self.minint = minint self.maxint = maxint self.tforms = tforms self.frameId = frameId - self.layout=layout + self.layout = layout self.inputfilters = inputfilters self.scale3Url = scale3Url self.scale2Url = scale2Url self.scale1Url = scale1Url - def to_dict(self): - thedict={} - thedict['tileId']=self.tileId - thedict['z']=self.z - thedict['width']=self.width - thedict['height']=self.height - thedict['minIntensity']=self.minint - thedict['maxIntensity']=self.maxint - thedict['frameId']=self.frameId + thedict = {} + thedict['tileId'] = self.tileId + thedict['z'] = self.z + thedict['width'] = self.width + thedict['height'] = self.height + thedict['minIntensity'] = self.minint + thedict['maxIntensity'] = self.maxint + thedict['frameId'] = self.frameId if self.layout is not None: - thedict['layout']=self.layout.to_dict() - mipmapdict={} - mipmapdict['0']={} - mipmapdict['0']['imageUrl']=self.imageUrl + thedict['layout'] = self.layout.to_dict() + mipmapdict = {} + mipmapdict['0'] = {} + mipmapdict['0']['imageUrl'] = self.imageUrl if self.scale1Url is not None: - mipmapdict['1']={} - mipmapdict['1']['imageUrl']=self.scale1Url + mipmapdict['1'] = {} + mipmapdict['1']['imageUrl'] = self.scale1Url if self.scale3Url is not None: - mipmapdict['3']={} - mipmapdict['3']['imageUrl']=self.scale3Url + mipmapdict['3'] = {} + mipmapdict['3']['imageUrl'] = self.scale3Url if self.scale2Url is not None: - mipmapdict['2']={} - mipmapdict['2']['imageUrl']=self.scale2Url + mipmapdict['2'] = {} + mipmapdict['2']['imageUrl'] = self.scale2Url if self.maskUrl is not None: - mipmapdict['0']['maskUrl']=self.maskUrl - thedict['mipmapLevels']=mipmapdict - thedict['transforms']={} - thedict['transforms']['type']='list' - #thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] - thedict['transforms']['specList'] = [] - for t in self.tforms: - strlist = {} - #added by sharmi - if your speclist contains a speclist (can happen if you run the optimization more than once) - if isinstance(t,list): - strlist['type'] = 'list' - strlist['specList'] = [tt.to_dict() for tt in t] - thedict['transforms']['specList'].append(strlist) - else: - thedict['transforms']['specList'].append(t.to_dict()) - ############################################################################################################ - - - thedict['inputfilters']={} - thedict['inputfilters']['type']='list' - thedict['inputfilters']['specList']=[f.to_dict() for f in self.inputfilters] + mipmapdict['0']['maskUrl'] = self.maskUrl + thedict['mipmapLevels'] = mipmapdict + thedict['transforms'] = {} + thedict['transforms']['type'] = 'list' + # thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] + thedict['transforms']['specList'] = [] + for t in self.tforms: + strlist = {} + # added by sharmi - if your speclist contains a speclist (can + # happen if you run the optimization more than once) + if isinstance(t, list): + strlist['type'] = 'list' + strlist['specList'] = [tt.to_dict() for tt in t] + thedict['transforms']['specList'].append(strlist) + else: + thedict['transforms']['specList'].append(t.to_dict()) + + thedict['inputfilters'] = {} + thedict['inputfilters']['type'] = 'list' + thedict['inputfilters']['specList'] = [f.to_dict() for f + in self.inputfilters] return thedict - - def from_dict(self,d): + + def from_dict(self, d): '''Method to load tilespec from json dictionary''' self.tileId = d['tileId'] self.z = d['z'] @@ -345,48 +363,43 @@ def from_dict(self,d): self.height = d['height'] self.minint = d['minIntensity'] self.maxint = d['maxIntensity'] - self.frameId = d.get('frameId',None) + self.frameId = d.get('frameId', None) self.layout = Layout() self.layout.from_dict(d['layout']) - self.minX = d.get('minX',None) - self.maxX = d.get('maxX',None) - self.maxY = d.get('maxY',None) - self.minY = d.get('minY',None) + self.minX = d.get('minX', None) + self.maxX = d.get('maxX', None) + self.maxY = d.get('maxY', None) + self.minY = d.get('minY', None) self.imageUrl = d['mipmapLevels']['0']['imageUrl'] - self.maskUrl = d['mipmapLevels']['0'].get('maskUrl',None) - if d['mipmapLevels'].get('2',None) is not None: - self.scale2Url = d['mipmapLevels']['2'].get('imageUrl',None) + self.maskUrl = d['mipmapLevels']['0'].get('maskUrl', None) + if d['mipmapLevels'].get('2', None) is not None: + self.scale2Url = d['mipmapLevels']['2'].get('imageUrl', None) else: self.scale2Url = None - if d['mipmapLevels'].get('1',None) is not None: - self.scale1Url = d['mipmapLevels']['1'].get('imageUrl',None) + if d['mipmapLevels'].get('1', None) is not None: + self.scale1Url = d['mipmapLevels']['1'].get('imageUrl', None) else: self.scale1Url = None - if d['mipmapLevels'].get('3',None) is not None: - self.scale3Url = d['mipmapLevels']['3'].get('imageUrl',None) + if d['mipmapLevels'].get('3', None) is not None: + self.scale3Url = d['mipmapLevels']['3'].get('imageUrl', None) else: self.scale3Url = None self.tforms = [] for t in d['transforms']['specList']: - if t['type']=='ref': + if t['type'] == 'ref': tf = ReferenceTransform(refId=t['refId']) - elif t['type']=='leaf': - if t['className']==AffineModel.className: + elif t['type'] == 'leaf': + if t['className'] == AffineModel.className: tf = AffineModel() tf.from_dict(t) else: tf = Transform(json=t) self.tforms.append(tf) self.inputfilters = [] - if d.get('inputfilters',None) is not None: + if d.get('inputfilters', None) is not None: for f in d['inputfilters']['specList']: f['type'] f = Filter() f.from_dict(f) self.inputfilters.append(f) - - - - - From 963da6ff283da8c0c747c9dd6657527bb94cf150 Mon Sep 17 00:00:00 2001 From: Russel Date: Wed, 18 Jan 2017 06:47:50 -0800 Subject: [PATCH 024/766] stack: move StacVersion to stack --- renderapi/stack.py | 36 +++++++++++++++++++++++++++++++++++- renderapi/tilespec.py | 35 ----------------------------------- 2 files changed, 35 insertions(+), 36 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 27ee1132..a6853478 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -1,9 +1,9 @@ #!/usr/bin/env python import json import logging +from time import strftime import requests from render import Render -from tilespec import StackVersion def format_baseurl(host, port): @@ -16,6 +16,40 @@ def format_preamble(host, port, owner, project, stack): return preamble +class StackVersion: + def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=1, + stackResolutionY=1, stackResolutionZ=1, + materializedBoxRootPath=None, versionNotes="", + createTimestamp=None): + self.cycleNumber = cycleNumber + self.cycleStepNumber = cycleStepNumber + self.stackResolutionX = stackResolutionX + self.stackResolutionY = stackResolutionY + self.stackResolutionZ = stackResolutionZ + self.materializedBoxRootPath = materializedBoxRootPath + if createTimestamp is None: + createTimestamp = strftime('%Y-%M-%dT%H:%M:%S.00Z') + self.createTimestamp = createTimestamp + self.versionNotes = versionNotes + + def to_dict(self): + d = {} + d['cycleNumber'] = self.cycleNumber + d['cycleStepNumber'] = self.cycleStepNumber + d['stackResolutionX'] = self.stackResolutionX + d['stackResolutionY'] = self.stackResolutionY + d['stackResolutionZ'] = self.stackResolutionZ + d['createTimestamp'] = self.createTimestamp + d["materializedBoxRootPath"] = "string" + d['mipmapPathBuilder'] = {'numberOfLevels': 0} + d['versionNotes'] = self.versionNotes + return d + + def from_dict(self, d): + for key in d.keys(): + eval('self.%s=d[%s]' % (key, key)) + + def set_stack_state(stack, render=None, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), verbose=False, **kwargs): diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 96899b56..f489c8ae 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,4 +1,3 @@ -import time import numpy as np @@ -60,40 +59,6 @@ def from_dict(self, d): self.transforms.append(tf) -class StackVersion: - def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=1, - stackResolutionY=1, stackResolutionZ=1, - materializedBoxRootPath=None, versionNotes="", - createTimestamp=None): - self.cycleNumber = cycleNumber - self.cycleStepNumber = cycleStepNumber - self.stackResolutionX = stackResolutionX - self.stackResolutionY = stackResolutionY - self.stackResolutionZ = stackResolutionZ - self.materializedBoxRootPath = materializedBoxRootPath - if createTimestamp is None: - createTimestamp = time.strftime('%Y-%M-%dT%H:%M:%S.00Z') - self.createTimestamp = createTimestamp - self.versionNotes = versionNotes - - def to_dict(self): - d = {} - d['cycleNumber'] = self.cycleNumber - d['cycleStepNumber'] = self.cycleStepNumber - d['stackResolutionX'] = self.stackResolutionX - d['stackResolutionY'] = self.stackResolutionY - d['stackResolutionZ'] = self.stackResolutionZ - d['createTimestamp'] = self.createTimestamp - d["materializedBoxRootPath"] = "string" - d['mipmapPathBuilder'] = {'numberOfLevels': 0} - d['versionNotes'] = self.versionNotes - return d - - def from_dict(self, d): - for key in d.keys(): - eval('self.%s=d[%s]' % (key, key)) - - class Filter: def __init__(self, classname, params={}): self.classname = classname From 2f9c7a1bf0b7a054b7182fc81fa300db10ca304d Mon Sep 17 00:00:00 2001 From: Russel Date: Wed, 18 Jan 2017 07:09:18 -0800 Subject: [PATCH 025/766] render: move format_ functions to render --- renderapi/image.py | 12 +----- renderapi/pointmatch.py | 6 +-- renderapi/render.py | 85 ++++++----------------------------------- renderapi/stack.py | 12 +----- 4 files changed, 15 insertions(+), 100 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index 4358c80e..834bb065 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -4,7 +4,7 @@ import requests from PIL import Image import numpy as np -from render import Render +from render import Render, format_baseurl, format_preamble # define acceptable image formats -- currently render generates png, jpeg, tiff IMAGE_FORMATS = {'png': 'png-image', @@ -18,16 +18,6 @@ None: 'png-image'} # Default to png -def format_baseurl(host, port): - return 'http://%s:%d/render-ws/v1' % (host, port) - - -def format_preamble(host, port, owner, project, stack): - preamble = "%s/owner/%s/project/%s/stack/%s" % ( - format_baseurl(host, port), owner, project, stack) - return preamble - - def get_bb_image(stack, z, x, y, width, height, render=None, scale=1.0, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), **kwargs): diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 2bf14da3..358208ac 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -3,11 +3,7 @@ Point Match APIs ''' import requests -from render import Render - - -def format_baseurl(host, port): - return 'http://%s:%d/render-ws/v1' % (host, port) +from render import Render, format_baseurl def get_matchcollection_owners(render=None, host=None, port=None, diff --git a/renderapi/render.py b/renderapi/render.py index ec00af90..31e4b2a4 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -324,16 +324,6 @@ def format_preamble(self, host, port, owner, project, stack): self.format_baseurl(host, port), owner, project, stack) return preamble - def process_simple_url_request(self, request_url, session): - r = session.get(request_url) - try: - #print(r.json()) - return r.json() - except: - #print e - print(r.text) - return None - def put_resolved_tilespecs(self, stack, data, host=None, port=None, owner=None, project=None, session=requests.session(), verbose=False): @@ -345,71 +335,10 @@ def put_resolved_tilespecs(self, stack, data, host=None, port=None, print request_url r = session.put(request_url, data=data, - headers={"content-type":"application/json", - "Accept":"text/plain"}) + headers={"content-type": "application/json", + "Accept": "text/plain"}) return r - - # http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/tile/140422184419060139 - def get_tile_spec(self, stack, tile, host=None, port=None, owner=None, - project=None, session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/tile/%s/render-parameters" % (tile) - - tilespec_json = self.process_simple_url_request(request_url, session) - - return TileSpec(json=tilespec_json['tileSpecs'][0]) - - def get_tile_specs_from_minmax_box(self, stack, z, xmin, xmax, ymin, ymax, - scale=1.0, host=None, port=None, - owner=None, project=None, - session=requests.session(), - verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - x = xmin - y = ymin - width = xmax - xmin - height = ymax - ymin - return self.get_tile_specs_from_box(stack, z, x, y, width, height, - scale, host, port, owner, project, - session, verbose) - - def get_tile_specs_from_box(self, stack, z, x, y, width, height, scale=1.0, - host=None, port=None, owner=None, project=None, - session=requests.session(), verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( - z, x, y, width, height, scale) - if verbose: - print request_url - tilespecs_json = self.process_simple_url_request( - request_url, session)['tileSpecs'] - return [TileSpec(json=tilespec_json) - for tilespec_json in tilespecs_json] - - def get_tile_specs_from_z(self, stack, z, host=None, port=None, owner=None, - project=None, session=requests.session(), - verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) - if verbose: - print request_url - tilespecs_json = self.process_simple_url_request(request_url, session) - if len(tilespecs_json) == 0: - return None - else: - return [TileSpec(json=tilespec_json) - for tilespec_json in tilespecs_json] - def get_bounds_from_z(self, stack, z, host=None, port=None, owner=None, project=None, session=requests.session()): (host, port, owner, project, client_scripts) = self.process_defaults( @@ -563,3 +492,13 @@ def connect(host=None, port=None, owner=None, project=None, return Render(host=host, port=port, owner=owner, project=project, client_scripts=client_scripts) + + +def format_baseurl(host, port): + return 'http://%s:%d/render-ws/v1' % (host, port) + + +def format_preamble(host, port, owner, project, stack): + preamble = "%s/owner/%s/project/%s/stack/%s" % ( + format_baseurl(host, port), owner, project, stack) + return preamble diff --git a/renderapi/stack.py b/renderapi/stack.py index a6853478..290d6779 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,17 +3,7 @@ import logging from time import strftime import requests -from render import Render - - -def format_baseurl(host, port): - return 'http://%s:%d/render-ws/v1' % (host, port) - - -def format_preamble(host, port, owner, project, stack): - preamble = "%s/owner/%s/project/%s/stack/%s" % ( - format_baseurl(host, port), owner, project, stack) - return preamble +from render import Render, format_baseurl, format_preamble class StackVersion: From 49dbd0421df4e3ef0ab6c2216b10a6e2ddf1342f Mon Sep 17 00:00:00 2001 From: Russel Date: Wed, 18 Jan 2017 07:37:13 -0800 Subject: [PATCH 026/766] tilespec: move tilespec functionality into tilespec --- renderapi/render.py | 1 - renderapi/tilespec.py | 100 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index 31e4b2a4..e2af5892 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -7,7 +7,6 @@ import tempfile import requests import numpy as np -from tilespec import TileSpec # GET http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/world-to-local-coordinates/{x},{y} # curl "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates/40000,40000" diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index f489c8ae..f81db0cd 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python +from render import Render, format_baseurl, format_preamble import numpy as np @@ -368,3 +370,101 @@ def from_dict(self, d): f = Filter() f.from_dict(f) self.inputfilters.append(f) + + +def get_tile_spec(stack, tile, render=None, host=None, port=None, owner=None, + project=None, session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_tile_spec( + stack, tile, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/render-parameters" % (tile) + r = session.get(request_url) + try: + tilespec_json = r.json() + except: + logging.error(r.text) + return TileSpec(json=tilespec_json['tileSpecs'][0]) + + +def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, + render=None, scale=1.0, host=None, + port=None, owner=None, project=None, + session=requests.session(), + verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_tile_specs_from_minmax_box( + stack, z, xmin, xmax, ymin, ymax, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session, 'verbose': verbose})) + + x = xmin + y = ymin + width = xmax - xmin + height = ymax - ymin + return get_tile_specs_from_box(stack, z, x, y, width, height, + scale, host, port, owner, project, + session, verbose) + + +def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, + scale=1.0, host=None, port=None, owner=None, + project=None, session=requests.session(), + verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_tile_specs_from_box( + stack, z, x, y, width, height, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session, 'verbose': verbose})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( + z, x, y, width, height, scale) + if verbose: + print request_url + r = session.get(request_url) + try: + tilespecs_json = r.json() + except: + logging.error(r.text) + return [TileSpec(json=tilespec_json) + for tilespec_json in tilespecs_json['tileSpecs']] + + +def get_tile_specs_from_z(stack, z, render=None, host=None, port=None, + owner=None, project=None, session=requests.session(), + verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_tile_specs_from_box( + stack, z, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session, 'verbose': verbose})) + + request_url = format_preamble( + host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) + if verbose: + print request_url + r = session.get(request_url) + try: + tilespecs_json = r.json() + except: + logging.error(r.text) + + if len(tilespecs_json) == 0: + return None + else: + return [TileSpec(json=tilespec_json) + for tilespec_json in tilespecs_json] From 31eb10ed637766173d3a27248f2ed6dd022225b6 Mon Sep 17 00:00:00 2001 From: Russel Date: Sun, 22 Jan 2017 15:18:48 -0800 Subject: [PATCH 027/766] image, pointmatch, stack, tilespec: relative imports --- renderapi/image.py | 2 +- renderapi/pointmatch.py | 2 +- renderapi/stack.py | 2 +- renderapi/tilespec.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index 834bb065..30fe5a5d 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -4,7 +4,7 @@ import requests from PIL import Image import numpy as np -from render import Render, format_baseurl, format_preamble +from .render import Render, format_baseurl, format_preamble # define acceptable image formats -- currently render generates png, jpeg, tiff IMAGE_FORMATS = {'png': 'png-image', diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 358208ac..4d860d23 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -3,7 +3,7 @@ Point Match APIs ''' import requests -from render import Render, format_baseurl +from .render import Render, format_baseurl def get_matchcollection_owners(render=None, host=None, port=None, diff --git a/renderapi/stack.py b/renderapi/stack.py index 290d6779..7844fb7c 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,7 +3,7 @@ import logging from time import strftime import requests -from render import Render, format_baseurl, format_preamble +from .render import Render, format_baseurl, format_preamble class StackVersion: diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index f81db0cd..58383f33 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from render import Render, format_baseurl, format_preamble +from .render import Render, format_baseurl, format_preamble import numpy as np From 346b34bae9b936a1c6fb3d155a95c3e8963995ff Mon Sep 17 00:00:00 2001 From: Russel Date: Sun, 22 Jan 2017 15:19:33 -0800 Subject: [PATCH 028/766] errors: docstring --- renderapi/errors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/renderapi/errors.py b/renderapi/errors.py index 94b5e6d8..dde1e282 100644 --- a/renderapi/errors.py +++ b/renderapi/errors.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +''' +Custom errors for render api +''' class ClientScriptError(Exception): From 36a673ddcdce472f39dbd624c894e4fa4c910b59 Mon Sep 17 00:00:00 2001 From: Russel Date: Sun, 22 Jan 2017 15:20:34 -0800 Subject: [PATCH 029/766] client: include java import/validate, relative imports --- renderapi/client.py | 52 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index f43cf333..58a4db15 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -6,8 +6,8 @@ from functools import partial import logging import subprocess -from render import Render -from stack import set_stack_state, make_stack_params +from .render import Render +from .stack import set_stack_state, make_stack_params try: from pathos.multiprocessing import ProcessingPool as Pool @@ -162,3 +162,51 @@ def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, print proc.stdout.read() if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +# FIXME paperweight for proper render_ws_client implemntation +# TODO this should allow you to set validator parameters +def import_jsonfiles_validate_client(stack, jsonfiles, render=None, + transformFile=None, client_scripts=None, + host=None, port=None, owner=None, + project=None, close_stack=True, mem=6, + **kwargs): + ''' + Uses java client for parallelization and validation + ''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return import_jsonfiles_validate_client( + stack, jsonfiles, **render.make_kwargs( + host=host, port=port, owner=owner, + project=project, client_scripts=client_scripts, + **{'close_stack': close_stack, 'mem'=mem, + 'transformFile': transformFile})) + + transform_params = (['--transformFile', transformFile] + if transformFile is not None else []) + validator_params = [ + '--validatorClass', + 'org.janelia.alignment.spec.validator.TemTileSpecValidator', + '--validatorData', + 'minCoordinate:-500,maxCoordinate:50000,minSize:500,maxSize:10000'] + my_env = os.environ.copy() + stack_params = make_stack_params(host, port, owner, project, stack) + cmd = [os.path.join(client_scripts, 'run_ws_client.sh')] + \ + ['{}G'.format(str(int(mem))), + 'org.janelia.render.client.ImportJsonClient'] + \ + stack_params + \ + validator_params + \ + transform_params + \ + jsonfiles + + set_stack_state(stack, 'LOADING', host, port, owner, project) + logging.debug(cmd) + + proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) + proc.wait() + logging.debug(proc.stdout.read()) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) From f14d42dda7764adb15d2751b31f9cdc3ea279ef1 Mon Sep 17 00:00:00 2001 From: Russel Date: Mon, 23 Jan 2017 13:56:24 -0800 Subject: [PATCH 030/766] coordinate: add corrdinate mapping module --- renderapi/__init__.py | 7 +- renderapi/coordinate.py | 163 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 renderapi/coordinate.py diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 611365b4..10be0385 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -2,12 +2,15 @@ from . import render from . import tilespec -from . import client from . import errors from . import stack +from . import client from . import image +from . import pointmatch +from . import coordinate from .render import connect from .render import Render __all__ = ['render', 'client', 'tilespec', 'errors', - 'stack', 'image', 'connect', 'Render'] + 'stack', 'image', 'pointmatch', 'coordinate', + 'connect', 'Render'] diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py new file mode 100644 index 00000000..d4758ff1 --- /dev/null +++ b/renderapi/coordinate.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +''' +coordinate mapping functions for render api +''' + +from .render import Render, format_preamble + + +def world_to_local_coordinates(stack, z, x, y, render=None, host=None, + port=None, owner=None, project=None, + session=requests.session(), **kwargs): + '''''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return world_to_local_coordinates(stack, z, x, y, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def local_to_world_coordinates(stack, tileId, x, y, render=None, + host=None, port=None, owner=None, project=None, + session=requests.session(), **kwargs): + '''''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return local_to_world_coordinates( + stack, tileId, x, y, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def world_to_local_coordinates_batch(stack, z, data, render=None, host=None, + port=None, owner=None, project=None, + session=requests.session(), **kwargs): + '''''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return world_to_local_coordinates_batch( + stack, z, data, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/world-to-local-coordinates" % (z) + r = session.put(request_url, data=data, + headers={"content-type": "application/json"}) + return r.json() + + +# FIXME different inputs than world_to_local? +def local_to_world_coordinates_batch(stack, data, z, render=None, host=None, + port=None, owner=None, project=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return local_to_world_coordinates_batch( + stack, data, z, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/local-to-world-coordinates" % (z) + r = session.put(request_url, data=data, + headers={"content-type": "application/json"}) + return r.json() + + +def world_to_local_coordinates_array(stack, dataarray, tileId, z=0, + render=None, host=None, port=None, + owner=None, project=None, + session=requests.session(), **kwargs): + '''''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return world_to_local_coordinates_array( + stack, dataarray, tileId, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session, 'z': z})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/world-to-local-coordinates" % (z) + dlist = [] + for i in range(dataarray.shape[0]): + d = {} + d['tileId'] = tileId + d['world'] = [dataarray[i, 0], dataarray[i, 1]] + dlist.append(d) + jsondata = json.dumps(dlist) + r = session.put(request_url, data=jsondata, + headers={"content-type": "application/json"}) + json_answer = r.json() + try: + answer = np.zeros(dataarray.shape) + for i, coord in enumerate(json_answer): + c = coord['local'] + answer[i, 0] = c[0] + answer[i, 1] = c[1] + return answer + except: + logging.error(json_answer) + + +def local_to_world_coordinates_array(stack, dataarray, tileId, z=0, + render=None, host=None, port=None, + owner=None, project=None, + session=requests.session(), **kwargs): + '''''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return local_to_world_coordinates_array( + stack, dataarray, tileId, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session, 'z': z})) + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/local-to-world-coordinates" % (z) + dlist = [] + for i in range(dataarray.shape[0]): + d = {} + d['tileId'] = tileId + d['local'] = [dataarray[i, 0], dataarray[i, 1]] + dlist.append(d) + jsondata = json.dumps(dlist) + r = session.put(request_url, data=jsondata, + headers={"content-type": "application/json"}) + json_answer = r.json() + try: + answer = np.zeros(dataarray.shape) + for i, coord in enumerate(json_answer): + c = coord['world'] + answer[i, 0] = c[0] + answer[i, 1] = c[1] + return answer + except: + logging.error(json_answer) From 68f881dcf67e16703556c437508cfcc6d4f05c6e Mon Sep 17 00:00:00 2001 From: Russel Date: Mon, 23 Jan 2017 13:57:13 -0800 Subject: [PATCH 031/766] setup: add Brett's simple_setup for setuptools develop --- other_setup.py | 370 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 other_setup.py diff --git a/other_setup.py b/other_setup.py new file mode 100644 index 00000000..49598c2b --- /dev/null +++ b/other_setup.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" distribute- and pip-enabled setup.py """ + +try: + import ConfigParser +except ImportError as E: + import configparser + +import logging +import os +import re +import sys + +# ----- overrides ----- + +# set these to anything but None to override the automatic defaults +author = 'Forrest Collman, Eric Perlman, Sharmi Seshamani' +author_email = 'forrest.collman@gmail.com' +dependency_links = None +long_description = ' a python API setup to interact via python with render'\ + ' databases see https://github.com/saalfeldlab/render' +packages = None +package_name = None +package_data = None +scripts = None +requirements_file = None +requirements = None +version = '0.01' +test_suite = None + +# --------------------- + + +# ----- control flags ----- + +# fallback to setuptools if distribute isn't found +setup_tools_fallback = False + +# don't include subdir named 'tests' in package_data +skip_tests = True + +# print some extra debugging info +debug = True + +# use numpy.distutils instead of setuptools +use_numpy = False + +# ------------------------- +update_url = "https://raw.githubusercontent.com/braingram/simple_setup/master/setup.py" + +# this next line is important for the 'fetch' option (see below) +# MARK +if (len(sys.argv) > 1) and sys.argv[1] == 'fetch': + _overrides = {} + _locals = locals() + for _k in _locals.keys(): + if (_k[0] != '_') and not isinstance(_locals[_k], type(sys)): + _overrides[_k] = _locals[_k] + if len(sys.argv) > 2: + target_fn = sys.argv[2] + else: + target_fn = __file__ + print("Fetching a new simple_setup.py to {}".format(target_fn)) + import urllib2 + new_ss = urllib2.urlopen(update_url) + with open(target_fn, 'w') as target: + found_mark = False + for l in new_ss: + if found_mark or len(l.strip()) == 0: + target.write(l) + else: + if l[0] == '#': + if l.strip() == '# MARK': + found_mark = True + target.write(l) + continue + lt = l.split('=') + key = lt[0].strip() + if (len(lt) == 2) and (key in _overrides): + # copy over the overrides + target.write("{} = {!r}\n".format(key, _overrides[key])) + else: + target.write(l) + continue + print("successfully fetched new setup.py") + sys.exit(0) + +if debug: + logging.basicConfig(level=logging.DEBUG) +# distribute import and testing +try: + import distribute_setup + distribute_setup.use_setuptools() + logging.debug("distribute_setup.py imported and used") +except ImportError: + # fallback to setuptools? + # distribute_setup.py was not in this directory + if not (setup_tools_fallback): + import setuptools + # check if setuptools is distribute + vt = setuptools.__version__.split('.') + if len(vt) == 1: + vmajor = int(vt[0]) + vminor = 0 + elif len(vt) > 1: + vmajor = int(vt[0]) + vminor = int(vt[1]) + if (hasattr(setuptools, '_distribute') and + setuptools._distribute) or (vmajor > 0 or vminor > 6): + logging.debug("distribute_setup.py not found, " + "defaulted to system distribute") + else: + raise ImportError( + "distribute was not found and fallback " + "to setuptools was not allowed") + else: + logging.debug("distribute_setup.py not found, " + "defaulting to system setuptools") + +import setuptools + + +def find_scripts(): + return [s for s in setuptools.findall('scripts/') + if os.path.splitext(s)[1] != '.pyc'] + + +def package_to_path(package): + """ + Convert a package (as found by setuptools.find_packages) + e.g. "foo.bar" to usable path + e.g. "foo/bar" + No idea if this works on windows + """ + return package.replace('.', '/') + + +def find_subdirectories(package): + """ + Get the subdirectories within a package + This will include resources (non-submodules) and submodules + """ + try: + subdirectories = next(os.walk(package_to_path(package)))[1] + except StopIteration: + subdirectories = [] + return subdirectories + + +def subdir_findall(dir, subdir): + """ + Find all files in a subdirectory and return paths relative to dir + This is similar to (and uses) setuptools.findall + However, the paths returned are in the form needed for package_data + """ + strip_n = len(dir.split('/')) + path = '/'.join((dir, subdir)) + return ['/'.join(s.split('/')[strip_n:]) for s in setuptools.findall(path)] + + +def find_package_data(packages): + """ + For a list of packages, find the package_data + This function scans the subdirectories of a package and considers all + non-submodule subdirectories as resources, including them in + the package_data + Returns a dictionary suitable for setup(package_data=) + """ + package_data = {} + for package in packages: + package_data[package] = [] + for subdir in find_subdirectories(package): + if '.'.join((package, subdir)) in packages: # skip submodules + logging.debug("skipping submodule %s/%s" % (package, subdir)) + continue + if skip_tests and (subdir == 'tests'): # skip tests + logging.debug("skipping tests %s/%s" % (package, subdir)) + continue + package_data[package] += \ + subdir_findall(package_to_path(package), subdir) + return package_data + + +def parse_requirements(file_name): + """ + from: + http://cburgmer.posterous.com/pip-requirementstxt-and-setuppy + """ + requirements = [] + with open(file_name, 'r') as f: + for line in f: + if re.match(r'(\s*#)|(\s*$)', line): + continue + if re.match(r'\s*-e\s+', line): + requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', + r'\1', line).strip()) + elif re.match(r'\s*-f\s+', line): + pass + else: + requirements.append(line.strip()) + return requirements + + +def parse_dependency_links(file_name): + """ + from: + http://cburgmer.posterous.com/pip-requirementstxt-and-setuppy + """ + dependency_links = [] + with open(file_name) as f: + for line in f: + if re.match(r'\s*-[ef]\s+', line): + dependency_links.append(re.sub(r'\s*-[ef]\s+', + '', line)) + return dependency_links + + +def detect_version(): + """ + Try to detect the main package/module version by looking at: + module.__version__ + otherwise, return 'dev' + """ + try: + m = __import__(package_name, fromlist=['__version__']) + if hasattr(m, '__version__'): + return m.__version__ + except ImportError: + pass + return 'dev' + + +def author_info_from_pypirc(): + """ + Try to read author name and email from ~/.pypirc (section simple). + In addition to the normal content for pypirc include the following to + allow this function to read your name and email + [simple_setup] + author: Joe + author_email: joe@schmo.org + """ + author = None + author_email = None + fn = os.path.expanduser('~/.pypirc') + if os.path.exists(fn): + c = ConfigParser.SafeConfigParser() + c.read(fn) + if c.has_section('simple_setup'): + if c.has_option('simple_setup', 'author'): + author = c.get('simple_setup', 'author') + if c.has_option('simple_setup', 'author_email'): + author_email = c.get('simple_setup', 'author_email') + return author, author_email + + +def long_description_from_readme(): + s = None + fn = os.path.join(os.path.dirname(__file__), 'README') + if os.path.exists(fn): + with open(fn, 'r') as f: + s = f.read() + return s + + +# ----------- Override defaults here ---------------- +if packages is None: + packages = setuptools.find_packages() + +if len(packages) == 0: + raise Exception("No valid packages found") + +if package_name is None: + package_name = packages[0] + +if package_data is None: + package_data = find_package_data(packages) + +if scripts is None: + scripts = find_scripts() + +if requirements_file is None: + requirements_file = 'requirements.txt' + +if os.path.exists(requirements_file): + if requirements is None: + requirements = parse_requirements(requirements_file) + if dependency_links is None: + dependency_links = parse_dependency_links(requirements_file) +else: + if requirements is None: + requirements = [] + if dependency_links is None: + dependency_links = [] + +if version is None: + version = detect_version() + +if author is None: + author, email = author_info_from_pypirc() # save email for later +else: + email = None + +if author_email is None: + if email is not None: # if email was previously gotten + author_email = email + else: + _, author_email = author_info_from_pypirc() + +if long_description is None: + long_description = long_description_from_readme() + +if test_suite is None: + if os.path.exists('%s/tests.py' % package_name): + test_suite = "%s.tests.suite" % package_name + + +if debug: + logging.debug("Module name: %s" % package_name) + for package in packages: + logging.debug("Package: %s" % package) + logging.debug("\tData: %s" % str(package_data[package])) + logging.debug("Scripts:") + for script in scripts: + logging.debug("\tScript: %s" % script) + logging.debug("Requirements:") + for req in requirements: + logging.debug("\t%s" % req) + logging.debug("Dependency links:") + for dl in dependency_links: + logging.debug("\t%s" % dl) + logging.debug("Version: %s" % version) + logging.debug("Author: %s" % author) + logging.debug("Author email: %s" % author_email) + logging.debug("Test Suite: %s" % test_suite) + +if __name__ == '__main__': + + sub_packages = packages + + if use_numpy: + from numpy.distutils.misc_util import Configuration + config = Configuration(package_name, '', None) + + for sub_package in sub_packages: + print('adding %s' % sub_package) + config.add_subpackage(sub_package) + + from numpy.distutils.core import setup + setup(**config.todict()) + + else: + setuptools.setup( + name=package_name, + version=version, + packages=packages, + scripts=scripts, + long_description=long_description, + + package_data=package_data, + include_package_data=True, + + install_requires=requirements, + dependency_links=dependency_links, + + author=author, + author_email=author_email, + test_suite=test_suite, + ) From f1e4d3cf76b514d1790edc197a91e4f0fa012197 Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 24 Jan 2017 00:53:34 -0800 Subject: [PATCH 032/766] render: pull functions out of render object, PEP docstring --- renderapi/render.py | 319 +++++++++++--------------------------------- 1 file changed, 75 insertions(+), 244 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index e2af5892..8f521156 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -8,21 +8,6 @@ import requests import numpy as np -# GET http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/world-to-local-coordinates/{x},{y} -# curl "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates/40000,40000" -# returns: -# [ -# { -# "tileId": "140422184419060139", -# "visible": true, -# "local": [ -# 1238.9023, -# 1044.9727, -# 2239.0 -# ] -# } -# ] - class Render(object): def __init__(self, host=None, port=None, owner=None, project=None, @@ -67,11 +52,13 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, def process_defaults(self, host, port, owner, project, client_scripts=None): - #def process_defaults(self,host,port,owner,project,client_scripts=DEFAULT_CLIENT_SCRIPTS): - #utility function which will convert arguments to default arguments if they are None - #allows Render object to be used with defaults if lazy, but allows projects/hosts/owners to be changed - #from call to call if desired. - #used by many functions convert default None arguments to default values. + ''' + utility function which will convert arguments to default arguments if + they are None allows Render object to be used with defaults if + lazy, but allows projects/hosts/owners to be changed from call + to call if desired. used by many functions convert default None + arguments to default values. + ''' if host is None: host = self.DEFAULT_HOST if port is None: @@ -84,123 +71,6 @@ def process_defaults(self, host, port, owner, project, client_scripts = self.DEFAULT_CLIENT_SCRIPTS return (host, port, owner, project, client_scripts) - def world_to_local_coordinates(self, stack, z, x, y, host=None, port=None, - owner=None, project=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) - - r = session.get(request_url) - try: - return r.json() - except: - print(r.text) - return None - - # GET http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/tile/{tileId}/local-to-world-coordinates/{x},{y} - # curl "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/tile/140422184419063136/local-to-world-coordinates/1244.0508,1433.8711" - # returns: - # { - # "tileId": "140422184419063136", - # "world": [ - # 40000.0, - # 40000.004, - # 2239.0 - # ] - # } - def local_to_world_coordinates(self, stack, tileId, x, y, host=None, - port=None, owner=None, project=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) - - r = session.get(request_url) - try: - return r.json() - except: - print(r.text) - return None - - def get_owners(self, host=None, port=None, session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, None, None) - request_url = "%s/owners/" % self.format_baseurl(host, port) - return self.process_simple_url_request(request_url, session) - - def get_projects_by_owner(self, owner=None, host=None, port=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - metadata = self.get_stack_metadata_by_owner(owner) - projects = list(set([m['stackId']['project'] for m in metadata])) - return projects - - def get_stacks_by_owner_project(self, owner=None, project=None, host=None, - port=None, session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - metadata = self.get_stack_metadata_by_owner(owner) - stacks = ([m['stackId']['stack'] for m in metadata - if m['stackId']['project'] == project]) - return stacks - - def get_stack_metadata_by_owner(self, owner=None, host=None, port=None, - session=requests.session(), verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, None) - request_url = "%s/owner/%s/stacks/" % ( - self.format_baseurl(host, port), owner) - if verbose: - request_url - return self.process_simple_url_request(request_url, session) - - # PUT http://{host}:{port}/render-ws/v1/owner/{owner}/project/{project}/stack/{stack}/z/{z}/local-to-world-coordinates - # with request body containing JSON array of local coordinate elements - # curl -H "Content-Type: application/json" -X PUT --data @coordinate-local.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/local-to-world-coordinates" - # [ - # { - # "tileId": "140422184419063136", - # "world": [ - # 40000.0, - # 40000.004, - # 2239.0 - # ] - # } - # ] - def world_to_local_coordinates_batch(self, stack, z, data, host=None, - port=None, owner=None, project=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/world-to-local-coordinates" % (z) - r = session.put(request_url, data=data, - headers={"content-type": "application/json"}) - return r.json() - - # curl -H "Content-Type: application/json" -X PUT --data @coordinate-world.json "http://renderer.int.janelia.org:8080/render-ws/v1/owner/flyTEM/project/fly_pilot/stack/20141107_863/z/2239/world-to-local-coordinates" - # [ - # [ - # { - # "tileId": "140422184419060139", - # "visible": true, - # "local": [ - # 1238.9023, - # 1044.9727, - # 2239.0 - # ] - # } - # ] - # ] def get_z_values_for_stack(self, stack, project=None, host=None, port=None, owner=None, session=requests.session(), verbose=False): @@ -231,98 +101,6 @@ def get_z_value_for_section(self, stack, sectionId, project=None, print(r.text) return None - def world_to_local_coordinates_array(self, stack, dataarray, tileId, z=0, - host=None, port=None, owner=None, - project=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/world-to-local-coordinates" % (z) - dlist = [] - for i in range(dataarray.shape[0]): - d = {} - d['tileId'] = tileId - d['world'] = [dataarray[i, 0], dataarray[i, 1]] - dlist.append(d) - jsondata = json.dumps(dlist) - - r = session.put(request_url, data=jsondata, - headers={"content-type": "application/json"}) - - json_answer = r.json() - try: - answer = np.zeros(dataarray.shape) - - for i, coord in enumerate(json_answer): - - c = coord['local'] - answer[i, 0] = c[0] - answer[i, 1] = c[1] - return answer - - except: - print json_answer - return None - - def local_to_world_coordinates_array(self, stack, dataarray, tileId, z=0, - host=None, port=None, owner=None, - project=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/local-to-world-coordinates" % (z) - dlist = [] - for i in range(dataarray.shape[0]): - d = {} - d['tileId'] = tileId - d['local'] = [dataarray[i, 0], dataarray[i, 1]] - dlist.append(d) - jsondata = json.dumps(dlist) - - r = session.put(request_url, data=jsondata, - headers={"content-type": "application/json"}) - - json_answer = r.json() - try: - answer = np.zeros(dataarray.shape) - - for i, coord in enumerate(json_answer): - - c = coord['world'] - answer[i, 0] = c[0] - answer[i, 1] = c[1] - return answer - - except: - print json_answer - return None - - def local_to_world_coordinates_batch(self, stack, data, z, host=None, - port=None, owner=None, project=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/local-to-world-coordinates" % (z) - - r = session.put(request_url, data=data, - headers={"content-type": "application/json"}) - - return r.json() - - def format_baseurl(self, host, port): - return 'http://%s:%d/render-ws/v1' % (host, port) - - def format_preamble(self, host, port, owner, project, stack): - preamble = "%s/owner/%s/project/%s/stack/%s" % ( - self.format_baseurl(host, port), owner, project, stack) - return preamble - def put_resolved_tilespecs(self, stack, data, host=None, port=None, owner=None, project=None, session=requests.session(), verbose=False): @@ -352,21 +130,6 @@ def get_bounds_from_z(self, stack, z, host=None, port=None, owner=None, # MAP_COORD_SCRIPT = "/groups/flyTEM/flyTEM/render/bin/map-coord.sh" - - def set_stack_state(self, stack, state='LOADING', host=None, port=None, - owner=None, project=None, session=requests.session(), - verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] - request_url = self.format_preamble( - host, port, owner, project, stack) + "/state/%s" % state - if verbose: - request_url - r = session.put(request_url, data=None, - headers={"content-type": "application/json"}) - return r - def batch_local_work(self, stack, z, data, host=None, port=None, owner=None, project=None, localToWorld=False, deleteTemp=True, threads=16): @@ -501,3 +264,71 @@ def format_preamble(host, port, owner, project, stack): preamble = "%s/owner/%s/project/%s/stack/%s" % ( format_baseurl(host, port), owner, project, stack) return preamble + + +def get_owners(host=None, port=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_owners(**render.make_kwargs( + host=host, port=port, + **{'session': session})) + + request_url = "%s/owners/" % format_baseurl(host, port) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, + session=requests.session(), + verbose=False, **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_stack_metadata_by_owner(**render.make_kwargs( + owner=owner, host=host, port=port, + **{'session': session, 'verbose': verbose})) + + request_url = "%s/owner/%s/stacks/" % ( + format_baseurl(host, port), owner) + if verbose: + logging.debug(request_url) + r = session.get(request_url) + try: + return r.json() + except: + logging.error(r.text) + + +def get_projects_by_owner(owner=None, host=None, port=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_projects_by_owner(**render.make_kwargs( + owner=owner, host=host, port=port, + **{'session': session})) + + metadata = get_stack_metadata_by_owner(owner) + projects = list(set([m['stackId']['project'] for m in metadata])) + return projects + + +def get_stacks_by_owner_project(owner=None, project=None, host=None, + port=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_projects_by_owner(**render.make_kwargs( + owner=owner, host=host, port=port, project=project, + **{'session': session})) + + metadata = get_stack_metadata_by_owner(owner) + stacks = ([m['stackId']['stack'] for m in metadata + if m['stackId']['project'] == project]) + return stacks From e04758e9706b535aa0117c4a2e3a90ef76fd4772 Mon Sep 17 00:00:00 2001 From: Russel Date: Tue, 24 Jan 2017 13:52:40 -0800 Subject: [PATCH 033/766] tilespec: forrest 1b5f6c8 -- transposed m01, m10 representation, affine concatenate --- renderapi/tilespec.py | 53 +++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 58383f33..0bb031d7 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -146,26 +146,26 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0): self.load_M() def load_M(self): - self.M = np.identity(4, np.double) + self.M = np.identity(3, np.double) self.M[0, 0] = self.M00 self.M[0, 1] = self.M01 self.M[1, 0] = self.M10 self.M[1, 1] = self.M11 - self.M[0, 3] = self.B0 - self.M[1, 3] = self.B1 + self.M[0, 2] = self.B0 + self.M[1, 2] = self.B1 def to_dict(self): d = {} d['type'] = 'leaf' d['className'] = self.className - d['dataString'] = "%f %f %f %f %f %f" % ( - self.M[0, 0], self.M[0, 1], self.M[1, 0], - self.M[1, 1], self.M[0, 3], self.M[1, 3]) + d['dataString'] = "%.10f %.10f %.10f %.10f %.10f %.10f" % ( + self.M[0, 0], self.M[1, 0], self.M[0, 1], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) return d def from_dict(self, d): ds = d['dataString'].split() - (self.M00, self.M01, self.M10, self.M11, self.B0, self.B1) = map( + (self.M00, self.M10, self.M01, self.M11, self.B0, self.B1) = map( float, ds) self.load_M() @@ -180,18 +180,13 @@ def convert_to_point_vector(self, points): zerovec = np.zeros((Np, 1), np.double) onevec = np.ones((Np, 1), np.double) - if points.shape[1] == 2: - Nd = 2 - points = np.concatenate((points, zerovec), axis=1) - points = np.concatenate((points, onevec), axis=1) - elif points.shape[1] == 3: - points = np.concatenate((points, onevec), axis=1) - Nd = 3 - assert(points.shape[1] == 4) + assert(points.shape[1] == 2) + Nd = 2 + points = np.concatenate((points, zerovec), axis=1) return points, Nd def convert_points_vector_to_array(self, points, Nd): - points = points[:, 0:Nd] / np.tile(points[:, 3], (Nd, 1)).T + points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T return points def tform(self, points): @@ -199,6 +194,30 @@ def tform(self, points): pt = np.dot(self.M, points.T).T return self.convert_points_vector_to_array(pt, Nd) + def concatenate(self, model): + ''' + concatenate a model to this model -- proted from trakEM2 below: + final double a00 = m00 * model.m00 + m01 * model.m10; + final double a01 = m00 * model.m01 + m01 * model.m11; + final double a02 = m00 * model.m02 + m01 * model.m12 + m02; + + final double a10 = m10 * model.m00 + m11 * model.m10; + final double a11 = m10 * model.m01 + m11 * model.m11; + final double a12 = m10 * model.m02 + m11 * model.m12 + m12; + ''' + a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] + a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] + a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + + self.M[0, 2]) + + a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] + a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] + a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + + self.M[1, 2]) + + newmodel = AffineModel(a00, a01, a10, a11, a02, a12) + return newmodel + def inverse_tform(self, points): points, Nd = self.convert_to_point_vector(points) pt = np.dot(np.linalg.inv(self.M), points.T).T @@ -207,7 +226,7 @@ def inverse_tform(self, points): def __str__(self): return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( self.M[0, 0], self.M[0, 1], self.M[1, 0], - self.M[1, 1], self.M[0, 3], self.M[1, 3]) + self.M[1, 1], self.M[0, 2], self.M[1, 2]) class Layout: From 580ff37f497ed3ca89a6e00f60dc509825272f19 Mon Sep 17 00:00:00 2001 From: Russel Date: Wed, 25 Jan 2017 17:21:03 -0800 Subject: [PATCH 034/766] render: adding draft RenderClient class; fixing imports on run --- renderapi/client.py | 2 +- renderapi/coordinate.py | 1 + renderapi/render.py | 11 +++++++++++ renderapi/tilespec.py | 1 + 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 58a4db15..de386212 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -181,7 +181,7 @@ def import_jsonfiles_validate_client(stack, jsonfiles, render=None, stack, jsonfiles, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, - **{'close_stack': close_stack, 'mem'=mem, + **{'close_stack': close_stack, 'mem': mem, 'transformFile': transformFile})) transform_params = (['--transformFile', transformFile] diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index d4758ff1..29fddb3a 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -4,6 +4,7 @@ ''' from .render import Render, format_preamble +import requests def world_to_local_coordinates(stack, z, x, y, render=None, host=None, diff --git a/renderapi/render.py b/renderapi/render.py index 8f521156..aab199df 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -197,6 +197,17 @@ def get_section_z_value(self, stack, sectionId, host=None, port=None, return float(self.process_simple_url_request(request_url, session)) +class RenderClient(Render): + '''Draft object for render_webservice_client.sh calls''' + def __init__(self, client_script=None, *args, **kwargs): + super(RenderClient, self).__init__(**kwargs) + self.client_script = client_script + + def make_kwargs(self, *args, **kwargs): + return super(RenderClient, self).make_kwargs( + client_script=self.client_script, *args, **kwargs) + + def connect(host=None, port=None, owner=None, project=None, client_scripts=None): '''helper function to connect to a render instance''' diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 0bb031d7..c0bcda53 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,5 +1,6 @@ #!/usr/bin/env python from .render import Render, format_baseurl, format_preamble +import requests import numpy as np From 86c97142d7560ad4939d2a536f7d5ea7d8bd609f Mon Sep 17 00:00:00 2001 From: Russel Date: Thu, 26 Jan 2017 19:09:09 -0800 Subject: [PATCH 035/766] stack: adding clone_stack & utils --- renderapi/stack.py | 40 ++++++++++++++++++++++++++++++++++++---- renderapi/utils.py | 8 ++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 renderapi/utils.py diff --git a/renderapi/stack.py b/renderapi/stack.py index 7844fb7c..321a085f 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -4,13 +4,14 @@ from time import strftime import requests from .render import Render, format_baseurl, format_preamble +from .utils import jbool class StackVersion: - def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=1, - stackResolutionY=1, stackResolutionZ=1, + def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=0, + stackResolutionY=0, stackResolutionZ=0, materializedBoxRootPath=None, versionNotes="", - createTimestamp=None): + createTimestamp=None, **kwargs): self.cycleNumber = cycleNumber self.cycleStepNumber = cycleStepNumber self.stackResolutionX = stackResolutionX @@ -103,7 +104,7 @@ def delete_stack(stack, render=None, host=None, port=None, owner=None, return r -def create_stack(self, stack, cycleNumber=1, cycleStepNumber=1, +def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, host=None, port=None, owner=None, project=None, verbose=False, session=requests.session(), **kwargs): if render is not None: @@ -128,3 +129,34 @@ def create_stack(self, stack, cycleNumber=1, cycleStepNumber=1, except: logging.error(r.text) return None + + +def clone_stack(inputstack, outputstack, render=None, host=None, port=None, + owner=None, project=None, skipTransforms=False, toProject=None, + z=None, session=None, **kwargs): + ''' + result: + cloned stack in LOADING state with tiles in layers specified by z' + ''' + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return clone_stack(inputstack, outputstack, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + session=session, skipTransforms=skipTransforms, + toProject=toProject, **kwargs)) + + if z is not None: + zs = [float(i) for i in z] # TODO test me + session = requests.session() if session is None else session + sv = StackVersion(**kwargs) + request_url = '{}/{}'.format(format_preamble( + host, port, owner, project, inputstack), outputstack) + + logging.debug(request_url) + r = session.put(request_url, params={ + 'z': zs, 'toProject': toProject, + 'skipTransforms': jbool(skipTransforms)}, + data=json.dumps(sv.to_dict())) + + return r diff --git a/renderapi/utils.py b/renderapi/utils.py new file mode 100644 index 00000000..ea8f3cba --- /dev/null +++ b/renderapi/utils.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +''' +utilities to make render/java/web interfacing easier +''' + + +def jbool(val): + return 'true' if val else 'false' From e2dc214963c9d05a0018e9417f55750b1a6024e6 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 6 Feb 2017 14:08:09 -0800 Subject: [PATCH 036/766] minor changes, import logging --- renderapi/coordinate.py | 1 + renderapi/image.py | 1 + renderapi/pointmatch.py | 1 + renderapi/stack.py | 7 +++---- renderapi/tilespec.py | 6 +++--- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 29fddb3a..a78ba336 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -4,6 +4,7 @@ ''' from .render import Render, format_preamble +import logging import requests diff --git a/renderapi/image.py b/renderapi/image.py index 30fe5a5d..1e76eec2 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -4,6 +4,7 @@ import requests from PIL import Image import numpy as np +import logging from .render import Render, format_baseurl, format_preamble # define acceptable image formats -- currently render generates png, jpeg, tiff diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 4d860d23..1e6e9a86 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -3,6 +3,7 @@ Point Match APIs ''' import requests +import logging from .render import Render, format_baseurl diff --git a/renderapi/stack.py b/renderapi/stack.py index 321a085f..380c4fae 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -18,9 +18,8 @@ def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=0, self.stackResolutionY = stackResolutionY self.stackResolutionZ = stackResolutionZ self.materializedBoxRootPath = materializedBoxRootPath - if createTimestamp is None: - createTimestamp = strftime('%Y-%M-%dT%H:%M:%S.00Z') - self.createTimestamp = createTimestamp + self.createTimestamp = (strftime('%Y-%M-%dT%H:%M:%S.00Z') if + createTimestamp is None else createTimestamp) self.versionNotes = versionNotes def to_dict(self): @@ -31,7 +30,7 @@ def to_dict(self): d['stackResolutionY'] = self.stackResolutionY d['stackResolutionZ'] = self.stackResolutionZ d['createTimestamp'] = self.createTimestamp - d["materializedBoxRootPath"] = "string" + d["materializedBoxRootPath"] = self.materializedBoxRootPath d['mipmapPathBuilder'] = {'numberOfLevels': 0} d['versionNotes'] = self.versionNotes return d diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index c0bcda53..d8284945 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,5 +1,6 @@ #!/usr/bin/env python from .render import Render, format_baseurl, format_preamble +import logging import requests import numpy as np @@ -77,7 +78,6 @@ def from_dict(self, d): self.classname = d['className'] self.params = d['params'] - class ReferenceTransform: def __init__(self, refId=None, json=None): if json is not None: @@ -197,7 +197,7 @@ def tform(self, points): def concatenate(self, model): ''' - concatenate a model to this model -- proted from trakEM2 below: + concatenate a model to this model -- ported from trakEM2 below: final double a00 = m00 * model.m00 + m01 * model.m10; final double a01 = m00 * model.m01 + m01 * model.m11; final double a02 = m00 * model.m02 + m01 * model.m12 + m02; @@ -468,7 +468,7 @@ def get_tile_specs_from_z(stack, z, render=None, host=None, port=None, if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - return get_tile_specs_from_box( + return get_tile_specs_from_z( stack, z, **render.make_kwargs( host=host, port=port, owner=owner, project=project, **{'session': session, 'verbose': verbose})) From c68b5818fe8fc0220b923fc20616d096e083d79b Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 6 Feb 2017 14:23:43 -0800 Subject: [PATCH 037/766] remove function-level verbosity: use logging.DEBUG instead --- renderapi/client.py | 23 +++++++++------------- renderapi/image.py | 6 ++---- renderapi/pointmatch.py | 43 +++++++++++++++++++---------------------- renderapi/stack.py | 15 ++++++-------- renderapi/tilespec.py | 22 ++++++++++----------- 5 files changed, 47 insertions(+), 62 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index de386212..e9bee074 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -20,7 +20,7 @@ def import_single_json_file(stack, jsonfile, render=None, transformFile=None, client_scripts=None, host=None, port=None, - owner=None, project=None, verbose=False, **kwargs): + owner=None, project=None, **kwargs): ''' calls client script to import given jsonfile: transformFile: ? @@ -31,8 +31,7 @@ def import_single_json_file(stack, jsonfile, render=None, transformFile=None, raise ValueError('invalid Render object specified!') return import_single_json_file(stack, jsonfile, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - client_scripts=client_scripts, **{'verbose': verbose, - 'transformFile': None})) + client_scripts=client_scripts, **{'transformFile': None})) if transformFile is None: transform_params = [] @@ -45,18 +44,16 @@ def import_single_json_file(stack, jsonfile, render=None, transformFile=None, stack_params + \ transform_params + \ [jsonfile] - if verbose: - print cmd + logging.debug(cmd) proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) proc.wait() - if verbose: - print proc.stdout.read() + logging.debug(proc.stdout.read()) def import_jsonfiles_and_transforms_parallel_by_z( stack, jsonfiles, transformfiles, render=None, poolsize=20, client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, verbose=False, **kwargs): + project=None, close_stack=True, **kwargs): ''' imports json files and transform files in parallel: jsonfiles: "list of tilespec" jsons to import @@ -71,16 +68,14 @@ def import_jsonfiles_and_transforms_parallel_by_z( return import_jsonfiles_and_transforms_parallel_by_z( stack, jsonfile, transformfiles, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - client_scripts=client_scripts, **{'verbose': verbose, - 'close_stack': close_stack, + client_scripts=client_scripts, **{'close_stack': close_stack, 'poolsize': poolsize})) set_stack_state(stack, 'LOADING', host, port, owner, project) pool = Pool(poolsize) partial_import = partial(import_single_json_file, stack, render=render, client_scripts=client_scripts, host=host, - port=port, owner=owner, project=project, - verbose=verbose) + port=port, owner=owner, project=project) rs = pool.amap(partial_import, jsonfiles, transformfiles) rs.wait() if close_stack: @@ -90,7 +85,7 @@ def import_jsonfiles_and_transforms_parallel_by_z( def import_jsonfiles_parallel( stack, jsonfiles, render=None, poolsize=20, transformFile=None, client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, verbose=False, **kwargs): + project=None, close_stack=True, **kwargs): ''' import jsons using client script in parallel jsonfiles: list of jsonfiles to upload @@ -105,7 +100,7 @@ def import_jsonfiles_parallel( stack, jsonfiles, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, - **{'verbose': verbose, 'close_stack': close_stack, + **{'close_stack': close_stack, 'poolsize': poolsize, 'transformFile': transformFile})) set_stack_state(stack, 'LOADING', host, port, owner, project) diff --git a/renderapi/image.py b/renderapi/image.py index 1e76eec2..1e98822e 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -60,7 +60,7 @@ def get_bb_image(stack, z, x, y, width, height, render=None, scale=1.0, def get_tile_image_data(stack, tileId, render=None, normalizeForMatching=True, host=None, port=None, owner=None, project=None, img_format=None, - session=requests.session(), verbose=False, **kwargs): + session=requests.session(), **kwargs): ''' render image from a tile with all transforms and return numpy array ''' @@ -81,8 +81,7 @@ def get_tile_image_data(stack, tileId, render=None, "/tile/%s/png-image" % (tileId) if normalizeForMatching: request_url += "?normalizeForMatching=true" - if verbose: - print request_url + logging.debug(request_url) r = session.get(request_url) try: img = Image.open(io.BytesIO(r.content)) @@ -90,4 +89,3 @@ def get_tile_image_data(stack, tileId, render=None, return array except: logging.error(r.text) - return None diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 1e6e9a86..30cd7cb2 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -8,14 +8,12 @@ def get_matchcollection_owners(render=None, host=None, port=None, - verbose=False, session=requests.session(), - **kwargs): + session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return get_matchcollection_owners(**render.make_kwargs( - host=host, port=port, **{'verbose': verbose, - 'session': session})) + host=host, port=port, **{'session': session})) request_url = format_baseurl(host, port) + \ "/matchCollectionOwners" @@ -27,13 +25,13 @@ def get_matchcollection_owners(render=None, host=None, port=None, def get_matchcollections(render=None, owner=None, host=None, port=None, - verbose=False, session=requests.session(), **kwargs): + session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return get_matchcollections(**render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollections" % owner @@ -45,14 +43,13 @@ def get_matchcollections(render=None, owner=None, host=None, port=None, def get_match_groupIds(matchCollection, render=None, owner=None, host=None, - port=None, verbose=False, - session=requests.session()): + port=None, session=requests.session()): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return get_match_groupIds(matchCollection, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) @@ -64,7 +61,7 @@ def get_match_groupIds(matchCollection, render=None, owner=None, host=None, def get_matches_outside_group(matchCollection, groupId, render=None, - owner=None, host=None, port=None, verbose=False, + owner=None, host=None, port=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -72,7 +69,7 @@ def get_matches_outside_group(matchCollection, groupId, render=None, return get_matches_outside_group( matchCollection, groupId, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( @@ -85,7 +82,7 @@ def get_matches_outside_group(matchCollection, groupId, render=None, def get_matches_within_group(matchCollection, groupId, owner=None, - host=None, port=None, verbose=False, + host=None, port=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -93,7 +90,7 @@ def get_matches_within_group(matchCollection, groupId, owner=None, return get_matches_within_group( matchCollection, groupId, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( @@ -107,7 +104,7 @@ def get_matches_within_group(matchCollection, groupId, owner=None, def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, render=None, owner=None, host=None, - port=None, verbose=False, + port=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -115,7 +112,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, return get_matches_from_group_to_group( matchCollection, pgroup, qgroup, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( @@ -129,7 +126,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, qgroup, qid, render=None, owner=None, - host=None, port=None, verbose=False, + host=None, port=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -137,7 +134,7 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, return get_matches_from_tile_to_tile( matchCollection, pgroup, pid, qgroup, qid, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ ("/owner/%s/matchCollection/%s/group/%s/id/%s/" @@ -151,7 +148,7 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, - host=None, port=None, verbose=False, + host=None, port=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -159,7 +156,7 @@ def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, return get_matches_with_group( matchCollection, pgroup, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( @@ -172,7 +169,7 @@ def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, def get_match_groupIds_from_only(matchCollection, render=None, owner=None, - host=None, port=None, verbose=False, + host=None, port=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -180,7 +177,7 @@ def get_match_groupIds_from_only(matchCollection, render=None, owner=None, return get_match_groupIds_from_only( matchCollection, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) @@ -192,7 +189,7 @@ def get_match_groupIds_from_only(matchCollection, render=None, owner=None, def get_match_groupIds_to_only(matchCollection, render=None, owner=None, - host=None, port=None, verbose=False, + host=None, port=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -200,7 +197,7 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, return get_match_groupIds_to_only( matchCollection, **render.make_kwargs( owner=owner, host=host, port=port, - **{'verbose': verbose, 'session': session})) + **{'session': session})) request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) diff --git a/renderapi/stack.py b/renderapi/stack.py index 380c4fae..ed7daa36 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -42,7 +42,7 @@ def from_dict(self, d): def set_stack_state(stack, render=None, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), - verbose=False, **kwargs): + **kwargs): if render is not None: if not isinstance(render, Render): @@ -50,13 +50,12 @@ def set_stack_state(stack, render=None, state='LOADING', host=None, port=None, return set_stack_state( stack, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - **{'verbose': verbose, 'state': state, 'session': session})) + **{'state': state, 'session': session})) assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] request_url = format_preamble( host, port, owner, project, stack) + "/state/%s" % state - if verbose: - request_url + logging.debug(request_url) r = session.put(request_url, data=None, headers={"content-type": "application/json"}) return r @@ -104,7 +103,7 @@ def delete_stack(stack, render=None, host=None, port=None, owner=None, def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, - host=None, port=None, owner=None, project=None, verbose=False, + host=None, port=None, owner=None, project=None, session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -112,13 +111,12 @@ def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, return create_stack(stack, **render.make_kwargs( host=host, port=port, owner=owner, project=project, **{'session': session, 'cycleNumber': cycleNumber, - 'cycleStepNumber': cycleStepNumber, 'verbose': verbose})) + 'cycleStepNumber': cycleStepNumber})) sv = StackVersion( cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber) request_url = format_preamble(host, port, owner, project, stack) - if verbose: - print "stack version2", request_url, sv.to_dict() + logging.debug("stack version {} {}".format(request_url, sv.to_dict())) payload = json.dumps(sv.to_dict()) r = session.post(request_url, data=payload, headers={"content-type": "application/json", @@ -127,7 +125,6 @@ def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, return r except: logging.error(r.text) - return None def clone_stack(inputstack, outputstack, render=None, host=None, port=None, diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index d8284945..0a45b689 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -78,6 +78,7 @@ def from_dict(self, d): self.classname = d['className'] self.params = d['params'] + class ReferenceTransform: def __init__(self, refId=None, json=None): if json is not None: @@ -416,15 +417,14 @@ def get_tile_spec(stack, tile, render=None, host=None, port=None, owner=None, def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, render=None, scale=1.0, host=None, port=None, owner=None, project=None, - session=requests.session(), - verbose=False, **kwargs): + session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return get_tile_specs_from_minmax_box( stack, z, xmin, xmax, ymin, ymax, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - **{'session': session, 'verbose': verbose})) + **{'session': session})) x = xmin y = ymin @@ -432,27 +432,26 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, height = ymax - ymin return get_tile_specs_from_box(stack, z, x, y, width, height, scale, host, port, owner, project, - session, verbose) + session) def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, scale=1.0, host=None, port=None, owner=None, project=None, session=requests.session(), - verbose=False, **kwargs): + **kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return get_tile_specs_from_box( stack, z, x, y, width, height, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - **{'session': session, 'verbose': verbose})) + **{'session': session})) request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( z, x, y, width, height, scale) - if verbose: - print request_url + logging.debug(request_url) r = session.get(request_url) try: tilespecs_json = r.json() @@ -464,19 +463,18 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, def get_tile_specs_from_z(stack, z, render=None, host=None, port=None, owner=None, project=None, session=requests.session(), - verbose=False, **kwargs): + **kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return get_tile_specs_from_z( stack, z, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - **{'session': session, 'verbose': verbose})) + **{'session': session})) request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) - if verbose: - print request_url + logging.debug(request_url) r = session.get(request_url) try: tilespecs_json = r.json() From 7aacfa3610bfb316e8223bd3d30e03be2ec931ce Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 6 Feb 2017 18:11:55 -0800 Subject: [PATCH 038/766] transform: adding transform module and Polynomial2D functionality which works occasionally --- renderapi/transform.py | 297 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 renderapi/transform.py diff --git a/renderapi/transform.py b/renderapi/transform.py new file mode 100644 index 00000000..078e167f --- /dev/null +++ b/renderapi/transform.py @@ -0,0 +1,297 @@ +#!/usr/bin/env python +''' +handling mpicbg transforms in python + +Currently only implemented to facilitate Affine, Polynomial2D, + and LensCorrection used in Khaled Khairy's EM aligner workflow +TODO: + interpolation functions + Affine as subset of Polynomial2D + approximation of other functions(TPS, meshtechniques) to Polynomial2D +''' +import json +import numpy as np + + +class EstimationError(Exception): + pass + + +def _load_dict(obj, d): + obj.__dict__.update({k: v for k, v in d.items()}) + + +def _load_json(obj, j): + with open(j, 'r') as f: + jd = json.load(f) + _load_dict(obj, jd) + + +class TransformList: + def __init__(self, tforms): + self.tforms = tforms + + def to_dict(self): + return {'type': 'list', + 'specList': [tform.to_dict() for tform in self.tforms]} + + def to_json(self): + return json.dumps(self.to_dict()) + + +class ReferenceTransform: + def __init__(self, refId=None, json=None): + if json is not None: + self.from_dict(json) + else: + self.refId = refId + + def to_dict(self): + d = {} + d['type'] = 'ref' + d['refId'] = self.refId + return d + + def from_dict(self, d): + self.refId = d['refId'] + + def __str__(self): + return 'ReferenceTransform(%s)' % self.refId + + def __repr__(self): + return self.__str__() + + +class Transform: + def __init__(self, className=None, dataString=None, + transformId=None, json=None): + if json is not None: + self.from_dict(json) + else: + self.className = className + self.dataString = dataString + self.transformId = transformId + + def to_dict(self): + d = {} + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = self.dataString + if self.transformId is not None: + d['transformId'] = self.transformId + return d + + def from_dict(self, d): + self.dataString = d['dataString'] + self.className = d['className'] + self.transformId = d.get('transformId', None) + + def __str__(self): + return 'className:%s\ndataString:%s' % ( + self.className, self.dataString) + + def __repr__(self): + return self.__str__() + + +class AffineModel(Transform): + className = 'mpicbg.trakem2.transform.AffineModel2D' + + def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0): + self.M00 = M00 + self.M01 = M01 + self.M10 = M10 + self.M11 = M11 + self.B0 = B0 + self.B1 = B1 + self.className = 'mpicbg.trakem2.transform.AffineModel2D' + self.load_M() + + def load_M(self): + self.M = np.identity(3, np.double) + self.M[0, 0] = self.M00 + self.M[0, 1] = self.M01 + self.M[1, 0] = self.M10 + self.M[1, 1] = self.M11 + self.M[0, 2] = self.B0 + self.M[1, 2] = self.B1 + + def to_dict(self): + d = {} + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = "%.10f %.10f %.10f %.10f %.10f %.10f" % ( + self.M[0, 0], self.M[1, 0], self.M[0, 1], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + return d + + def from_dict(self, d): + ds = d['dataString'].split() + (self.M00, self.M10, self.M01, self.M11, self.B0, self.B1) = map( + float, ds) + self.load_M() + + def invert(self): + Ai = AffineModel() + Ai.M = np.linalg.inv(self.M) + return Ai + + def convert_to_point_vector(self, points): + Np = points.shape[0] + + zerovec = np.zeros((Np, 1), np.double) + onevec = np.ones((Np, 1), np.double) + + assert(points.shape[1] == 2) + Nd = 2 + points = np.concatenate((points, zerovec), axis=1) + return points, Nd + + def convert_points_vector_to_array(self, points, Nd): + points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T + return points + + def tform(self, points): + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(self.M, points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + def concatenate(self, model): + ''' + concatenate a model to this model -- ported from trakEM2 below: + final double a00 = m00 * model.m00 + m01 * model.m10; + final double a01 = m00 * model.m01 + m01 * model.m11; + final double a02 = m00 * model.m02 + m01 * model.m12 + m02; + + final double a10 = m10 * model.m00 + m11 * model.m10; + final double a11 = m10 * model.m01 + m11 * model.m11; + final double a12 = m10 * model.m02 + m11 * model.m12 + m12; + ''' + a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] + a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] + a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + + self.M[0, 2]) + + a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] + a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] + a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + + self.M[1, 2]) + + newmodel = AffineModel(a00, a01, a10, a11, a02, a12) + return newmodel + + def inverse_tform(self, points): + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(np.linalg.inv(self.M), points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + def __str__(self): + return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( + self.M[0, 0], self.M[0, 1], self.M[1, 0], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + + +class Polynomial2DTransform(Transform): + ''' + Polynomial2DTransform implemented as in skimage + TODO: + fall back to Affine Model in special cases + robustness in estimation + ''' + className = 'mpicbg.trakEM2.transform.PolynomialTransform2D' + + def __init__(self, dataString=None, src=None, dst=None, order=2, + force_polynomial=True): + self.className = 'mpicbg.trakEM2.transform.PolynomialTransform2D' + if dataString is not None: + self._process_dataString(dataString) + if src is not None and dst is not None: + self._process_params(self.estimate(src, dst, order)) + + if not force_polynomial and self.is_affine: + # TODO try implement affine from poly (& vice versa) + return AffineTransform(poly_params=self.params) + + @property + def is_affine(self): + '''TODO allow default to Affine''' + return False + + def estimate(self, src, dst, order=2, convergence_test=None): + '''This is unreliable -- add tests to ensure repeatability''' + xs = src[:, 0] + ys = src[:, 1] + xd = dst[:, 0] + yd = dst[:, 1] + rows = src.shape[0] + no_coeff = (order + 1) * (order + 2) + + if len(src) != len(dst): + raise EstimationError( + 'source has {} points, but dest has {}!'.format( + len(src), len(dst))) + if no_coeff > len(src): + raise EstimationError( + 'order {} is too large to fit {} points!'.format( + order, len(src))) + + A = np.empty([rows * 2, no_coeff + 1]) + pidx = 0 + for j in range(order + 1): + for i in range(j + 1): + A[:rows, pidx] = xs ** (j - i) * ys ** i + A[rows:, pidx + no_coeff // 2] = xs ** (j - i) * ys ** i + pidx += 1 + + A[:rows, -1] = xd + A[rows:, -1] = yd + + # right singular vector corresponding to smallest singular value + # TODO implement tests for this + _, _, V = np.linalg.svd(A) + return (-V[-1, :-1] / V[-1, -1]).reshape((2, no_coeff // 2)) + + def _process_params(self, params): + ''' + generate datastring and param attributes from params + ''' + self.params = params + self.dataString = self._dataStringfromParams(params) + + def _dataStringfromParams(self, params=None): + return ' '.join([str(i) for i in params.flatten()]).replace('e', 'E') + + def _process_dataString(self, datastring): + ''' + generate datastring and param attributes from datastring + ''' + dsList = datastring.split(' ') + self.params = numpy.array( + [[float(d) for d in dsList[:len(dsList)/2]], + [float(d) for d in dsList[len(dsList)/2]]]) + self.dataString = datastring + + def tform(self, points): + dst = np.zeros(points.shape) + x = points[:, 0] + y = points[:, 1] + + o = int((-3 + np.sqrt(9 - 4 * (2 - len(self.params.ravel())))) / 2) + pidx = 0 + for j in range(o + 1): + for i in range(j + 1): + dst[:, 0] += self.params[0, pidx] * x ** (j - i) * y ** i + dst[:, 1] += self.params[1, pidx] * x ** (j - i) * y ** i + pidx += 1 + return dst + + +def transformsum(transformlist): + ''' + summation of all transforms in a list of transforms. + Will force affines as polynomials. Does not support LC. + Returns: + AffineTransform or Polynomial2DTransform representing the sum of the + input list + ''' + pass From b23bca9aafb9e1f068c615cf6c6356ff29017ef9 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 7 Feb 2017 16:07:55 -0800 Subject: [PATCH 039/766] transform: pseudo-composition to concatenate polynomials --- renderapi/transform.py | 76 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 6 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 078e167f..bc16c814 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -11,6 +11,7 @@ ''' import json import numpy as np +from .errors import ConversionError class EstimationError(Exception): @@ -201,11 +202,15 @@ class Polynomial2DTransform(Transform): className = 'mpicbg.trakEM2.transform.PolynomialTransform2D' def __init__(self, dataString=None, src=None, dst=None, order=2, - force_polynomial=True): + force_polynomial=True, params=None, identity=False): self.className = 'mpicbg.trakEM2.transform.PolynomialTransform2D' if dataString is not None: self._process_dataString(dataString) - if src is not None and dst is not None: + elif identity: + self._process_params(np.array([[0, 1, 0], [0, 0, 1]])) + elif params is not None: + self._process_params(params) + elif src is not None and dst is not None: self._process_params(self.estimate(src, dst, order)) if not force_polynomial and self.is_affine: @@ -216,6 +221,12 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, def is_affine(self): '''TODO allow default to Affine''' return False + # return self.order + + @property + def order(self): + no_coeffs = len(self.params.ravel()) + return (abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2 def estimate(self, src, dst, order=2, convergence_test=None): '''This is unreliable -- add tests to ensure repeatability''' @@ -266,11 +277,16 @@ def _process_dataString(self, datastring): generate datastring and param attributes from datastring ''' dsList = datastring.split(' ') - self.params = numpy.array( + self.params = np.array( [[float(d) for d in dsList[:len(dsList)/2]], [float(d) for d in dsList[len(dsList)/2]]]) self.dataString = datastring + def _format_raveled_params(self, raveled_params): + return np.array( + [[float(d) for d in dsList[:len(raveled_params)/2]], + [float(d) for d in dsList[len(raveled_params)/2]]]) + def tform(self, points): dst = np.zeros(points.shape) x = points[:, 0] @@ -285,13 +301,61 @@ def tform(self, points): pidx += 1 return dst + def coefficients(self, order=None): + if order is None: + order = self.order + return (order + 1) * (order + 2) + + def asorder(self, order): + '''''' + if self.order > order: + raise ConversionError( + 'transformation {} is order {} -- conversion to ' + 'order {} not supported'.format( + self.dataString, self.order, order)) + new_params_raveled = self.params.ravel() + [ + 0 for i in range(self.coefficients(order) - self.coefficients())] + new_params = self._format_raveled_params(new_params_raveled) + return Polynomial2DTransform(params=new_params) + + def _fromAffine(self, aff): + if not isinstance(aff, AffineModel): + raise ConversionError('attempting to convert a nonaffine model!') + return Polynomial2DTransform(params=np.array([ + [aff.M[0, 2], aff.M[0, 0], aff.M[0, 1]], + [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) + + def concatenate(self, othertform, order=None, srcpts=None): + ''' + currently uses an estimation to represent composition of transforms + ''' + if isinstance(othertform, AffineModel): + othertform = self._fromAffine(othertform) + if order is None: + order = max([self.order, othertform.order]) + # TODO define srcpts and dstpts + if srcpts is not None: + dstpts = othertform.tform(self.tform(srcpts)) + else: + raise NotImplementedError('default source points unavailable!') + return Polynomial2DTransform(src=srcpts, dst=dstpts, order=order) + -def transformsum(transformlist): +def transformsum(transformlist, src=None): ''' summation of all transforms in a list of transforms. Will force affines as polynomials. Does not support LC. + input: + src -- test points representing the Returns: - AffineTransform or Polynomial2DTransform representing the sum of the + Polynomial2DTransform representing the sum of the input list ''' - pass + sumtform = Polynomial2DTransform(identity=True) + for tform in transformlist: + if isinstance(tform, list): + logging.debug('found transformlist!') + sumtform.concatenate(transformsum(tform, src=src), srcpts=src) + else: + sumtform.concatenate(tform, srcpts=src) + return tform From 1b679c809d9317771ea95a7dffe319d8d71d583e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 7 Feb 2017 16:11:51 -0800 Subject: [PATCH 040/766] transform: fix polynomial transformsum --- renderapi/transform.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index bc16c814..6849cf5e 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -355,7 +355,8 @@ def transformsum(transformlist, src=None): for tform in transformlist: if isinstance(tform, list): logging.debug('found transformlist!') - sumtform.concatenate(transformsum(tform, src=src), srcpts=src) + sumtform = sumtform.concatenate( + transformsum(tform, src=src), srcpts=src) else: - sumtform.concatenate(tform, srcpts=src) - return tform + sumtform = sumtform.concatenate(tform, srcpts=src) + return sumtform From fb31699dddfb0af916d989fe219c8464dba23763 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 9 Feb 2017 12:10:18 -0800 Subject: [PATCH 041/766] errors: add ConversionError --- renderapi/errors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/renderapi/errors.py b/renderapi/errors.py index dde1e282..232ca508 100644 --- a/renderapi/errors.py +++ b/renderapi/errors.py @@ -6,3 +6,7 @@ class ClientScriptError(Exception): pass + + +class ConversionError(Exception): + pass From 0d512f5ef6d632897378ba69ee221a7b51853f40 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 9 Feb 2017 12:52:07 -0800 Subject: [PATCH 042/766] logging changes --- renderapi/client.py | 46 +++++++++++++++++++++++++---------------- renderapi/coordinate.py | 10 +++++---- renderapi/image.py | 8 ++++--- renderapi/pointmatch.py | 22 +++++++++++--------- renderapi/stack.py | 25 ++++++++++++---------- renderapi/tilespec.py | 12 ++++++----- 6 files changed, 72 insertions(+), 51 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index e9bee074..aa617829 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -2,6 +2,7 @@ ''' render functions relying on render-ws client scripts ''' +import os import json from functools import partial import logging @@ -9,6 +10,9 @@ from .render import Render from .stack import set_stack_state, make_stack_params +# setup logger +logger = logging.getLogger(__name__) + try: from pathos.multiprocessing import ProcessingPool as Pool has_pathos = True @@ -44,10 +48,10 @@ def import_single_json_file(stack, jsonfile, render=None, transformFile=None, stack_params + \ transform_params + \ [jsonfile] - logging.debug(cmd) + logger.debug(cmd) proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) proc.wait() - logging.debug(proc.stdout.read()) + logger.debug(proc.stdout.read()) def import_jsonfiles_and_transforms_parallel_by_z( @@ -109,7 +113,7 @@ def import_jsonfiles_parallel( transformFile=transformFile, client_scripts=client_scripts, host=host, port=port, owner=owner, - project=project, verbose=verbose) + project=project) partial_import(jsonfiles[0]) rs = pool.amap(partial_import, jsonfiles) rs.wait() @@ -120,7 +124,7 @@ def import_jsonfiles_parallel( def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, - verbose=False, **kwargs): + **kwargs): ''' import jsons using client script serially jsonfiles: iterator of filenames to be uploaded @@ -134,7 +138,7 @@ def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, stack, jsonfiles, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, - **{'verbose': verbose, 'close_stack': close_stack, + **{'close_stack': close_stack, 'transformFile': transformFile})) set_stack_state(stack, 'LOADING', host, port, owner, project) @@ -149,22 +153,20 @@ def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, stack_params + \ transform_params + \ jsonfiles - if verbose: - print cmd + logger.debug(cmd) proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) proc.wait() - if verbose: - print proc.stdout.read() + logger.debug(proc.stdout.read()) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) # FIXME paperweight for proper render_ws_client implemntation -# TODO this should allow you to set validator parameters def import_jsonfiles_validate_client(stack, jsonfiles, render=None, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, mem=6, + validator=None, **kwargs): ''' Uses java client for parallelization and validation @@ -181,11 +183,16 @@ def import_jsonfiles_validate_client(stack, jsonfiles, render=None, transform_params = (['--transformFile', transformFile] if transformFile is not None else []) - validator_params = [ - '--validatorClass', - 'org.janelia.alignment.spec.validator.TemTileSpecValidator', - '--validatorData', - 'minCoordinate:-500,maxCoordinate:50000,minSize:500,maxSize:10000'] + if validator is None: + validator_params = [ + '--validatorClass', + 'org.janelia.alignment.spec.validator.TemTileSpecValidator', + '--validatorData', + 'minCoordinate:-500,maxCoordinate:100000,' + 'minSize:500,maxSize:10000'] + else: + raise NotImplementedError('No custom validation handling!') + my_env = os.environ.copy() stack_params = make_stack_params(host, port, owner, project, stack) cmd = [os.path.join(client_scripts, 'run_ws_client.sh')] + \ @@ -197,11 +204,14 @@ def import_jsonfiles_validate_client(stack, jsonfiles, render=None, jsonfiles set_stack_state(stack, 'LOADING', host, port, owner, project) - logging.debug(cmd) + logger.debug(cmd) + subprocess.call(cmd, env=my_env) + + ''' proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) proc.wait() - logging.debug(proc.stdout.read()) - + logger.debug(proc.stdout.read()) + ''' if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index a78ba336..ea5800f4 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -7,6 +7,8 @@ import logging import requests +logger = logging.getLogger(__name__) + def world_to_local_coordinates(stack, z, x, y, render=None, host=None, port=None, owner=None, project=None, @@ -26,7 +28,7 @@ def world_to_local_coordinates(stack, z, x, y, render=None, host=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def local_to_world_coordinates(stack, tileId, x, y, render=None, @@ -48,7 +50,7 @@ def local_to_world_coordinates(stack, tileId, x, y, render=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def world_to_local_coordinates_batch(stack, z, data, render=None, host=None, @@ -125,7 +127,7 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z=0, answer[i, 1] = c[1] return answer except: - logging.error(json_answer) + logger.error(json_answer) def local_to_world_coordinates_array(stack, dataarray, tileId, z=0, @@ -162,4 +164,4 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z=0, answer[i, 1] = c[1] return answer except: - logging.error(json_answer) + logger.error(json_answer) diff --git a/renderapi/image.py b/renderapi/image.py index 1e98822e..17544cb7 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -7,6 +7,8 @@ import logging from .render import Render, format_baseurl, format_preamble +logger = logging.getLogger(__name__) + # define acceptable image formats -- currently render generates png, jpeg, tiff IMAGE_FORMATS = {'png': 'png-image', '.png': 'png-image', @@ -54,7 +56,7 @@ def get_bb_image(stack, z, x, y, width, height, render=None, scale=1.0, image = np.asarray(Image.open(io.BytesIO(r.content))) return image except: - logging.error(r.text) + logger.error(r.text) def get_tile_image_data(stack, tileId, render=None, @@ -81,11 +83,11 @@ def get_tile_image_data(stack, tileId, render=None, "/tile/%s/png-image" % (tileId) if normalizeForMatching: request_url += "?normalizeForMatching=true" - logging.debug(request_url) + logger.debug(request_url) r = session.get(request_url) try: img = Image.open(io.BytesIO(r.content)) array = np.asarray(img) return array except: - logging.error(r.text) + logger.error(r.text) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 30cd7cb2..25e255ef 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -6,6 +6,8 @@ import logging from .render import Render, format_baseurl +logger = logging.getLogger(__name__) + def get_matchcollection_owners(render=None, host=None, port=None, session=requests.session(), **kwargs): @@ -21,7 +23,7 @@ def get_matchcollection_owners(render=None, host=None, port=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_matchcollections(render=None, owner=None, host=None, port=None, @@ -39,7 +41,7 @@ def get_matchcollections(render=None, owner=None, host=None, port=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_match_groupIds(matchCollection, render=None, owner=None, host=None, @@ -57,7 +59,7 @@ def get_match_groupIds(matchCollection, render=None, owner=None, host=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_matches_outside_group(matchCollection, groupId, render=None, @@ -78,7 +80,7 @@ def get_matches_outside_group(matchCollection, groupId, render=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_matches_within_group(matchCollection, groupId, owner=None, @@ -99,7 +101,7 @@ def get_matches_within_group(matchCollection, groupId, owner=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, @@ -121,7 +123,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, @@ -144,7 +146,7 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, @@ -165,7 +167,7 @@ def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_match_groupIds_from_only(matchCollection, render=None, owner=None, @@ -185,7 +187,7 @@ def get_match_groupIds_from_only(matchCollection, render=None, owner=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_match_groupIds_to_only(matchCollection, render=None, owner=None, @@ -205,4 +207,4 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) diff --git a/renderapi/stack.py b/renderapi/stack.py index ed7daa36..2d3b6041 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -6,6 +6,8 @@ from .render import Render, format_baseurl, format_preamble from .utils import jbool +logger = logging.getLogger(__name__) + class StackVersion: def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=0, @@ -40,9 +42,9 @@ def from_dict(self, d): eval('self.%s=d[%s]' % (key, key)) -def set_stack_state(stack, render=None, state='LOADING', host=None, port=None, - owner=None, project=None, session=requests.session(), - **kwargs): +def set_stack_state(stack, state='LOADING', host=None, port=None, + owner=None, project=None, render=None, + session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): @@ -55,13 +57,13 @@ def set_stack_state(stack, render=None, state='LOADING', host=None, port=None, assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] request_url = format_preamble( host, port, owner, project, stack) + "/state/%s" % state - logging.debug(request_url) + logger.debug(request_url) r = session.put(request_url, data=None, headers={"content-type": "application/json"}) return r -def likelyUniqueId(render=None, host=None, port=None, +def likelyUniqueId(host=None, port=None, render=None, session=requests.session(), **kwargs): '''return hex-code nearly-unique id from render server''' if render is not None: @@ -71,8 +73,9 @@ def likelyUniqueId(render=None, host=None, port=None, **{'session': session})) request_url = '{}/likelyUniqueId'.format(format_baseurl(host, port)) - return session.get(request_url, data=None, - headers={"content-type": "application/json"}) + r = session.get(request_url, data=None, + headers={"content-type": "text/plain"}) + return r.text def make_stack_params(host, port, owner, project, stack): @@ -98,7 +101,7 @@ def delete_stack(stack, render=None, host=None, port=None, owner=None, request_url = format_preamble(host, port, owner, project, stack) r = session.delete(request_url) - logging.debug(r.text) + logger.debug(r.text) return r @@ -116,7 +119,7 @@ def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, sv = StackVersion( cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber) request_url = format_preamble(host, port, owner, project, stack) - logging.debug("stack version {} {}".format(request_url, sv.to_dict())) + logger.debug("stack version {} {}".format(request_url, sv.to_dict())) payload = json.dumps(sv.to_dict()) r = session.post(request_url, data=payload, headers={"content-type": "application/json", @@ -124,7 +127,7 @@ def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, try: return r except: - logging.error(r.text) + logger.error(r.text) def clone_stack(inputstack, outputstack, render=None, host=None, port=None, @@ -149,7 +152,7 @@ def clone_stack(inputstack, outputstack, render=None, host=None, port=None, request_url = '{}/{}'.format(format_preamble( host, port, owner, project, inputstack), outputstack) - logging.debug(request_url) + logger.debug(request_url) r = session.put(request_url, params={ 'z': zs, 'toProject': toProject, 'skipTransforms': jbool(skipTransforms)}, diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 0a45b689..114ee54d 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -4,6 +4,8 @@ import requests import numpy as np +logger = logging.getLogger(__name__) + class ResolvedTileSpecMap: def __init__(self, tilespecs=[], transforms=[]): @@ -410,7 +412,7 @@ def get_tile_spec(stack, tile, render=None, host=None, port=None, owner=None, try: tilespec_json = r.json() except: - logging.error(r.text) + logger.error(r.text) return TileSpec(json=tilespec_json['tileSpecs'][0]) @@ -451,12 +453,12 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( z, x, y, width, height, scale) - logging.debug(request_url) + logger.debug(request_url) r = session.get(request_url) try: tilespecs_json = r.json() except: - logging.error(r.text) + logger.error(r.text) return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json['tileSpecs']] @@ -474,12 +476,12 @@ def get_tile_specs_from_z(stack, z, render=None, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) - logging.debug(request_url) + logger.debug(request_url) r = session.get(request_url) try: tilespecs_json = r.json() except: - logging.error(r.text) + logger.error(r.text) if len(tilespecs_json) == 0: return None From 4e419b9401f175deff0d0272a11083c7182286f7 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 9 Feb 2017 12:52:47 -0800 Subject: [PATCH 043/766] transform; utils: make transforms hashable, add function to streamline logging imports --- renderapi/transform.py | 12 +++++++++++- renderapi/utils.py | 9 +++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 6849cf5e..5c98b27d 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -8,11 +8,15 @@ interpolation functions Affine as subset of Polynomial2D approximation of other functions(TPS, meshtechniques) to Polynomial2D + ^ would this be better in Java using mpicbg implementation? ''' import json +import logging import numpy as np from .errors import ConversionError +logger = logging.getLogger(__name__) + class EstimationError(Exception): pass @@ -94,6 +98,12 @@ def __str__(self): def __repr__(self): return self.__str__() + def __eq__(self, other): + return self.__str__() == other.__str__() + + def __hash__(self): + return hash((self.__str__())) + class AffineModel(Transform): className = 'mpicbg.trakem2.transform.AffineModel2D' @@ -354,7 +364,7 @@ def transformsum(transformlist, src=None): sumtform = Polynomial2DTransform(identity=True) for tform in transformlist: if isinstance(tform, list): - logging.debug('found transformlist!') + logger.debug('found transformlist!') sumtform = sumtform.concatenate( transformsum(tform, src=src), srcpts=src) else: diff --git a/renderapi/utils.py b/renderapi/utils.py index ea8f3cba..bc8083c7 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -2,7 +2,16 @@ ''' utilities to make render/java/web interfacing easier ''' +import logging def jbool(val): + '''return string representing json string values of py booleans''' return 'true' if val else 'false' + + +def stripLogger(logger): + '''remove all handlers from a logger -- useful for redefining''' + if logger.handlers: + for handler in logger.handlers: + logger.removeHandler(handler) From 236b2d9f12d6a31fc53cd6bfca18f5d40e9e974d Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 15 Feb 2017 12:39:49 -0800 Subject: [PATCH 044/766] utils: doctring and logging --- renderapi/utils.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index bc8083c7..301cca23 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -1,17 +1,26 @@ #!/usr/bin/env python ''' -utilities to make render/java/web interfacing easier +utilities to make render/java/web/life interfacing easier ''' import logging +logger = logging.getLogger(__name__) + def jbool(val): - '''return string representing json string values of py booleans''' + '''return string representing java string values of py booleans''' + if not isinstance(val, bool): + logger.warning('Evaluating javastring of non-boolean {} {}'.format( + type(val), val)) return 'true' if val else 'false' def stripLogger(logger): - '''remove all handlers from a logger -- useful for redefining''' + ''' + remove all handlers from a logger -- useful for redefining + input: + logger: logging logger as from logging.getLogger + ''' if logger.handlers: for handler in logger.handlers: logger.removeHandler(handler) From 3198b21359449c017b6d29b1a7d332bc94165e33 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 15 Feb 2017 12:40:36 -0800 Subject: [PATCH 045/766] utils: fix local logger --- renderapi/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 301cca23..902ba67c 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -15,12 +15,12 @@ def jbool(val): return 'true' if val else 'false' -def stripLogger(logger): +def stripLogger(logger_tostrip): ''' remove all handlers from a logger -- useful for redefining input: - logger: logging logger as from logging.getLogger + logger_tostrip: logging logger as from logging.getLogger ''' - if logger.handlers: - for handler in logger.handlers: - logger.removeHandler(handler) + if logger_tostrip.handlers: + for handler in logger_tostrip.handlers: + logger_tostrip.removeHandler(handler) From a8c0ba083f48e748353c54fd4157496197a796f3 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 20 Feb 2017 09:23:36 -0800 Subject: [PATCH 046/766] utils: functions for loading jsons and dictionaries as objects --- renderapi/transform.py | 17 ++--------------- renderapi/utils.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 5c98b27d..03811279 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -13,25 +13,12 @@ import json import logging import numpy as np -from .errors import ConversionError +from .errors import ConversionError, EstimationError +from .utils import _load_dict, _load_json logger = logging.getLogger(__name__) -class EstimationError(Exception): - pass - - -def _load_dict(obj, d): - obj.__dict__.update({k: v for k, v in d.items()}) - - -def _load_json(obj, j): - with open(j, 'r') as f: - jd = json.load(f) - _load_dict(obj, jd) - - class TransformList: def __init__(self, tforms): self.tforms = tforms diff --git a/renderapi/utils.py b/renderapi/utils.py index 902ba67c..b83cb0b2 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -24,3 +24,14 @@ def stripLogger(logger_tostrip): if logger_tostrip.handlers: for handler in logger_tostrip.handlers: logger_tostrip.removeHandler(handler) + + +def _load_dict(obj, d): + obj.__dict__.update({k: v for k, v in d.items()}) + + +def _load_json(obj, j): + '''load object from dictionary-style json''' + with open(j, 'r') as f: + jd = json.load(f) + _load_dict(obj, jd) From 1d37f7cd16a0dd4d859415fd0fff349a49f1e02a Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 20 Feb 2017 15:36:53 -0800 Subject: [PATCH 047/766] requirements: add pathos requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b88e5f3d..e316b911 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ requests numpy pillow +pathos From d08ead7ae72c1e42390da690a9e5f5bb644c48ea Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 20 Feb 2017 15:49:55 -0800 Subject: [PATCH 048/766] stack: simple port of functions from render.py --- renderapi/stack.py | 99 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/renderapi/stack.py b/renderapi/stack.py index 2d3b6041..778a3ad2 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -130,6 +130,7 @@ def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, logger.error(r.text) +# FIXME multiple z indices require multiple params? def clone_stack(inputstack, outputstack, render=None, host=None, port=None, owner=None, project=None, skipTransforms=False, toProject=None, z=None, session=None, **kwargs): @@ -159,3 +160,101 @@ def clone_stack(inputstack, outputstack, render=None, host=None, port=None, data=json.dumps(sv.to_dict())) return r + + +def get_z_values_for_stack(stack, project=None, host=None, port=None, + owner=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_z_values_for_stack(stack, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + session=session, **kwargs)) + + request_url = format_preamble( + host, port, owner, project, stack) + "/zValues/" + if verbose: + print request_url + r = session.get(request_url) + try: + return r.json() + except: + logger.error(r.text) + + +def get_z_value_for_section(stack, sectionId, project=None, + host=None, port=None, owner=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_z_value_for_section(stack, sectionId, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + session=session, **kwargs)) + + request_url = format_preamble( + host, port, owner, project, stack) + "/section/%s/z" % (sectionId) + r = session.get(request_url) + try: + return r.json() + except: + logger.error(r.text) + + +def put_resolved_tilespecs(stack, data, host=None, port=None, + owner=None, project=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return put_resolved_tilespecs(stack, data, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + session=session, **kwargs)) + + request_url = format_preamble( + host, port, owner, project, stack) + "/resolvedTiles" + r = session.put(request_url, data=data, + headers={"content-type": "application/json", + "Accept": "text/plain"}) + return r + + +def get_bounds_from_z(stack, z, host=None, port=None, owner=None, + project=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_bounds_from_z(stack, z, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + session=session, **kwargs)) + + request_url = format_preamble( + host, port, owner, project, stack) + '/z/%f/bounds' % (z) + + r = session.get(request_url) + try: + return r.json() + except: + logger.error(r.text) + + +def get_section_z_value(stack, sectionId, host=None, port=None, + owner=None, project=None, render=None, + session=requests.session(), **kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_section_z_value(stack, sectionId, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + session=session, **kwargs)) + + request_url = format_preamble( + host, port, owner, project, stack) + "/section/%s/z" % sectionId + r = session.get(request_url) + try: + return float(r.json()) + except: + logger.error(r.text) + return float(process_simple_url_request(request_url, session)) From 1c54396bcf284e8fb03884c6ec9fab4049a295de Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 20 Feb 2017 15:50:54 -0800 Subject: [PATCH 049/766] errors: add Estimation and Spec Exceptions --- renderapi/errors.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/renderapi/errors.py b/renderapi/errors.py index 232ca508..845028f7 100644 --- a/renderapi/errors.py +++ b/renderapi/errors.py @@ -10,3 +10,11 @@ class ClientScriptError(Exception): class ConversionError(Exception): pass + + +class EstimationError(Exception): + pass + + +class SpecError(Exception): + pass From 12f69cf296d9a881b49858b0204fc2c077e319a9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 20 Feb 2017 16:17:39 -0800 Subject: [PATCH 050/766] fixing setup.py install script for module --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da0c5414..7a7aa0cf 100644 --- a/setup.py +++ b/setup.py @@ -5,5 +5,4 @@ author='Forrest Collman,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', - packages = [''], - py_modules=['tilespec','renderapi']) + packages = ['renderapi']) From ca44f7fd8a4d720a3abd4ad9aa315b3508a67b38 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 20 Feb 2017 16:18:01 -0800 Subject: [PATCH 051/766] tilespec: default to 16-bit --- renderapi/tilespec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 114ee54d..5932cdb2 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -275,7 +275,7 @@ def from_dict(self, d): class TileSpec: def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, frameId=None, maskUrl=None, - minint=0, maxint=65000, layout=Layout(), tforms=[], + minint=0, maxint=65535, layout=Layout(), tforms=[], inputfilters=[], scale3Url=None, scale2Url=None, scale1Url=None, json=None): if json is not None: From 42874efc9003a21962f92fb1a3e46b7779cb04ba Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 20 Feb 2017 16:39:37 -0800 Subject: [PATCH 052/766] utils: missing json import --- renderapi/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/renderapi/utils.py b/renderapi/utils.py index b83cb0b2..d2da3c9b 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -3,6 +3,7 @@ utilities to make render/java/web/life interfacing easier ''' import logging +import json logger = logging.getLogger(__name__) From ff88554f7311fc5d16134433ba5b4494e4a99f64 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 20 Feb 2017 16:46:46 -0800 Subject: [PATCH 053/766] setup: PEP8 --- setup.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 7a7aa0cf..69845f07 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ from distutils.core import setup setup(name='render-python', - version='1.0', - description=' a python API setup to interact via python with render databases see https://github.com/saalfeldlab/render', - author='Forrest Collman,Eric Perlman,Sharmi Seshamani', - author_email='forrest.collman@gmail.com', - url='https://github.com/fcollman/render-python', - packages = ['renderapi']) + version='1.0', + description=' a python API setup to interact via python with render ' + 'databases see https://github.com/saalfeldlab/render', + author='Forrest Collman,Eric Perlman,Sharmi Seshamani', + author_email='forrest.collman@gmail.com', + url='https://github.com/fcollman/render-python', + packages=['renderapi']) From ec50528b16b82eb5133036e377126874422c237f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 09:45:33 -0800 Subject: [PATCH 054/766] adding requirements to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7a7aa0cf..bbd7a86b 100644 --- a/setup.py +++ b/setup.py @@ -5,4 +5,5 @@ author='Forrest Collman,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', + install_requires=[requests,numpy,pillow,pathos], packages = ['renderapi']) From db78f51ef67ebbfd289ba3baadffa414a675a6c8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 09:47:53 -0800 Subject: [PATCH 055/766] fixing requirements to setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bbd7a86b..16dead2c 100644 --- a/setup.py +++ b/setup.py @@ -5,5 +5,5 @@ author='Forrest Collman,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', - install_requires=[requests,numpy,pillow,pathos], + install_requires=['requests','numpy','pillow','pathos'], packages = ['renderapi']) From c064ac22cff54b234deb365abfe09d8e27b5600a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 09:53:07 -0800 Subject: [PATCH 056/766] fixing requirements to setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 16dead2c..7a7aa0cf 100644 --- a/setup.py +++ b/setup.py @@ -5,5 +5,4 @@ author='Forrest Collman,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', - install_requires=['requests','numpy','pillow','pathos'], packages = ['renderapi']) From 49c7328c8c33b292e15b4a6a07c10161d778acc9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 10:01:23 -0800 Subject: [PATCH 057/766] fixing requirements to setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a7aa0cf..eb75b0d4 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ -from distutils.core import setup +from setuptools import setup setup(name='render-python', version='1.0', description=' a python API setup to interact via python with render databases see https://github.com/saalfeldlab/render', author='Forrest Collman,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', + install_requires=['requests','pillow','numpy','pathos'], packages = ['renderapi']) From af14dafa2e4021c5ecb7b2077a05c9d8c6d8336a Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 21 Feb 2017 11:48:18 -0800 Subject: [PATCH 058/766] tilespec: add basic mipMapping classes --- renderapi/tilespec.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 5932cdb2..e874af17 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,5 +1,6 @@ #!/usr/bin/env python from .render import Render, format_baseurl, format_preamble +from collections import OrderedDict import logging import requests import numpy as np @@ -395,6 +396,60 @@ def from_dict(self, d): self.inputfilters.append(f) +class MipMapLevel: + def __init__(self, level, imageUrl=None, maskUrl=None): + self.level = level + self.imageUrl = imageUrl + self.maskUrl = maskUrl + + def to_dict(self): + return dict(self.__iter__()) + + def _formatUrls(self): + d = {} + if self.imageUrl is not None: + d.update({'imageUrl': self.imageUrl}) + if self.maskUrl is not None: + d.update({'maskUrl': self.maskUrl}) + return d + + def __iter__(self): + return iter([(self.level, self._formatUrls())]) + + +class ImagePyramid: + def __init__(self, mipMapLevels=[]): + self.mipMapLevels = mipMapLevels + + def to_dict(self): + return dict(self.__iter__()) + + def to_ordered_dict(self, key=None): + '''defaults to order by mipmapLevel''' + return OrderedDict(sorted( + self.__iter__(), key=((lambda x: x[0]) if key + is None else key))) + + def append(self, mmL): + self.mipMapLevels.append(mmL) + + def update(self, mmL): + self.mipMapLevels = [ + l for l in self.mipMapLevels if l.level != mmL.level] + self.append(mmL) + + def get(self, to_get): + return self.to_dict()[to_get] # TODO should this default + + @property + def levels(self): + return [int(i.level) for i in self.mipMapLevels] + + def __iter__(self): + return iter([ + l for sl in [list(mmL) for mmL in self.mipMapLevels] for l in sl]) + + def get_tile_spec(stack, tile, render=None, host=None, port=None, owner=None, project=None, session=requests.session(), **kwargs): if render is not None: From aa40dc04ae7d57d94db1ffc14b7b8df3dcd4748c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 12:28:37 -0800 Subject: [PATCH 059/766] fixing minmax call --- renderapi/tilespec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 5932cdb2..395725f5 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -433,8 +433,8 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, width = xmax - xmin height = ymax - ymin return get_tile_specs_from_box(stack, z, x, y, width, height, - scale, host, port, owner, project, - session) + scale=scale, host=host, port=port, + owner=owner, project=project, session=session) def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, From e61ce35bc17b241b989e2f7ce09c5ae2370e6577 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 21 Feb 2017 12:56:16 -0800 Subject: [PATCH 060/766] tilespec: substitute imageXURL for ImagePyramid --- renderapi/tilespec.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index e874af17..3e3ad861 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -278,7 +278,7 @@ def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, frameId=None, maskUrl=None, minint=0, maxint=65535, layout=Layout(), tforms=[], inputfilters=[], scale3Url=None, scale2Url=None, - scale1Url=None, json=None): + scale1Url=None, json=None, mipMapLevels=[]): if json is not None: self.from_dict(json) else: @@ -287,18 +287,31 @@ def __init__(self, tileId=None, z=None, width=None, height=None, self.width = width self.height = height self.layout = layout - self.imageUrl = imageUrl - self.maskUrl = maskUrl self.minint = minint self.maxint = maxint self.tforms = tforms self.frameId = frameId self.layout = layout self.inputfilters = inputfilters + + self.ip = ImagePyramid(mipMapLevels=mipMapLevels) + # legacy scaleXUrl + self.maskUrl = maskUrl + self.imageUrl = imageUrl self.scale3Url = scale3Url self.scale2Url = scale2Url self.scale1Url = scale1Url + if imageUrl is not None: + self.ip.update(MipMapLevel( + 0, imageUrl=imageUrl, maskUrl=maskUrl)) + if scale1Url is not None: + self.ip.update(MipMapLevel(1, imageUrl=scale1Url)) + if scale2Url is not None: + self.ip.update(MipMapLevel(2, imageUrl=scale2Url)) + if scale3Url is not None: + self.ip.update(MipMapLevel(3, imageUrl=scale3Url)) + def to_dict(self): thedict = {} thedict['tileId'] = self.tileId @@ -310,6 +323,7 @@ def to_dict(self): thedict['frameId'] = self.frameId if self.layout is not None: thedict['layout'] = self.layout.to_dict() + ''' mipmapdict = {} mipmapdict['0'] = {} mipmapdict['0']['imageUrl'] = self.imageUrl @@ -324,7 +338,8 @@ def to_dict(self): mipmapdict['2']['imageUrl'] = self.scale2Url if self.maskUrl is not None: mipmapdict['0']['maskUrl'] = self.maskUrl - thedict['mipmapLevels'] = mipmapdict + ''' + thedict['mipmapLevels'] = self.ip.to_ordered_dict() thedict['transforms'] = {} thedict['transforms']['type'] = 'list' # thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] @@ -361,6 +376,7 @@ def from_dict(self, d): self.maxX = d.get('maxX', None) self.maxY = d.get('maxY', None) self.minY = d.get('minY', None) + # legacy scaleXUrl self.imageUrl = d['mipmapLevels']['0']['imageUrl'] self.maskUrl = d['mipmapLevels']['0'].get('maskUrl', None) if d['mipmapLevels'].get('2', None) is not None: @@ -376,6 +392,11 @@ def from_dict(self, d): else: self.scale3Url = None + self.ip = ImagePyramid(mipMapLevels=[ + MipMapLevel( + int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) + for l, v in d['mipmapLevels'].items()]) + self.tforms = [] for t in d['transforms']['specList']: if t['type'] == 'ref': @@ -488,8 +509,9 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, width = xmax - xmin height = ymax - ymin return get_tile_specs_from_box(stack, z, x, y, width, height, - scale, host, port, owner, project, - session) + scale=scale, host=host, port=port, + owner=owner, project=project, + session=session) def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, From c72113859690f97645f0fd6d1e3d53b4fbbdb8b7 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 13:20:56 -0800 Subject: [PATCH 061/766] adding kwargs to connect call to allow extra keyword args to get passed into function --- renderapi/render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index aab199df..97b8d3df 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -209,7 +209,8 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, - client_scripts=None): + client_scripts=None,json_dict=None,**kwargs): + '''helper function to connect to a render instance''' if host is None: if 'RENDER_HOST' not in os.environ: From ffb723c2ea1f59a4d6e08636755ca78a0cf0bf74 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 13:43:31 -0800 Subject: [PATCH 062/766] fixed to allow dicts with no layouts --- renderapi/tilespec.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 395725f5..33129cf5 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -261,15 +261,16 @@ def to_dict(self): return d def from_dict(self, d): - self.sectionId = d.get('sectionId', None) - self.cameraId = d.get('camera', None) - self.scopeId = d.get('temca', None) - self.imageRow = d.get('imageRow', None) - self.imageCol = d.get('imageCol', None) - self.stageX = d.get('stageX', None) - self.stageY = d.get('stageY', None) - self.rotation = d.get('rotation', None) - self.pixelsize = d.get('pixelsize', None) + if d is not None: + self.sectionId = d.get('sectionId', None) + self.cameraId = d.get('camera', None) + self.scopeId = d.get('temca', None) + self.imageRow = d.get('imageRow', None) + self.imageCol = d.get('imageCol', None) + self.stageX = d.get('stageX', None) + self.stageY = d.get('stageY', None) + self.rotation = d.get('rotation', None) + self.pixelsize = d.get('pixelsize', None) class TileSpec: @@ -355,7 +356,7 @@ def from_dict(self, d): self.maxint = d['maxIntensity'] self.frameId = d.get('frameId', None) self.layout = Layout() - self.layout.from_dict(d['layout']) + self.layout.from_dict(d.get('layout',None)) self.minX = d.get('minX', None) self.maxX = d.get('maxX', None) self.maxY = d.get('maxY', None) From cdc9d2e91a9ed2ab3287e02630621dba2e3d8b66 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 21 Feb 2017 16:19:47 -0800 Subject: [PATCH 063/766] tilespec: replace self.scaleXUrl with ImagePyramid attribute --- renderapi/tilespec.py | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 3e3ad861..60a83905 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -323,22 +323,6 @@ def to_dict(self): thedict['frameId'] = self.frameId if self.layout is not None: thedict['layout'] = self.layout.to_dict() - ''' - mipmapdict = {} - mipmapdict['0'] = {} - mipmapdict['0']['imageUrl'] = self.imageUrl - if self.scale1Url is not None: - mipmapdict['1'] = {} - mipmapdict['1']['imageUrl'] = self.scale1Url - if self.scale3Url is not None: - mipmapdict['3'] = {} - mipmapdict['3']['imageUrl'] = self.scale3Url - if self.scale2Url is not None: - mipmapdict['2'] = {} - mipmapdict['2']['imageUrl'] = self.scale2Url - if self.maskUrl is not None: - mipmapdict['0']['maskUrl'] = self.maskUrl - ''' thedict['mipmapLevels'] = self.ip.to_ordered_dict() thedict['transforms'] = {} thedict['transforms']['type'] = 'list' @@ -376,22 +360,6 @@ def from_dict(self, d): self.maxX = d.get('maxX', None) self.maxY = d.get('maxY', None) self.minY = d.get('minY', None) - # legacy scaleXUrl - self.imageUrl = d['mipmapLevels']['0']['imageUrl'] - self.maskUrl = d['mipmapLevels']['0'].get('maskUrl', None) - if d['mipmapLevels'].get('2', None) is not None: - self.scale2Url = d['mipmapLevels']['2'].get('imageUrl', None) - else: - self.scale2Url = None - if d['mipmapLevels'].get('1', None) is not None: - self.scale1Url = d['mipmapLevels']['1'].get('imageUrl', None) - else: - self.scale1Url = None - if d['mipmapLevels'].get('3', None) is not None: - self.scale3Url = d['mipmapLevels']['3'].get('imageUrl', None) - else: - self.scale3Url = None - self.ip = ImagePyramid(mipMapLevels=[ MipMapLevel( int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) From 31d202464b19b9661d12c0b2da4bba9a4460478e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 18:26:44 -0800 Subject: [PATCH 064/766] fixed default stack resolutions to be 1.0 and allowed stackresolutions to be specified --- renderapi/stack.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 778a3ad2..98833447 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -10,8 +10,8 @@ class StackVersion: - def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=0, - stackResolutionY=0, stackResolutionZ=0, + def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=1.0, + stackResolutionY=1.0, stackResolutionZ=1.0, materializedBoxRootPath=None, versionNotes="", createTimestamp=None, **kwargs): self.cycleNumber = cycleNumber @@ -105,7 +105,9 @@ def delete_stack(stack, render=None, host=None, port=None, owner=None, return r -def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, +def create_stack(stack, cycleNumber=1, cycleStepNumber=1, + stackResolutionX=1.0, stackResolutionY=1.0,stackResolutionZ=1.0, + render=None, host=None, port=None, owner=None, project=None, session=requests.session(), **kwargs): if render is not None: @@ -114,10 +116,13 @@ def create_stack(stack, cycleNumber=1, cycleStepNumber=1, render=None, return create_stack(stack, **render.make_kwargs( host=host, port=port, owner=owner, project=project, **{'session': session, 'cycleNumber': cycleNumber, - 'cycleStepNumber': cycleStepNumber})) + 'cycleStepNumber': cycleStepNumber,'stackResolutionX':stackResolutionX, + 'stackResolutionY':stackResolutionY,'stackResolutionZ':stackResolutionZ})) sv = StackVersion( - cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber) + cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, + stackResolutionX=stackResolutionX, stackResolutionY=stackResolutionY, + stackResolutionZ=stackResolutionZ) request_url = format_preamble(host, port, owner, project, stack) logger.debug("stack version {} {}".format(request_url, sv.to_dict())) payload = json.dumps(sv.to_dict()) From 813b7028f845fadfb1e8ce550cc98733c1668e61 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 18:34:15 -0800 Subject: [PATCH 065/766] fixed import parallel to do more than first json file in list --- renderapi/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index aa617829..afacee5b 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -94,7 +94,7 @@ def import_jsonfiles_parallel( import jsons using client script in parallel jsonfiles: list of jsonfiles to upload poolsize: number of upload processes spawned by multiprocessing pool - transformFile: ? + transformFile: a single json file containing transforms referenced in the jsonfiles ''' # process render-based default configuration if render is not None: @@ -114,7 +114,7 @@ def import_jsonfiles_parallel( client_scripts=client_scripts, host=host, port=port, owner=owner, project=project) - partial_import(jsonfiles[0]) + partial_import(jsonfiles) rs = pool.amap(partial_import, jsonfiles) rs.wait() if close_stack: From c1fd5fb209886b34e12ff2562ab3684dbdd40ea2 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 18:35:26 -0800 Subject: [PATCH 066/766] fixed import parallel to do more than first json file in list --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index afacee5b..01616bd5 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -114,7 +114,7 @@ def import_jsonfiles_parallel( client_scripts=client_scripts, host=host, port=port, owner=owner, project=project) - partial_import(jsonfiles) + rs = pool.amap(partial_import, jsonfiles) rs.wait() if close_stack: From 02ace042624fb9379e6f9ce3c731dbc4ba48df1e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 18:39:02 -0800 Subject: [PATCH 067/766] fixed import parallel to do more than first json file in list --- renderapi/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 01616bd5..e0cb253d 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -115,8 +115,8 @@ def import_jsonfiles_parallel( host=host, port=port, owner=owner, project=project) - rs = pool.amap(partial_import, jsonfiles) - rs.wait() + rs = pool.map(partial_import, jsonfiles) + if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From d0bf2895e3c64e21329fbd4821f790286112d83a Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 21 Feb 2017 18:40:09 -0800 Subject: [PATCH 068/766] render, utils: RenderClient class additions --- renderapi/render.py | 234 ++++++++++++-------------------------------- renderapi/utils.py | 4 + 2 files changed, 69 insertions(+), 169 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index aab199df..3ef0f89f 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -1,12 +1,11 @@ #!/usr/bin/env python import logging import os -import json -import subprocess -import sys -import tempfile import requests -import numpy as np +from .utils import defaultifNone +from .errors import ClientScriptError + +logger = logging.getLogger(__name__) class Render(object): @@ -18,9 +17,9 @@ def __init__(self, host=None, port=None, owner=None, project=None, self.DEFAULT_OWNER = owner self.DEFAULT_CLIENT_SCRIPTS = client_scripts - logging.debug('Render object created with ' - 'host={h}, port={p}, project={pr}, ' - 'owner={o}, scripts={s}'.format( + logger.debug('Render object created with ' + 'host={h}, port={p}, project={pr}, ' + 'owner={o}, scripts={s}'.format( h=self.DEFAULT_HOST, p=self.DEFAULT_PORT, pr=self.DEFAULT_PROJECT, o=self.DEFAULT_OWNER, s=self.DEFAULT_CLIENT_SCRIPTS)) @@ -50,173 +49,49 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, processed_kwargs.update(kwargs) return processed_kwargs - def process_defaults(self, host, port, owner, project, - client_scripts=None): - ''' - utility function which will convert arguments to default arguments if - they are None allows Render object to be used with defaults if - lazy, but allows projects/hosts/owners to be changed from call - to call if desired. used by many functions convert default None - arguments to default values. - ''' - if host is None: - host = self.DEFAULT_HOST - if port is None: - port = self.DEFAULT_PORT - if owner is None: - owner = self.DEFAULT_OWNER - if project is None: - project = self.DEFAULT_PROJECT - if client_scripts is None: - client_scripts = self.DEFAULT_CLIENT_SCRIPTS - return (host, port, owner, project, client_scripts) - - def get_z_values_for_stack(self, stack, project=None, host=None, port=None, - owner=None, session=requests.session(), - verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + "/zValues/" - if verbose: - print request_url - r = session.get(request_url) - try: - return r.json() - except: - print(r.text) - return None - - def get_z_value_for_section(self, stack, sectionId, project=None, - host=None, port=None, owner=None, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + "/section/%s/z" % (sectionId) - r = session.get(request_url) - try: - return r.json() - except: - print(r.text) - return None - - def put_resolved_tilespecs(self, stack, data, host=None, port=None, - owner=None, project=None, - session=requests.session(), verbose=False): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + "/resolvedTiles" - if verbose: - print request_url - - r = session.put(request_url, data=data, - headers={"content-type": "application/json", - "Accept": "text/plain"}) - return r - - def get_bounds_from_z(self, stack, z, host=None, port=None, owner=None, - project=None, session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + '/z/%f/bounds' % (z) - return self.process_simple_url_request(request_url, session) - - # - # API for doing the bulk requests locally (i.e., to be run on the cluster) - # Full documentation here: http://wiki.int.janelia.org/wiki/display/flyTEM/Coordinate+Mapping+Tools - # - - MAP_COORD_SCRIPT = "/groups/flyTEM/flyTEM/render/bin/map-coord.sh" - def batch_local_work(self, stack, z, data, host=None, port=None, - owner=None, project=None, localToWorld=False, - deleteTemp=True, threads=16): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - fromJson = tempfile.NamedTemporaryFile( - suffix=".json", mode='w', delete=False) - fromJson.write(data) - fromJson.flush() - fromJson.close() - - toJson = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - toJson.close() - - #cmd = "%s --owner %s --project %s --stack %s --z %d --fromJson %s --toJson %s --baseDataUrl http://tem-services.int.janelia.org:8080/render-ws/v1 --numberOfThreads %d" % (MAP_COORD_SCRIPT, owner, project, stack, z, fromJson.name, toJson.name, threads) - cmd = ("%s --owner %s --project %s --stack %s --z %d --fromJson %s " - "--toJson %s --baseDataUrl http://10.40.3.162:8080/render-ws/v1 " - "--numberOfThreads %d") % ( - MAP_COORD_SCRIPT, owner, project, stack, z, - fromJson.name, toJson.name, threads) - - if localToWorld: - cmd = cmd + " --localToWorld" - try: - rc = subprocess.call(cmd, shell="True") - if rc != 0: - raise Exception("Invalid return code (%d): %s" % (rc, cmd)) - - with open(toJson.name) as f: - outdata = json.load(f) - except: - print("Unexpected error:", sys.exc_info()[0]) - return json.loads("{}") - - if deleteTemp: - os.unlink(fromJson.name) - os.unlink(toJson.name) - - return outdata - - def world_to_local_coordinates_batch_local(self, stack, z, data, host=None, - port=None, owner=None, - project=None): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - return batch_local_work(stack, z, data, host, port, owner, project, - localToWorld=False) - - def local_to_world_coordinates_batch_local(self, stack, z, data, host=None, - port=None, owner=None, - project=None): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - return batch_local_work(stack, z, data, host, port, owner, project, - localToWorld=True) - - def get_section_z_value(self, stack, sectionId, host=None, port=None, - owner=None, project=None, verbose=False, - session=requests.session()): - (host, port, owner, project, client_scripts) = self.process_defaults( - host, port, owner, project) - request_url = self.format_preamble( - host, port, owner, project, stack) + "/section/%s/z" % sectionId - return float(self.process_simple_url_request(request_url, session)) - class RenderClient(Render): - '''Draft object for render_webservice_client.sh calls''' - def __init__(self, client_script=None, *args, **kwargs): + '''Draft object for run_ws_client.sh calls''' + def __init__(self, client_script=None, memGB=None, *args, **kwargs): super(RenderClient, self).__init__(**kwargs) + # FIXME remove this when completed + logger.error('Client functionality not implemented!') + if client_script is None: + raise ClientScriptError('No RenderClient script specified!') + elif not os.path.isfile(client_script): + raise ClientScriptError('Client script {} not found!') + if 'run_ws_client.sh' not in os.path.basename(client_script): + logger.warning( + 'Unrecognized client script {}!'.format(client_script)) self.client_script = client_script + if memGB is None: + logger.warning( + 'No default Java heap specified -- defaulting to 1G') + memGB = '1G' + self.memGB = memGB + def make_kwargs(self, *args, **kwargs): + # hack to get dictionary defaults to work + client_script = defaultifNone( + kwargs.pop('client_script', None), self.client_script) + memGB = defaultifNone(kwargs.pop('memGB', None), self.memGB) return super(RenderClient, self).make_kwargs( - client_script=self.client_script, *args, **kwargs) + client_script=client_script, + memGB=memGB, + *args, **kwargs) def connect(host=None, port=None, owner=None, project=None, - client_scripts=None): + client_scripts=None, client_script=None, memGB=None, **kwargs): '''helper function to connect to a render instance''' if host is None: if 'RENDER_HOST' not in os.environ: host = str(raw_input("Enter Render Host: ")) if host == '': - logging.critical('Render Host must not be empty!') + logger.critical('Render Host must not be empty!') raise ValueError('Render Host must not be empty!') + # TODO more flexible server input # host = (host if host.startswith('http') # else 'http://{}'.format(host)) else: @@ -227,7 +102,7 @@ def connect(host=None, port=None, owner=None, project=None, port = str(int(raw_input("Enter Render Port: "))) if port == '': # TODO better (no) port handling - logging.critical('Render Port must not be empty!') + logger.critical('Render Port must not be empty!') raise ValueError('Render Port must not be empty!') else: port = int(os.environ['RENDER_PORT']) @@ -238,7 +113,7 @@ def connect(host=None, port=None, owner=None, project=None, else: project = str(os.environ['RENDER_PROJECT']) if project == '': - logging.critical('Render Project must not be empty!') + logger.critical('Render Project must not be empty!') raise ValueError('Render Project must not be empty!') if owner is None: @@ -247,7 +122,7 @@ def connect(host=None, port=None, owner=None, project=None, else: owner = str(os.environ['RENDER_OWNER']) if owner == '': - logging.critical('Render Owner must not be empty!') + logger.critical('Render Owner must not be empty!') raise ValueError('Render Owner must not be empty!') # TODO should client_scripts be required? @@ -258,13 +133,34 @@ def connect(host=None, port=None, owner=None, project=None, else: client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) if client_scripts == '': - logging.critical('Render Client Scripts must ' - 'not be empty!') + logger.critical('Render Client Scripts must ' + 'not be empty!') raise ValueError('Render Client Scripts must ' 'not be empty!') + if client_script is None: + if 'RENDER_CLIENT_SCRIPT' not in os.environ: + # client_script = str(raw_input("Enter Render Client Script: ")) + pass + else: + client_script = str(os.environ['RENDER_CLIENT_SCRIPT']) - return Render(host=host, port=port, owner=owner, project=project, - client_scripts=client_scripts) + if memGB is None: + if 'RENDER_CLIENT_HEAP' not in os.environ: + pass + else: + memGB = str(os.environ['RENDER_CLIENT_HEAP']) + + try: + return RenderClient(client_script=client_script, memGB=memGB, + host=host, port=port, + owner=owner, project=project, + client_scripts=client_scripts) + except ClientScriptError as e: + logger.info(e) + logger.warning( + 'Could not initiate render Client -- falling back to web') + return Render(host=host, port=port, owner=owner, project=project, + client_scripts=client_scripts) def format_baseurl(host, port): @@ -291,7 +187,7 @@ def get_owners(host=None, port=None, render=None, try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, @@ -307,12 +203,12 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) if verbose: - logging.debug(request_url) + logger.debug(request_url) r = session.get(request_url) try: return r.json() except: - logging.error(r.text) + logger.error(r.text) def get_projects_by_owner(owner=None, host=None, port=None, render=None, diff --git a/renderapi/utils.py b/renderapi/utils.py index d2da3c9b..fe3d1aaf 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -36,3 +36,7 @@ def _load_json(obj, j): with open(j, 'r') as f: jd = json.load(f) _load_dict(obj, jd) + + +def defaultifNone(val, default=None): + return val if val is not None else default From 95c6c23ecc777380874fdcf4875422b58cf68a56 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 21 Feb 2017 18:46:47 -0800 Subject: [PATCH 069/766] fixed import parallel to do more than first json file in list --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb75b0d4..e0b5e064 100644 --- a/setup.py +++ b/setup.py @@ -5,5 +5,5 @@ author='Forrest Collman,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', - install_requires=['requests','pillow','numpy','pathos'], + install_requires=['requests','pillow','numpy','pathos','multiprocess'], packages = ['renderapi']) From 66a1fdbb1733bc3772f65d50d15f2f6f26d57266 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 21 Feb 2017 19:01:28 -0800 Subject: [PATCH 070/766] render: add Render.run() method to opt out of "render=Render". bug squash --- renderapi/render.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 3ef0f89f..01b05c44 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -49,6 +49,13 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, processed_kwargs.update(kwargs) return processed_kwargs + def run(self, f, *args, **kwargs): + ''' + run function from object + technically shorter than adding render=Render to kwargs + ''' + return f(*args, **self.make_kwargs(**kwargs)) + class RenderClient(Render): '''Draft object for run_ws_client.sh calls''' @@ -59,7 +66,8 @@ def __init__(self, client_script=None, memGB=None, *args, **kwargs): if client_script is None: raise ClientScriptError('No RenderClient script specified!') elif not os.path.isfile(client_script): - raise ClientScriptError('Client script {} not found!') + raise ClientScriptError('Client script {} not found!'.format( + client_script)) if 'run_ws_client.sh' not in os.path.basename(client_script): logger.warning( 'Unrecognized client script {}!'.format(client_script)) @@ -191,8 +199,7 @@ def get_owners(host=None, port=None, render=None, def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, - session=requests.session(), - verbose=False, **kwargs): + session=requests.session(), **kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') @@ -202,8 +209,7 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) - if verbose: - logger.debug(request_url) + logger.debug(request_url) r = session.get(request_url) try: return r.json() @@ -220,7 +226,8 @@ def get_projects_by_owner(owner=None, host=None, port=None, render=None, owner=owner, host=host, port=port, **{'session': session})) - metadata = get_stack_metadata_by_owner(owner) + metadata = get_stack_metadata_by_owner(owner=owner, host=host, + port=port, session=session) projects = list(set([m['stackId']['project'] for m in metadata])) return projects @@ -235,7 +242,8 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, owner=owner, host=host, port=port, project=project, **{'session': session})) - metadata = get_stack_metadata_by_owner(owner) + metadata = get_stack_metadata_by_owner(owner=owner, host=host, + port=port, session=session) stacks = ([m['stackId']['stack'] for m in metadata if m['stackId']['project'] == project]) return stacks From 8a02719886996247a82f6e799628ea4726c0d1f5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 08:06:33 -0800 Subject: [PATCH 071/766] adding dockerfile for render-python --- Dockerfile | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..376e0e33 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +FROM continuumio/anaconda +MAINTAINER Forrest Collman (forrest.collman@gmail.com) + +#install java +# auto validate license +RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections +# update repos +RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list +RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 +RUN apt-get update +RUN apt-get install oracle-java8-installer -y +ENV JAVA_HOME /usr/lib/jvm/java-8-oracle +#install gcc for pathos +RUN apt-get install gcc -y +RUN apt-get clean + +#install render python using pip from github +RUN pip install -e git+https://github.com/fcollman/render-python.git@module#egg=render-python + From 992d38d92e243ab89442926e0d8ced0bfcc7d175 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 08:31:09 -0800 Subject: [PATCH 072/766] adding example script to build and tag docker image --- docker_setup.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 docker_setup.sh diff --git a/docker_setup.sh b/docker_setup.sh new file mode 100755 index 00000000..6be10aaf --- /dev/null +++ b/docker_setup.sh @@ -0,0 +1,2 @@ +docker build -t renderpython:latest . +docker tag renderpython:latest fcollman/renderpython \ No newline at end of file From 465a6b722a08a53ea77954898d586f6e7a0ca957 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 09:40:47 -0800 Subject: [PATCH 073/766] fixing getting projects and stacks --- renderapi/render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 97b8d3df..a3e6595b 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -325,7 +325,7 @@ def get_projects_by_owner(owner=None, host=None, port=None, render=None, owner=owner, host=host, port=port, **{'session': session})) - metadata = get_stack_metadata_by_owner(owner) + metadata = get_stack_metadata_by_owner(owner=owner,host=host,port=port,session=session) projects = list(set([m['stackId']['project'] for m in metadata])) return projects @@ -340,7 +340,7 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, owner=owner, host=host, port=port, project=project, **{'session': session})) - metadata = get_stack_metadata_by_owner(owner) + metadata = get_stack_metadata_by_owner(owner=owner,host=host,port=port,session=session) stacks = ([m['stackId']['stack'] for m in metadata if m['stackId']['project'] == project]) return stacks From 47d09acb2488b0b2ae6e0e086ce265b398c29ae5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 09:46:39 -0800 Subject: [PATCH 074/766] fixing typo in get_stacks_by_owner_project --- renderapi/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index a3e6595b..30af2d6f 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -340,7 +340,7 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, owner=owner, host=host, port=port, project=project, **{'session': session})) - metadata = get_stack_metadata_by_owner(owner=owner,host=host,port=port,session=session) + metadata = get_stacks_by_owner_project(owner=owner,host=host,port=port,session=session) stacks = ([m['stackId']['stack'] for m in metadata if m['stackId']['project'] == project]) return stacks From 2bec086ab3e4db7c30c4316a6389b283e81c690a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 09:49:24 -0800 Subject: [PATCH 075/766] fixing typo in get_stacks_by_owner_project --- renderapi/render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 30af2d6f..6a286014 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -336,11 +336,11 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - return get_projects_by_owner(**render.make_kwargs( + return get_stacks_by_owner_project(**render.make_kwargs( owner=owner, host=host, port=port, project=project, **{'session': session})) - metadata = get_stacks_by_owner_project(owner=owner,host=host,port=port,session=session) + metadata = get_stack_metadata_by_owner(owner=owner,host=host,port=port,session=session) stacks = ([m['stackId']['stack'] for m in metadata if m['stackId']['project'] == project]) return stacks From c4c9c5a8c8dee5df63aa6fae880db2e0122979e2 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 10:02:46 -0800 Subject: [PATCH 076/766] fixing verbose logging in get_z_values_for_stack --- renderapi/stack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 98833447..98353c07 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -179,8 +179,7 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" - if verbose: - print request_url + logger.debug(request_url) r = session.get(request_url) try: return r.json() From 53eda81d8e3d7338282e0a993928111f0470c8c2 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 22 Feb 2017 10:38:12 -0800 Subject: [PATCH 077/766] render: bug fix --- renderapi/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index 01b05c44..361e19ef 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -238,7 +238,7 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - return get_projects_by_owner(**render.make_kwargs( + return get_stacks_by_owner_project(**render.make_kwargs( owner=owner, host=host, port=port, project=project, **{'session': session})) From c5a09a66dc329ac41a9bacfaee84b011833c5d06 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 22 Feb 2017 10:38:34 -0800 Subject: [PATCH 078/766] client: support RenderClient for ImportJsonClient, TilePairClient, ImportTransformChangeClient, and CoordinateClient --- renderapi/client.py | 224 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 223 insertions(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index aa617829..ceb048bd 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -7,7 +7,9 @@ from functools import partial import logging import subprocess -from .render import Render +import tempfile +from .errors import ClientScriptError +from .render import Render, RenderClient from .stack import set_stack_state, make_stack_params # setup logger @@ -215,3 +217,223 @@ def import_jsonfiles_validate_client(stack, jsonfiles, render=None, ''' if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +def call_run_ws_client(className, add_args=[], renderclient=None, + memGB=None, client_script=None, subprocess_mode=None, + **kwargs): + ''' + simple call for run_ws_client.sh -- all arguments set in add_args + ''' + if renderclient is not None: + if isinstance(renderclient, RenderClient): + return call_run_ws_client(className, add_args=add_args, + subprocess_mode=subprocess_mode, + **renderclient.make_kwargs( + memGB=memGB, + client_script=client_script)) + + subprocess_modes = {'call': subprocess.call, + 'check_call': subprocess.check_call, + 'check_output': subprocess.check_output} + if subprocess_mode not in subprocess_modes: + logging.warning( + 'Unknown subprocess mode {} specified -- ' + 'using default subprocess.call'.format(subprocess_mode)) + return subprocess_modes.get( + subprocess_mode, subprocess.call)( + map(str, [client_script, memGB, className] + add_args)) + + +def get_param(var, flag): + return ([flag, var] if var is not None else []) + + +def importJsonClient(stack, tileFiles=None, transformFile=None, + subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + '''run ImportJsonClient.java''' + if render is not None: + if isinstance(render, RenderClient): + return importJsonClient( + stack, tileFiles=tileFiles, transformFile=transformFile, + subprocess_mode=subprocess_mode, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs)) + elif not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + + argvs = (make_stack_params(host, port, owner, project, stack) + + (['--transformFile', transformFile] if transformFile else []) + + (tileFiles if isinstance(tileFiles, list) + else [tileFiles])) + call_run_ws_client('org.janelia.render.client.ImportJsonClient', + add_args=argvs, subprocess_mode=subprocess_mode, + client_script=client_script, memGB=memGB) + + +def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, + baseowner=None, baseproject=None, basestack=None, + xyNeighborFactor=None, zNeighborDistance=None, + excludeCornerNeighbors=None, + excludeCompletelyObscuredTiles=None, + excludeSameLayerNeighbors=None, + excludeSameSectionNeighbors=None, + excludePairsInMatchCollection=None, + minx=None, maxx=None, miny=None, maxy=None, + subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + '''run TilePairClient.java''' + if render is not None: + if isinstance(render, RenderClient): + return tilePairClient( + stack, minz, maxz, outjson=outjson, delete_json=delete_json, + baseowner=baseowner, baseproject=baseproject, + basestack=basestack, + xyNeighborFactor=xyNeighborFactor, + zNeighborDistance=zNeighborDistance, + excludeCornerNeighbors=excludeCornerNeighbors, + excludeCompletelyObscuredTiles=excludeCompletelyObscuredTiles, + excludeSameLayerNeighbors=excludeSameLayerNeighbors, + excludeSameSectionNeighbors=excludeSameSectionNeighbors, + excludePairsInMatchCollection=excludePairsInMatchCollection, + minx=minx, maxx=maxx, miny=miny, maxy=maxy, + subprocess_mode=subprocess_mode, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs)) + elif not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + + if outjson is None: + tempjson = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) + tempjson.close() + delete_json = True + outjson = tempjson.name + + argvs = (make_stack_params(host, port, owner, project, stack) + + get_param(baseowner, '--baseOwner') + + get_param(baseproject, '--baseProject') + + get_param(basestack, '--baseStack') + + ['--minZ', minz, '--maxZ', maxz] + + get_param(xyNeighborFactor, '--xyNeighborFactor') + + get_param(zNeighborDistance, '--zNeighborDistance') + + get_param(excludeCornerNeighbors, '--excludeCornerNeighbors') + + get_param(excludeCompletelyObscuredTiles, + '--excludeCompletelyObscuredTiles') + + get_param(excludeSameLayerNeighbors, + '--excludeSameLayerNeighbors') + + get_param(excludeSameSectionNeighbors, + '--excludeSameSectionNeighbors') + + get_param(excludePairsInMatchCollection, + '--excludePairsInMatchCollection') + + ['--toJson', outjson] + + get_param(minx, '--minX') + get_param(maxx, '--maxX') + + get_param(miny, '--minY') + get_param(maxy, '--maxY')) + + call_run_ws_client('org.janelia.render.client.TilePairClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, + add_args=argvs) + + with open(outjson, 'r') as f: + jsondata = json.load(f) + + if delete_json: + os.remove(outjson) + return jsondata + + +def importTransformChangesClient(stack, targetStack, transformFile, + targetOwner=None, targetProject=None, + changeMode=None, subprocess_mode=None, + host=None, port=None, owner=None, + project=None, client_script=None, memGB=None, + render=None, **kwargs): + ''' + run ImportTransformChangesClient.java + ''' + if render is not None: + if isinstance(render, RenderClient): + return importTransformChangesClient( + stack, targetStack, transformFile, targetOwner=targetOwner, + targetProject=targetProject, changeMode=changeMode, + subprocess_mode=subprocess_mode, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs)) + elif not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + + if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: + raise ClientScriptError( + 'changeMode {} is not valid!'.format(changeMode)) + + argvs = (make_stack_params(host, port, owner, project, stack) + + ['--stack', stack, '--targetStack', targetStack] + + ['--transformFile', transformFile] + + get_param(targetOwner, '--targetOwner') + + get_param(targetProject, '--targetProject') + + get_param(changeMode, '--changeMode')) + call_run_ws_client( + 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, + client_script=client_script, subprocess_mode=subprocess_mode, + add_args=argv) + + +def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, + numberOfThreads=None, delete_fromJson=False, + delete_toJson=False, subprocess_mode=None, + host=None, port=None, owner=None, + project=None, client_script=None, memGB=None, + render=None, **kwargs): + ''' + run CoordinateClient.java + ''' + if render is not None: + if isinstance(render, RenderClient): + return coordinateClient( + stack, z, fromJson=fromJson, toJson=toJson, + localToWorld=localToWorld, numberOfThreads=numberOfThreads, + delete_toJson=delete_toJson, delete_fromJson=delete_fromJson, + subprocess_mode=subprocess_mode, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs)) + elif not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + + # TODO allow using array as input for mapping + if toJson is None: + tempjson = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) + tempjson.close() + delete_toJson = True + toJson = tempjson.name + if fromJson is None: + tempjson = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) + tempjson.write(input_array) + tempjson.flush() + tempjson.close() + delete_fromJson = True + fromJson = tempjson.name + + argvs = (make_stack_params(host, port, owner, project, stack) + + ['--z', z, '--fromJson', fromJson, '--toJson', toJson] + + (['--localToWorld', jbool(localToWorld)] + if localToWorld is not None else []) + + get_param(numberOfThreads, '--numberOfThreads')) + call_run_ws_client('org.janelia.render.client.CoordinateClient') + + with open(toJson, 'r') as f: + jsondata = json.load(f) + + if delete_toJson: + os.remove(toJson) + if delete_fromJson: + os.remove(fromJson) + + return jsondata From 9eaf81eea9ceeb4e3069e85f517d78df0bb2a512 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 11:06:05 -0800 Subject: [PATCH 079/766] pre-installing pathos and multiprocess to speed build time --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 376e0e33..b9af3ec6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,10 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-oracle #install gcc for pathos RUN apt-get install gcc -y RUN apt-get clean - +RUN pip install multiprocess +RUN pip install pathos +RUN uptime&&uptime&&uptime&&uptime&&uptime #install render python using pip from github + RUN pip install -e git+https://github.com/fcollman/render-python.git@module#egg=render-python From 698322213cafc5c5dca9aa603db02253b29c39e2 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 22 Feb 2017 13:02:30 -0800 Subject: [PATCH 080/766] client: fix for running with Render rather than RenderClient --- renderapi/client.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index d8eddcb8..fa849116 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -257,13 +257,13 @@ def importJsonClient(stack, tileFiles=None, transformFile=None, render=None, **kwargs): '''run ImportJsonClient.java''' if render is not None: - if isinstance(render, RenderClient): + if isinstance(render, Render): return importJsonClient( stack, tileFiles=tileFiles, transformFile=transformFile, subprocess_mode=subprocess_mode, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_script=client_script, memGB=memGB, **kwargs)) - elif not isinstance(render, Render): + else: raise ValueError('invalid Render object specified!') argvs = (make_stack_params(host, port, owner, project, stack) + @@ -290,7 +290,7 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, render=None, **kwargs): '''run TilePairClient.java''' if render is not None: - if isinstance(render, RenderClient): + if isinstance(render, Render): return tilePairClient( stack, minz, maxz, outjson=outjson, delete_json=delete_json, baseowner=baseowner, baseproject=baseproject, @@ -306,7 +306,7 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, subprocess_mode=subprocess_mode, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_script=client_script, memGB=memGB, **kwargs)) - elif not isinstance(render, Render): + else: raise ValueError('invalid Render object specified!') if outjson is None: @@ -359,14 +359,14 @@ def importTransformChangesClient(stack, targetStack, transformFile, run ImportTransformChangesClient.java ''' if render is not None: - if isinstance(render, RenderClient): + if isinstance(render, Render): return importTransformChangesClient( stack, targetStack, transformFile, targetOwner=targetOwner, targetProject=targetProject, changeMode=changeMode, subprocess_mode=subprocess_mode, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_script=client_script, memGB=memGB, **kwargs)) - elif not isinstance(render, Render): + else: raise ValueError('invalid Render object specified!') if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: @@ -395,7 +395,7 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, run CoordinateClient.java ''' if render is not None: - if isinstance(render, RenderClient): + if isinstance(render, Render): return coordinateClient( stack, z, fromJson=fromJson, toJson=toJson, localToWorld=localToWorld, numberOfThreads=numberOfThreads, @@ -403,7 +403,7 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, subprocess_mode=subprocess_mode, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_script=client_script, memGB=memGB, **kwargs)) - elif not isinstance(render, Render): + else: raise ValueError('invalid Render object specified!') # TODO allow using array as input for mapping From b88b342a26a8bc140b84b09a7a45d2dab500a5fb Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 15:08:23 -0800 Subject: [PATCH 081/766] fixed verbose flag that isn't passed --- renderapi/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index ffd63cda..470da0b1 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -206,7 +206,7 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, raise ValueError('invalid Render object specified!') return get_stack_metadata_by_owner(**render.make_kwargs( owner=owner, host=host, port=port, - **{'session': session, 'verbose': verbose})) + **{'session': session})) request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) From 329463128bdb9ccb24695633379812a7ae453a65 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 16:58:05 -0800 Subject: [PATCH 082/766] trying to make tilespecs more easily serializable --- renderapi/tilespec.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index b53a3054..c0083a90 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -346,6 +346,9 @@ def to_dict(self): in self.inputfilters] return thedict + def __dict__(self): + return self.to_dict() + def from_dict(self, d): '''Method to load tilespec from json dictionary''' self.tileId = d['tileId'] From 0a85723ff6107032518e318254cc87f5b27a7134 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 17:16:39 -0800 Subject: [PATCH 083/766] adding an encoder to improve json serialization --- renderapi/tilespec.py | 3 --- renderapi/utils.py | 7 +++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index c0083a90..b53a3054 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -346,9 +346,6 @@ def to_dict(self): in self.inputfilters] return thedict - def __dict__(self): - return self.to_dict() - def from_dict(self, d): '''Method to load tilespec from json dictionary''' self.tileId = d['tileId'] diff --git a/renderapi/utils.py b/renderapi/utils.py index fe3d1aaf..370d4180 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -7,6 +7,13 @@ logger = logging.getLogger(__name__) +class RenderEncoder(json.JSONencoder): + def default(self,obj): + to_dict = getattr(obj, "to_dict", None) + if callable(to_dict): + return obj.to_dict() + else: + return obj.__dict__ def jbool(val): '''return string representing java string values of py booleans''' From b8bcbfa1944629681ca55505cb2a14883b1c757a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 22 Feb 2017 17:19:27 -0800 Subject: [PATCH 084/766] fixed typo --- renderapi/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 370d4180..6393170c 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) -class RenderEncoder(json.JSONencoder): +class RenderEncoder(json.JSONEncoder): def default(self,obj): to_dict = getattr(obj, "to_dict", None) if callable(to_dict): From 0797425aa14f38b12b7df2aaca049ed68465f2b4 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 23 Feb 2017 09:29:03 -0800 Subject: [PATCH 085/766] setup: read from requirements.txt --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c26be222..9815317d 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,8 @@ from setuptools import setup + +with open('requirements.txt', 'r') as f: + required = f.read().splitlines() + setup(name='render-python', version='1.0', description=' a python API setup to interact via python with render ' @@ -6,4 +10,5 @@ author='Forrest Collman,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', - packages=['renderapi']) + packages=['renderapi'], + install_requires=required) From 662ffd2adbcf6a8be72089d844794fdb6ab54aa0 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 23 Feb 2017 09:35:46 -0800 Subject: [PATCH 086/766] utils, tilespec: PEP8 --- renderapi/tilespec.py | 2 +- renderapi/utils.py | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index b53a3054..621aa0c6 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -356,7 +356,7 @@ def from_dict(self, d): self.maxint = d['maxIntensity'] self.frameId = d.get('frameId', None) self.layout = Layout() - self.layout.from_dict(d.get('layout',None)) + self.layout.from_dict(d.get('layout', None)) self.minX = d.get('minX', None) self.maxX = d.get('maxX', None) self.maxY = d.get('maxY', None) diff --git a/renderapi/utils.py b/renderapi/utils.py index 6393170c..a847e945 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -7,13 +7,15 @@ logger = logging.getLogger(__name__) + class RenderEncoder(json.JSONEncoder): - def default(self,obj): - to_dict = getattr(obj, "to_dict", None) - if callable(to_dict): - return obj.to_dict() - else: - return obj.__dict__ + def default(self, obj): + to_dict = getattr(obj, "to_dict", None) + if callable(to_dict): + return obj.to_dict() + else: + return obj.__dict__ + def jbool(val): '''return string representing java string values of py booleans''' From 9dcd7e944f7d18f7dfe82b4dcd766c204d8920b7 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 23 Feb 2017 10:53:32 -0800 Subject: [PATCH 087/766] utils: add renderdump and renderdumps functionality --- renderapi/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/renderapi/utils.py b/renderapi/utils.py index a847e945..902527be 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -17,6 +17,16 @@ def default(self, obj): return obj.__dict__ +def renderdumps(obj, *args, **kwargs): + cls_ = kwargs.pop('cls', RenderEncoder) + return json.dumps(obj, *args, cls=cls_, **kwargs) + + +def renderdump(obj, *args, **kwargs): + cls_ = kwargs.pop('cls', RenderEncoder) + return json.dump(obj, *args, cls=cls_, **kwargs) + + def jbool(val): '''return string representing java string values of py booleans''' if not isinstance(val, bool): From d7e4c645aff66d7e74c0b67b6e6be5221d42fb60 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 23 Feb 2017 11:00:34 -0800 Subject: [PATCH 088/766] utils: remove poorly-named and unused load json/dict functions --- renderapi/utils.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 902527be..63c34ecf 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -46,16 +46,5 @@ def stripLogger(logger_tostrip): logger_tostrip.removeHandler(handler) -def _load_dict(obj, d): - obj.__dict__.update({k: v for k, v in d.items()}) - - -def _load_json(obj, j): - '''load object from dictionary-style json''' - with open(j, 'r') as f: - jd = json.load(f) - _load_dict(obj, jd) - - def defaultifNone(val, default=None): return val if val is not None else default From 95faca1a22c7d8af7d5207b86b0b9a368dbdde49 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 23 Feb 2017 11:25:03 -0800 Subject: [PATCH 089/766] utils: add NullHandler for straightforward logging support, remove unsupported imports --- renderapi/render.py | 3 ++- renderapi/transform.py | 1 - renderapi/utils.py | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 470da0b1..deb13943 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -2,10 +2,11 @@ import logging import os import requests -from .utils import defaultifNone +from .utils import defaultifNone, NullHandler from .errors import ClientScriptError logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) class Render(object): diff --git a/renderapi/transform.py b/renderapi/transform.py index 03811279..d1b779a6 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -14,7 +14,6 @@ import logging import numpy as np from .errors import ConversionError, EstimationError -from .utils import _load_dict, _load_json logger = logging.getLogger(__name__) diff --git a/renderapi/utils.py b/renderapi/utils.py index 63c34ecf..fc5e9eae 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -35,6 +35,11 @@ def jbool(val): return 'true' if val else 'false' +class NullHandler(logging.Handler): + def emit(self, record): + pass + + def stripLogger(logger_tostrip): ''' remove all handlers from a logger -- useful for redefining From 8a7536fda83a7ff76c4fda0a7266cf962b32c78e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 23 Feb 2017 12:02:24 -0800 Subject: [PATCH 090/766] stack: allow defaults for stack version --- renderapi/stack.py | 59 ++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 98353c07..c6534c88 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -10,15 +10,18 @@ class StackVersion: - def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=1.0, - stackResolutionY=1.0, stackResolutionZ=1.0, - materializedBoxRootPath=None, versionNotes="", + def __init__(self, cycleNumber=None, cycleStepNumber=None, + stackResolutionX=None, stackResolutionY=None, + stackResolutionZ=None, + materializedBoxRootPath=None, mipmapPathBuilder=None, + versionNotes=None, createTimestamp=None, **kwargs): self.cycleNumber = cycleNumber self.cycleStepNumber = cycleStepNumber self.stackResolutionX = stackResolutionX self.stackResolutionY = stackResolutionY self.stackResolutionZ = stackResolutionZ + self.mipmapPathBuilder = mipmapPathBuilder self.materializedBoxRootPath = materializedBoxRootPath self.createTimestamp = (strftime('%Y-%M-%dT%H:%M:%S.00Z') if createTimestamp is None else createTimestamp) @@ -26,20 +29,28 @@ def __init__(self, cycleNumber=1, cycleStepNumber=1, stackResolutionX=1.0, def to_dict(self): d = {} - d['cycleNumber'] = self.cycleNumber - d['cycleStepNumber'] = self.cycleStepNumber - d['stackResolutionX'] = self.stackResolutionX - d['stackResolutionY'] = self.stackResolutionY - d['stackResolutionZ'] = self.stackResolutionZ - d['createTimestamp'] = self.createTimestamp - d["materializedBoxRootPath"] = self.materializedBoxRootPath - d['mipmapPathBuilder'] = {'numberOfLevels': 0} - d['versionNotes'] = self.versionNotes + d.update(({'cycleNumber': self.cycleNumber} + if self.cycleNumber is not None else {})) + d.update(({'cycleStepNumber': self.cycleStepNumber} + if self.cycleStepNumber is not None else {})) + d.update(({'stackResolutionX': self.stackResolutionX} + if self.stackResolutionX is not None else {})) + d.update(({'stackResolutionY': self.stackResolutionY} + if self.stackResolutionY is not None else {})) + d.update(({'stackResolutionZ': self.stackResolutionZ} + if self.stackResolutionZ is not None else {})) + d.update(({'createTimestamp': self.createTimestamp} + if self.createTimestamp is not None else {})) + d.update(({'mipmapPathBuilder': self.mipmapPathBuilder} + if self.mipmapPathBuilder is not None else {})) + d.update(({'versionNotes': self.versionNotes} + if self.versionNotes is not None else {})) + d.update(({'materializedBoxRootPath': self.materializedBoxRootPath} + if self.materializedBoxRootPath is not None else {})) return d def from_dict(self, d): - for key in d.keys(): - eval('self.%s=d[%s]' % (key, key)) + self.__dict__.update({k: v for k, v in d.items()}) def set_stack_state(stack, state='LOADING', host=None, port=None, @@ -105,19 +116,21 @@ def delete_stack(stack, render=None, host=None, port=None, owner=None, return r -def create_stack(stack, cycleNumber=1, cycleStepNumber=1, - stackResolutionX=1.0, stackResolutionY=1.0,stackResolutionZ=1.0, - render=None, +def create_stack(stack, cycleNumber=None, cycleStepNumber=None, + stackResolutionX=None, stackResolutionY=None, + stackResolutionZ=None, host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): + session=requests.session(), render=None, **kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') - return create_stack(stack, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session, 'cycleNumber': cycleNumber, - 'cycleStepNumber': cycleStepNumber,'stackResolutionX':stackResolutionX, - 'stackResolutionY':stackResolutionY,'stackResolutionZ':stackResolutionZ})) + return create_stack( + stack, cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, + stackResolutionX=stackResolutionX, + stackResolutionY=stackResolutionY, + stackResolutionZ=stackResolutionZ, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + **{'session': session})) sv = StackVersion( cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, From b890a72467492b24d298b429b6479feed9b80884 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 23 Feb 2017 14:34:40 -0800 Subject: [PATCH 091/766] fixed get_match_groupIds --- renderapi/pointmatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 25e255ef..bb101b51 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -45,7 +45,7 @@ def get_matchcollections(render=None, owner=None, host=None, port=None, def get_match_groupIds(matchCollection, render=None, owner=None, host=None, - port=None, session=requests.session()): + port=None, session=requests.session(),**kwargs): if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') From 31326669df66e8a75f76b28f15aa1ba38f898281 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 23 Feb 2017 14:47:46 -0800 Subject: [PATCH 092/766] fixed json dependancy missing --- renderapi/coordinate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index ea5800f4..07fc4cc1 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -6,7 +6,7 @@ from .render import Render, format_preamble import logging import requests - +import json logger = logging.getLogger(__name__) @@ -98,6 +98,7 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z=0, owner=None, project=None, session=requests.session(), **kwargs): '''''' + if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') From 8b1a4b47139b791c81f3ecf4c64fd830c28e0126 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 23 Feb 2017 15:02:55 -0800 Subject: [PATCH 093/766] fixed json dependancy missing --- renderapi/coordinate.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 07fc4cc1..67fa45a9 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -159,6 +159,8 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z=0, json_answer = r.json() try: answer = np.zeros(dataarray.shape) + print dataarray.shape + print len(json_answer) for i, coord in enumerate(json_answer): c = coord['world'] answer[i, 0] = c[0] From 5109b90fa1bc411bda8f5f65c950d814af66fb4a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 23 Feb 2017 15:31:56 -0800 Subject: [PATCH 094/766] fixed json dependancy missing --- renderapi/coordinate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 67fa45a9..07fc4cc1 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -159,8 +159,6 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z=0, json_answer = r.json() try: answer = np.zeros(dataarray.shape) - print dataarray.shape - print len(json_answer) for i, coord in enumerate(json_answer): c = coord['world'] answer[i, 0] = c[0] From 6bffdf34e1ea8690c565a848548aa8d52d3d08d8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 23 Feb 2017 15:33:12 -0800 Subject: [PATCH 095/766] fixed numpy dependancy missing --- renderapi/coordinate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 07fc4cc1..38b4baae 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -7,6 +7,7 @@ import logging import requests import json +import numpy as np logger = logging.getLogger(__name__) From c9ac62410d000f7825375a9202631d8f93e24568 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 23 Feb 2017 18:12:58 -0800 Subject: [PATCH 096/766] adding default fallback for client_script --- renderapi/render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index 470da0b1..cc99f2cc 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -149,9 +149,10 @@ def connect(host=None, port=None, owner=None, project=None, if client_script is None: if 'RENDER_CLIENT_SCRIPT' not in os.environ: # client_script = str(raw_input("Enter Render Client Script: ")) + client_script=os.path.join(client_scripts,'run_ws_client.sh') pass else: - client_script = str(os.environ['RENDER_CLIENT_SCRIPT']) + client_script = str(os.environ['RENDER_CLIENT_SCRIPT']) if memGB is None: if 'RENDER_CLIENT_HEAP' not in os.environ: From fda2648defd8e7987447d9c5dc6304ec227a4e51 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 08:12:45 -0800 Subject: [PATCH 097/766] adding import matches call --- renderapi/pointmatch.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index bb101b51..bdde1f47 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -208,3 +208,17 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, return r.json() except: logger.error(r.text) + +def import_matches(matchCollection,data,render=None,owner=None,host=None,port=None,session=requests.session(),**kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return import_matches( + matchCollection, data, **render.make_kwargs( + owner=owner, host=host, port=port, + **{'session': session})) + + request_url =self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/matches"%(owner,matchCollection) + logger.debug(request_url) + r = session.put(request_url, data=data, headers={"content-type":"application/json","Accept":"application/json"}) + return r \ No newline at end of file From 8be39e4e5f28229e3067f8a409492618c5533441 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 08:17:53 -0800 Subject: [PATCH 098/766] new dockerfile and fixed typo in import_matches --- Dockerfile | 18 +++++++++++++++--- renderapi/client.py | 2 +- renderapi/pointmatch.py | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index b9af3ec6..15817d3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,20 +4,32 @@ MAINTAINER Forrest Collman (forrest.collman@gmail.com) #install java # auto validate license RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections -# update repos RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 RUN apt-get update RUN apt-get install oracle-java8-installer -y ENV JAVA_HOME /usr/lib/jvm/java-8-oracle -#install gcc for pathos + +#install pathos,multiprocess with gcc RUN apt-get install gcc -y +RUN apt-get install build-essential -y RUN apt-get clean RUN pip install multiprocess RUN pip install pathos + +#install components for common render-python apps +#jupyter notebook, shapely with geos +RUN /opt/conda/bin/conda install jupyter -y +RUN apt-get install libgeos-dev -y +RUN pip install shapely==1.6b2 + +#stupid uptime calls to bypass caching on github pull +#only here for development purposes +RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime +RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime RUN uptime&&uptime&&uptime&&uptime&&uptime -#install render python using pip from github +#install render python using pip from github RUN pip install -e git+https://github.com/fcollman/render-python.git@module#egg=render-python diff --git a/renderapi/client.py b/renderapi/client.py index fa849116..0414667a 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -233,7 +233,7 @@ def call_run_ws_client(className, add_args=[], renderclient=None, **renderclient.make_kwargs( memGB=memGB, client_script=client_script)) - + subprocess_modes = {'call': subprocess.call, 'check_call': subprocess.check_call, 'check_output': subprocess.check_output} diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index bdde1f47..09340579 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -218,7 +218,7 @@ def import_matches(matchCollection,data,render=None,owner=None,host=None,port=No owner=owner, host=host, port=port, **{'session': session})) - request_url =self.format_baseurl(host, port)+"/owner/%s/matchCollection/%s/matches"%(owner,matchCollection) + request_url =format_baseurl(host, port)+"/owner/%s/matchCollection/%s/matches"%(owner,matchCollection) logger.debug(request_url) r = session.put(request_url, data=data, headers={"content-type":"application/json","Accept":"application/json"}) return r \ No newline at end of file From acf32330dc736ed41a2e80fd95f9de4d1f22b4cc Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 09:07:40 -0800 Subject: [PATCH 099/766] adding in client side coordinate mapping options --- renderapi/coordinate.py | 157 +++++++++++++++++++++++++++------------- 1 file changed, 107 insertions(+), 50 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 38b4baae..1476421e 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -4,7 +4,7 @@ ''' from .render import Render, format_preamble -import logging +from .client import call_run_ws_client import requests import json import numpy as np @@ -55,7 +55,7 @@ def local_to_world_coordinates(stack, tileId, x, y, render=None, def world_to_local_coordinates_batch(stack, z, data, render=None, host=None, - port=None, owner=None, project=None, + port=None, owner=None, project=None, execute_local=True, session=requests.session(), **kwargs): '''''' if render is not None: @@ -93,10 +93,27 @@ def local_to_world_coordinates_batch(stack, data, z, render=None, host=None, headers={"content-type": "application/json"}) return r.json() +def package_point_match_data_into_json(dataarray,tileId,local_or_world='local'): + dlist = [] + for i in range(dataarray.shape[0]): + d = {} + d['tileId'] = tileId + d[local_or_world] = [dataarray[i, 0], dataarray[i, 1]] + dlist.append(d) + return = json.dumps(dlist) + +def unpackage_point_match_data_from_json(json_answer,local_or_world='local'): + answer = np.zeros((len(json_answer,2))) + for i, coord in enumerate(json_answer): + c = coord[local_or_world] + answer[i, 0] = c[0] + answer[i, 1] = c[1] + return answer -def world_to_local_coordinates_array(stack, dataarray, tileId, z=0, +def world_to_local_coordinates_array(stack, dataarray, tileId, z, render=None, host=None, port=None, - owner=None, project=None, + owner=None, project=None, client_script = None, + doClientSide = False, session=requests.session(), **kwargs): '''''' @@ -104,66 +121,106 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z=0, if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return world_to_local_coordinates_array( - stack, dataarray, tileId, **render.make_kwargs( + stack, dataarray, tileId, z, **render.make_kwargs( host=host, port=port, owner=owner, project=project, - **{'session': session, 'z': z})) + client_script = client_script, + **{'session': session,'doClientSide':doClientSide})) + + jsondata = package_point_match_data_into_json(dataarray,tileId,'world') + if doClientSide: + json_answer = world_to_local_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,client_script,number_of_threads=number_of_threads) + else: + json_answer = world_to_local_coordinates_batch(stack,z,jsondata,host,port,owner,project,session) - request_url = format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/world-to-local-coordinates" % (z) - dlist = [] - for i in range(dataarray.shape[0]): - d = {} - d['tileId'] = tileId - d['world'] = [dataarray[i, 0], dataarray[i, 1]] - dlist.append(d) - jsondata = json.dumps(dlist) - r = session.put(request_url, data=jsondata, - headers={"content-type": "application/json"}) - json_answer = r.json() try: - answer = np.zeros(dataarray.shape) - for i, coord in enumerate(json_answer): - c = coord['local'] - answer[i, 0] = c[0] - answer[i, 1] = c[1] - return answer + return unpackage_point_match_data_from_json(json_answer,'local') except: logger.error(json_answer) - -def local_to_world_coordinates_array(stack, dataarray, tileId, z=0, +def local_to_world_coordinates_array(stack, dataarray, tileId, z, render=None, host=None, port=None, - owner=None, project=None, + owner=None, project=None, client_script = None, + doClientSide = False, number_of_threads=20, session=requests.session(), **kwargs): '''''' if render is not None: if not isinstance(render, Render): raise ValueError('invalid Render object specified!') return local_to_world_coordinates_array( - stack, dataarray, tileId, **render.make_kwargs( + stack, dataarray, tileId, z,**render.make_kwargs( host=host, port=port, owner=owner, project=project, - **{'session': session, 'z': z})) - - request_url = format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/local-to-world-coordinates" % (z) - dlist = [] - for i in range(dataarray.shape[0]): - d = {} - d['tileId'] = tileId - d['local'] = [dataarray[i, 0], dataarray[i, 1]] - dlist.append(d) - jsondata = json.dumps(dlist) - r = session.put(request_url, data=jsondata, - headers={"content-type": "application/json"}) - json_answer = r.json() + client_script = client_script, + **{'session': session,'doClientSide':doClientSide,'number_of_threads':number_of_threads})) + + jsondata = package_point_match_data_into_json(dataarray,tileId,'local') + if doClientSide: + json_answer = local_to_world_coordinates_clientside(stack,jsondata,tileId,z,host,owner,project,client_script,number_of_threads=number_of_threads) + else: + json_answer = local_to_world_coordinates_batch(stack,z,jsondata,host,port,owner,project,session) try: - answer = np.zeros(dataarray.shape) - for i, coord in enumerate(json_answer): - c = coord['world'] - answer[i, 0] = c[0] - answer[i, 1] = c[1] - return answer + return unpackage_point_match_data_from_json(json_answer,'world') except: logger.error(json_answer) + +def map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=False,number_of_threads=20): + #write point match json to temp file on disk + json_infile,json_inpath = tempfile.mkstemp(prefix='render_coordinates_in_',suffix='.json') + json.dump(jsondata,json_infile) + close(json_infile) + + #get a temporary location for the output + json_outpath = tempfile.mktemp(prefix='render_coordinates_out_',suffix='.json') + + if isLocalToWorld: + localToWorld = 'true' + else: + localToWorld = 'false' + + #define arguments + args = ['--baseDataUrl','https://%s:%d/render-ws/v1'%(host,port), + '--owner',owner, + '--project',project, + '--stack',stack, + '--z',str(z), + '--fromJson',json_inpath, + '--toJson',json_outpath, + '--localToWorld',localToWorld, + '--numberOfThreads',str(number_of_threads)] + + #call the java client + call_run_ws_client('org.janelia.render.client', add_args=args, client_script=client_script) + + #return the json results + return json.load(open(json_outpath,'r')) + +def world_to_local_coordinates_clientside(stack,jsondata,tileId,z,render=None, + host=None,port=None,owner=None, + project=None,client_script=None, + number_of_threads=20, + session=requests.session(),**kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return world_to_local_coordinates_clientside( + stack, dataarray, tileId, z,**render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_script=client_script, + **{'session': session,'number_of_threads':20})) + + return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=False,number_of_threads=number_of_threads): + +def local_to_world_coordinates_clientside(stack,jsondata,tileId,z,render=None, + host=None,port=None,owner=None, + project=None,client_script=None, + number_of_threads=20, + session=requests.session(),**kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return local_to_world_coordinates_clientside( + stack, dataarray, tileId, z,**render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_script=client_script, + **{'session': session,'number_of_threads':20})) + + return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=True,number_of_threads=number_of_threads): \ No newline at end of file From b713d83b98b2cc5722b9c3ccff6f3b2e741db7b2 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 09:13:21 -0800 Subject: [PATCH 100/766] adding in client side coordinate mapping options --- renderapi/coordinate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 1476421e..9d235663 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -191,7 +191,10 @@ def map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,c call_run_ws_client('org.janelia.render.client', add_args=args, client_script=client_script) #return the json results - return json.load(open(json_outpath,'r')) + try: + return json.load(open(json_outpath,'r')) + except: + logger.error('failed to load json after map_coordinates_clientside:\n%s'%json.dumps(jsondata)) def world_to_local_coordinates_clientside(stack,jsondata,tileId,z,render=None, host=None,port=None,owner=None, From e8434ba71ff8e4ccaa2c6235463704afeabe5cf3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 09:14:31 -0800 Subject: [PATCH 101/766] adding in client side coordinate mapping options --- renderapi/coordinate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 9d235663..1dadfa40 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -100,7 +100,7 @@ def package_point_match_data_into_json(dataarray,tileId,local_or_world='local'): d['tileId'] = tileId d[local_or_world] = [dataarray[i, 0], dataarray[i, 1]] dlist.append(d) - return = json.dumps(dlist) + return json.dumps(dlist) def unpackage_point_match_data_from_json(json_answer,local_or_world='local'): answer = np.zeros((len(json_answer,2))) From 21d35069f0a701a9e18f65fd8eba289b5881c477 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 09:15:37 -0800 Subject: [PATCH 102/766] adding in client side coordinate mapping options --- renderapi/coordinate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 1dadfa40..76a58a3c 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -210,7 +210,7 @@ def world_to_local_coordinates_clientside(stack,jsondata,tileId,z,render=None, client_script=client_script, **{'session': session,'number_of_threads':20})) - return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=False,number_of_threads=number_of_threads): + return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=False,number_of_threads=number_of_threads) def local_to_world_coordinates_clientside(stack,jsondata,tileId,z,render=None, host=None,port=None,owner=None, @@ -226,4 +226,4 @@ def local_to_world_coordinates_clientside(stack,jsondata,tileId,z,render=None, client_script=client_script, **{'session': session,'number_of_threads':20})) - return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=True,number_of_threads=number_of_threads): \ No newline at end of file + return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=True,number_of_threads=number_of_threads) \ No newline at end of file From 22cc3c8867bd39cbe986ea545be49325649d1291 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 09:16:31 -0800 Subject: [PATCH 103/766] adding in client side coordinate mapping options --- renderapi/coordinate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 76a58a3c..3d490401 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -8,6 +8,7 @@ import requests import json import numpy as np +import logging logger = logging.getLogger(__name__) From 88dcc9f6f4103fcab87b062f328c72c316f6c5b8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 10:27:31 -0800 Subject: [PATCH 104/766] adding in client side coordinate mapping options --- renderapi/coordinate.py | 96 ++++++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 3d490401..3cb48811 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -9,6 +9,7 @@ import json import numpy as np import logging +import tempfile logger = logging.getLogger(__name__) @@ -55,7 +56,7 @@ def local_to_world_coordinates(stack, tileId, x, y, render=None, logger.error(r.text) -def world_to_local_coordinates_batch(stack, z, data, render=None, host=None, +def world_to_local_coordinates_batch(stack, data,z, render=None, host=None, port=None, owner=None, project=None, execute_local=True, session=requests.session(), **kwargs): '''''' @@ -69,7 +70,7 @@ def world_to_local_coordinates_batch(stack, z, data, render=None, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ - "/z/%d/world-to-local-coordinates" % (z) + "/z/%s/world-to-local-coordinates" % (str(z)) r = session.put(request_url, data=data, headers={"content-type": "application/json"}) return r.json() @@ -89,10 +90,13 @@ def local_to_world_coordinates_batch(stack, data, z, render=None, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ - "/z/%d/local-to-world-coordinates" % (z) + "/z/%s/local-to-world-coordinates" % (str(z)) r = session.put(request_url, data=data, headers={"content-type": "application/json"}) - return r.json() + try: + return r.json() + except: + logger.error(r.text) def package_point_match_data_into_json(dataarray,tileId,local_or_world='local'): dlist = [] @@ -104,7 +108,9 @@ def package_point_match_data_into_json(dataarray,tileId,local_or_world='local'): return json.dumps(dlist) def unpackage_point_match_data_from_json(json_answer,local_or_world='local'): - answer = np.zeros((len(json_answer,2))) + answer = np.zeros((len(json_answer),2)) + if type(json_answer[0]) == list: + json_answer = json_answer[0] for i, coord in enumerate(json_answer): c = coord[local_or_world] answer[i, 0] = c[0] @@ -114,7 +120,7 @@ def unpackage_point_match_data_from_json(json_answer,local_or_world='local'): def world_to_local_coordinates_array(stack, dataarray, tileId, z, render=None, host=None, port=None, owner=None, project=None, client_script = None, - doClientSide = False, + doClientSide = False,number_of_threads=20, session=requests.session(), **kwargs): '''''' @@ -125,18 +131,25 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, stack, dataarray, tileId, z, **render.make_kwargs( host=host, port=port, owner=owner, project=project, client_script = client_script, - **{'session': session,'doClientSide':doClientSide})) + **{'session': session,'doClientSide':doClientSide,'number_of_threads':20})) jsondata = package_point_match_data_into_json(dataarray,tileId,'world') if doClientSide: - json_answer = world_to_local_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,client_script,number_of_threads=number_of_threads) + json_answer = world_to_local_coordinates_clientside(stack,jsondata,tileId,z, + host=host,port=port,owner=owner, + project=project, + client_script=client_script, + number_of_threads=number_of_threads) else: - json_answer = world_to_local_coordinates_batch(stack,z,jsondata,host,port,owner,project,session) - - try: - return unpackage_point_match_data_from_json(json_answer,'local') - except: - logger.error(json_answer) + json_answer = world_to_local_coordinates_batch(stack,jsondata,z, + host=host,port=port, + owner=owner,project=project, + session=session) + #try: + return unpackage_point_match_data_from_json(json_answer,'local') + #except: + # logger.error('could not unpackage world_to_local answer') + # logger.error(json_answer) def local_to_world_coordinates_array(stack, dataarray, tileId, z, render=None, host=None, port=None, @@ -155,47 +168,54 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, jsondata = package_point_match_data_into_json(dataarray,tileId,'local') if doClientSide: - json_answer = local_to_world_coordinates_clientside(stack,jsondata,tileId,z,host,owner,project,client_script,number_of_threads=number_of_threads) + json_answer = local_to_world_coordinates_clientside(stack,jsondata,tileId,z, + host=host,port=port,owner=owner,project=project, + client_script=client_script, + number_of_threads=number_of_threads) else: - json_answer = local_to_world_coordinates_batch(stack,z,jsondata,host,port,owner,project,session) - try: - return unpackage_point_match_data_from_json(json_answer,'world') - except: - logger.error(json_answer) + json_answer = local_to_world_coordinates_batch(stack,jsondata,z, + host=host,port=port,owner=owner, + project=project,session=session) + #try: + return unpackage_point_match_data_from_json(json_answer,'world') + #except: + # logger.error('could not unpackage local_to_world answer') + # logger.error(json_answer) def map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=False,number_of_threads=20): #write point match json to temp file on disk json_infile,json_inpath = tempfile.mkstemp(prefix='render_coordinates_in_',suffix='.json') - json.dump(jsondata,json_infile) - close(json_infile) + fp = open(json_inpath,'w') + fp.write(jsondata) + fp.close() + + #json.dump(jsondata,open(json_inpath,'w')) #get a temporary location for the output json_outpath = tempfile.mktemp(prefix='render_coordinates_out_',suffix='.json') - if isLocalToWorld: - localToWorld = 'true' - else: - localToWorld = 'false' + #define arguments - args = ['--baseDataUrl','https://%s:%d/render-ws/v1'%(host,port), + args = ['--baseDataUrl','http://%s:%d/render-ws/v1'%(host,port), '--owner',owner, '--project',project, '--stack',stack, '--z',str(z), '--fromJson',json_inpath, '--toJson',json_outpath, - '--localToWorld',localToWorld, '--numberOfThreads',str(number_of_threads)] + if isLocalToWorld: + args+=['--localToWorld'] #call the java client - call_run_ws_client('org.janelia.render.client', add_args=args, client_script=client_script) + call_run_ws_client('org.janelia.render.client.CoordinateClient', add_args=args, client_script=client_script) #return the json results - try: - return json.load(open(json_outpath,'r')) - except: - logger.error('failed to load json after map_coordinates_clientside:\n%s'%json.dumps(jsondata)) + #try: + return json.load(open(json_outpath,'r')) + #except: + # logger.error('failed to load json after map_coordinates_clientside:\n%s'%json.dumps(jsondata)) def world_to_local_coordinates_clientside(stack,jsondata,tileId,z,render=None, host=None,port=None,owner=None, @@ -211,7 +231,10 @@ def world_to_local_coordinates_clientside(stack,jsondata,tileId,z,render=None, client_script=client_script, **{'session': session,'number_of_threads':20})) - return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=False,number_of_threads=number_of_threads) + return map_coordinates_clientside(stack,jsondata,tileId,z, + host=host,port=port,owner=owner, + project=project,client_script=client_script, + isLocalToWorld=False,number_of_threads=number_of_threads) def local_to_world_coordinates_clientside(stack,jsondata,tileId,z,render=None, host=None,port=None,owner=None, @@ -227,4 +250,7 @@ def local_to_world_coordinates_clientside(stack,jsondata,tileId,z,render=None, client_script=client_script, **{'session': session,'number_of_threads':20})) - return map_coordinates_clientside(stack,jsondata,tileId,z,host,port,owner,project,client_script,isLocalToWorld=True,number_of_threads=number_of_threads) \ No newline at end of file + return map_coordinates_clientside(stack,jsondata,tileId,z, + host=host,port=port,owner=owner, + project=project,client_script=client_script, + isLocalToWorld=True,number_of_threads=number_of_threads) \ No newline at end of file From 35ca6c01008fbff098278facb00de8ad41f649cb Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Feb 2017 13:19:20 -0800 Subject: [PATCH 105/766] fixed world to local calls --- renderapi/coordinate.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 3cb48811..bb9e4e92 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -107,12 +107,20 @@ def package_point_match_data_into_json(dataarray,tileId,local_or_world='local'): dlist.append(d) return json.dumps(dlist) -def unpackage_point_match_data_from_json(json_answer,local_or_world='local'): +def unpackage_world_to_local_point_match_from_json(json_answer,tileId): + answer = np.zeros((len(json_answer),2)) + for i, local_answer in enumerate(json_answer): + coord = next(ans for ans in local_answer if ans['tileId']==tileId) + c=coord['local'] + answer[i, 0] = c[0] + answer[i, 1] = c[1] + return answer + +def unpackage_local_to_world_point_match_from_json(json_answer): + logger.debug("json_answer_length %d"%len(json_answer)) answer = np.zeros((len(json_answer),2)) - if type(json_answer[0]) == list: - json_answer = json_answer[0] for i, coord in enumerate(json_answer): - c = coord[local_or_world] + c = coord['world'] answer[i, 0] = c[0] answer[i, 1] = c[1] return answer @@ -145,8 +153,9 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, host=host,port=port, owner=owner,project=project, session=session) + #print json_answer #try: - return unpackage_point_match_data_from_json(json_answer,'local') + return unpackage_world_to_local_point_match_from_json(json_answer,tileId) #except: # logger.error('could not unpackage world_to_local answer') # logger.error(json_answer) @@ -177,7 +186,7 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, host=host,port=port,owner=owner, project=project,session=session) #try: - return unpackage_point_match_data_from_json(json_answer,'world') + return unpackage_local_to_world_point_match_from_json(json_answer) #except: # logger.error('could not unpackage local_to_world answer') # logger.error(json_answer) From aafc618a6cbee44078ae03fbfd99246a36626bca Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 24 Feb 2017 13:32:54 -0800 Subject: [PATCH 106/766] render: add renderaccess for decorator --- renderapi/render.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/renderapi/render.py b/renderapi/render.py index deb13943..9427302c 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import logging import os +from functools import wraps import requests from .utils import defaultifNone, NullHandler from .errors import ClientScriptError @@ -173,6 +174,17 @@ def connect(host=None, port=None, owner=None, project=None, client_scripts=client_scripts) +def renderaccess(f): + @wraps(f) + def wrapper(*args, **kwargs): + render = kwargs.pop('render', None) + if render is not None: + return (f(*args, **render.make_kwargs(**kwargs))) + else: + return f(*args, **kwargs) + return wrapper + + def format_baseurl(host, port): return 'http://%s:%d/render-ws/v1' % (host, port) From f95cba7a7c000aa24b9cc0d6693cea6216903574 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 24 Feb 2017 13:45:13 -0800 Subject: [PATCH 107/766] render: add value checking for renderaccess --- renderapi/render.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index 9427302c..29b28c08 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -179,7 +179,12 @@ def renderaccess(f): def wrapper(*args, **kwargs): render = kwargs.pop('render', None) if render is not None: - return (f(*args, **render.make_kwargs(**kwargs))) + if isinstance(render, Render): + return (f(*args, **render.make_kwargs(**kwargs))) + else: + raise ValueError( + 'invalid Render object type {} specified!'.format( + type(render))) else: return f(*args, **kwargs) return wrapper From 03e708106a9705f68fcd2e29b6a8c842ad63ecc7 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 24 Feb 2017 15:05:16 -0800 Subject: [PATCH 108/766] renderapi: add render.renderaccess to make life better --- renderapi/coordinate.py | 85 ++++++++------------------- renderapi/image.py | 34 ++++------- renderapi/pointmatch.py | 112 ++++++++--------------------------- renderapi/render.py | 2 +- renderapi/stack.py | 125 ++++++++++------------------------------ renderapi/tilespec.py | 56 +++++------------- 6 files changed, 105 insertions(+), 309 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 67fa45a9..bce461d2 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -3,24 +3,19 @@ coordinate mapping functions for render api ''' -from .render import Render, format_preamble +from .render import Render, format_preamble, renderaccess import logging import requests import json logger = logging.getLogger(__name__) -def world_to_local_coordinates(stack, z, x, y, render=None, host=None, +@renderaccess +def world_to_local_coordinates(stack, z, x, y, host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): + session=requests.session(), + render=None, **kwargs): '''''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return world_to_local_coordinates(stack, z, x, y, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) @@ -31,18 +26,12 @@ def world_to_local_coordinates(stack, z, x, y, render=None, host=None, logger.error(r.text) -def local_to_world_coordinates(stack, tileId, x, y, render=None, +@renderaccess +def local_to_world_coordinates(stack, tileId, x, y, host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): + session=requests.session(), + render=None, **kwargs): '''''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return local_to_world_coordinates( - stack, tileId, x, y, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) @@ -53,18 +42,12 @@ def local_to_world_coordinates(stack, tileId, x, y, render=None, logger.error(r.text) -def world_to_local_coordinates_batch(stack, z, data, render=None, host=None, +@renderaccess +def world_to_local_coordinates_batch(stack, z, data, host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): + session=requests.session(), + render=None, **kwargs): '''''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return world_to_local_coordinates_batch( - stack, z, data, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/world-to-local-coordinates" % (z) @@ -74,17 +57,11 @@ def world_to_local_coordinates_batch(stack, z, data, render=None, host=None, # FIXME different inputs than world_to_local? -def local_to_world_coordinates_batch(stack, data, z, render=None, host=None, +@renderaccess +def local_to_world_coordinates_batch(stack, data, z, host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return local_to_world_coordinates_batch( - stack, data, z, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - + session=requests.session(), + render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/local-to-world-coordinates" % (z) @@ -93,20 +70,14 @@ def local_to_world_coordinates_batch(stack, data, z, render=None, host=None, return r.json() +@renderaccess def world_to_local_coordinates_array(stack, dataarray, tileId, z=0, - render=None, host=None, port=None, + host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): + session=requests.session(), + render=None, **kwargs): '''''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return world_to_local_coordinates_array( - stack, dataarray, tileId, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session, 'z': z})) - request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/world-to-local-coordinates" % (z) @@ -131,19 +102,13 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z=0, logger.error(json_answer) +@renderaccess def local_to_world_coordinates_array(stack, dataarray, tileId, z=0, - render=None, host=None, port=None, + host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): + session=requests.session(), + render=None, **kwargs): '''''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return local_to_world_coordinates_array( - stack, dataarray, tileId, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session, 'z': z})) - request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/local-to-world-coordinates" % (z) diff --git a/renderapi/image.py b/renderapi/image.py index 17544cb7..e0666947 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -5,7 +5,7 @@ from PIL import Image import numpy as np import logging -from .render import Render, format_baseurl, format_preamble +from .render import Render, format_baseurl, format_preamble, renderaccess logger = logging.getLogger(__name__) @@ -21,9 +21,11 @@ None: 'png-image'} # Default to png -def get_bb_image(stack, z, x, y, width, height, render=None, scale=1.0, +@renderaccess +def get_bb_image(stack, z, x, y, width, height, scale=1.0, host=None, port=None, owner=None, project=None, - img_format=None, session=requests.session(), **kwargs): + img_format=None, session=requests.session(), + render=None, **kwargs): ''' render image from a bounding box defined in xy and return numpy array: z: layer @@ -32,16 +34,6 @@ def get_bb_image(stack, z, x, y, width, height, render=None, scale=1.0, width: extent to right in x height: extent down in y ''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_bb_image( - stack, z, x, y, width, height, - **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'scale': scale, 'img_format': img_format, - 'session': session})) - try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: @@ -59,20 +51,14 @@ def get_bb_image(stack, z, x, y, width, height, render=None, scale=1.0, logger.error(r.text) -def get_tile_image_data(stack, tileId, render=None, - normalizeForMatching=True, host=None, port=None, - owner=None, project=None, img_format=None, - session=requests.session(), **kwargs): +@renderaccess +def get_tile_image_data(stack, tileId, normalizeForMatching=True, + host=None, port=None, owner=None, project=None, + img_format=None, session=requests.session(), + render=None, **kwargs): ''' render image from a tile with all transforms and return numpy array ''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_tile_image_data(stack, tileId, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'img_format': img_format, 'session': session})) - try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index bb101b51..86efae3a 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -4,19 +4,15 @@ ''' import requests import logging -from .render import Render, format_baseurl +from .render import Render, format_baseurl, renderaccess logger = logging.getLogger(__name__) -def get_matchcollection_owners(render=None, host=None, port=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_matchcollection_owners(**render.make_kwargs( - host=host, port=port, **{'session': session})) - +@renderaccess +def get_matchcollection_owners(host=None, port=None, + session=requests.session(), + render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/matchCollectionOwners" r = session.get(request_url) @@ -26,15 +22,9 @@ def get_matchcollection_owners(render=None, host=None, port=None, logger.error(r.text) -def get_matchcollections(render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_matchcollections(**render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - +@renderaccess +def get_matchcollections(owner=None, host=None, port=None, + session=requests.session(), render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollections" % owner r = session.get(request_url) @@ -44,15 +34,10 @@ def get_matchcollections(render=None, owner=None, host=None, port=None, logger.error(r.text) -def get_match_groupIds(matchCollection, render=None, owner=None, host=None, - port=None, session=requests.session(),**kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_match_groupIds(matchCollection, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - +@renderaccess +def get_match_groupIds(matchCollection, owner=None, host=None, + port=None, session=requests.session(), + render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) r = session.get(request_url) @@ -62,17 +47,10 @@ def get_match_groupIds(matchCollection, render=None, owner=None, host=None, logger.error(r.text) -def get_matches_outside_group(matchCollection, groupId, render=None, - owner=None, host=None, port=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_matches_outside_group( - matchCollection, groupId, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - +@renderaccess +def get_matches_outside_group(matchCollection, groupId, owner=None, host=None, + port=None, session=requests.session(), + render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( owner, matchCollection, groupId) @@ -83,17 +61,10 @@ def get_matches_outside_group(matchCollection, groupId, render=None, logger.error(r.text) +@renderaccess def get_matches_within_group(matchCollection, groupId, owner=None, - host=None, port=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_matches_within_group( - matchCollection, groupId, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - + host=None, port=None, session=requests.session(), + render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( owner, matchCollection, groupId) @@ -104,18 +75,11 @@ def get_matches_within_group(matchCollection, groupId, owner=None, logger.error(r.text) +@renderaccess def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_matches_from_group_to_group( - matchCollection, pgroup, qgroup, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( owner, matchCollection, pgroup, qgroup) @@ -126,18 +90,11 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, logger.error(r.text) +@renderaccess def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, qgroup, qid, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_matches_from_tile_to_tile( - matchCollection, pgroup, pid, qgroup, qid, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - request_url = format_baseurl(host, port) + \ ("/owner/%s/matchCollection/%s/group/%s/id/%s/" "matchesWith/%s/id/%s" % ( @@ -149,17 +106,10 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, logger.error(r.text) +@renderaccess def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_matches_with_group( - matchCollection, pgroup, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( owner, matchCollection, pgroup) @@ -170,17 +120,10 @@ def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, logger.error(r.text) +@renderaccess def get_match_groupIds_from_only(matchCollection, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_match_groupIds_from_only( - matchCollection, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) r = session.get(request_url) @@ -190,17 +133,10 @@ def get_match_groupIds_from_only(matchCollection, render=None, owner=None, logger.error(r.text) +@renderaccess def get_match_groupIds_to_only(matchCollection, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_match_groupIds_to_only( - matchCollection, **render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) r = session.get(request_url) diff --git a/renderapi/render.py b/renderapi/render.py index 29b28c08..2098ac02 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -180,7 +180,7 @@ def wrapper(*args, **kwargs): render = kwargs.pop('render', None) if render is not None: if isinstance(render, Render): - return (f(*args, **render.make_kwargs(**kwargs))) + return f(*args, **render.make_kwargs(**kwargs)) else: raise ValueError( 'invalid Render object type {} specified!'.format( diff --git a/renderapi/stack.py b/renderapi/stack.py index c6534c88..df5b534b 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,7 +3,7 @@ import logging from time import strftime import requests -from .render import Render, format_baseurl, format_preamble +from .render import Render, format_baseurl, format_preamble, renderaccess from .utils import jbool logger = logging.getLogger(__name__) @@ -53,18 +53,10 @@ def from_dict(self, d): self.__dict__.update({k: v for k, v in d.items()}) +@renderaccess def set_stack_state(stack, state='LOADING', host=None, port=None, - owner=None, project=None, render=None, - session=requests.session(), **kwargs): - - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return set_stack_state( - stack, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'state': state, 'session': session})) - + owner=None, project=None, + session=requests.session(), render=None, **kwargs): assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] request_url = format_preamble( host, port, owner, project, stack) + "/state/%s" % state @@ -74,15 +66,10 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, return r -def likelyUniqueId(host=None, port=None, render=None, - session=requests.session(), **kwargs): +@renderaccess +def likelyUniqueId(host=None, port=None, + session=requests.session(), render=None, **kwargs): '''return hex-code nearly-unique id from render server''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return likelyUniqueId(**render.make_kwargs(host=host, port=port, - **{'session': session})) - request_url = '{}/likelyUniqueId'.format(format_baseurl(host, port)) r = session.get(request_url, data=None, headers={"content-type": "text/plain"}) @@ -101,37 +88,22 @@ def make_stack_params(host, port, owner, project, stack): return stack_params -def delete_stack(stack, render=None, host=None, port=None, owner=None, - project=None, session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return delete_stack(stack, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - +@renderaccess +def delete_stack(stack, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): request_url = format_preamble(host, port, owner, project, stack) r = session.delete(request_url) logger.debug(r.text) return r +@renderaccess def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, stackResolutionZ=None, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return create_stack( - stack, cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, - stackResolutionX=stackResolutionX, - stackResolutionY=stackResolutionY, - stackResolutionZ=stackResolutionZ, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - sv = StackVersion( cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, stackResolutionX=stackResolutionX, stackResolutionY=stackResolutionY, @@ -149,21 +121,14 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, # FIXME multiple z indices require multiple params? -def clone_stack(inputstack, outputstack, render=None, host=None, port=None, +@renderaccess +def clone_stack(inputstack, outputstack, host=None, port=None, owner=None, project=None, skipTransforms=False, toProject=None, - z=None, session=None, **kwargs): + z=None, session=None, render=None, **kwargs): ''' result: cloned stack in LOADING state with tiles in layers specified by z' ''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return clone_stack(inputstack, outputstack, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - session=session, skipTransforms=skipTransforms, - toProject=toProject, **kwargs)) - if z is not None: zs = [float(i) for i in z] # TODO test me session = requests.session() if session is None else session @@ -180,16 +145,10 @@ def clone_stack(inputstack, outputstack, render=None, host=None, port=None, return r +@renderaccess def get_z_values_for_stack(stack, project=None, host=None, port=None, - owner=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_z_values_for_stack(stack, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - session=session, **kwargs)) - + owner=None, session=requests.session(), + render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" logger.debug(request_url) @@ -200,16 +159,10 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, logger.error(r.text) +@renderaccess def get_z_value_for_section(stack, sectionId, project=None, - host=None, port=None, owner=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_z_value_for_section(stack, sectionId, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - session=session, **kwargs)) - + host=None, port=None, owner=None, + session=requests.session(), render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % (sectionId) r = session.get(request_url) @@ -219,16 +172,10 @@ def get_z_value_for_section(stack, sectionId, project=None, logger.error(r.text) +@renderaccess def put_resolved_tilespecs(stack, data, host=None, port=None, - owner=None, project=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return put_resolved_tilespecs(stack, data, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - session=session, **kwargs)) - + owner=None, project=None, + session=requests.session(), render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + "/resolvedTiles" r = session.put(request_url, data=data, @@ -237,16 +184,10 @@ def put_resolved_tilespecs(stack, data, host=None, port=None, return r +@renderaccess def get_bounds_from_z(stack, z, host=None, port=None, owner=None, - project=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_bounds_from_z(stack, z, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - session=session, **kwargs)) - + project=None, session=requests.session(), + render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/bounds' % (z) @@ -257,16 +198,10 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, logger.error(r.text) +@renderaccess def get_section_z_value(stack, sectionId, host=None, port=None, - owner=None, project=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_section_z_value(stack, sectionId, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - session=session, **kwargs)) - + owner=None, project=None, session=requests.session(), + render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % sectionId r = session.get(request_url) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 621aa0c6..95d5dd56 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from .render import Render, format_baseurl, format_preamble +from .render import Render, format_baseurl, format_preamble, renderaccess from collections import OrderedDict import logging import requests @@ -440,16 +440,10 @@ def __iter__(self): l for sl in [list(mmL) for mmL in self.mipMapLevels] for l in sl]) -def get_tile_spec(stack, tile, render=None, host=None, port=None, owner=None, - project=None, session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_tile_spec( - stack, tile, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - +@renderaccess +def get_tile_spec(stack, tile, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/render-parameters" % (tile) @@ -461,18 +455,12 @@ def get_tile_spec(stack, tile, render=None, host=None, port=None, owner=None, return TileSpec(json=tilespec_json['tileSpecs'][0]) +@renderaccess def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, - render=None, scale=1.0, host=None, + scale=1.0, host=None, port=None, owner=None, project=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_tile_specs_from_minmax_box( - stack, z, xmin, xmax, ymin, ymax, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - + session=requests.session(), + render=None, **kwargs): x = xmin y = ymin width = xmax - xmin @@ -483,18 +471,11 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, session=session) -def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, +@renderaccess +def get_tile_specs_from_box(stack, z, x, y, width, height, scale=1.0, host=None, port=None, owner=None, project=None, session=requests.session(), - **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_tile_specs_from_box( - stack, z, x, y, width, height, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - + render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( @@ -509,17 +490,10 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, render=None, for tilespec_json in tilespecs_json['tileSpecs']] -def get_tile_specs_from_z(stack, z, render=None, host=None, port=None, +@renderaccess +def get_tile_specs_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), - **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_tile_specs_from_z( - stack, z, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - **{'session': session})) - + render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) logger.debug(request_url) From 3233ab5eb58715b7c036559f6ccd6b6d03c3add5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 24 Feb 2017 15:43:15 -0800 Subject: [PATCH 109/766] render: include renderaccess --- renderapi/render.py | 49 ++++++++++++--------------------------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 2098ac02..ad5dff1c 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -200,15 +200,9 @@ def format_preamble(host, port, owner, project, stack): return preamble -def get_owners(host=None, port=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_owners(**render.make_kwargs( - host=host, port=port, - **{'session': session})) - +@renderaccess +def get_owners(host=None, port=None, session=requests.session(), + render=None, **kwargs): request_url = "%s/owners/" % format_baseurl(host, port) r = session.get(request_url) try: @@ -217,15 +211,10 @@ def get_owners(host=None, port=None, render=None, logger.error(r.text) -def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_stack_metadata_by_owner(**render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - +@renderaccess +def get_stack_metadata_by_owner(owner=None, host=None, port=None, + session=requests.session(), + render=None, **kwargs): request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) logger.debug(request_url) @@ -236,31 +225,19 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, render=None, logger.error(r.text) -def get_projects_by_owner(owner=None, host=None, port=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_projects_by_owner(**render.make_kwargs( - owner=owner, host=host, port=port, - **{'session': session})) - +@renderaccess +def get_projects_by_owner(owner=None, host=None, port=None, + session=requests.session(), render=None, **kwargs): metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) projects = list(set([m['stackId']['project'] for m in metadata])) return projects +@renderaccess def get_stacks_by_owner_project(owner=None, project=None, host=None, - port=None, render=None, - session=requests.session(), **kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_stacks_by_owner_project(**render.make_kwargs( - owner=owner, host=host, port=port, project=project, - **{'session': session})) - + port=None, session=requests.session(), + render=None, **kwargs): metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) stacks = ([m['stackId']['stack'] for m in metadata From 0eb6069486ef563044b0886c24b435d3e73c6086 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 24 Feb 2017 16:52:35 -0800 Subject: [PATCH 110/766] client, transform: renderaccess, draft importtilespec --- renderapi/client.py | 242 ++++++++++++++++++----------------------- renderapi/transform.py | 5 +- 2 files changed, 105 insertions(+), 142 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index fa849116..7dadcc1f 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,8 +8,9 @@ import logging import subprocess import tempfile +from .utils import jbool, renderdump from .errors import ClientScriptError -from .render import Render, RenderClient +from .render import Render, RenderClient, renderaccess from .stack import set_stack_state, make_stack_params # setup logger @@ -19,26 +20,19 @@ from pathos.multiprocessing import ProcessingPool as Pool has_pathos = True except ImportError as e: - logging.warning(e) + logger.warning(e) has_pathos = False from multiprocessing import Pool -def import_single_json_file(stack, jsonfile, render=None, transformFile=None, +@renderaccess +def import_single_json_file(stack, jsonfile, transformFile=None, client_scripts=None, host=None, port=None, - owner=None, project=None, **kwargs): + owner=None, project=None, render=None, **kwargs): ''' calls client script to import given jsonfile: transformFile: ? ''' - # process render-based default configuration - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return import_single_json_file(stack, jsonfile, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - client_scripts=client_scripts, **{'transformFile': None})) - if transformFile is None: transform_params = [] else: @@ -56,10 +50,11 @@ def import_single_json_file(stack, jsonfile, render=None, transformFile=None, logger.debug(proc.stdout.read()) +@renderaccess def import_jsonfiles_and_transforms_parallel_by_z( - stack, jsonfiles, transformfiles, render=None, poolsize=20, + stack, jsonfiles, transformfiles, poolsize=20, client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, **kwargs): + project=None, close_stack=True, render=None, **kwargs): ''' imports json files and transform files in parallel: jsonfiles: "list of tilespec" jsons to import @@ -67,16 +62,6 @@ def import_jsonfiles_and_transforms_parallel_by_z( poolsize: number of processes for multiprocessing pool close_stack: mark render stack as COMPLETE after successful import ''' - # process render-based default configuration - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return import_jsonfiles_and_transforms_parallel_by_z( - stack, jsonfile, transformfiles, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - client_scripts=client_scripts, **{'close_stack': close_stack, - 'poolsize': poolsize})) - set_stack_state(stack, 'LOADING', host, port, owner, project) pool = Pool(poolsize) partial_import = partial(import_single_json_file, stack, render=render, @@ -88,10 +73,11 @@ def import_jsonfiles_and_transforms_parallel_by_z( set_stack_state(stack, 'COMPLETE', host, port, owner, project) +@renderaccess def import_jsonfiles_parallel( - stack, jsonfiles, render=None, poolsize=20, transformFile=None, + stack, jsonfiles, poolsize=20, transformFile=None, client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, **kwargs): + project=None, close_stack=True, render=None, **kwargs): ''' import jsons using client script in parallel jsonfiles: list of jsonfiles to upload @@ -99,17 +85,6 @@ def import_jsonfiles_parallel( transformFile: a single json file containing transforms referenced in the jsonfiles ''' - # process render-based default configuration - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return import_jsonfiles_parallel( - stack, jsonfiles, **render.make_kwargs( - host=host, port=port, owner=owner, - project=project, client_scripts=client_scripts, - **{'close_stack': close_stack, - 'poolsize': poolsize, 'transformFile': transformFile})) - set_stack_state(stack, 'LOADING', host, port, owner, project) pool = Pool(poolsize) partial_import = partial(import_single_json_file, stack, render=render, @@ -124,26 +99,17 @@ def import_jsonfiles_parallel( set_stack_state(stack, 'COMPLETE', host, port, owner, project) -def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, +@renderaccess +def import_jsonfiles(stack, jsonfiles, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, - **kwargs): + render=None, **kwargs): ''' import jsons using client script serially jsonfiles: iterator of filenames to be uploaded transformFile: ? close_stack: ? ''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return import_jsonfiles( - stack, jsonfiles, **render.make_kwargs( - host=host, port=port, owner=owner, - project=project, client_scripts=client_scripts, - **{'close_stack': close_stack, - 'transformFile': transformFile})) - set_stack_state(stack, 'LOADING', host, port, owner, project) if transformFile is None: transform_params = [] @@ -164,26 +130,16 @@ def import_jsonfiles(stack, jsonfiles, render=None, transformFile=None, set_stack_state(stack, 'COMPLETE', host, port, owner, project) -# FIXME paperweight for proper render_ws_client implemntation -def import_jsonfiles_validate_client(stack, jsonfiles, render=None, +@renderaccess +def import_jsonfiles_validate_client(stack, jsonfiles, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, mem=6, validator=None, - **kwargs): + render=None, **kwargs): ''' Uses java client for parallelization and validation ''' - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return import_jsonfiles_validate_client( - stack, jsonfiles, **render.make_kwargs( - host=host, port=port, owner=owner, - project=project, client_scripts=client_scripts, - **{'close_stack': close_stack, 'mem': mem, - 'transformFile': transformFile})) - transform_params = (['--transformFile', transformFile] if transformFile is not None else []) if validator is None: @@ -220,6 +176,78 @@ def import_jsonfiles_validate_client(stack, jsonfiles, render=None, set_stack_state(stack, 'COMPLETE', host, port, owner, project) +@renderaccess +def import_tilespecs(stack, tilespecs, sharedTransforms=None, + subprocess_mode=None, host=None, port=None, + owner=None, project=None, client_script=None, + memGB=None, render=None, **kwargs): + ''' + input: + tilespecs -- list of tilespecs + ''' + tempjson = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) + tempjson.close() + tsjson = tempjson.name + with open(tsjson, 'w') as f: + renderdump(tilespecs, f) + + if sharedTransforms is not None: + tempjson = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) + tempjson.close() + trjson = tempjson.name + with open(trjson, 'w') as f: + renderdump(sharedTransforms, f) + + importJsonClient(stack, tileFiles=[tsjson], transformFile=( + trjson if sharedTransforms is not None else None), + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, + client_script=client_script, memGB=memGB) + + os.remove(tsjson) + os.remove(trjson) + + +@renderaccess +def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, + subprocess_mode=None, poolsize=20, + close_stack=True, host=None, port=None, + owner=None, project=None, + client_script=None, memGB=None, render=None, + **kwargs): + set_stack_state(stack, 'LOADING', host, port, owner, project) + pool = Pool(poolsize) + partial_import = partial( + import_tilespecs, stack, sharedTransforms=sharedTransforms, + subprocess_mode=subprocess_mode, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs)) + + # TODO this is a weird way to do splits.... is that okay? + tilespec_groups = [tilespecs[i::poolsize] for i in xrange(poolsize)] + rs = pool.map(partial_import, tilespec_groups) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +# TODO handle fromJson and toJson persistence in these calls +@renderaccess +def local_to_world_array(): + '''placeholder function for coordinateClient localtoworld''' + raise NotImplementedError('Whoops') + pass + + +@renderaccess +def world_to_local_array(): + '''placeholder function for coordinateClient worldtolocal''' + raise NotImplementedError('Whoops.') + pass + + def call_run_ws_client(className, add_args=[], renderclient=None, memGB=None, client_script=None, subprocess_mode=None, **kwargs): @@ -238,7 +266,7 @@ def call_run_ws_client(className, add_args=[], renderclient=None, 'check_call': subprocess.check_call, 'check_output': subprocess.check_output} if subprocess_mode not in subprocess_modes: - logging.warning( + logger.warning( 'Unknown subprocess mode {} specified -- ' 'using default subprocess.call'.format(subprocess_mode)) return subprocess_modes.get( @@ -250,22 +278,13 @@ def get_param(var, flag): return ([flag, var] if var is not None else []) +@renderaccess def importJsonClient(stack, tileFiles=None, transformFile=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): '''run ImportJsonClient.java''' - if render is not None: - if isinstance(render, Render): - return importJsonClient( - stack, tileFiles=tileFiles, transformFile=transformFile, - subprocess_mode=subprocess_mode, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - client_script=client_script, memGB=memGB, **kwargs)) - else: - raise ValueError('invalid Render object specified!') - argvs = (make_stack_params(host, port, owner, project, stack) + (['--transformFile', transformFile] if transformFile else []) + (tileFiles if isinstance(tileFiles, list) @@ -275,6 +294,7 @@ def importJsonClient(stack, tileFiles=None, transformFile=None, client_script=client_script, memGB=memGB) +@renderaccess def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, baseowner=None, baseproject=None, basestack=None, xyNeighborFactor=None, zNeighborDistance=None, @@ -289,26 +309,6 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, client_script=None, memGB=None, render=None, **kwargs): '''run TilePairClient.java''' - if render is not None: - if isinstance(render, Render): - return tilePairClient( - stack, minz, maxz, outjson=outjson, delete_json=delete_json, - baseowner=baseowner, baseproject=baseproject, - basestack=basestack, - xyNeighborFactor=xyNeighborFactor, - zNeighborDistance=zNeighborDistance, - excludeCornerNeighbors=excludeCornerNeighbors, - excludeCompletelyObscuredTiles=excludeCompletelyObscuredTiles, - excludeSameLayerNeighbors=excludeSameLayerNeighbors, - excludeSameSectionNeighbors=excludeSameSectionNeighbors, - excludePairsInMatchCollection=excludePairsInMatchCollection, - minx=minx, maxx=maxx, miny=miny, maxy=maxy, - subprocess_mode=subprocess_mode, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - client_script=client_script, memGB=memGB, **kwargs)) - else: - raise ValueError('invalid Render object specified!') - if outjson is None: tempjson = tempfile.NamedTemporaryFile( suffix=".json", mode='r', delete=False) @@ -349,6 +349,7 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, return jsondata +@renderaccess def importTransformChangesClient(stack, targetStack, transformFile, targetOwner=None, targetProject=None, changeMode=None, subprocess_mode=None, @@ -358,17 +359,6 @@ def importTransformChangesClient(stack, targetStack, transformFile, ''' run ImportTransformChangesClient.java ''' - if render is not None: - if isinstance(render, Render): - return importTransformChangesClient( - stack, targetStack, transformFile, targetOwner=targetOwner, - targetProject=targetProject, changeMode=changeMode, - subprocess_mode=subprocess_mode, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - client_script=client_script, memGB=memGB, **kwargs)) - else: - raise ValueError('invalid Render object specified!') - if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: raise ClientScriptError( 'changeMode {} is not valid!'.format(changeMode)) @@ -385,43 +375,22 @@ def importTransformChangesClient(stack, targetStack, transformFile, add_args=argv) +@renderaccess def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, - numberOfThreads=None, delete_fromJson=False, - delete_toJson=False, subprocess_mode=None, + numberOfThreads=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): ''' run CoordinateClient.java + expects: + fromJson -- json in format defined by list of + coordinate dictionaries (world) or + list of list of coordinate dictionaries (local) + toJson -- json to save results of mapping + localToWorld -- flag defaults to accepting world coordinates + numberOfThreads -- java-based threads for client script ''' - if render is not None: - if isinstance(render, Render): - return coordinateClient( - stack, z, fromJson=fromJson, toJson=toJson, - localToWorld=localToWorld, numberOfThreads=numberOfThreads, - delete_toJson=delete_toJson, delete_fromJson=delete_fromJson, - subprocess_mode=subprocess_mode, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - client_script=client_script, memGB=memGB, **kwargs)) - else: - raise ValueError('invalid Render object specified!') - - # TODO allow using array as input for mapping - if toJson is None: - tempjson = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - tempjson.close() - delete_toJson = True - toJson = tempjson.name - if fromJson is None: - tempjson = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - tempjson.write(input_array) - tempjson.flush() - tempjson.close() - delete_fromJson = True - fromJson = tempjson.name - argvs = (make_stack_params(host, port, owner, project, stack) + ['--z', z, '--fromJson', fromJson, '--toJson', toJson] + (['--localToWorld', jbool(localToWorld)] @@ -432,9 +401,4 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, with open(toJson, 'r') as f: jsondata = json.load(f) - if delete_toJson: - os.remove(toJson) - if delete_fromJson: - os.remove(fromJson) - return jsondata diff --git a/renderapi/transform.py b/renderapi/transform.py index d1b779a6..b27e8b43 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -330,10 +330,9 @@ def concatenate(self, othertform, order=None, srcpts=None): if order is None: order = max([self.order, othertform.order]) # TODO define srcpts and dstpts - if srcpts is not None: - dstpts = othertform.tform(self.tform(srcpts)) - else: + if srcpts is None: raise NotImplementedError('default source points unavailable!') + dstpts = othertform.tform(self.tform(srcpts)) return Polynomial2DTransform(src=srcpts, dst=dstpts, order=order) From cc166f3e64813a25b5547e97aa284f2d32997339 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 25 Feb 2017 07:33:43 -0800 Subject: [PATCH 111/766] render: functions no longer recurse if render is Render instance -- can safely pass --- renderapi/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index 7bb52ccc..35db81d5 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -177,7 +177,7 @@ def connect(host=None, port=None, owner=None, project=None, def renderaccess(f): @wraps(f) def wrapper(*args, **kwargs): - render = kwargs.pop('render', None) + render = kwargs.get('render') if render is not None: if isinstance(render, Render): return f(*args, **render.make_kwargs(**kwargs)) From 4763112982d70016276155f9084d734529aeecd5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 25 Feb 2017 07:39:36 -0800 Subject: [PATCH 112/766] client: correct localtoworld arity in coordinateclient --- renderapi/client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 077bdf72..8dba8e0e 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -261,7 +261,7 @@ def call_run_ws_client(className, add_args=[], renderclient=None, **renderclient.make_kwargs( memGB=memGB, client_script=client_script)) - + subprocess_modes = {'call': subprocess.call, 'check_call': subprocess.check_call, 'check_output': subprocess.check_output} @@ -393,8 +393,7 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, ''' argvs = (make_stack_params(host, port, owner, project, stack) + ['--z', z, '--fromJson', fromJson, '--toJson', toJson] + - (['--localToWorld', jbool(localToWorld)] - if localToWorld is not None else []) + + (['--localToWorld'] if localToWorld else []) + get_param(numberOfThreads, '--numberOfThreads')) call_run_ws_client('org.janelia.render.client.CoordinateClient') From 0e2adff146146b32a605fe1834d59f5f7284d2db Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 25 Feb 2017 07:42:34 -0800 Subject: [PATCH 113/766] coordinateclient: fix call_run_ws_client call --- renderapi/client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 8dba8e0e..ad70fc0c 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -395,7 +395,9 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, ['--z', z, '--fromJson', fromJson, '--toJson', toJson] + (['--localToWorld'] if localToWorld else []) + get_param(numberOfThreads, '--numberOfThreads')) - call_run_ws_client('org.janelia.render.client.CoordinateClient') + call_run_ws_client('org.janelia.render.client.CoordinateClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs) with open(toJson, 'r') as f: jsondata = json.load(f) From f3c9bdac7f9becb2facfd5b8263c720459d1394e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 25 Feb 2017 07:45:07 -0800 Subject: [PATCH 114/766] readme: add example environment variables and link to render-python-apps --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index ecf50132..3ed89fcc 100644 --- a/README.md +++ b/README.md @@ -12,3 +12,15 @@ modify the tilespecs to include paths to downsampled images, and then write thos [create_mipmaps.py](docs/examples/create_mipmaps.py) is a simple python program for using Pillow to create downsampled images from a single image that is included simply for reference. +Render connection objects created with `renderapi.connect()` can default to environment variables. Below is an example of the variables which can be sourced and added to ~/.bashrc or ~/.bash_profile. +``` + export RENDER_HOST="localhost" + export RENDER_PORT="8080" + export RENDER_PROJECT="YOURPROJECT" + export RENDER_OWNER="YOURNAME" + export RENDER_CLIENT_SCRIPTS=".../render/render-ws-java-client/src/main/scripts" + export RENDER_CLIENT_SCRIPT="$RENDER_CLIENT_SCRIPTS/run_ws_client.sh" + export RENDER_CLIENT_HEAP="1G" +``` + +[Usage examples for an Array Tomography workflow](https://github.com/fcollman/render-python-apps/tree/newrender) are available. From 0d35b559997ed50fdb60f3b891f27e94d93b4eef Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 25 Feb 2017 08:14:33 -0800 Subject: [PATCH 115/766] render: loosen strict host and port requirements --- renderapi/render.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 35db81d5..5d5bfe28 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -102,9 +102,8 @@ def connect(host=None, port=None, owner=None, project=None, if host == '': logger.critical('Render Host must not be empty!') raise ValueError('Render Host must not be empty!') - # TODO more flexible server input - # host = (host if host.startswith('http') - # else 'http://{}'.format(host)) + host = (host if host.startswith('http') + else 'http://{}'.format(host)) else: host = os.environ['RENDER_HOST'] @@ -191,7 +190,9 @@ def wrapper(*args, **kwargs): def format_baseurl(host, port): - return 'http://%s:%d/render-ws/v1' % (host, port) + # return 'http://%s:%d/render-ws/v1' % (host, port) + server = '{}{}'.format(host, ('' if port is None else ':{}'.format(port))) + return '{}/render-ws/v1'.format(server) def format_preamble(host, port, owner, project, stack): From 423c39b714ee2d6326591cf992b7653aa1dcf5c8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 25 Feb 2017 11:29:53 -0800 Subject: [PATCH 116/766] removing no longer present and unused load_dict imports --- renderapi/transform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 03811279..d1b779a6 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -14,7 +14,6 @@ import logging import numpy as np from .errors import ConversionError, EstimationError -from .utils import _load_dict, _load_json logger = logging.getLogger(__name__) From 2bb9ee45b6e3d248dd198cf4ca3b67ba214a9aec Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 25 Feb 2017 11:30:36 -0800 Subject: [PATCH 117/766] adding default 1G memory to render client as it isn't an optional parameter and will fail if not specified --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 0414667a..1803a2a0 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -221,7 +221,7 @@ def import_jsonfiles_validate_client(stack, jsonfiles, render=None, def call_run_ws_client(className, add_args=[], renderclient=None, - memGB=None, client_script=None, subprocess_mode=None, + memGB='1G', client_script=None, subprocess_mode=None, **kwargs): ''' simple call for run_ws_client.sh -- all arguments set in add_args From b5b40c261eb061a1b9f1e0068731ee8f877afc46 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 25 Feb 2017 18:32:56 -0800 Subject: [PATCH 118/766] stack: add force_resolution flag for easier ndviz interfacing --- renderapi/stack.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index df5b534b..7a07707e 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -101,9 +101,16 @@ def delete_stack(stack, host=None, port=None, owner=None, @renderaccess def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, - stackResolutionZ=None, + stackResolutionZ=None, force_resolution=True, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + if force_resolution: + stackResolutionX, stackResolutionY, stackResolutionZ = [ + (1.0 if res is None else res) + for res in [stackResolutionX, stackResolutionY, stackResolutionZ]] + logger.debug('forcing resolution x:{}, y:{}, z:{}'.format( + stackResolutionX, stackResolutionY, stackResolutionZ)) + sv = StackVersion( cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, stackResolutionX=stackResolutionX, stackResolutionY=stackResolutionY, From 4e3e8411d98c82fb85461174c43608759eea5b27 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 26 Feb 2017 00:36:25 -0800 Subject: [PATCH 119/766] transform: fix tpyo --- renderapi/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index b27e8b43..5471551a 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -275,13 +275,13 @@ def _process_dataString(self, datastring): dsList = datastring.split(' ') self.params = np.array( [[float(d) for d in dsList[:len(dsList)/2]], - [float(d) for d in dsList[len(dsList)/2]]]) + [float(d) for d in dsList[len(dsList)/2:]]]) self.dataString = datastring def _format_raveled_params(self, raveled_params): return np.array( [[float(d) for d in dsList[:len(raveled_params)/2]], - [float(d) for d in dsList[len(raveled_params)/2]]]) + [float(d) for d in dsList[len(raveled_params)/2:]]]) def tform(self, points): dst = np.zeros(points.shape) From 46298dca2b0857ac96675457f8cc898b9076b223 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 26 Feb 2017 01:30:41 -0800 Subject: [PATCH 120/766] stack: add READ_ONLY state --- renderapi/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 7a07707e..eb3d6004 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -57,7 +57,7 @@ def from_dict(self, d): def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] + assert state in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'] request_url = format_preamble( host, port, owner, project, stack) + "/state/%s" % state logger.debug(request_url) From 77de298e9fbc88fca76d624d31177c210435bf86 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 08:54:52 -0800 Subject: [PATCH 121/766] adding getting section data call --- renderapi/stack.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/renderapi/stack.py b/renderapi/stack.py index eb3d6004..cd01a955 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -151,6 +151,25 @@ def clone_stack(inputstack, outputstack, host=None, port=None, return r +def get_sectionData_for_stack(stack,project = None, + host = None,port = None,owner = None,render =None, + session=requests.session(),**kwargs): + if render is not None: + if not isinstance(render, Render): + raise ValueError('invalid Render object specified!') + return get_sectionData_for_stack(stack, **render.make_kwargs( + host=host, port=port, owner=owner, project=project, + session=session, **kwargs)) + + request_url = format_preamble( + host,port,owner,project,stack)+"/sectionData" + logger.debug(request_url) + r = session.get(request_url) + try: + return r.json() + except: + logger.error(r.text) + @renderaccess def get_z_values_for_stack(stack, project=None, host=None, port=None, From e1ab2abae07d2dbb1b4b1d8f82bcc4d5cbe149f7 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 08:55:52 -0800 Subject: [PATCH 122/766] conforming to new renderaccess style --- renderapi/stack.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index cd01a955..064c98dd 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -151,15 +151,10 @@ def clone_stack(inputstack, outputstack, host=None, port=None, return r +@renderaccess def get_sectionData_for_stack(stack,project = None, - host = None,port = None,owner = None,render =None, - session=requests.session(),**kwargs): - if render is not None: - if not isinstance(render, Render): - raise ValueError('invalid Render object specified!') - return get_sectionData_for_stack(stack, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - session=session, **kwargs)) + host = None,port = None,owner = None, + session=requests.session(),render =None,**kwargs): request_url = format_preamble( host,port,owner,project,stack)+"/sectionData" From 4c48ae0a4f5af2f641eb567134c4a3875d11955e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 27 Feb 2017 10:36:21 -0800 Subject: [PATCH 123/766] render, utils: explicit argspec matching makes coding style requirements less strict --- renderapi/render.py | 4 +- renderapi/utils.py | 123 ++++++++++++++++++++++++-------------------- 2 files changed, 71 insertions(+), 56 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 5d5bfe28..8f1d1d8e 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -3,7 +3,7 @@ import os from functools import wraps import requests -from .utils import defaultifNone, NullHandler +from .utils import defaultifNone, NullHandler, fitargspec from .errors import ClientScriptError logger = logging.getLogger(__name__) @@ -56,6 +56,7 @@ def run(self, f, *args, **kwargs): run function from object technically shorter than adding render=Render to kwargs ''' + args, kwargs = fitargspec(f, args, kwargs) return f(*args, **self.make_kwargs(**kwargs)) @@ -176,6 +177,7 @@ def connect(host=None, port=None, owner=None, project=None, def renderaccess(f): @wraps(f) def wrapper(*args, **kwargs): + args, kwargs = fitargspec(f, args, kwargs) render = kwargs.get('render') if render is not None: if isinstance(render, Render): diff --git a/renderapi/utils.py b/renderapi/utils.py index fc5e9eae..96d51e85 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -1,55 +1,68 @@ -#!/usr/bin/env python -''' -utilities to make render/java/web/life interfacing easier -''' -import logging -import json - -logger = logging.getLogger(__name__) - - -class RenderEncoder(json.JSONEncoder): - def default(self, obj): - to_dict = getattr(obj, "to_dict", None) - if callable(to_dict): - return obj.to_dict() - else: - return obj.__dict__ - - -def renderdumps(obj, *args, **kwargs): - cls_ = kwargs.pop('cls', RenderEncoder) - return json.dumps(obj, *args, cls=cls_, **kwargs) - - -def renderdump(obj, *args, **kwargs): - cls_ = kwargs.pop('cls', RenderEncoder) - return json.dump(obj, *args, cls=cls_, **kwargs) - - -def jbool(val): - '''return string representing java string values of py booleans''' - if not isinstance(val, bool): - logger.warning('Evaluating javastring of non-boolean {} {}'.format( - type(val), val)) - return 'true' if val else 'false' - - -class NullHandler(logging.Handler): - def emit(self, record): - pass - - -def stripLogger(logger_tostrip): - ''' - remove all handlers from a logger -- useful for redefining - input: - logger_tostrip: logging logger as from logging.getLogger - ''' - if logger_tostrip.handlers: - for handler in logger_tostrip.handlers: - logger_tostrip.removeHandler(handler) - - -def defaultifNone(val, default=None): - return val if val is not None else default +#!/usr/bin/env python +''' +utilities to make render/java/web/life interfacing easier +''' +import logging +import inspect +import copy +import json + +logger = logging.getLogger(__name__) + + +class RenderEncoder(json.JSONEncoder): + def default(self, obj): + to_dict = getattr(obj, "to_dict", None) + if callable(to_dict): + return obj.to_dict() + else: + return obj.__dict__ + + +def renderdumps(obj, *args, **kwargs): + cls_ = kwargs.pop('cls', RenderEncoder) + return json.dumps(obj, *args, cls=cls_, **kwargs) + + +def renderdump(obj, *args, **kwargs): + cls_ = kwargs.pop('cls', RenderEncoder) + return json.dump(obj, *args, cls=cls_, **kwargs) + + +def jbool(val): + '''return string representing java string values of py booleans''' + if not isinstance(val, bool): + logger.warning('Evaluating javastring of non-boolean {} {}'.format( + type(val), val)) + return 'true' if val else 'false' + + +class NullHandler(logging.Handler): + def emit(self, record): + pass + + +def stripLogger(logger_tostrip): + ''' + remove all handlers from a logger -- useful for redefining + input: + logger_tostrip: logging logger as from logging.getLogger + ''' + if logger_tostrip.handlers: + for handler in logger_tostrip.handlers: + logger_tostrip.removeHandler(handler) + + +def defaultifNone(val, default=None): + return val if val is not None else default + + +def fitargspec(f, oldargs, oldkwargs): + ''' fit function argspec given input args tuple and kwargs dict''' + args, varargs, keywords, defaults = inspect.getargspec(f) + num_expected_args = len(args) - len(defaults) + new_args = tuple(oldargs[:num_expected_args]) + new_kwargs = copy.copy(oldkwargs) + for i, arg in enumerate(oldargs[num_expected_args:]): + new_kwargs.update({args[i + num_expected_args]: arg}) + return new_args, new_kwargs From 781e916951c230d39c5c1063e8bd2abc187d098d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 10:41:43 -0800 Subject: [PATCH 124/766] removing unused tileId from coordinate mapping calls --- renderapi/coordinate.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index b9b1cea4..050b46d9 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -151,7 +151,7 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, jsondata = package_point_match_data_into_json(dataarray, tileId, 'world') if doClientSide: json_answer = world_to_local_coordinates_clientside( - stack, jsondata, tileId, z, host=host, port=port, owner=owner, + stack, jsondata, z, host=host, port=port, owner=owner, project=project, client_script=client_script, number_of_threads=number_of_threads) else: @@ -205,7 +205,7 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, jsondata = package_point_match_data_into_json(dataarray, tileId, 'local') if doClientSide: json_answer = local_to_world_coordinates_clientside( - stack, jsondata, tileId, z, host=host, port=port, owner=owner, + stack, jsondata, z, host=host, port=port, owner=owner, project=project, client_script=client_script, number_of_threads=number_of_threads) else: @@ -215,9 +215,9 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, return unpackage_local_to_world_point_match_from_json(json_answer) -def map_coordinates_clientside(stack, jsondata, tileId, z, host, port, owner, +def map_coordinates_clientside(stack, jsondata, z, host, port, owner, project, client_script, isLocalToWorld=False, - number_of_threads=20): + number_of_threads=20,memGB=1): # write point match json to temp file on disk json_infile, json_inpath = tempfile.mkstemp( prefix='render_coordinates_in_', suffix='.json') @@ -230,35 +230,24 @@ def map_coordinates_clientside(stack, jsondata, tileId, z, host, port, owner, json_outpath = tempfile.mktemp( prefix='render_coordinates_out_', suffix='.json') - # define arguments - args = ['--baseDataUrl', 'http://%s:%d/render-ws/v1' % (host, port), - '--owner', owner, - '--project', project, - '--stack', stack, - '--z', str(z), - '--fromJson', json_inpath, - '--toJson', json_outpath, - '--numberOfThreads', str(number_of_threads)] - if isLocalToWorld: - args += ['--localToWorld'] - # call the java client - call_run_ws_client('org.janelia.render.client.CoordinateClient', - add_args=args, client_script=client_script) + renderapi.client.coordinateClient(stack, z, fromJson=json_inpath, toJson=json_outpath, localToWorld=isLocalToWorld, + numberOfThreads=number_of_threads, host=host,port=port, owner=owner,project=project, client_script=client_script, + memGB=memGB) # return the json results return json.load(open(json_outpath, 'r')) @renderaccess -def world_to_local_coordinates_clientside(stack, jsondata, tileId, z, +def world_to_local_coordinates_clientside(stack, jsondata, z, host=None, port=None, owner=None, project=None, client_script=None, number_of_threads=20, session=requests.session(), render=None, **kwargs): - return map_coordinates_clientside(stack, jsondata, tileId, z, + return map_coordinates_clientside(stack, jsondata, z, host=host, port=port, owner=owner, project=project, client_script=client_script, From 2ce888fa2f776fded81265535cdc9ee44fbc65bd Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 27 Feb 2017 10:53:44 -0800 Subject: [PATCH 125/766] client: call_run_ws_client memGB=None issues warning, some docstrings --- renderapi/client.py | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 7d14fb47..a11ce540 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -183,7 +183,10 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, memGB=None, render=None, **kwargs): ''' input: + stack -- stack to which tilespecs will be added tilespecs -- list of tilespecs + sharedTransforms -- list of shared + referenced transforms to be ingested ''' tempjson = tempfile.NamedTemporaryFile( suffix=".json", mode='r', delete=False) @@ -235,21 +238,45 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, # TODO handle fromJson and toJson persistence in these calls @renderaccess -def local_to_world_array(): - '''placeholder function for coordinateClient localtoworld''' +def local_to_world_array(stack, points, tileId, subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + ''' + placeholder function for coordinateClient localtoworld + + inputs: + stack -- stack to which world coordinates are mapped + points -- local points to map to world + tileId -- tileId to which points correspond + outputs: + list of points in world coordinates corresponding to local points + ''' raise NotImplementedError('Whoops') pass @renderaccess -def world_to_local_array(): - '''placeholder function for coordinateClient worldtolocal''' +def world_to_local_array(stack, points, subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + ''' + placeholder function for coordinateClient worldtolocal + + inputs: + stack -- stack to which world coordinates are mapped + points -- world points in stack to map to local + outputs: + list of list of dictionaries defining local coordinates + and tileIds corresponding to world point + ''' raise NotImplementedError('Whoops.') pass def call_run_ws_client(className, add_args=[], renderclient=None, - memGB='1G', client_script=None, subprocess_mode=None, + memGB=None, client_script=None, subprocess_mode=None, **kwargs): ''' simple call for run_ws_client.sh -- all arguments set in add_args @@ -261,6 +288,10 @@ def call_run_ws_client(className, add_args=[], renderclient=None, **renderclient.make_kwargs( memGB=memGB, client_script=client_script)) + if memGB is None: + logger.warning('call_run_ws_client requires memory specification -- ' + 'defaulting to 1G') + memGB = '1G' subprocess_modes = {'call': subprocess.call, 'check_call': subprocess.check_call, From 765755c93efb8314f9b12120fe6819e94059037a Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 27 Feb 2017 11:10:02 -0800 Subject: [PATCH 126/766] logging: update loggers with NullHandler --- renderapi/client.py | 3 ++- renderapi/coordinate.py | 3 +++ renderapi/image.py | 2 ++ renderapi/pointmatch.py | 2 ++ renderapi/render.py | 2 +- renderapi/stack.py | 28 ++++------------------------ renderapi/tilespec.py | 2 ++ renderapi/transform.py | 2 ++ renderapi/utils.py | 11 ++++++----- 9 files changed, 24 insertions(+), 31 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index a11ce540..8b69d9de 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,13 +8,14 @@ import logging import subprocess import tempfile -from .utils import jbool, renderdump +from .utils import jbool, renderdump, NullHandler from .errors import ClientScriptError from .render import Render, RenderClient, renderaccess from .stack import set_stack_state, make_stack_params # setup logger logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) try: from pathos.multiprocessing import ProcessingPool as Pool diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 050b46d9..be107f02 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -4,13 +4,16 @@ ''' from .render import Render, format_preamble, renderaccess from .client import call_run_ws_client +from .utils import NullHandler import requests import json import numpy as np import logging import tempfile + logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) @renderaccess diff --git a/renderapi/image.py b/renderapi/image.py index e0666947..9d6ab3b1 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -6,8 +6,10 @@ import numpy as np import logging from .render import Render, format_baseurl, format_preamble, renderaccess +from .utils import NullHandler logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) # define acceptable image formats -- currently render generates png, jpeg, tiff IMAGE_FORMATS = {'png': 'png-image', diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 8784270f..ce980795 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -5,8 +5,10 @@ import requests import logging from .render import Render, format_baseurl, renderaccess +from .utils import NullHandler logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) @renderaccess diff --git a/renderapi/render.py b/renderapi/render.py index 8f1d1d8e..166cb7ac 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -184,7 +184,7 @@ def wrapper(*args, **kwargs): return f(*args, **render.make_kwargs(**kwargs)) else: raise ValueError( - 'invalid Render object type {} specified!'.format( + 'invalid Render object type {} specified!'.format( type(render))) else: return f(*args, **kwargs) diff --git a/renderapi/stack.py b/renderapi/stack.py index 064c98dd..64eddf58 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -4,9 +4,10 @@ from time import strftime import requests from .render import Render, format_baseurl, format_preamble, renderaccess -from .utils import jbool +from .utils import jbool, NullHandler logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) class StackVersion: @@ -57,7 +58,7 @@ def from_dict(self, d): def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - assert state in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'] + assert state in ['LOADING', 'COMPLETE', 'OFFLINE'] request_url = format_preamble( host, port, owner, project, stack) + "/state/%s" % state logger.debug(request_url) @@ -101,16 +102,9 @@ def delete_stack(stack, host=None, port=None, owner=None, @renderaccess def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, - stackResolutionZ=None, force_resolution=True, + stackResolutionZ=None, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - if force_resolution: - stackResolutionX, stackResolutionY, stackResolutionZ = [ - (1.0 if res is None else res) - for res in [stackResolutionX, stackResolutionY, stackResolutionZ]] - logger.debug('forcing resolution x:{}, y:{}, z:{}'.format( - stackResolutionX, stackResolutionY, stackResolutionZ)) - sv = StackVersion( cycleNumber=cycleNumber, cycleStepNumber=cycleStepNumber, stackResolutionX=stackResolutionX, stackResolutionY=stackResolutionY, @@ -151,20 +145,6 @@ def clone_stack(inputstack, outputstack, host=None, port=None, return r -@renderaccess -def get_sectionData_for_stack(stack,project = None, - host = None,port = None,owner = None, - session=requests.session(),render =None,**kwargs): - - request_url = format_preamble( - host,port,owner,project,stack)+"/sectionData" - logger.debug(request_url) - r = session.get(request_url) - try: - return r.json() - except: - logger.error(r.text) - @renderaccess def get_z_values_for_stack(stack, project=None, host=None, port=None, diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 95d5dd56..6082523d 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,11 +1,13 @@ #!/usr/bin/env python from .render import Render, format_baseurl, format_preamble, renderaccess +from .utils import NullHandler from collections import OrderedDict import logging import requests import numpy as np logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) class ResolvedTileSpecMap: diff --git a/renderapi/transform.py b/renderapi/transform.py index 5471551a..95faa925 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -14,8 +14,10 @@ import logging import numpy as np from .errors import ConversionError, EstimationError +from .utils import NullHandler logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) class TransformList: diff --git a/renderapi/utils.py b/renderapi/utils.py index 96d51e85..5499b632 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -7,7 +7,13 @@ import copy import json + +class NullHandler(logging.Handler): + def emit(self, record): + pass + logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) class RenderEncoder(json.JSONEncoder): @@ -37,11 +43,6 @@ def jbool(val): return 'true' if val else 'false' -class NullHandler(logging.Handler): - def emit(self, record): - pass - - def stripLogger(logger_tostrip): ''' remove all handlers from a logger -- useful for redefining From 45f6e1e69f0070e8636411d07ab9f4601118be49 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 11:21:16 -0800 Subject: [PATCH 127/766] removed fitargspec for render.run --- renderapi/render.py | 2 +- renderapi/stack.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 8f1d1d8e..673bbade 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -56,7 +56,7 @@ def run(self, f, *args, **kwargs): run function from object technically shorter than adding render=Render to kwargs ''' - args, kwargs = fitargspec(f, args, kwargs) + #args, kwargs = fitargspec(f, args, kwargs) return f(*args, **self.make_kwargs(**kwargs)) diff --git a/renderapi/stack.py b/renderapi/stack.py index 064c98dd..00409ef4 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -155,7 +155,9 @@ def clone_stack(inputstack, outputstack, host=None, port=None, def get_sectionData_for_stack(stack,project = None, host = None,port = None,owner = None, session=requests.session(),render =None,**kwargs): - + '''result: + json dictionary metadata about sectionData for this stack + ''' request_url = format_preamble( host,port,owner,project,stack)+"/sectionData" logger.debug(request_url) From a8cf8f65b43a18259b8a27bd0b2f622658b78219 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 11:29:29 -0800 Subject: [PATCH 128/766] adding force_http option defaulting to true, to allow non-http host definitions, but assume you want http://[host] --- renderapi/render.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 673bbade..fda79370 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -94,20 +94,20 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, - client_scripts=None, client_script=None, memGB=None, - json_dict=None, **kwargs): - '''helper function to connect to a render instance''' - if host is None: - if 'RENDER_HOST' not in os.environ: - host = str(raw_input("Enter Render Host: ")) - if host == '': - logger.critical('Render Host must not be empty!') - raise ValueError('Render Host must not be empty!') - host = (host if host.startswith('http') - else 'http://{}'.format(host)) - else: - host = os.environ['RENDER_HOST'] - + client_scripts=None, client_script=None, memGB=None, + json_dict=None, force_http=True, **kwargs): + '''helper function to connect to a render instance''' + if host is None: + if 'RENDER_HOST' not in os.environ: + host = str(raw_input("Enter Render Host: ")) + if host == '': + logger.critical('Render Host must not be empty!') + raise ValueError('Render Host must not be empty!') + else: + host = os.environ['RENDER_HOST'] + if force_http: + host = (host if host.startswith('http') + else 'http://{}'.format(host)) if port is None: if 'RENDER_PORT' not in os.environ: port = str(int(raw_input("Enter Render Port: "))) From ef79869265a651234eb09c4a92c2ece3df23600f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 11:30:20 -0800 Subject: [PATCH 129/766] fixing tabs --- renderapi/render.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index fda79370..6292875e 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -96,16 +96,16 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, json_dict=None, force_http=True, **kwargs): - '''helper function to connect to a render instance''' - if host is None: - if 'RENDER_HOST' not in os.environ: - host = str(raw_input("Enter Render Host: ")) - if host == '': - logger.critical('Render Host must not be empty!') - raise ValueError('Render Host must not be empty!') - else: - host = os.environ['RENDER_HOST'] - if force_http: + '''helper function to connect to a render instance''' + if host is None: + if 'RENDER_HOST' not in os.environ: + host = str(raw_input("Enter Render Host: ")) + if host == '': + logger.critical('Render Host must not be empty!') + raise ValueError('Render Host must not be empty!') + else: + host = os.environ['RENDER_HOST'] + if force_http: host = (host if host.startswith('http') else 'http://{}'.format(host)) if port is None: From b2c80e943ee88792aba535fc6dfe717cda4972f6 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 11:33:29 -0800 Subject: [PATCH 130/766] missed removing a tileid from client side coordinate mapping --- renderapi/coordinate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 050b46d9..7a0055b0 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -256,7 +256,7 @@ def world_to_local_coordinates_clientside(stack, jsondata, z, @renderaccess -def local_to_world_coordinates_clientside(stack, jsondata, tileId, z, +def local_to_world_coordinates_clientside(stack, jsondata, z, host=None, port=None, owner=None, project=None, client_script=None, number_of_threads=20, From b042974a1c53f12d51379fe5992c3bf7f8124f52 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 11:34:32 -0800 Subject: [PATCH 131/766] missed removing a tileid from client side coordinate mapping --- renderapi/coordinate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 7a0055b0..ce025bbf 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -262,7 +262,7 @@ def local_to_world_coordinates_clientside(stack, jsondata, z, number_of_threads=20, session=requests.session(), render=None, **kwargs): - return map_coordinates_clientside(stack, jsondata, tileId, z, + return map_coordinates_clientside(stack, jsondata, z, host=host, port=port, owner=owner, project=project, client_script=client_script, From 7d8d77b37a26c36a0a214235b429ff9c0d91586a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 11:35:27 -0800 Subject: [PATCH 132/766] fixed coordinate client --- renderapi/coordinate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index ce025bbf..1a524c3f 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -3,7 +3,7 @@ coordinate mapping functions for render api ''' from .render import Render, format_preamble, renderaccess -from .client import call_run_ws_client +from .client import call_run_ws_client,coordinateClient import requests import json import numpy as np @@ -231,7 +231,7 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, prefix='render_coordinates_out_', suffix='.json') # call the java client - renderapi.client.coordinateClient(stack, z, fromJson=json_inpath, toJson=json_outpath, localToWorld=isLocalToWorld, + coordinateClient(stack, z, fromJson=json_inpath, toJson=json_outpath, localToWorld=isLocalToWorld, numberOfThreads=number_of_threads, host=host,port=port, owner=owner,project=project, client_script=client_script, memGB=memGB) From 4f471ee313989b525923c92e4c97017afde4b781 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 27 Feb 2017 11:49:15 -0800 Subject: [PATCH 133/766] render, utils, setup: "fixing" bug in Render.run(), dumb try-except in fitargspec, setup shebang --- renderapi/render.py | 15 ++++++++------- renderapi/utils.py | 19 ++++++++++++------- setup.py | 1 + 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 166cb7ac..b71a9a8c 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -56,16 +56,16 @@ def run(self, f, *args, **kwargs): run function from object technically shorter than adding render=Render to kwargs ''' - args, kwargs = fitargspec(f, args, kwargs) - return f(*args, **self.make_kwargs(**kwargs)) + # TODO WARNING I think renderaccess can default to + # another render if defined in args (test/squash) + kwargs['render'] = self + return f(*args, **kwargs) class RenderClient(Render): '''Draft object for run_ws_client.sh calls''' def __init__(self, client_script=None, memGB=None, *args, **kwargs): super(RenderClient, self).__init__(**kwargs) - # FIXME remove this when completed - logger.error('Client functionality not implemented!') if client_script is None: raise ClientScriptError('No RenderClient script specified!') elif not os.path.isfile(client_script): @@ -95,7 +95,7 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, - json_dict=None, **kwargs): + json_dict=None, force_http=True, **kwargs): '''helper function to connect to a render instance''' if host is None: if 'RENDER_HOST' not in os.environ: @@ -103,10 +103,11 @@ def connect(host=None, port=None, owner=None, project=None, if host == '': logger.critical('Render Host must not be empty!') raise ValueError('Render Host must not be empty!') - host = (host if host.startswith('http') - else 'http://{}'.format(host)) else: host = os.environ['RENDER_HOST'] + if force_http: + host = (host if host.startswith('http') + else 'http://{}'.format(host)) if port is None: if 'RENDER_PORT' not in os.environ: diff --git a/renderapi/utils.py b/renderapi/utils.py index 5499b632..fb4da013 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -60,10 +60,15 @@ def defaultifNone(val, default=None): def fitargspec(f, oldargs, oldkwargs): ''' fit function argspec given input args tuple and kwargs dict''' - args, varargs, keywords, defaults = inspect.getargspec(f) - num_expected_args = len(args) - len(defaults) - new_args = tuple(oldargs[:num_expected_args]) - new_kwargs = copy.copy(oldkwargs) - for i, arg in enumerate(oldargs[num_expected_args:]): - new_kwargs.update({args[i + num_expected_args]: arg}) - return new_args, new_kwargs + try: + args, varargs, keywords, defaults = inspect.getargspec(f) + num_expected_args = len(args) - len(defaults) + new_args = tuple(oldargs[:num_expected_args]) + new_kwargs = copy.copy(oldkwargs) + for i, arg in enumerate(oldargs[num_expected_args:]): + new_kwargs.update({args[i + num_expected_args]: arg}) + return new_args, new_kwargs + except Exception as e: + logger.error('Cannot fit argspec for {}'.format(f)) + logger.error(e) + return oldargs, oldkwargs diff --git a/setup.py b/setup.py index 9815317d..a5d643b6 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python from setuptools import setup with open('requirements.txt', 'r') as f: From a5c5d77ef9b89e29ce41d6569c393c50c27e99a1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 13:56:23 -0800 Subject: [PATCH 134/766] making the calls consistent in passing in and out json dumpable objects but not json strings --- renderapi/coordinate.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 1a524c3f..89f9ada2 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -46,7 +46,7 @@ def local_to_world_coordinates(stack, tileId, x, y, @renderaccess -def world_to_local_coordinates_batch(stack, data, z, host=None, +def world_to_local_coordinates_batch(stack, d, z, host=None, port=None, owner=None, project=None, execute_local=True, session=requests.session(), @@ -56,20 +56,20 @@ def world_to_local_coordinates_batch(stack, data, z, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%s/world-to-local-coordinates" % (str(z)) - r = session.put(request_url, data=data, + r = session.put(request_url, data=json.dumps(data), headers={"content-type": "application/json"}) return r.json() @renderaccess -def local_to_world_coordinates_batch(stack, data, z, host=None, +def local_to_world_coordinates_batch(stack, d, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%s/local-to-world-coordinates" % (str(z)) - r = session.put(request_url, data=data, + r = session.put(request_url, data=json.dumps(d), headers={"content-type": "application/json"}) try: return r.json() @@ -85,7 +85,7 @@ def package_point_match_data_into_json(dataarray, tileId, d['tileId'] = tileId d[local_or_world] = [dataarray[i, 0], dataarray[i, 1]] dlist.append(d) - return json.dumps(dlist) + return dlist def unpackage_world_to_local_point_match_from_json(json_answer, tileId): @@ -217,12 +217,15 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, def map_coordinates_clientside(stack, jsondata, z, host, port, owner, project, client_script, isLocalToWorld=False, - number_of_threads=20,memGB=1): + number_of_threads=20,memGB='1G'): # write point match json to temp file on disk json_infile, json_inpath = tempfile.mkstemp( prefix='render_coordinates_in_', suffix='.json') with open(json_inpath, 'w') as fp: - fp.write(jsondata) + #d = json.loads(jsondata) + json.dump(jsondata,fp) + #fp.close() + #fp.write(jsondata) # json.dump(jsondata,open(json_inpath,'w')) From 676d0b9a71a01f6c10bf100b660e5185ecc47467 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 16:11:30 -0800 Subject: [PATCH 135/766] making Polynomial2DTransform transformId None to allow inherited to_dict to work --- renderapi/transform.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 5471551a..056a7f7c 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -103,6 +103,7 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0): self.B1 = B1 self.className = 'mpicbg.trakem2.transform.AffineModel2D' self.load_M() + self.transformId = None def load_M(self): self.M = np.identity(3, np.double) @@ -212,7 +213,8 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, if not force_polynomial and self.is_affine: # TODO try implement affine from poly (& vice versa) return AffineTransform(poly_params=self.params) - + self.transformId = None + @property def is_affine(self): '''TODO allow default to Affine''' From b874eef7accb45fadb7affec453a01b756e4a611 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 16:12:19 -0800 Subject: [PATCH 136/766] making Polynomial2DTransform transformId None to allow inherited to_dict to work --- renderapi/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 056a7f7c..868023be 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -213,8 +213,8 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, if not force_polynomial and self.is_affine: # TODO try implement affine from poly (& vice versa) return AffineTransform(poly_params=self.params) - self.transformId = None - + self.transformId = None + @property def is_affine(self): '''TODO allow default to Affine''' From 097e56c94db1d3e823ccab95174000b09bb528bb Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 16:14:32 -0800 Subject: [PATCH 137/766] fixed it from trying to remove sharedTransforms file that doesn't exist --- renderapi/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index a11ce540..d5b2e903 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -210,7 +210,8 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, client_script=client_script, memGB=memGB) os.remove(tsjson) - os.remove(trjson) + if sharedTransforms is not None: + os.remove(trjson) @renderaccess From 53e54ec777119b8f114c741122978de4e6055c01 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Feb 2017 16:17:37 -0800 Subject: [PATCH 138/766] fixed typo in polynomial transform class name --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 868023be..08e591f2 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -200,7 +200,7 @@ class Polynomial2DTransform(Transform): def __init__(self, dataString=None, src=None, dst=None, order=2, force_polynomial=True, params=None, identity=False): - self.className = 'mpicbg.trakEM2.transform.PolynomialTransform2D' + self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' if dataString is not None: self._process_dataString(dataString) elif identity: From bcdc814d3e09a44ce07ec189892e0fb9e287b9d0 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 27 Feb 2017 16:41:11 -0800 Subject: [PATCH 139/766] transform: fix other className string --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index c71f3dcb..139b5ca7 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -198,7 +198,7 @@ class Polynomial2DTransform(Transform): fall back to Affine Model in special cases robustness in estimation ''' - className = 'mpicbg.trakEM2.transform.PolynomialTransform2D' + className = 'mpicbg.trakem2.transform.PolynomialTransform2D' def __init__(self, dataString=None, src=None, dst=None, order=2, force_polynomial=True, params=None, identity=False): From dcad65dd05e872afb05c1ac148b30855ad94586e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 1 Mar 2017 12:09:52 -0800 Subject: [PATCH 140/766] transform: ignore svd ordering for polynomial estimate, test numpy/scipy linalg, integer polynomial order --- renderapi/transform.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 139b5ca7..45ffa081 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -19,6 +19,14 @@ logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) +# TODO preference for svd version? +try: + from scipy.linalg import svd +except ImportError as e: + logger.info(e) + logger.info('scipy-based linalg may lead to better parameter fitting') + from numpy.linalg import svd + class TransformList: def __init__(self, tforms): @@ -226,7 +234,7 @@ def is_affine(self): @property def order(self): no_coeffs = len(self.params.ravel()) - return (abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2 + return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) def estimate(self, src, dst, order=2, convergence_test=None): '''This is unreliable -- add tests to ensure repeatability''' @@ -259,8 +267,10 @@ def estimate(self, src, dst, order=2, convergence_test=None): # right singular vector corresponding to smallest singular value # TODO implement tests for this - _, _, V = np.linalg.svd(A) - return (-V[-1, :-1] / V[-1, -1]).reshape((2, no_coeff // 2)) + _, s, V = svd(A) + Vsm = V[np.argmin(s), :] # never trust computers + return (-Vsm[:-1] / Vsm[-1]).reshape((2, no_coeff // 2)) + # return (-V[-1, :-1] / V[-1, -1]).reshape((2, no_coeff // 2)) def _process_params(self, params): ''' From 8f40adfd370441cf75088aadc96adab5679b7e4e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 1 Mar 2017 16:17:02 -0800 Subject: [PATCH 141/766] transform: fix order casting.... whoops. --- renderapi/transform.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 45ffa081..e0f68ac6 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -294,8 +294,8 @@ def _process_dataString(self, datastring): def _format_raveled_params(self, raveled_params): return np.array( - [[float(d) for d in dsList[:len(raveled_params)/2]], - [float(d) for d in dsList[len(raveled_params)/2:]]]) + [[float(d) for d in raveled_params[:len(raveled_params)/2]], + [float(d) for d in raveled_params[len(raveled_params)/2:]]]) def tform(self, points): dst = np.zeros(points.shape) @@ -323,9 +323,8 @@ def asorder(self, order): 'transformation {} is order {} -- conversion to ' 'order {} not supported'.format( self.dataString, self.order, order)) - new_params_raveled = self.params.ravel() + [ - 0 for i in range(self.coefficients(order) - self.coefficients())] - new_params = self._format_raveled_params(new_params_raveled) + new_params = np.zeros([2, self.coefficients(order)]) + new_params[:self.params.shape[0], :self.params.shape[1]] = self.params return Polynomial2DTransform(params=new_params) def _fromAffine(self, aff): From a66fb55bc9393f6ebf0fa1efa41f51b24cba11d6 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 6 Mar 2017 15:29:49 -0800 Subject: [PATCH 142/766] not sure --- renderapi/transform.py | 57 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 08e591f2..4bb2547a 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -90,7 +90,6 @@ def __eq__(self, other): def __hash__(self): return hash((self.__str__())) - class AffineModel(Transform): className = 'mpicbg.trakem2.transform.AffineModel2D' @@ -131,8 +130,38 @@ def from_dict(self, d): def invert(self): Ai = AffineModel() - Ai.M = np.linalg.inv(self.M) - return Ai + + def fit(self,A,B): + assert A.shape[0] == B.shape[0] + assert A.shape[1] == 2 + assert B.shape[1] == 2 + + N = A.shape[0]; # total points + + M = np.zeros((2*N,6)) + Y = np.zeros((2*N,1)) + for i in range(N): + M[2*i,:]=[A[i,0],A[i,1],0,0,1,0] + M[2*i+1,:]=[0,0,A[i,0],A[i,1],0,1] + Y[2*i]=B[i,0] + Y[2*i+1]=B[i,1] + + (Tvec,residuals,rank,s)=np.linalg.lstsq(M,Y) + return Tvec + + def estimate(self, A,B): + + Tvec = self.fit(A,B) + #t = numpy.array([Tvec[4,0],Tvec[5,0]]) + #R = numpy.array([[Tvec[0,0],Tvec[1,0]],[Tvec[2,0],Tvec[3,0]]]) + self.M00=Tvec[0,0] + self.M10=Tvec[2,0] + self.M01=Tvec[1,0] + self.M11=Tvec[3,0] + self.B0=Tvec[4,0] + self.B1=Tvec[5,0] + self.load_M() + return self.M def convert_to_point_vector(self, points): Np = points.shape[0] @@ -226,8 +255,9 @@ def order(self): no_coeffs = len(self.params.ravel()) return (abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2 - def estimate(self, src, dst, order=2, convergence_test=None): + def fit(self,src,dst,order=2): '''This is unreliable -- add tests to ensure repeatability''' + xs = src[:, 0] ys = src[:, 1] xd = dst[:, 0] @@ -260,6 +290,25 @@ def estimate(self, src, dst, order=2, convergence_test=None): _, _, V = np.linalg.svd(A) return (-V[-1, :-1] / V[-1, -1]).reshape((2, no_coeff // 2)) + def estimate(self, src, dst, order=2,convergence_test=None,max_tries =100): + old_params = self.params + + params = fit(src,dst,**kwargs) + self._process_params(params) + if convergence_test is not None: + if(convergence_test(self)): + return params + + for i in range(max_tries): + params = fit(src,dst,**kwargs) + self._process_params(params) + if convergence_test(self): + return params + + raise EstimationError('could not find a converged estimate in %d tries'%max_tries) + else: + return params + def _process_params(self, params): ''' generate datastring and param attributes from params From 29ecd83879eeb786f53e35a031990a77885e5998 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 7 Mar 2017 10:18:49 -0800 Subject: [PATCH 143/766] importing transform.py --- renderapi/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 10be0385..f9118189 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -6,6 +6,7 @@ from . import stack from . import client from . import image +from . import transform from . import pointmatch from . import coordinate from .render import connect @@ -13,4 +14,4 @@ __all__ = ['render', 'client', 'tilespec', 'errors', 'stack', 'image', 'pointmatch', 'coordinate', - 'connect', 'Render'] + 'connect', 'transform','Render'] From 9e8824ae7ee715cbcd221168287a3b6122e2cb77 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 7 Mar 2017 10:19:12 -0800 Subject: [PATCH 144/766] updating dockerfile --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 15817d3d..147575e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,9 @@ RUN pip install shapely==1.6b2 #only here for development purposes RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime -RUN uptime&&uptime&&uptime&&uptime&&uptime +RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime #install render python using pip from github -RUN pip install -e git+https://github.com/fcollman/render-python.git@module#egg=render-python +RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python + From 70f567f32eddf926f5a5c0aae29e467cc059bd79 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 7 Mar 2017 10:27:02 -0800 Subject: [PATCH 145/766] fixing tform, should have been ones --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 4bb2547a..333049f3 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -171,7 +171,7 @@ def convert_to_point_vector(self, points): assert(points.shape[1] == 2) Nd = 2 - points = np.concatenate((points, zerovec), axis=1) + points = np.concatenate((points, onevec), axis=1) return points, Nd def convert_points_vector_to_array(self, points, Nd): From 03ec0a30d78341c755defec13a32dbd71ba27fe8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 7 Mar 2017 10:37:32 -0800 Subject: [PATCH 146/766] removing AffineModel and Tranform from tilespec --- renderapi/tilespec.py | 128 +----------------------------------------- 1 file changed, 1 insertion(+), 127 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 95d5dd56..b3b71ebc 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,5 +1,6 @@ #!/usr/bin/env python from .render import Render, format_baseurl, format_preamble, renderaccess +from .transform import Transform,AffineModel from collections import OrderedDict import logging import requests @@ -105,133 +106,6 @@ def __repr__(self): return self.__str__() -class Transform: - def __init__(self, className=None, dataString=None, - transformId=None, json=None): - if json is not None: - self.from_dict(json) - else: - self.className = className - self.dataString = dataString - self.transformId = transformId - - def to_dict(self): - d = {} - d['type'] = 'leaf' - d['className'] = self.className - d['dataString'] = self.dataString - if self.transformId is not None: - d['transformId'] = self.transformId - return d - - def from_dict(self, d): - self.dataString = d['dataString'] - self.className = d['className'] - self.transformId = d.get('transformId', None) - - def __str__(self): - return 'className:%s\ndataString:%s' % ( - self.className, self.dataString) - - def __repr__(self): - return self.__str__() - - -class AffineModel(Transform): - className = 'mpicbg.trakem2.transform.AffineModel2D' - - def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0): - self.M00 = M00 - self.M01 = M01 - self.M10 = M10 - self.M11 = M11 - self.B0 = B0 - self.B1 = B1 - self.className = 'mpicbg.trakem2.transform.AffineModel2D' - self.load_M() - - def load_M(self): - self.M = np.identity(3, np.double) - self.M[0, 0] = self.M00 - self.M[0, 1] = self.M01 - self.M[1, 0] = self.M10 - self.M[1, 1] = self.M11 - self.M[0, 2] = self.B0 - self.M[1, 2] = self.B1 - - def to_dict(self): - d = {} - d['type'] = 'leaf' - d['className'] = self.className - d['dataString'] = "%.10f %.10f %.10f %.10f %.10f %.10f" % ( - self.M[0, 0], self.M[1, 0], self.M[0, 1], - self.M[1, 1], self.M[0, 2], self.M[1, 2]) - return d - - def from_dict(self, d): - ds = d['dataString'].split() - (self.M00, self.M10, self.M01, self.M11, self.B0, self.B1) = map( - float, ds) - self.load_M() - - def invert(self): - Ai = AffineModel() - Ai.M = np.linalg.inv(self.M) - return Ai - - def convert_to_point_vector(self, points): - Np = points.shape[0] - - zerovec = np.zeros((Np, 1), np.double) - onevec = np.ones((Np, 1), np.double) - - assert(points.shape[1] == 2) - Nd = 2 - points = np.concatenate((points, zerovec), axis=1) - return points, Nd - - def convert_points_vector_to_array(self, points, Nd): - points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T - return points - - def tform(self, points): - points, Nd = self.convert_to_point_vector(points) - pt = np.dot(self.M, points.T).T - return self.convert_points_vector_to_array(pt, Nd) - - def concatenate(self, model): - ''' - concatenate a model to this model -- ported from trakEM2 below: - final double a00 = m00 * model.m00 + m01 * model.m10; - final double a01 = m00 * model.m01 + m01 * model.m11; - final double a02 = m00 * model.m02 + m01 * model.m12 + m02; - - final double a10 = m10 * model.m00 + m11 * model.m10; - final double a11 = m10 * model.m01 + m11 * model.m11; - final double a12 = m10 * model.m02 + m11 * model.m12 + m12; - ''' - a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] - a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] - a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + - self.M[0, 2]) - - a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] - a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] - a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + - self.M[1, 2]) - - newmodel = AffineModel(a00, a01, a10, a11, a02, a12) - return newmodel - - def inverse_tform(self, points): - points, Nd = self.convert_to_point_vector(points) - pt = np.dot(np.linalg.inv(self.M), points.T).T - return self.convert_points_vector_to_array(pt, Nd) - - def __str__(self): - return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( - self.M[0, 0], self.M[0, 1], self.M[1, 0], - self.M[1, 1], self.M[0, 2], self.M[1, 2]) class Layout: From 5948d8e801f5b45a3809b5772aeb5eb297d00c81 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 7 Mar 2017 10:53:41 -0800 Subject: [PATCH 147/766] adding dummy classes for backward compatibility and removing ReferenceTransform --- renderapi/tilespec.py | 32 +++++++------------------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index b3b71ebc..2fbfbfc7 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,6 +1,6 @@ #!/usr/bin/env python from .render import Render, format_baseurl, format_preamble, renderaccess -from .transform import Transform,AffineModel +from .transform import Transform,AffineModel,ReferenceTransform from collections import OrderedDict import logging import requests @@ -82,30 +82,12 @@ def from_dict(self, d): self.classname = d['className'] self.params = d['params'] - -class ReferenceTransform: - def __init__(self, refId=None, json=None): - if json is not None: - self.from_dict(json) - else: - self.refId = refId - - def to_dict(self): - d = {} - d['type'] = 'ref' - d['refId'] = self.refId - return d - - def from_dict(self, d): - self.refId = d['refId'] - - def __str__(self): - return 'ReferenceTransform(%s)' % self.refId - - def __repr__(self): - return self.__str__() - - +class AffineModel(AffineModel): + pass +class Transform(Transform): + pass +class ReferenceTransform(ReferenceTransform): + pass class Layout: From 816f0782c406ed8d4df081d9f39b221d409a58e2 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 13:11:31 -0800 Subject: [PATCH 148/766] changing the way package is installed in Docker --- Dockerfile | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 147575e0..7277ebf6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,13 +24,10 @@ RUN /opt/conda/bin/conda install jupyter -y RUN apt-get install libgeos-dev -y RUN pip install shapely==1.6b2 -#stupid uptime calls to bypass caching on github pull -#only here for development purposes -RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime -RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime&&uptime -RUN uptime&&uptime&&uptime&&uptime&&uptime&&uptime - #install render python using pip from github -RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python - +#RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python +RUN mkdir -p /usr/local/render-python +COPY . /usr/local/render-python +WORKDIR /usr/local/render-python +RUN python setup.py install From f98f6061261f418990bcbc2152e97804789d37b4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 13:16:27 -0800 Subject: [PATCH 149/766] removing notebook --- explore_point_match_database-large.ipynb | 8440 ---------------------- 1 file changed, 8440 deletions(-) delete mode 100644 explore_point_match_database-large.ipynb diff --git a/explore_point_match_database-large.ipynb b/explore_point_match_database-large.ipynb deleted file mode 100644 index 3d8903c9..00000000 --- a/explore_point_match_database-large.ipynb +++ /dev/null @@ -1,8440 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "import renderapi\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from matplotlib.lines import Line2D\n", - "import time\n", - "\n", - "from matplotlib.patches import FancyArrowPatch, Circle, ConnectionStyle\n", - "\n", - "%matplotlib notebook" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "host = 'ibs-forrestc-ux1.corp.alleninstitute.org'\n", - "port = 8080\n", - "owner = 'Sharmishtaas'\n", - "project = 'M270907_Scnn1aTg2Tdt_13'\n", - "stack = 'ALIGNEDSTACK_JAN3_DAPI_1_NORM'\n", - "matchcollection = 'large_rigid_run'\n", - "render = renderapi.Render(host,port,owner,project)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[u'Forrest', u'Sharmishtaas']" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "render.get_matchcollection_owners()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[{u'collectionId': {u'name': u'rigid_sift_half_scale_fullrun',\n", - " u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 349937},\n", - " {u'collectionId': {u'name': u'rigid_sift_run2', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 3817},\n", - " {u'collectionId': {u'name': u'rigid_sift_half_scale_run2',\n", - " u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 193240},\n", - " {u'collectionId': {u'name': u'norm_sift_run1', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 384058},\n", - " {u'collectionId': {u'name': u'rigid_sift_run1', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 16679},\n", - " {u'collectionId': {u'name': u'tilepairtest', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 2},\n", - " {u'collectionId': {u'name': u'large_rigid_run', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 3508512},\n", - " {u'collectionId': {u'name': u'rigid_sift_half_scale_run1',\n", - " u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 15845},\n", - " {u'collectionId': {u'name': u'rigid_sift_run3', u'owner': u'Sharmishtaas'},\n", - " u'pairCount': 19313}]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "render.get_matchcollections()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "allmatches=render.get_matches_from_group_to_group(matchcollection,'0','1')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sections_per_row = 200\n", - "rows = int(np.ceil(maxz/sections_per_row))\n", - "\n", - "\n", - "f,ax = plt.subplots(rows,1,figsize=(12,2*rows))\n", - "first_section_indices=np.concatenate([np.array([0]),np.where(np.diff(groups%1000)<0)[0]+1])\n", - "first_sections=groups[first_section_indices]\n", - "first_section_zs = [zvalues[section] for section in first_sections]\n", - "\n", - "for row in range(rows):\n", - " startz = row*sections_per_row\n", - " endz = row*sections_per_row + sections_per_row\n", - " ax[row].imshow(match_matrix[startz:endz,:].T,interpolation='nearest',cmap=plt.cm.viridis,extent=(startz-.5,endz-.5,maxdz,-maxdz),vmin=0,vmax=200)\n", - " ax[row].autoscale(tight=True)\n", - " for z in first_section_zs:\n", - " if (z>=startz)&(z<=endz):\n", - " ax[row].plot([z-.5,z-.5],[-maxdz,maxdz],c='w',linewidth=2,linestyle='--')\n", - "\n", - "# img=ax.imshow(match_matrix,interpolation='nearest',extent =(-maxdz,maxdz,maxz+.5,-.5))\n", - "# plt.xlabel('dz')\n", - "# plt.ylabel('z values')\n", - "# ax.set_title('matches')\n", - "# plt.colorbar(img)\n", - "plt.tight_layout()\n", - "\n", - "\n", - "# ax.set_yticks(np.arange(0,maxz,100))\n", - "#ax.autoscale(tight=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 194, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 194, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "plt.figure(figsize=(12,6))\n", - "plt.plot(np.sum(match_matrix,axis=1))\n", - "plt.xlabel('section z')\n", - "plt.ylabel('num_matches')\n", - "plt.figure()\n", - "plt.errorbar(np.arange(-maxdz,maxdz+1).T,np.median(match_matrix,axis=0),np.std(match_matrix,axis=0))\n", - "plt.xlabel('dz')\n", - "plt.ylabel('median tile matches')" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "0.011663597298956415" - ] - }, - "execution_count": 29, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "np.sum(np.sum(match_matrix,axis=1)==0)*1.0/match_matrix.shape[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 198, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import networkx as nx\n", - "G=nx.Graph()\n", - "\n", - "\n", - "#subgroup =[group for i, group in enumerate(groups) if (zvalues[group]>1200) & (zvalues[group]<1300)]\n", - "#subanswers =[answers[i] for i, group in enumerate(groups) if (zvalues[group]>1200) & (zvalues[group]<1300)]\n", - "\n", - "G.add_nodes_from(groups)\n", - "for group in groups:\n", - " G.node[group]['z']=zvalues[group]\n", - " # if zvalues[group]<100:\n", - "# nx.set_node_attributes(G,'z',zvalues)\n", - "\n", - "for answer,group in zip(answers,groups):\n", - " for group2 in answer.keys():\n", - "\n", - " if G.has_edge(group,group2):\n", - " w = G[group][group2]['weight']\n", - " G[group][group2]['weight']=w+answer[group2]\n", - " else:\n", - " if group2=='z':\n", - " print group,group2\n", - " if group=='z':\n", - " print group,group2\n", - " G.add_edge(group,group2,weight=answer[group2])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 199, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0 - 3145\n", - "3146 - 3257\n" - ] - } - ], - "source": [ - "subgraphs=nx.connected_components(G)\n", - "for sG in subgraphs:\n", - " zs = np.array([G.node[node]['z'] for node in sG])\n", - " print np.min(zs),'-',np.max(zs)" - ] - }, - { - "cell_type": "code", - "execution_count": 186, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{0,\n", - " 1,\n", - " 2,\n", - " 3,\n", - " 4,\n", - " 5,\n", - " 6,\n", - " 7,\n", - " 8,\n", - " 9,\n", - " 10,\n", - " 11,\n", - " 12,\n", - " 13,\n", - " 14,\n", - " 15,\n", - " 16,\n", - " 17,\n", - " 18,\n", - " 19,\n", - " 20,\n", - " 21,\n", - " 22,\n", - " 23,\n", - " 24,\n", - " 25,\n", - " 26,\n", - " 43013,\n", - " 45010,\n", - " 43014,\n", - " 41000,\n", - " 41001,\n", - " 41002,\n", - " 41003,\n", - " 41004,\n", - " 41005,\n", - " 41006,\n", - " 41007,\n", - " 41008,\n", - " 41009,\n", - " 41010,\n", - " 41011,\n", - " 41012,\n", - " 41013,\n", - " 41014,\n", - " 41015,\n", - " 41016,\n", - " 41017,\n", - " 41018,\n", - " 41019,\n", - " 41020,\n", - " 41021,\n", - " 41022,\n", - " 41023,\n", - " 41024,\n", - " 41025,\n", - " 41026,\n", - " 41027,\n", - " 43020,\n", - " 43021,\n", - " 43022,\n", - " 39000,\n", - " 39001,\n", - " 39002,\n", - " 39003,\n", - " 39004,\n", - " 39005,\n", - " 39006,\n", - " 39007,\n", - " 39008,\n", - " 39009,\n", - " 39010,\n", - " 39011,\n", - " 39012,\n", - " 39013,\n", - " 39014,\n", - " 39015,\n", - " 39016,\n", - " 39017,\n", - " 39018,\n", - " 39019,\n", - " 39020,\n", - " 39021,\n", - " 39022,\n", - " 39023,\n", - " 39024,\n", - " 39025,\n", - " 39026,\n", - " 39027,\n", - " 39028,\n", - " 39029,\n", - " 39030,\n", - " 45017,\n", - " 37000,\n", - " 37001,\n", - " 37002,\n", - " 37003,\n", - " 37004,\n", - " 37005,\n", - " 37006,\n", - " 37007,\n", - " 37008,\n", - " 37009,\n", - " 37010,\n", - " 37011,\n", - " 37012,\n", - " 37013,\n", - " 37014,\n", - " 37015,\n", - " 37016,\n", - " 37017,\n", - " 37018,\n", - " 37019,\n", - " 37020,\n", - " 37021,\n", - " 37022,\n", - " 37023,\n", - " 37024,\n", - " 37025,\n", - " 37026,\n", - " 35000,\n", - " 35001,\n", - " 35002,\n", - " 35003,\n", - " 35004,\n", - " 35005,\n", - " 35006,\n", - " 35007,\n", - " 35008,\n", - " 35009,\n", - " 35010,\n", - " 35011,\n", - " 35012,\n", - " 35013,\n", - " 35014,\n", - " 35015,\n", - " 35016,\n", - " 35017,\n", - " 35018,\n", - " 35019,\n", - " 35020,\n", - " 35021,\n", - " 35022,\n", - " 35023,\n", - " 35024,\n", - " 35025,\n", - " 35026,\n", - " 35027,\n", - " 45016,\n", - " 43011,\n", - " 33000,\n", - " 33001,\n", - " 33002,\n", - " 33003,\n", - " 33004,\n", - " 33005,\n", - " 33006,\n", - " 33007,\n", - " 33008,\n", - " 33009,\n", - " 33010,\n", - " 33011,\n", - " 33012,\n", - " 33013,\n", - " 33014,\n", - " 33015,\n", - " 33016,\n", - " 33017,\n", - " 33018,\n", - " 33019,\n", - " 33020,\n", - " 33021,\n", - " 33022,\n", - " 33023,\n", - " 33024,\n", - " 33025,\n", - " 33026,\n", - " 45009,\n", - " 31000,\n", - " 31001,\n", - " 31002,\n", - " 31003,\n", - " 31004,\n", - " 31005,\n", - " 31006,\n", - " 31007,\n", - " 31008,\n", - " 31009,\n", - " 31010,\n", - " 31011,\n", - " 31012,\n", - " 31013,\n", - " 31014,\n", - " 31015,\n", - " 31016,\n", - " 31017,\n", - " 31018,\n", - " 31019,\n", - " 31020,\n", - " 31021,\n", - " 31022,\n", - " 31023,\n", - " 31024,\n", - " 31025,\n", - " 31026,\n", - " 29000,\n", - " 29002,\n", - " 29003,\n", - " 29004,\n", - " 29005,\n", - " 29006,\n", - " 29007,\n", - " 29008,\n", - " 29009,\n", - " 29010,\n", - " 29011,\n", - " 29012,\n", - " 29013,\n", - " 29014,\n", - " 29015,\n", - " 29016,\n", - " 29017,\n", - " 29018,\n", - " 29019,\n", - " 29020,\n", - " 29021,\n", - " 29022,\n", - " 29023,\n", - " 29024,\n", - " 29025,\n", - " 29026,\n", - " 29027,\n", - " 27001,\n", - " 27002,\n", - " 27003,\n", - " 27004,\n", - " 27005,\n", - " 27006,\n", - " 27007,\n", - " 27008,\n", - " 27009,\n", - " 27010,\n", - " 27011,\n", - " 27012,\n", - " 27013,\n", - " 27014,\n", - " 27015,\n", - " 27016,\n", - " 27017,\n", - " 27018,\n", - " 27019,\n", - " 27020,\n", - " 27021,\n", - " 27022,\n", - " 27023,\n", - " 27024,\n", - " 27025,\n", - " 27026,\n", - " 27027,\n", - " 27028,\n", - " 45008,\n", - " 25000,\n", - " 25001,\n", - " 25002,\n", - " 25003,\n", - " 25004,\n", - " 25005,\n", - " 25006,\n", - " 25007,\n", - " 25008,\n", - " 25009,\n", - " 25010,\n", - " 25011,\n", - " 25012,\n", - " 25013,\n", - " 25014,\n", - " 25015,\n", - " 25016,\n", - " 25017,\n", - " 25018,\n", - " 25019,\n", - " 25020,\n", - " 25021,\n", - " 25022,\n", - " 25023,\n", - " 25024,\n", - " 25025,\n", - " 25026,\n", - " 45024,\n", - " 43024,\n", - " 23000,\n", - " 23001,\n", - " 23002,\n", - " 23003,\n", - " 23004,\n", - " 23005,\n", - " 23006,\n", - " 23007,\n", - " 23008,\n", - " 23009,\n", - " 23010,\n", - " 23011,\n", - " 23012,\n", - " 23013,\n", - " 23014,\n", - " 23015,\n", - " 23016,\n", - " 23017,\n", - " 23018,\n", - " 23019,\n", - " 23020,\n", - " 23021,\n", - " 23022,\n", - " 23023,\n", - " 23024,\n", - " 23025,\n", - " 23026,\n", - " 45012,\n", - " 21000,\n", - " 21001,\n", - " 21002,\n", - " 21003,\n", - " 21004,\n", - " 21005,\n", - " 21006,\n", - " 21007,\n", - " 21008,\n", - " 21009,\n", - " 21010,\n", - " 21011,\n", - " 21012,\n", - " 21013,\n", - " 21014,\n", - " 21015,\n", - " 21016,\n", - " 21017,\n", - " 21018,\n", - " 21019,\n", - " 21020,\n", - " 21021,\n", - " 21022,\n", - " 21023,\n", - " 21024,\n", - " 19000,\n", - " 19001,\n", - " 19002,\n", - " 19003,\n", - " 19004,\n", - " 19005,\n", - " 19006,\n", - " 19007,\n", - " 19008,\n", - " 19009,\n", - " 19010,\n", - " 19011,\n", - " 19012,\n", - " 19013,\n", - " 19014,\n", - " 19015,\n", - " 19016,\n", - " 19017,\n", - " 19018,\n", - " 19019,\n", - " 19020,\n", - " 19021,\n", - " 19022,\n", - " 19023,\n", - " 19024,\n", - " 19025,\n", - " 19026,\n", - " 19027,\n", - " 44017,\n", - " 17000,\n", - " 17001,\n", - " 17002,\n", - " 17003,\n", - " 17004,\n", - " 17005,\n", - " 17006,\n", - " 17007,\n", - " 17008,\n", - " 17009,\n", - " 17010,\n", - " 17011,\n", - " 17012,\n", - " 17013,\n", - " 17014,\n", - " 17015,\n", - " 17016,\n", - " 17017,\n", - " 17018,\n", - " 17019,\n", - " 17020,\n", - " 17021,\n", - " 17022,\n", - " 17023,\n", - " 17024,\n", - " 43008,\n", - " 44008,\n", - " 15000,\n", - " 15001,\n", - " 15002,\n", - " 15003,\n", - " 15004,\n", - " 15005,\n", - " 15006,\n", - " 15007,\n", - " 15008,\n", - " 15009,\n", - " 15010,\n", - " 15011,\n", - " 15012,\n", - " 15013,\n", - " 15014,\n", - " 15015,\n", - " 15016,\n", - " 15017,\n", - " 15018,\n", - " 15019,\n", - " 15020,\n", - " 15021,\n", - " 15022,\n", - " 15023,\n", - " 15024,\n", - " 15025,\n", - " 15026,\n", - " 44009,\n", - " 43019,\n", - " 13000,\n", - " 13001,\n", - " 13002,\n", - " 13003,\n", - " 13004,\n", - " 13005,\n", - " 13006,\n", - " 13007,\n", - " 13008,\n", - " 13009,\n", - " 13010,\n", - " 13011,\n", - " 13012,\n", - " 13013,\n", - " 13014,\n", - " 13015,\n", - " 13016,\n", - " 13017,\n", - " 13018,\n", - " 13019,\n", - " 13020,\n", - " 13021,\n", - " 13022,\n", - " 13023,\n", - " 13024,\n", - " 13025,\n", - " 13026,\n", - " 44011,\n", - " 44018,\n", - " 11000,\n", - " 11001,\n", - " 11002,\n", - " 11003,\n", - " 11004,\n", - " 11005,\n", - " 11006,\n", - " 11007,\n", - " 11008,\n", - " 11009,\n", - " 11010,\n", - " 11011,\n", - " 11012,\n", - " 11013,\n", - " 11014,\n", - " 11015,\n", - " 11016,\n", - " 11017,\n", - " 11018,\n", - " 11019,\n", - " 11020,\n", - " 11021,\n", - " 11022,\n", - " 11023,\n", - " 11024,\n", - " 11025,\n", - " 11026,\n", - " 11027,\n", - " 43009,\n", - " 45003,\n", - " 9000,\n", - " 9001,\n", - " 9002,\n", - " 9003,\n", - " 9004,\n", - " 9005,\n", - " 9006,\n", - " 9007,\n", - " 9008,\n", - " 9009,\n", - " 9010,\n", - " 9011,\n", - " 9012,\n", - " 9013,\n", - " 9014,\n", - " 9015,\n", - " 9016,\n", - " 9017,\n", - " 9018,\n", - " 9019,\n", - " 9020,\n", - " 9021,\n", - " 9022,\n", - " 9023,\n", - " 9024,\n", - " 9025,\n", - " 9026,\n", - " 7000,\n", - " 7001,\n", - " 7002,\n", - " 7003,\n", - " 7004,\n", - " 7005,\n", - " 7006,\n", - " 7007,\n", - " 7008,\n", - " 7009,\n", - " 7010,\n", - " 7011,\n", - " 7012,\n", - " 7013,\n", - " 7014,\n", - " 7015,\n", - " 7016,\n", - " 7017,\n", - " 7018,\n", - " 7019,\n", - " 7020,\n", - " 7021,\n", - " 7022,\n", - " 7023,\n", - " 7024,\n", - " 7025,\n", - " 7026,\n", - " 44016,\n", - " 5001,\n", - " 5002,\n", - " 5003,\n", - " 5004,\n", - " 5005,\n", - " 5006,\n", - " 5007,\n", - " 5008,\n", - " 5009,\n", - " 5010,\n", - " 5011,\n", - " 5012,\n", - " 5013,\n", - " 5014,\n", - " 5015,\n", - " 5016,\n", - " 5017,\n", - " 5018,\n", - " 5019,\n", - " 5020,\n", - " 5021,\n", - " 5022,\n", - " 5023,\n", - " 5024,\n", - " 5025,\n", - " 5026,\n", - " 5027,\n", - " 5028,\n", - " 3001,\n", - " 3002,\n", - " 3003,\n", - " 3004,\n", - " 3005,\n", - " 3006,\n", - " 3007,\n", - " 3008,\n", - " 3009,\n", - " 3010,\n", - " 3011,\n", - " 3012,\n", - " 3013,\n", - " 3014,\n", - " 3015,\n", - " 3016,\n", - " 3017,\n", - " 3018,\n", - " 3019,\n", - " 3020,\n", - " 3021,\n", - " 3022,\n", - " 3023,\n", - " 3024,\n", - " 3025,\n", - " 3026,\n", - " 3027,\n", - " 44019,\n", - " 44000,\n", - " 44001,\n", - " 44002,\n", - " 44003,\n", - " 44004,\n", - " 44005,\n", - " 44006,\n", - " 44007,\n", - " 1000,\n", - " 1001,\n", - " 1002,\n", - " 1003,\n", - " 1004,\n", - " 1005,\n", - " 1006,\n", - " 1007,\n", - " 1008,\n", - " 1009,\n", - " 1010,\n", - " 1011,\n", - " 1012,\n", - " 1013,\n", - " 1014,\n", - " 1015,\n", - " 1016,\n", - " 1017,\n", - " 1018,\n", - " 1019,\n", - " 1020,\n", - " 1021,\n", - " 1022,\n", - " 1023,\n", - " 1024,\n", - " 1025,\n", - " 1026,\n", - " 42000,\n", - " 42001,\n", - " 42002,\n", - " 42003,\n", - " 42004,\n", - " 42005,\n", - " 42006,\n", - " 42007,\n", - " 42008,\n", - " 42009,\n", - " 42010,\n", - " 42011,\n", - " 42012,\n", - " 42013,\n", - " 42014,\n", - " 42015,\n", - " 42016,\n", - " 42017,\n", - " 42018,\n", - " 42019,\n", - " 42020,\n", - " 42021,\n", - " 42022,\n", - " 42023,\n", - " 42024,\n", - " 42025,\n", - " 42026,\n", - " 40000,\n", - " 40001,\n", - " 40002,\n", - " 40003,\n", - " 40004,\n", - " 40005,\n", - " 40006,\n", - " 40007,\n", - " 40008,\n", - " 40009,\n", - " 40010,\n", - " 40011,\n", - " 40012,\n", - " 40013,\n", - " 40014,\n", - " 40015,\n", - " 40016,\n", - " 40017,\n", - " 40018,\n", - " 40019,\n", - " 40020,\n", - " 40021,\n", - " 40022,\n", - " 40023,\n", - " 40024,\n", - " 40025,\n", - " 40026,\n", - " 40027,\n", - " 38000,\n", - " 38001,\n", - " 38002,\n", - " 38003,\n", - " 38004,\n", - " 38005,\n", - " 38006,\n", - " 38007,\n", - " 38008,\n", - " 38009,\n", - " 38010,\n", - " 38011,\n", - " 38012,\n", - " 38013,\n", - " 38014,\n", - " 38015,\n", - " 38016,\n", - " 38017,\n", - " 38018,\n", - " 38019,\n", - " 38020,\n", - " 38021,\n", - " 38022,\n", - " 38023,\n", - " 38024,\n", - " 38025,\n", - " 38026,\n", - " 44020,\n", - " 44025,\n", - " 36000,\n", - " 36001,\n", - " 36002,\n", - " 36003,\n", - " 36004,\n", - " 36005,\n", - " 36006,\n", - " 36007,\n", - " 36008,\n", - " 36009,\n", - " 36010,\n", - " 36011,\n", - " 36012,\n", - " 36013,\n", - " 36014,\n", - " 36015,\n", - " 36016,\n", - " 36017,\n", - " 36018,\n", - " 36019,\n", - " 36020,\n", - " 36021,\n", - " 36022,\n", - " 36023,\n", - " 36024,\n", - " 36025,\n", - " 36026,\n", - " 44027,\n", - " 34000,\n", - " 34001,\n", - " 34002,\n", - " 34003,\n", - " 34004,\n", - " 34005,\n", - " 34006,\n", - " 34007,\n", - " 34008,\n", - " 34009,\n", - " 34010,\n", - " 34011,\n", - " 34012,\n", - " 34013,\n", - " 34014,\n", - " 34015,\n", - " 34016,\n", - " 34017,\n", - " 34018,\n", - " 34019,\n", - " 34020,\n", - " 34021,\n", - " 34022,\n", - " 34023,\n", - " 34024,\n", - " 34025,\n", - " 34026,\n", - " 32000,\n", - " 32001,\n", - " 32002,\n", - " 32003,\n", - " 32004,\n", - " 32005,\n", - " 32006,\n", - " 32007,\n", - " 32008,\n", - " 32009,\n", - " 32010,\n", - " 32011,\n", - " 32012,\n", - " 32013,\n", - " 32014,\n", - " 32015,\n", - " 32016,\n", - " 32017,\n", - " 32018,\n", - " 32019,\n", - " 32020,\n", - " 32021,\n", - " 32022,\n", - " 32023,\n", - " 32024,\n", - " 32025,\n", - " 32026,\n", - " 30000,\n", - " 30001,\n", - " 30002,\n", - " 30003,\n", - " 30004,\n", - " 30005,\n", - " 30006,\n", - " 30007,\n", - " 30008,\n", - " 30009,\n", - " 30010,\n", - " 30011,\n", - " 30012,\n", - " 30013,\n", - " 30014,\n", - " 30015,\n", - " 30016,\n", - " 30017,\n", - " 30018,\n", - " 30019,\n", - " 30020,\n", - " 30021,\n", - " 30022,\n", - " 30023,\n", - " 30024,\n", - " 30025,\n", - " 28000,\n", - " 28001,\n", - " 28002,\n", - " 28003,\n", - " 28004,\n", - " 28005,\n", - " 28006,\n", - " 28007,\n", - " 28008,\n", - " 28009,\n", - " 28010,\n", - " 28011,\n", - " 28012,\n", - " 28013,\n", - " 28014,\n", - " 28015,\n", - " 28016,\n", - " 28017,\n", - " 28018,\n", - " 28019,\n", - " 28020,\n", - " 28021,\n", - " 28022,\n", - " 28023,\n", - " 28024,\n", - " 28025,\n", - " 28026,\n", - " 45018,\n", - " 43010,\n", - " 26000,\n", - " 26001,\n", - " 26002,\n", - " 26003,\n", - " 26004,\n", - " 26005,\n", - " 26006,\n", - " 26007,\n", - " 26008,\n", - " 26009,\n", - " 26010,\n", - " 26011,\n", - " 26012,\n", - " 26013,\n", - " 26014,\n", - " 26015,\n", - " 26016,\n", - " 26017,\n", - " 26018,\n", - " 26019,\n", - " 26020,\n", - " 26021,\n", - " 26022,\n", - " 26023,\n", - " 26024,\n", - " 26025,\n", - " 26026,\n", - " 26027,\n", - " 43012,\n", - " 24000,\n", - " 24001,\n", - " 24002,\n", - " 24003,\n", - " 24004,\n", - " 24005,\n", - " 24006,\n", - " 24007,\n", - " 24008,\n", - " 24009,\n", - " 24010,\n", - " 24011,\n", - " 24012,\n", - " 24013,\n", - " 24014,\n", - " 24015,\n", - " 24016,\n", - " 24017,\n", - " 24018,\n", - " 24019,\n", - " 24020,\n", - " 24021,\n", - " 24022,\n", - " 24023,\n", - " 24024,\n", - " 24025,\n", - " 24026,\n", - " 22000,\n", - " 22001,\n", - " 22002,\n", - " 22003,\n", - " 22004,\n", - " 22005,\n", - " 22006,\n", - " 22007,\n", - " 22008,\n", - " 22009,\n", - " 22010,\n", - " 22011,\n", - " 22012,\n", - " 22013,\n", - " 22014,\n", - " 22015,\n", - " 22016,\n", - " 22017,\n", - " 22018,\n", - " 22019,\n", - " 22020,\n", - " 22021,\n", - " 22022,\n", - " 22023,\n", - " 22024,\n", - " 22025,\n", - " 22026,\n", - " 45019,\n", - " 43015,\n", - " 20000,\n", - " 20001,\n", - " 20002,\n", - " 20003,\n", - " 20004,\n", - " 20005,\n", - " 20006,\n", - " 20007,\n", - " 20008,\n", - " 20009,\n", - " 20010,\n", - " 20011,\n", - " 20012,\n", - " 20013,\n", - " 20014,\n", - " 20015,\n", - " 20016,\n", - " 20017,\n", - " 20018,\n", - " 20019,\n", - " 20020,\n", - " 43016,\n", - " 44010,\n", - " 18000,\n", - " 18001,\n", - " 18002,\n", - " 18003,\n", - " 18004,\n", - " 18005,\n", - " 18006,\n", - " 18007,\n", - " 18008,\n", - " 18009,\n", - " 18010,\n", - " 18011,\n", - " 18012,\n", - " 18013,\n", - " 18014,\n", - " 18015,\n", - " 18016,\n", - " 18017,\n", - " 18018,\n", - " 18019,\n", - " 18020,\n", - " 18021,\n", - " 18022,\n", - " 18023,\n", - " 18024,\n", - " 18025,\n", - " 43018,\n", - " 44024,\n", - " 43025,\n", - " 16000,\n", - " 16001,\n", - " 16002,\n", - " 16003,\n", - " 16004,\n", - " 16005,\n", - " 16006,\n", - " 16007,\n", - " 16008,\n", - " ...}" - ] - }, - "execution_count": 186, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sG" - ] - }, - { - "cell_type": "code", - "execution_count": 200, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "import networkx as nx\n", - "G=nx.Graph()\n", - "\n", - "subgroup =[group for i, group in enumerate(groups) if (zvalues[group]>3125) & (zvalues[group]<3175)]\n", - "subanswers =[answers[i] for i, group in enumerate(groups) if (zvalues[group]>3125) & (zvalues[group]<3175)]\n", - "\n", - "G.add_nodes_from(subgroup)\n", - "#for group in groups:\n", - "# if zvalues[group]<100:\n", - "# nx.set_node_attributes(G,'z',zvalues)\n", - "\n", - "for answer,group in zip(subanswers,subgroup):\n", - " for group2 in answer.keys():\n", - "\n", - " if G.has_edge(group,group2):\n", - " w = G[group][group2]['weight']\n", - " G[group][group2]['weight']=w+answer[group2]\n", - " else:\n", - " if group2=='z':\n", - " print group,group2\n", - " if group=='z':\n", - " print group,group2\n", - " G.add_edge(group,group2,weight=answer[group2])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 201, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "def draw_network(G,pos,ax,sg=None):\n", - " \n", - " \n", - " for n in G:\n", - " c=Circle(pos[n],radius=.25,alpha=0.5)\n", - " #ax.add_patch(c)\n", - " p=ax.plot(pos[n][0],pos[n][1],'bo')\n", - " G.node[n]['patch']=c\n", - " x,y=pos[n]\n", - " #print x,y\n", - " seen={}\n", - " #return\n", - " for (u,v,d) in G.edges(data=True):\n", - " x1,y1 = pos[u]\n", - " x2,y2 = pos[v]\n", - " n1=G.node[u]['patch']\n", - " if v=='z':\n", - " print u,v,d\n", - " n2=G.node[v]['patch']\n", - " dz=y2-y1\n", - " rad = np.sqrt(dz)\n", - " if (u,v) in seen:\n", - " rad=seen.get((u,v))\n", - " rad=(rad+np.sign(rad)*0.1)*-1\n", - " alpha=0.5\n", - " color='k'\n", - " \n", - " cs = ConnectionStyle('Angle',angleA=45*(-1)**(dz%2), angleB=-45*(-1)**(dz%2), rad=10)\n", - " \n", - " linew = d['weight']*.1\n", - " linew = min(linew,10)\n", - " e = FancyArrowPatch(n1.center,n2.center,patchA=n1,patchB=n2,\n", - " arrowstyle='-|>',\n", - "# connectionstyle='arc3,rad=%s'%rad,\n", - " connectionstyle=cs,\n", - " mutation_scale=10.0,\n", - " lw=linew,\n", - " alpha=alpha,\n", - " color=color)\n", - " seen[(u,v)]=rad\n", - " ax.add_patch(e)\n", - " return e" - ] - }, - { - "cell_type": "code", - "execution_count": 202, - "metadata": { - "collapsed": false, - "scrolled": false - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "f,ax=plt.subplots(1,2,figsize=(12,12))\n", - "#imgjoin = np.concatenate([lowmag_tile_p,lowmag_tile_q],axis=1)\n", - "ax[0].imshow(lowmag_tile_p,extent=(bounds_p['minX'],bounds_p['maxX'],bounds_p['maxY'],bounds_p['minY']))\n", - "ax[1].imshow(lowmag_tile_q,extent=(bounds_q['minX'],bounds_q['maxX'],bounds_q['maxY'],bounds_q['minY']))\n", - "ax[0].scatter(all_points_global_p[:,0],all_points_global_p[:,1],c='r',zorder=8)\n", - "ax[1].scatter(all_points_global_q[:,0],all_points_global_q[:,1],c='r',zorder=8)\n", - "ax[0].autoscale(tight=True)\n", - "ax[1].autoscale(tight=True)\n", - "ax[0].set_title(z_p)\n", - "ax[1].set_title(z_q)\n", - "#transFigure = f.transFigure.inverted()\n", - "#coord1 = transFigure.transform(ax[0].transData.transform(all_points_global_p))\n", - "#coord2 = transFigure.transform(ax[1].transData.transform(all_points_global_q))\n", - "#coords=np.concatenate([coord1,coord2],axis=1)\n", - "#for coor in coords:\n", - "# line = Line2D(coor[0:5:2]+.019,coor[1:5:2],\n", - "# transform=f.transFigure,zorder=9)\n", - "# f.lines.append(line)\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "z_p=0\n", - "z_q=1\n", - "section_p=[key for key in zvalues.keys() if zvalues[key]==z_p][0]\n", - "section_q=[key for key in zvalues.keys() if zvalues[key]==z_q][0]\n", - "bounds_p = render.get_bounds_from_z(stack,z_p)\n", - "bounds_q = render.get_bounds_from_z(stack,z_q)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "scale=.1\n", - "lowmag_tile_p = render.get_png_tile(stack,z_p,bounds_p['minX'],bounds_p['minY'],bounds_p['maxX']-bounds_p['minX'],bounds_p['maxY']-bounds_p['minY'],scale=scale)\n", - "lowmag_tile_q = render.get_png_tile(stack,z_q,bounds_p['minX'],bounds_p['minY'],bounds_p['maxX']-bounds_p['minX'],bounds_p['maxY']-bounds_p['minY'],scale=scale)\n", - "lowmag_rg = np.copy(lowmag_tile_p)\n", - "lowmag_rg[:,:,2]=lowmag_tile_q[:,:,1]\n", - "lowmag_rg[:,:,0]=0" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "allmatches = render.get_matches_from_group_to_group(matchcollection,section_p,section_q)\n", - "all_points_global_p = np.zeros((1,2))\n", - "all_points_global_q = np.zeros((1,2))\n", - "for matchobj in allmatches:\n", - " points_local_p = np.array(matchobj['matches']['p'])\n", - " points_local_q = np.array(matchobj['matches']['q'])\n", - " #print matchobj,section_p,section_q\n", - " \n", - " t_p = render.local_to_world_coordinates_array(stack,points_local_p.T,matchobj['pId'],z_p)\n", - " all_points_global_p=np.append(all_points_global_p,t_p,axis=0)\n", - " t_q = render.local_to_world_coordinates_array(stack,points_local_q.T,matchobj['qId'],z_q)\n", - " all_points_global_q=np.append(all_points_global_q,t_q,axis=0)\n", - "\n", - " #break\n", - "all_points_global_p = all_points_global_p[1:,:]\n", - "all_points_global_q = all_points_global_q[1:,:]\n" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "all_points=np.concatenate([all_points_global_p,all_points_global_q],axis=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(6347, 4)" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_points.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 206, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/javascript": [ - "/* Put everything inside the global mpl namespace */\n", - "window.mpl = {};\n", - "\n", - "mpl.get_websocket_type = function() {\n", - " if (typeof(WebSocket) !== 'undefined') {\n", - " return WebSocket;\n", - " } else if (typeof(MozWebSocket) !== 'undefined') {\n", - " return MozWebSocket;\n", - " } else {\n", - " alert('Your browser does not have WebSocket support.' +\n", - " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", - " 'Firefox 4 and 5 are also supported but you ' +\n", - " 'have to enable WebSockets in about:config.');\n", - " };\n", - "}\n", - "\n", - "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", - " this.id = figure_id;\n", - "\n", - " this.ws = websocket;\n", - "\n", - " this.supports_binary = (this.ws.binaryType != undefined);\n", - "\n", - " if (!this.supports_binary) {\n", - " var warnings = document.getElementById(\"mpl-warnings\");\n", - " if (warnings) {\n", - " warnings.style.display = 'block';\n", - " warnings.textContent = (\n", - " \"This browser does not support binary websocket messages. \" +\n", - " \"Performance may be slow.\");\n", - " }\n", - " }\n", - "\n", - " this.imageObj = new Image();\n", - "\n", - " this.context = undefined;\n", - " this.message = undefined;\n", - " this.canvas = undefined;\n", - " this.rubberband_canvas = undefined;\n", - " this.rubberband_context = undefined;\n", - " this.format_dropdown = undefined;\n", - "\n", - " this.image_mode = 'full';\n", - "\n", - " this.root = $('
');\n", - " this._root_extra_style(this.root)\n", - " this.root.attr('style', 'display: inline-block');\n", - "\n", - " $(parent_element).append(this.root);\n", - "\n", - " this._init_header(this);\n", - " this._init_canvas(this);\n", - " this._init_toolbar(this);\n", - "\n", - " var fig = this;\n", - "\n", - " this.waiting = false;\n", - "\n", - " this.ws.onopen = function () {\n", - " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", - " fig.send_message(\"send_image_mode\", {});\n", - " fig.send_message(\"refresh\", {});\n", - " }\n", - "\n", - " this.imageObj.onload = function() {\n", - " if (fig.image_mode == 'full') {\n", - " // Full images could contain transparency (where diff images\n", - " // almost always do), so we need to clear the canvas so that\n", - " // there is no ghosting.\n", - " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", - " }\n", - " fig.context.drawImage(fig.imageObj, 0, 0);\n", - " };\n", - "\n", - " this.imageObj.onunload = function() {\n", - " this.ws.close();\n", - " }\n", - "\n", - " this.ws.onmessage = this._make_on_message_function(this);\n", - "\n", - " this.ondownload = ondownload;\n", - "}\n", - "\n", - "mpl.figure.prototype._init_header = function() {\n", - " var titlebar = $(\n", - " '
');\n", - " var titletext = $(\n", - " '
');\n", - " titlebar.append(titletext)\n", - " this.root.append(titlebar);\n", - " this.header = titletext[0];\n", - "}\n", - "\n", - "\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._init_canvas = function() {\n", - " var fig = this;\n", - "\n", - " var canvas_div = $('
');\n", - "\n", - " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", - "\n", - " function canvas_keyboard_event(event) {\n", - " return fig.key_event(event, event['data']);\n", - " }\n", - "\n", - " canvas_div.keydown('key_press', canvas_keyboard_event);\n", - " canvas_div.keyup('key_release', canvas_keyboard_event);\n", - " this.canvas_div = canvas_div\n", - " this._canvas_extra_style(canvas_div)\n", - " this.root.append(canvas_div);\n", - "\n", - " var canvas = $('');\n", - " canvas.addClass('mpl-canvas');\n", - " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", - "\n", - " this.canvas = canvas[0];\n", - " this.context = canvas[0].getContext(\"2d\");\n", - "\n", - " var rubberband = $('');\n", - " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", - "\n", - " var pass_mouse_events = true;\n", - "\n", - " canvas_div.resizable({\n", - " start: function(event, ui) {\n", - " pass_mouse_events = false;\n", - " },\n", - " resize: function(event, ui) {\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " stop: function(event, ui) {\n", - " pass_mouse_events = true;\n", - " fig.request_resize(ui.size.width, ui.size.height);\n", - " },\n", - " });\n", - "\n", - " function mouse_event_fn(event) {\n", - " if (pass_mouse_events)\n", - " return fig.mouse_event(event, event['data']);\n", - " }\n", - "\n", - " rubberband.mousedown('button_press', mouse_event_fn);\n", - " rubberband.mouseup('button_release', mouse_event_fn);\n", - " // Throttle sequential mouse events to 1 every 20ms.\n", - " rubberband.mousemove('motion_notify', mouse_event_fn);\n", - "\n", - " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", - " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", - "\n", - " canvas_div.on(\"wheel\", function (event) {\n", - " event = event.originalEvent;\n", - " event['data'] = 'scroll'\n", - " if (event.deltaY < 0) {\n", - " event.step = 1;\n", - " } else {\n", - " event.step = -1;\n", - " }\n", - " mouse_event_fn(event);\n", - " });\n", - "\n", - " canvas_div.append(canvas);\n", - " canvas_div.append(rubberband);\n", - "\n", - " this.rubberband = rubberband;\n", - " this.rubberband_canvas = rubberband[0];\n", - " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", - " this.rubberband_context.strokeStyle = \"#000000\";\n", - "\n", - " this._resize_canvas = function(width, height) {\n", - " // Keep the size of the canvas, canvas container, and rubber band\n", - " // canvas in synch.\n", - " canvas_div.css('width', width)\n", - " canvas_div.css('height', height)\n", - "\n", - " canvas.attr('width', width);\n", - " canvas.attr('height', height);\n", - "\n", - " rubberband.attr('width', width);\n", - " rubberband.attr('height', height);\n", - " }\n", - "\n", - " // Set the figure to an initial 600x600px, this will subsequently be updated\n", - " // upon first draw.\n", - " this._resize_canvas(600, 600);\n", - "\n", - " // Disable right mouse context menu.\n", - " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", - " return false;\n", - " });\n", - "\n", - " function set_focus () {\n", - " canvas.focus();\n", - " canvas_div.focus();\n", - " }\n", - "\n", - " window.setTimeout(set_focus, 100);\n", - "}\n", - "\n", - "mpl.figure.prototype._init_toolbar = function() {\n", - " var fig = this;\n", - "\n", - " var nav_element = $('
')\n", - " nav_element.attr('style', 'width: 100%');\n", - " this.root.append(nav_element);\n", - "\n", - " // Define a callback function for later on.\n", - " function toolbar_event(event) {\n", - " return fig.toolbar_button_onclick(event['data']);\n", - " }\n", - " function toolbar_mouse_event(event) {\n", - " return fig.toolbar_button_onmouseover(event['data']);\n", - " }\n", - "\n", - " for(var toolbar_ind in mpl.toolbar_items) {\n", - " var name = mpl.toolbar_items[toolbar_ind][0];\n", - " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", - " var image = mpl.toolbar_items[toolbar_ind][2];\n", - " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", - "\n", - " if (!name) {\n", - " // put a spacer in here.\n", - " continue;\n", - " }\n", - " var button = $('');\n", - " button.click(method_name, toolbar_event);\n", - " button.mouseover(tooltip, toolbar_mouse_event);\n", - " nav_element.append(button);\n", - " }\n", - "\n", - " // Add the status bar.\n", - " var status_bar = $('');\n", - " nav_element.append(status_bar);\n", - " this.message = status_bar[0];\n", - "\n", - " // Add the close button to the window.\n", - " var buttongrp = $('
');\n", - " var button = $('');\n", - " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", - " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", - " buttongrp.append(button);\n", - " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", - " titlebar.prepend(buttongrp);\n", - "}\n", - "\n", - "mpl.figure.prototype._root_extra_style = function(el){\n", - " var fig = this\n", - " el.on(\"remove\", function(){\n", - "\tfig.close_ws(fig, {});\n", - " });\n", - "}\n", - "\n", - "mpl.figure.prototype._canvas_extra_style = function(el){\n", - " // this is important to make the div 'focusable\n", - " el.attr('tabindex', 0)\n", - " // reach out to IPython and tell the keyboard manager to turn it's self\n", - " // off when our div gets focus\n", - "\n", - " // location in version 3\n", - " if (IPython.notebook.keyboard_manager) {\n", - " IPython.notebook.keyboard_manager.register_events(el);\n", - " }\n", - " else {\n", - " // location in version 2\n", - " IPython.keyboard_manager.register_events(el);\n", - " }\n", - "\n", - "}\n", - "\n", - "mpl.figure.prototype._key_event_extra = function(event, name) {\n", - " var manager = IPython.notebook.keyboard_manager;\n", - " if (!manager)\n", - " manager = IPython.keyboard_manager;\n", - "\n", - " // Check for shift+enter\n", - " if (event.shiftKey && event.which == 13) {\n", - " this.canvas_div.blur();\n", - " event.shiftKey = false;\n", - " // Send a \"J\" for go to next cell\n", - " event.which = 74;\n", - " event.keyCode = 74;\n", - " manager.command_mode();\n", - " manager.handle_keydown(event);\n", - " }\n", - "}\n", - "\n", - "mpl.figure.prototype.handle_save = function(fig, msg) {\n", - " fig.ondownload(fig, null);\n", - "}\n", - "\n", - "\n", - "mpl.find_output_cell = function(html_output) {\n", - " // Return the cell and output element which can be found *uniquely* in the notebook.\n", - " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", - " // IPython event is triggered only after the cells have been serialised, which for\n", - " // our purposes (turning an active figure into a static one), is too late.\n", - " var cells = IPython.notebook.get_cells();\n", - " var ncells = cells.length;\n", - " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", - " data = data.data;\n", - " }\n", - " if (data['text/html'] == html_output) {\n", - " return [cell, data, j];\n", - " }\n", - " }\n", - " }\n", - " }\n", - "}\n", - "\n", - "// Register the function which deals with the matplotlib target/channel.\n", - "// The kernel may be null if the page has been refreshed.\n", - "if (IPython.notebook.kernel != null) {\n", - " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", - "}\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "f,ax=plt.subplots(1,1,figsize=(14,40))\n", - "ax.imshow(lowmag_rg,extent=(bounds_p['minX'],bounds_p['maxX'],bounds_p['maxY'],bounds_p['minY']))\n", - "ax.scatter(all_points[:,0],all_points[:,1],c='m',marker='o',s=5,linewidth=0)\n", - "ax.quiver(all_points[:,0].T,all_points[:,1].T,\n", - " all_points[:,2].T-all_points[:,0].T,\n", - " all_points[:,3].T-all_points[:,1].T,\n", - " color='m',\n", - " angles='xy', scale_units='xy', scale=1)\n", - "ax.set_xlim((bounds_p['minX'],bounds_p['maxX']))\n", - "ax.set_ylim((bounds_p['maxY'],bounds_p['minY']))\n", - "#ax.autoscale(tight=True)\n", - "plt.tight_layout()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "f,ax=plt.subplots(1,1,figsize=(10,30))\n", - "ax.imshow(lowmag_rg,extent=(bounds_p['minX'],bounds_p['maxX'],bounds_p['maxY'],bounds_p['minY']))\n", - "ax.plot(all_points[:,0:5:2].T,all_points[:,1:5:2].T,c='m')\n", - "ax.autoscale(tight=True)\n", - "plt.tight_layout()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "print np.where(np.sum(match_matrix,axis=1)==0)\n", - "plt.figure()\n", - "plt.plot(np.sum(match_matrix,axis=1))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "plt.scatter?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true, - "scrolled": true - }, - "outputs": [], - "source": [ - "ax[0].scatter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "f,ax = plt.subplots(1,1,figsize=(12,12))\n", - "\n", - "img=ax.imshow(match_matrix,interpolation='nearest')\n", - "plt.xlabel('section a')\n", - "plt.ylabel('section b')\n", - "ax.set_title('matches')\n", - "plt.colorbar(img)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "(z2-z1)" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [], - "source": [ - "ts=render.get_tile_spec(stack,145000023044000)" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "1242.0" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ts.z" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": { - "collapsed": false, - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "(6347, 4)" - ] - }, - "execution_count": 18, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "all_points.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import json\n", - "jsonlist = []\n", - "for i in range(all_points.shape[0]):\n", - " jsonlist.append([(all_points[i,0],all_points[i,1],z_p),(all_points[i,2],all_points[i,3],z_q)])\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "outfile = 'z0_z1_point_matches_global.json'\n", - "json.dump(jsonlist,open(outfile,'w'))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} From d7f735057bc10ce24833c45accd2d3782ba354ff Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 13:20:03 -0800 Subject: [PATCH 150/766] adding testing environment --- setup.cfg | 5 +++++ setup.py | 7 ++++++- test/test_requirements.txt | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 setup.cfg create mode 100644 test/test_requirements.txt diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..b293454a --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[tool:pytest] +addopts = --boxed --cov=json_module --cov-report html --junitxml=test-reports/test.xml + +[aliases] +test=pytest \ No newline at end of file diff --git a/setup.py b/setup.py index 9815317d..1c632b98 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,8 @@ from setuptools import setup +with open('test/requirements.txt','r') as f: + test_required = f.read().splitlines() + with open('requirements.txt', 'r') as f: required = f.read().splitlines() @@ -11,4 +14,6 @@ author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', packages=['renderapi'], - install_requires=required) + install_requires=required, + setup_requires=['pytest-runner'], + test_require = test_required) diff --git a/test/test_requirements.txt b/test/test_requirements.txt new file mode 100644 index 00000000..9fbb4148 --- /dev/null +++ b/test/test_requirements.txt @@ -0,0 +1,9 @@ +coverage==4.1 +mock==2.0.0 +pep8==1.7.0 +pytest==3.0.5 +pytest-cov==2.2.1 +pytest-pep8==1.0.6 +pytest-xdist==1.14 +flake8>=3.0.4 +pylint>=1.5.4 \ No newline at end of file From ee4e362f6cfca1a2a17fc84d72d603d3d89ac9fa Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 13:36:46 -0800 Subject: [PATCH 151/766] modifying test environment --- setup.cfg | 1 + setup.py | 2 +- test/test_requirements.txt | 9 --------- 3 files changed, 2 insertions(+), 10 deletions(-) delete mode 100644 test/test_requirements.txt diff --git a/setup.cfg b/setup.cfg index b293454a..bf0f77e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,6 @@ [tool:pytest] addopts = --boxed --cov=json_module --cov-report html --junitxml=test-reports/test.xml + [aliases] test=pytest \ No newline at end of file diff --git a/setup.py b/setup.py index 1c632b98..db5c0a61 100644 --- a/setup.py +++ b/setup.py @@ -16,4 +16,4 @@ packages=['renderapi'], install_requires=required, setup_requires=['pytest-runner'], - test_require = test_required) + tests_require = test_required) diff --git a/test/test_requirements.txt b/test/test_requirements.txt deleted file mode 100644 index 9fbb4148..00000000 --- a/test/test_requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -coverage==4.1 -mock==2.0.0 -pep8==1.7.0 -pytest==3.0.5 -pytest-cov==2.2.1 -pytest-pep8==1.0.6 -pytest-xdist==1.14 -flake8>=3.0.4 -pylint>=1.5.4 \ No newline at end of file From e19393472ebde44c9e6aedfd997ecf0730291ea6 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 13:40:38 -0800 Subject: [PATCH 152/766] adding test requirements --- test/requirements.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/requirements.txt diff --git a/test/requirements.txt b/test/requirements.txt new file mode 100644 index 00000000..d2a75c7d --- /dev/null +++ b/test/requirements.txt @@ -0,0 +1,9 @@ +coverage==4.1 +mock==2.0.0 +pep8==1.7.0 +pytest==3.0.6 +pytest-cov==2.2.1 +pytest-pep8==1.0.6 +pytest-xdist==1.14 +flake8>=3.0.4 +pylint>=1.5.4 \ No newline at end of file From 7cc2391a30d54a0134107ac5a124dde8fe73e55c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 13:50:06 -0800 Subject: [PATCH 153/766] upgrading pytest to 3.0.6 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7277ebf6..134cc487 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-oracle RUN apt-get install gcc -y RUN apt-get install build-essential -y RUN apt-get clean +RUN pip install --upgrade pytest==3.0.6 RUN pip install multiprocess RUN pip install pathos From 8b4d3c6139e41959bdeee606dca0ecdb78fe20e5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 14:01:45 -0800 Subject: [PATCH 154/766] altering testing environment --- Dockerfile | 1 - test/requirements.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 134cc487..7277ebf6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,6 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-oracle RUN apt-get install gcc -y RUN apt-get install build-essential -y RUN apt-get clean -RUN pip install --upgrade pytest==3.0.6 RUN pip install multiprocess RUN pip install pathos diff --git a/test/requirements.txt b/test/requirements.txt index d2a75c7d..9fbb4148 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,7 +1,7 @@ coverage==4.1 mock==2.0.0 pep8==1.7.0 -pytest==3.0.6 +pytest==3.0.5 pytest-cov==2.2.1 pytest-pep8==1.0.6 pytest-xdist==1.14 From 79979d76127c8716c5d14479453c359a7cdb10b9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 14:03:36 -0800 Subject: [PATCH 155/766] fixing coverage mistake --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index bf0f77e8..6cc0e0b8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,5 @@ [tool:pytest] -addopts = --boxed --cov=json_module --cov-report html --junitxml=test-reports/test.xml - +addopts = --boxed --cov=renderapi --cov-report html --junitxml=test-reports/test.xml [aliases] test=pytest \ No newline at end of file From 9f2aeda3db4deafa1597ffac8d8054d982f9e2f2 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 14:10:04 -0800 Subject: [PATCH 156/766] a first test --- test/test_client.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 test/test_client.py diff --git a/test/test_client.py b/test/test_client.py new file mode 100644 index 00000000..64568d96 --- /dev/null +++ b/test/test_client.py @@ -0,0 +1,11 @@ +import renderapi + +def test_render_client(): + args={ + 'host':'renderhost', + 'port':8080, + 'owner':'renderowner', + 'project':'renderproject', + 'client_scripts':'/path/to/client_scripts' + } + r = renderapi.client.render(**args) From 85728e0de749840c6d176b8964ac9bab615cf81a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 14:11:56 -0800 Subject: [PATCH 157/766] fixing test! --- test/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_client.py b/test/test_client.py index 64568d96..353d79ec 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -8,4 +8,4 @@ def test_render_client(): 'project':'renderproject', 'client_scripts':'/path/to/client_scripts' } - r = renderapi.client.render(**args) + r = renderapi.render.client(**args) From fa78af2759ca551dc86cd1b998ae29297db0cc10 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 14:13:28 -0800 Subject: [PATCH 158/766] fixing tests for real --- test/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_client.py b/test/test_client.py index 353d79ec..65a58a2b 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -8,4 +8,4 @@ def test_render_client(): 'project':'renderproject', 'client_scripts':'/path/to/client_scripts' } - r = renderapi.render.client(**args) + r = renderapi.render.connect(**args) From 7b6ed7650335329d5886ca68bc301e5e84452808 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 8 Mar 2017 14:34:47 -0800 Subject: [PATCH 159/766] changing flake8 --- setup.cfg | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6cc0e0b8..909b560f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,4 +2,8 @@ addopts = --boxed --cov=renderapi --cov-report html --junitxml=test-reports/test.xml [aliases] -test=pytest \ No newline at end of file +test=pytest + +[flake8] +ignore = E201,E202,E226 +max-line-length = 200 diff --git a/setup.py b/setup.py index db5c0a61..7e7d436d 100644 --- a/setup.py +++ b/setup.py @@ -15,5 +15,5 @@ url='https://github.com/fcollman/render-python', packages=['renderapi'], install_requires=required, - setup_requires=['pytest-runner'], + setup_requires=['pytest-runner','flake8'], tests_require = test_required) From c079c3982040c92e503f783e9e256f9d1621c790 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 8 Mar 2017 18:20:38 -0800 Subject: [PATCH 160/766] transform: Affine extends Transform, has basic properties for quantifying shear, translation, rotation, scale --- renderapi/transform.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index e0f68ac6..56888a56 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -9,6 +9,7 @@ Affine as subset of Polynomial2D approximation of other functions(TPS, meshtechniques) to Polynomial2D ^ would this be better in Java using mpicbg implementation? + Allow reading datastring for Affine, Rigid, Translation into Affine ''' import json import logging @@ -24,7 +25,8 @@ from scipy.linalg import svd except ImportError as e: logger.info(e) - logger.info('scipy-based linalg may lead to better parameter fitting') + logger.info('scipy-based linalg may or may not lead ' + 'to better parameter fitting') from numpy.linalg import svd @@ -115,6 +117,12 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0): self.load_M() self.transformId = None + @property + def dataString(self): + return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( + self.M[0, 0], self.M[1, 0], self.M[0, 1], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + def load_M(self): self.M = np.identity(3, np.double) self.M[0, 0] = self.M00 @@ -124,6 +132,7 @@ def load_M(self): self.M[0, 2] = self.B0 self.M[1, 2] = self.B1 + ''' def to_dict(self): d = {} d['type'] = 'leaf' @@ -132,6 +141,7 @@ def to_dict(self): self.M[0, 0], self.M[1, 0], self.M[0, 1], self.M[1, 1], self.M[0, 2], self.M[1, 2]) return d + ''' def from_dict(self, d): ds = d['dataString'].split() @@ -193,6 +203,27 @@ def inverse_tform(self, points): pt = np.dot(np.linalg.inv(self.M), points.T).T return self.convert_points_vector_to_array(pt, Nd) + @property + def scale(self): + '''tuple of scale for x, y''' + return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) + for j in self.M.shape[1]])[:2] + + @property + def shear(self): + '''counter-clockwise shear angle''' + return np.atan2(-self.M[0, 1], self.M[1, 1]) - self.rotation + + @property + def translation(self): + '''tuple of translation in x, y''' + return tuple(self.M[:2, 2]) + + @property + def rotation(self): + '''counter-clockwise rotation''' + return numpy.atan2(self.M[1, 0], self.M[0, 0]) + def __str__(self): return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( self.M[0, 0], self.M[0, 1], self.M[1, 0], @@ -205,6 +236,8 @@ class Polynomial2DTransform(Transform): TODO: fall back to Affine Model in special cases robustness in estimation + + there is a hierarchy in the __init__ that should probably be defined ''' className = 'mpicbg.trakem2.transform.PolynomialTransform2D' From 6924520809074ae54431540c4657d0f441e9dfcf Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 8 Mar 2017 18:22:53 -0800 Subject: [PATCH 161/766] tilespec: get_tile_specs_from_stack function --- renderapi/tilespec.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 6082523d..212d649a 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from .render import Render, format_baseurl, format_preamble, renderaccess from .utils import NullHandler +from .stack import get_z_values_for_stack from collections import OrderedDict import logging import requests @@ -315,6 +316,15 @@ def __init__(self, tileId=None, z=None, width=None, height=None, if scale3Url is not None: self.ip.update(MipMapLevel(3, imageUrl=scale3Url)) + @property + def bbox(self): + '''bbox defined to fit shapely call''' + box = (self.minX, self.minY, self.maxX, self.maxY) + if any([v is None for v in box]): + logger.error( + 'undefined bounding box for tile {}'.format(self.tileId)) + return box + def to_dict(self): thedict = {} thedict['tileId'] = self.tileId @@ -510,3 +520,17 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, else: return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json] + + +@renderaccess +def get_tile_specs_from_stack(stack, host=None, port=None, + owner=None, project=None, + session=requests.session(), + render=None, **kwargs): + '''get flat list of tilespecs for stack using i for sl in l for i in sl''' + return [i for sl in [ + get_tile_specs_from_z(stack, z, host=host, port=port, + owner=owner, project=project, session=session) + for z in get_z_values_for_stack(stack, host=host, port=port, + owner=owner, project=project, + session=session)] for i in sl] From 3c179197f9a8c8deb9957a251bc559d9f7e0998e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 12 Mar 2017 14:15:56 -0700 Subject: [PATCH 162/766] errors: custom errors now extend RenderError --- renderapi/errors.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/renderapi/errors.py b/renderapi/errors.py index 845028f7..e519793f 100644 --- a/renderapi/errors.py +++ b/renderapi/errors.py @@ -4,17 +4,21 @@ ''' -class ClientScriptError(Exception): +class RenderError(Exception): pass -class ConversionError(Exception): +class ClientScriptError(RenderError): pass -class EstimationError(Exception): +class ConversionError(RenderError): pass -class SpecError(Exception): +class EstimationError(RenderError): + pass + + +class SpecError(RenderError): pass From c1db82fb706b9171204cb0a0f84e3376b0f0827c Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 12 Mar 2017 15:09:22 -0700 Subject: [PATCH 163/766] stack: add get_stack_bounds --- renderapi/stack.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/renderapi/stack.py b/renderapi/stack.py index da3cfe4f..4e3e1214 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -206,6 +206,18 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, logger.error(r.text) +@renderaccess +def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, + session=requests.session(), render=None, **kwargs): + request_url = format_preamble( + host, port, owner, project, stack) + '/bounds' + r = session.get(request_url) + try: + return r.json() + except: + logger.error(r.text) + + @renderaccess def get_section_z_value(stack, sectionId, host=None, port=None, owner=None, project=None, session=requests.session(), From e663d0e8ee8e7a033ddb14d23206e2dcf5ae8362 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 12 Mar 2017 15:10:35 -0700 Subject: [PATCH 164/766] tilespec: do not default to layout --- renderapi/tilespec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 0404ad22..2b027c51 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -144,7 +144,6 @@ def __init__(self, tileId=None, z=None, width=None, height=None, self.maxint = maxint self.tforms = tforms self.frameId = frameId - self.layout = layout self.inputfilters = inputfilters self.ip = ImagePyramid(mipMapLevels=mipMapLevels) From ac4ec7e41051b84cc7b9b5a54793974d6663705d Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 12 Mar 2017 15:24:44 -0700 Subject: [PATCH 165/766] tilespec: on second thought, layout is probably necessary --- renderapi/tilespec.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 2b027c51..d10cc28c 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -89,7 +89,7 @@ def from_dict(self, d): class Layout: def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, - rotation=None, pixelsize=0.100): + rotation=None, pixelsize=0.100, **kwargs): self.sectionId = str(sectionId) self.scopeId = str(scopeId) self.cameraId = str(cameraId) @@ -129,9 +129,9 @@ def from_dict(self, d): class TileSpec: def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, frameId=None, maskUrl=None, - minint=0, maxint=65535, layout=Layout(), tforms=[], + minint=0, maxint=65535, layout=None, tforms=[], inputfilters=[], scale3Url=None, scale2Url=None, - scale1Url=None, json=None, mipMapLevels=[]): + scale1Url=None, json=None, mipMapLevels=[], **kwargs): if json is not None: self.from_dict(json) else: @@ -145,6 +145,7 @@ def __init__(self, tileId=None, z=None, width=None, height=None, self.tforms = tforms self.frameId = frameId self.inputfilters = inputfilters + self.layout = Layout(**kwargs) if layout is None else layout self.ip = ImagePyramid(mipMapLevels=mipMapLevels) # legacy scaleXUrl From 0a873265cd03e21ba0fd6815472e09b82a0cf65b Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 12 Mar 2017 16:17:31 -0700 Subject: [PATCH 166/766] setup: remove magic setup in favor of setup.py --- other_setup.py | 370 ------------------------------------------------- 1 file changed, 370 deletions(-) delete mode 100644 other_setup.py diff --git a/other_setup.py b/other_setup.py deleted file mode 100644 index 49598c2b..00000000 --- a/other_setup.py +++ /dev/null @@ -1,370 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -""" distribute- and pip-enabled setup.py """ - -try: - import ConfigParser -except ImportError as E: - import configparser - -import logging -import os -import re -import sys - -# ----- overrides ----- - -# set these to anything but None to override the automatic defaults -author = 'Forrest Collman, Eric Perlman, Sharmi Seshamani' -author_email = 'forrest.collman@gmail.com' -dependency_links = None -long_description = ' a python API setup to interact via python with render'\ - ' databases see https://github.com/saalfeldlab/render' -packages = None -package_name = None -package_data = None -scripts = None -requirements_file = None -requirements = None -version = '0.01' -test_suite = None - -# --------------------- - - -# ----- control flags ----- - -# fallback to setuptools if distribute isn't found -setup_tools_fallback = False - -# don't include subdir named 'tests' in package_data -skip_tests = True - -# print some extra debugging info -debug = True - -# use numpy.distutils instead of setuptools -use_numpy = False - -# ------------------------- -update_url = "https://raw.githubusercontent.com/braingram/simple_setup/master/setup.py" - -# this next line is important for the 'fetch' option (see below) -# MARK -if (len(sys.argv) > 1) and sys.argv[1] == 'fetch': - _overrides = {} - _locals = locals() - for _k in _locals.keys(): - if (_k[0] != '_') and not isinstance(_locals[_k], type(sys)): - _overrides[_k] = _locals[_k] - if len(sys.argv) > 2: - target_fn = sys.argv[2] - else: - target_fn = __file__ - print("Fetching a new simple_setup.py to {}".format(target_fn)) - import urllib2 - new_ss = urllib2.urlopen(update_url) - with open(target_fn, 'w') as target: - found_mark = False - for l in new_ss: - if found_mark or len(l.strip()) == 0: - target.write(l) - else: - if l[0] == '#': - if l.strip() == '# MARK': - found_mark = True - target.write(l) - continue - lt = l.split('=') - key = lt[0].strip() - if (len(lt) == 2) and (key in _overrides): - # copy over the overrides - target.write("{} = {!r}\n".format(key, _overrides[key])) - else: - target.write(l) - continue - print("successfully fetched new setup.py") - sys.exit(0) - -if debug: - logging.basicConfig(level=logging.DEBUG) -# distribute import and testing -try: - import distribute_setup - distribute_setup.use_setuptools() - logging.debug("distribute_setup.py imported and used") -except ImportError: - # fallback to setuptools? - # distribute_setup.py was not in this directory - if not (setup_tools_fallback): - import setuptools - # check if setuptools is distribute - vt = setuptools.__version__.split('.') - if len(vt) == 1: - vmajor = int(vt[0]) - vminor = 0 - elif len(vt) > 1: - vmajor = int(vt[0]) - vminor = int(vt[1]) - if (hasattr(setuptools, '_distribute') and - setuptools._distribute) or (vmajor > 0 or vminor > 6): - logging.debug("distribute_setup.py not found, " - "defaulted to system distribute") - else: - raise ImportError( - "distribute was not found and fallback " - "to setuptools was not allowed") - else: - logging.debug("distribute_setup.py not found, " - "defaulting to system setuptools") - -import setuptools - - -def find_scripts(): - return [s for s in setuptools.findall('scripts/') - if os.path.splitext(s)[1] != '.pyc'] - - -def package_to_path(package): - """ - Convert a package (as found by setuptools.find_packages) - e.g. "foo.bar" to usable path - e.g. "foo/bar" - No idea if this works on windows - """ - return package.replace('.', '/') - - -def find_subdirectories(package): - """ - Get the subdirectories within a package - This will include resources (non-submodules) and submodules - """ - try: - subdirectories = next(os.walk(package_to_path(package)))[1] - except StopIteration: - subdirectories = [] - return subdirectories - - -def subdir_findall(dir, subdir): - """ - Find all files in a subdirectory and return paths relative to dir - This is similar to (and uses) setuptools.findall - However, the paths returned are in the form needed for package_data - """ - strip_n = len(dir.split('/')) - path = '/'.join((dir, subdir)) - return ['/'.join(s.split('/')[strip_n:]) for s in setuptools.findall(path)] - - -def find_package_data(packages): - """ - For a list of packages, find the package_data - This function scans the subdirectories of a package and considers all - non-submodule subdirectories as resources, including them in - the package_data - Returns a dictionary suitable for setup(package_data=) - """ - package_data = {} - for package in packages: - package_data[package] = [] - for subdir in find_subdirectories(package): - if '.'.join((package, subdir)) in packages: # skip submodules - logging.debug("skipping submodule %s/%s" % (package, subdir)) - continue - if skip_tests and (subdir == 'tests'): # skip tests - logging.debug("skipping tests %s/%s" % (package, subdir)) - continue - package_data[package] += \ - subdir_findall(package_to_path(package), subdir) - return package_data - - -def parse_requirements(file_name): - """ - from: - http://cburgmer.posterous.com/pip-requirementstxt-and-setuppy - """ - requirements = [] - with open(file_name, 'r') as f: - for line in f: - if re.match(r'(\s*#)|(\s*$)', line): - continue - if re.match(r'\s*-e\s+', line): - requirements.append(re.sub(r'\s*-e\s+.*#egg=(.*)$', - r'\1', line).strip()) - elif re.match(r'\s*-f\s+', line): - pass - else: - requirements.append(line.strip()) - return requirements - - -def parse_dependency_links(file_name): - """ - from: - http://cburgmer.posterous.com/pip-requirementstxt-and-setuppy - """ - dependency_links = [] - with open(file_name) as f: - for line in f: - if re.match(r'\s*-[ef]\s+', line): - dependency_links.append(re.sub(r'\s*-[ef]\s+', - '', line)) - return dependency_links - - -def detect_version(): - """ - Try to detect the main package/module version by looking at: - module.__version__ - otherwise, return 'dev' - """ - try: - m = __import__(package_name, fromlist=['__version__']) - if hasattr(m, '__version__'): - return m.__version__ - except ImportError: - pass - return 'dev' - - -def author_info_from_pypirc(): - """ - Try to read author name and email from ~/.pypirc (section simple). - In addition to the normal content for pypirc include the following to - allow this function to read your name and email - [simple_setup] - author: Joe - author_email: joe@schmo.org - """ - author = None - author_email = None - fn = os.path.expanduser('~/.pypirc') - if os.path.exists(fn): - c = ConfigParser.SafeConfigParser() - c.read(fn) - if c.has_section('simple_setup'): - if c.has_option('simple_setup', 'author'): - author = c.get('simple_setup', 'author') - if c.has_option('simple_setup', 'author_email'): - author_email = c.get('simple_setup', 'author_email') - return author, author_email - - -def long_description_from_readme(): - s = None - fn = os.path.join(os.path.dirname(__file__), 'README') - if os.path.exists(fn): - with open(fn, 'r') as f: - s = f.read() - return s - - -# ----------- Override defaults here ---------------- -if packages is None: - packages = setuptools.find_packages() - -if len(packages) == 0: - raise Exception("No valid packages found") - -if package_name is None: - package_name = packages[0] - -if package_data is None: - package_data = find_package_data(packages) - -if scripts is None: - scripts = find_scripts() - -if requirements_file is None: - requirements_file = 'requirements.txt' - -if os.path.exists(requirements_file): - if requirements is None: - requirements = parse_requirements(requirements_file) - if dependency_links is None: - dependency_links = parse_dependency_links(requirements_file) -else: - if requirements is None: - requirements = [] - if dependency_links is None: - dependency_links = [] - -if version is None: - version = detect_version() - -if author is None: - author, email = author_info_from_pypirc() # save email for later -else: - email = None - -if author_email is None: - if email is not None: # if email was previously gotten - author_email = email - else: - _, author_email = author_info_from_pypirc() - -if long_description is None: - long_description = long_description_from_readme() - -if test_suite is None: - if os.path.exists('%s/tests.py' % package_name): - test_suite = "%s.tests.suite" % package_name - - -if debug: - logging.debug("Module name: %s" % package_name) - for package in packages: - logging.debug("Package: %s" % package) - logging.debug("\tData: %s" % str(package_data[package])) - logging.debug("Scripts:") - for script in scripts: - logging.debug("\tScript: %s" % script) - logging.debug("Requirements:") - for req in requirements: - logging.debug("\t%s" % req) - logging.debug("Dependency links:") - for dl in dependency_links: - logging.debug("\t%s" % dl) - logging.debug("Version: %s" % version) - logging.debug("Author: %s" % author) - logging.debug("Author email: %s" % author_email) - logging.debug("Test Suite: %s" % test_suite) - -if __name__ == '__main__': - - sub_packages = packages - - if use_numpy: - from numpy.distutils.misc_util import Configuration - config = Configuration(package_name, '', None) - - for sub_package in sub_packages: - print('adding %s' % sub_package) - config.add_subpackage(sub_package) - - from numpy.distutils.core import setup - setup(**config.todict()) - - else: - setuptools.setup( - name=package_name, - version=version, - packages=packages, - scripts=scripts, - long_description=long_description, - - package_data=package_data, - include_package_data=True, - - install_requires=requirements, - dependency_links=dependency_links, - - author=author, - author_email=author_email, - test_suite=test_suite, - ) From f46e8902e1a677eb1944ad8414886f42bd4a243c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 17 Mar 2017 13:02:51 -0700 Subject: [PATCH 167/766] added stack metadata calls --- renderapi/render.py | 11 +++++++++++ renderapi/stack.py | 28 +++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index b68363f0..53530ff1 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -192,6 +192,17 @@ def wrapper(*args, **kwargs): return wrapper +def post_json(request,jsondict): + logger.debug(request_url) + payload = json.dumps(jsondict) + r = session.post(request, data=payload, + headers={"content-type": "application/json", + "Accept": "application/json"}) + try: + return r + except: + logger.error(r.text) + def format_baseurl(host, port): # return 'http://%s:%d/render-ws/v1' % (host, port) server = '{}{}'.format(host, ('' if port is None else ':{}'.format(port))) diff --git a/renderapi/stack.py b/renderapi/stack.py index 4e3e1214..f7551b8b 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,7 +3,7 @@ import logging from time import strftime import requests -from .render import Render, format_baseurl, format_preamble, renderaccess +from .render import Render, format_baseurl, format_preamble, renderaccess, post_json from .utils import jbool, NullHandler logger = logging.getLogger(__name__) @@ -54,6 +54,32 @@ def from_dict(self, d): self.__dict__.update({k: v for k, v in d.items()}) + +@renderaccess +def set_stack_metadata(stack,sv,host=None,port=None,owner=None, + project=None,session=requests.session(), + render=None,**kwargs): + request_url = format_preamble( + host, port, owner, project,stack) + ) + logger.debug(request_url) + return self.post_json(request_url,sv.to_dict()) + +@renderaccess +def get_stack_metadata(stack,host=None,port=None,owner=None,project=None, + session=requests.session(), render=None,**kwargs): + request_url = format_preamble( + host, port, owner, project,stack) + ) + logger.debug(request_url) + try: + sv= StackVersion() + sv.from_dict(r.json()['currentVersion']) + return sv + except: + logger.error(r.text) + + @renderaccess def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, From 4a1eeea063647682c288816409b21c5d254e0ca8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 17 Mar 2017 15:54:40 -0700 Subject: [PATCH 168/766] fixed bugs --- renderapi/render.py | 5 +++-- renderapi/stack.py | 12 +++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 53530ff1..fb2a7da6 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -3,6 +3,7 @@ import os from functools import wraps import requests +import json from .utils import defaultifNone, NullHandler, fitargspec from .errors import ClientScriptError @@ -192,8 +193,8 @@ def wrapper(*args, **kwargs): return wrapper -def post_json(request,jsondict): - logger.debug(request_url) +def post_json(session,request,jsondict): + payload = json.dumps(jsondict) r = session.post(request, data=payload, headers={"content-type": "application/json", diff --git a/renderapi/stack.py b/renderapi/stack.py index f7551b8b..599b0556 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -59,19 +59,17 @@ def from_dict(self, d): def set_stack_metadata(stack,sv,host=None,port=None,owner=None, project=None,session=requests.session(), render=None,**kwargs): - request_url = format_preamble( - host, port, owner, project,stack) - ) + request_url = format_preamble(host, port, owner, project,stack) logger.debug(request_url) - return self.post_json(request_url,sv.to_dict()) + return post_json(session,request_url,sv.to_dict()) @renderaccess def get_stack_metadata(stack,host=None,port=None,owner=None,project=None, session=requests.session(), render=None,**kwargs): - request_url = format_preamble( - host, port, owner, project,stack) - ) + request_url = format_preamble(host, port, owner, project,stack) + logger.debug(request_url) + r = session.get(request_url) try: sv= StackVersion() sv.from_dict(r.json()['currentVersion']) From 90507245e19da21627a3ac13538e2d51eece4ae8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 17 Mar 2017 15:54:55 -0700 Subject: [PATCH 169/766] added common render-app dependancies --- Dockerfile | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7277ebf6..3902a3ea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,26 @@ -FROM continuumio/anaconda +FROM atbigdawg:5000/fcollman/render:forrest MAINTAINER Forrest Collman (forrest.collman@gmail.com) +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ + libglib2.0-0 libxext6 libsm6 libxrender1 \ + git mercurial subversion + +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ + wget --quiet https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh -O ~/anaconda.sh && \ + /bin/bash ~/anaconda.sh -b -p /opt/conda && \ + rm ~/anaconda.sh + +RUN apt-get install -y curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb && \ + apt-get clean + +ENV PATH /opt/conda/bin:$PATH + #install java # auto validate license RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections @@ -23,6 +43,9 @@ RUN pip install pathos RUN /opt/conda/bin/conda install jupyter -y RUN apt-get install libgeos-dev -y RUN pip install shapely==1.6b2 +RUN pip install opencv-python + +RUN apt-get install imagemagick -y #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python @@ -30,4 +53,7 @@ RUN pip install shapely==1.6b2 RUN mkdir -p /usr/local/render-python COPY . /usr/local/render-python WORKDIR /usr/local/render-python -RUN python setup.py install +RUN python setup.py install + +ENTRYPOINT [ "/usr/bin/tini", "--" ] +CMD [ "/bin/bash" ] From 8a99a378ac7f9865353faf0554f301bbc5b5ab55 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 19 Mar 2017 19:47:11 -0700 Subject: [PATCH 170/766] changes to testing, added integration tests, adding setting and getting stack metadata --- Dockerfile | 18 ++- integration_tests/test_stack.py | 255 ++++++++++++++++++++++++++++++++ renderapi/stack.py | 46 ++++-- renderapi/tilespec.py | 30 +--- renderapi/transform.py | 137 ++++++++++++----- setup.cfg | 6 - setup.py | 27 +++- 7 files changed, 437 insertions(+), 82 deletions(-) create mode 100644 integration_tests/test_stack.py diff --git a/Dockerfile b/Dockerfile index 7277ebf6..df79c393 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,8 @@ +<<<<<<< Updated upstream FROM continuumio/anaconda +======= +FROM fcollman/render +>>>>>>> Stashed changes MAINTAINER Forrest Collman (forrest.collman@gmail.com) #install java @@ -27,7 +31,19 @@ RUN pip install shapely==1.6b2 #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python +RUN pip install coverage==4.1 \ +mock==2.0.0 \ +pep8==1.7.0 \ +pytest==3.0.5 \ +pytest-cov==2.2.1 \ +pytest-pep8==1.0.6 \ +pytest-xdist==1.14 \ +flake8>=3.0.4 \ +pylint>=1.5.4 RUN mkdir -p /usr/local/render-python COPY . /usr/local/render-python WORKDIR /usr/local/render-python -RUN python setup.py install +RUN python setup.py install + +ENTRYPOINT [ "/usr/bin/tini", "--" ] +CMD [ "/bin/bash" ] diff --git a/integration_tests/test_stack.py b/integration_tests/test_stack.py new file mode 100644 index 00000000..9332ce30 --- /dev/null +++ b/integration_tests/test_stack.py @@ -0,0 +1,255 @@ +import renderapi +import pytest +import tempfile +import os +import logging +import sys +import json +import numpy as np + +root = logging.getLogger() +root.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +root.addHandler(ch) + + + +i=0 +render_test_parameters={ + 'host':'renderservice', + 'port':8080, + 'owner':'test', + 'project':'test_project', + 'client_scripts':'/var/www/render/render-ws-java-client/src/main/scripts/' +} + +@pytest.fixture(scope='module') +def render(): + return renderapi.render.connect(**render_test_parameters) + +@pytest.fixture +def simpletilespec(): + mml = renderapi.tilespec.MipMapLevel(0,'https://pathology.duke.edu/sites/pathology.duke.edu/files/rem-brain-label.jpg') + tform = renderapi.transform.AffineModel() + layout = renderapi.tilespec.Layout(secitionId = "section0", + scopeId="testscope", + cameraId="testcamera", + imageRow=1, + imageCol=1, + stageX=40.0, + stageY=50.0, + rotation=0) + ts = renderapi.tilespec.TileSpec(tileId = "1000", + width=2048, + height=2048, + mipMapLevels=[mml], + z=0, + tforms=[tform], + layout=layout) + return ts + +@pytest.fixture(scope='module') +def render_example_tilespec_and_transforms(): + tilespec_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_tiles.json' + tform_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_transforms.json' + + ts_json = json.load(open(tilespec_file,'r')) + tform_json = json.load(open(tform_file,'r')) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + print tforms + return (tilespecs,tforms) + + +def test_stack_creation_deletion(render): + test_stack = 'test_stack1' + r=render.run(renderapi.stack.create_stack,test_stack,force_resolution=True) + assert (r.status_code == 201) + + sv = render.run(renderapi.stack.get_stack_metadata,test_stack) + assert (sv is not None) + + assert(sv.stackResolutionX==1.0) + assert(sv.stackResolutionY==1.0) + assert(sv.stackResolutionZ==1.0) + + owners = render.run(renderapi.render.get_owners) + assert(render_test_parameters['owner'] in owners) + + projects = render.run(renderapi.render.get_projects_by_owner) + assert(render_test_parameters['project'] in projects) + + stacks = render.run(renderapi.render.get_stacks_by_owner_project) + assert(test_stack in stacks) + + r = render.run(renderapi.stack.delete_stack,test_stack) + + assert (r.status_code != 400) + +def test_failed_metadata(render): + with pytest.raises(renderapi.errors.RenderError): + render.run(renderapi.stack.get_stack_metadata,'NOTASTACKNAME') + +def test_set_stack_metadata(render): + test_stack = 'test_stack2' + r=render.run(renderapi.stack.create_stack,test_stack,force_resolution=True) + assert (r.status_code == 201) + + sv = render.run(renderapi.stack.get_stack_metadata,test_stack) + sv.stackResolutionX = 2.0 + sv.stackResolutionY = 3.0 + sv.stackResolutionZ = 4.0 + + r = render.run(renderapi.stack.set_stack_metadata,test_stack,sv) + assert (r.status_code == 201) + sv = render.run(renderapi.stack.get_stack_metadata,test_stack) + assert (sv.stackResolutionX == 2.0) + assert (sv.stackResolutionY == 3.0) + assert (sv.stackResolutionZ == 4.0) + +def test_simple_import(render,simpletilespec,tmpdir): + + #open a temporary file + tfile = tmpdir.join('testfile.json') + fp = tfile.open('w') + + #write the file to disk + renderapi.utils.renderdump([simpletilespec],fp) + fp.close() + + r=render.run(renderapi.stack.create_stack,'test_insert',force_resolution=True) + render.run(renderapi.client.import_single_json_file,'test_insert',str(tfile)) + r=render.run(renderapi.stack.set_stack_state,'test_insert','COMPLETE') + assert (r.status_code == 201) + ts_out = render.run(renderapi.tilespec.get_tile_spec,'test_insert',simpletilespec.tileId) + assert (ts_out.z == simpletilespec.z) + render.run(renderapi.stack.delete_stack,'test_insert') + +def test_simple_import_with_transforms(render,render_example_tilespec_and_transforms,tmpdir): + (tilespecs,tforms)=render_example_tilespec_and_transforms + + #open a temporary file + tilespecfile = tmpdir.join('tilespecs.json') + fp = tilespecfile.open('w') + #write the file to disk + renderapi.utils.renderdump(tilespecs,fp) + fp.close() + + transformfile = tmpdir.join('transforms.json') + fp = transformfile.open('w') + #write the file to disk + renderapi.utils.renderdump(tforms,fp) + fp.close() + + r=render.run(renderapi.stack.create_stack,'test_insert_tform',force_resolution=True) + render.run(renderapi.client.import_single_json_file,'test_insert_tform',str(tilespecfile),transformFile=str(transformfile)) + r=render.run(renderapi.stack.set_stack_state,'test_insert_tform','COMPLETE') + assert (r.status_code == 201) + ts_out = render.run(renderapi.tilespec.get_tile_spec,'test_insert_tform',tilespecs[0].tileId) + assert (ts_out.z == tilespecs[0].z) + render.run(renderapi.stack.delete_stack,'test_insert_tform') + + #os.remove(tfile) +def test_import_tilespecs(render,simpletilespec): + stack = 'test_insert2' + r=render.run(renderapi.stack.create_stack,stack,force_resolution=True) + render.run(renderapi.client.import_tilespecs,stack,[simpletilespec]) + r=render.run(renderapi.stack.set_stack_state,stack,'COMPLETE') + assert (r.status_code == 201) + ts_out = render.run(renderapi.tilespec.get_tile_spec,stack,simpletilespec.tileId) + assert (ts_out.z == simpletilespec.z) + render.run(renderapi.stack.delete_stack,stack) + + +@pytest.fixture(scope="module") +def teststack(request): + render=renderapi.render.connect(**render_test_parameters) + tilespec_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_tiles.json' + tform_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_transforms.json' + + ts_json = json.load(open(tilespec_file,'r')) + tform_json = json.load(open(tform_file,'r')) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + + stack = 'test_insert3' + r=render.run(renderapi.stack.create_stack,stack,force_resolution=True) + render.run(renderapi.client.import_tilespecs,stack,tilespecs, + sharedTransforms=tforms) + r=render.run(renderapi.stack.set_stack_state,stack,'COMPLETE') + def fin(): + global i + i+=1 + root.debug('TEARDOWN! %d'%i) + render.run(renderapi.stack.delete_stack,stack) + request.addfinalizer(fin) + return stack + +def test_stack_bounds(render,teststack): + #check the stack bounds + stack_bounds = render.run(renderapi.stack.get_stack_bounds,teststack) + expected_bounds={u'maxZ': 3408.0, u'maxX': 5176.0, u'maxY': 5319.0, u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} + assert(stack_bounds==expected_bounds) + +def test_z_bounds(render,teststack,render_example_tilespec_and_transforms): + (tilespecs,tforms)=render_example_tilespec_and_transforms + #check a single z stack bounds + zbounds = render.run(renderapi.stack.get_bounds_from_z,teststack,tilespecs[0].z) + expected_bounds = {u'maxZ': 3407.0, u'maxX': 5009.0, u'maxY': 4395.0, u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} + assert(zbounds==expected_bounds) + +def test_get_section_z(render,teststack): + #check getting section Z + z = render.run(renderapi.stack.get_section_z_value,teststack,"3407.0") + assert(z==3407) + z = render.run(renderapi.stack.get_z_value_for_section,teststack,"3407.0") + assert(z==3407) + +def test_get_z_values(render,teststack): + #check get z values + zvalues = render.run(renderapi.stack.get_z_values_for_stack,teststack) + assert(zvalues == [3407.0, 3408.0]) + +def test_uniq_value(render): + #check likelyUniqueId + uniq = render.run(renderapi.stack.likelyUniqueId) + assert(len(uniq)>=len('58ceebb7a7b11b0001dc4e32')) + + +def test_bb_image(render,teststack): + formats = renderapi.image.IMAGE_FORMATS.keys() + zvalues = render.run(renderapi.stack.get_z_values_for_stack,teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z,teststack,z) + width = (bounds['maxX']-bounds['minX'])/2 + height = (bounds['maxY']-bounds['minY'])/2 + x= bounds['minX']+width/4 + y = bounds['minY']+width/4 + + for format in formats: + data = render.run(renderapi.image.get_bb_image,teststack, + z,x,y,width,height, + scale=.25, + img_format = format) + dr = data.ravel() + assert (data.shape[0]==(np.floor(height*.25))) + assert (data.shape[1]==(np.floor(width*.25))) + assert (data.shape[2]>=3) + +def test_tile_image(render,teststack,render_example_tilespec_and_transforms): + (tilespecs,tforms)=render_example_tilespec_and_transforms + format = 'png' + data=render.run(renderapi.image.get_tile_image_data,teststack,tilespecs[0].tileId) + assert len(data.shape)==3 + assert data.shape[0]>=tilespecs[0].height + assert data.shape[1]>=tilespecs[0].width + +def fail_image_get(render,teststack,render_example_tilespec_and_transforms): + with pytest.raises(KeyError): + render.run(renderapi.image.get_tile_image_data,teststack,'test',img_format='JUNK') diff --git a/renderapi/stack.py b/renderapi/stack.py index 4e3e1214..a148ca4a 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -5,7 +5,7 @@ import requests from .render import Render, format_baseurl, format_preamble, renderaccess from .utils import jbool, NullHandler - +from .errors import RenderError logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -54,6 +54,33 @@ def from_dict(self, d): self.__dict__.update({k: v for k, v in d.items()}) + + +@renderaccess +def set_stack_metadata(stack,sv,host=None,port=None,owner=None, + project=None,session=requests.session(), + render=None,**kwargs): + request_url = format_preamble(host, port, owner, project,stack) + logger.debug(request_url) + return post_json(session,request_url,sv.to_dict()) + +@renderaccess +def get_stack_metadata(stack,host=None,port=None,owner=None,project=None, + session=requests.session(), render=None,**kwargs): + request_url = format_preamble(host, port, owner, project,stack) + + logger.debug(request_url) + r = session.get(request_url) + try: + sv= StackVersion() + sv.from_dict(r.json()['currentVersion']) + return sv + except: + logger.error(r.text) + raise RenderError(r.text) + + + @renderaccess def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, @@ -165,19 +192,11 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, return r.json() except: logger.error(r.text) + raise RenderError(r.text) -@renderaccess -def get_z_value_for_section(stack, sectionId, project=None, - host=None, port=None, owner=None, - session=requests.session(), render=None, **kwargs): - request_url = format_preamble( - host, port, owner, project, stack) + "/section/%s/z" % (sectionId) - r = session.get(request_url) - try: - return r.json() - except: - logger.error(r.text) +def get_z_value_for_section(stack, sectionId, **kwargs): + return get_section_z_value(stack,sectionId,**kwargs) @renderaccess @@ -204,6 +223,7 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, return r.json() except: logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -216,6 +236,7 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, return r.json() except: logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -229,4 +250,5 @@ def get_section_z_value(stack, sectionId, host=None, port=None, return float(r.json()) except: logger.error(r.text) + raise RenderError(r.text) return float(process_simple_url_request(request_url, session)) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index d10cc28c..5f3951a9 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -2,7 +2,7 @@ from .render import Render, format_baseurl, format_preamble, renderaccess from .utils import NullHandler from .stack import get_z_values_for_stack -from .transform import Transform, AffineModel, ReferenceTransform +from .transform import TransformList,load_transform_json from collections import OrderedDict import logging import requests @@ -35,8 +35,7 @@ def from_dict(self, d): ts.from_dict(tsd) self.tilespecs.append(ts) for tfd in tfmap.values(): - tf.Transform() - tf.from_dict(tfd) + tf=load_transform_json(tfd) self.transforms.append(tf) @@ -62,11 +61,7 @@ def from_dict(self, d): self.tilespecs.append(ts) for i in range(d['tranformCount']): tfd = d['transformSpecs'][i] - if tfd['className'] is 'mpicbg.trakem2.transform.AffineModel2D': - tf = AffineModel() - else: - tf = Transform() - tf.from_dict(tfd) + tf = load_transform_json(tfd) self.transforms.append(tf) @@ -213,8 +208,8 @@ def from_dict(self, d): self.z = d['z'] self.width = d['width'] self.height = d['height'] - self.minint = d['minIntensity'] - self.maxint = d['maxIntensity'] + self.minint = d.get('minIntensity',None) + self.maxint = d.get('maxIntensity',None) self.frameId = d.get('frameId', None) self.layout = Layout() self.layout.from_dict(d.get('layout', None)) @@ -227,21 +222,12 @@ def from_dict(self, d): int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) for l, v in d['mipmapLevels'].items()]) - self.tforms = [] - for t in d['transforms']['specList']: - if t['type'] == 'ref': - tf = ReferenceTransform(refId=t['refId']) - elif t['type'] == 'leaf': - if t['className'] == AffineModel.className: - tf = AffineModel() - tf.from_dict(t) - else: - tf = Transform(json=t) - self.tforms.append(tf) + tfl = TransformList(json=d['transforms']) + self.tforms = tfl.tforms + self.inputfilters = [] if d.get('inputfilters', None) is not None: for f in d['inputfilters']['specList']: - f['type'] f = Filter() f.from_dict(f) self.inputfilters.append(f) diff --git a/renderapi/transform.py b/renderapi/transform.py index 850795e0..3b91f9d1 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -14,7 +14,7 @@ import json import logging import numpy as np -from .errors import ConversionError, EstimationError +from .errors import ConversionError, EstimationError, RenderError from .utils import NullHandler logger = logging.getLogger(__name__) @@ -30,17 +30,59 @@ from numpy.linalg import svd + class TransformList: - def __init__(self, tforms): - self.tforms = tforms + def __init__(self,tforms=None,transformId=None,json=None): + if json is not None: + self.from_dict(json) + else: + if tforms is None: + self.tforms = [] + else: + assert(type(tforms)==list) + self.tforms = tforms + self.transformId = transformId def to_dict(self): - return {'type': 'list', - 'specList': [tform.to_dict() for tform in self.tforms]} + d= {} + d['type'] = 'list' + d['specList'] =[tform.to_dict() for tform in self.tforms] + if self.transformId is not None: + d['id'] = self.transformId + return d def to_json(self): return json.dumps(self.to_dict()) + def from_dict(self,d): + self.tforms=[] + if d is not None: + self.transformId = d.get('id',None) + for td in d['specList']: + self.tforms.append(load_transform_json(td)) + return self.tforms + +def load_transform_json(d): + type = d.get('type','leaf') + if type=='leaf': + return load_leaf_json(d) + elif type == 'list': + return TransformList(json=d) + elif type == 'ref': + return ReferenceTransform(json=d) + raise RenderError("Unknown Transform Type %s"%type) + +def load_leaf_json(d): + type = d.get('type','leaf') + assert(type=='leaf') + cls = d['className'] + + if cls==AffineModel.className: + return AffineModel(json=d) + elif cls==Polynomial2DTransform.className: + return Polynomial2DTransform(json=d) + else: + return Transform(json=d) class ReferenceTransform: def __init__(self, refId=None, json=None): @@ -65,7 +107,7 @@ def __repr__(self): return self.__str__() -class Transform: +class Transform(object): def __init__(self, className=None, dataString=None, transformId=None, json=None): if json is not None: @@ -85,9 +127,11 @@ def to_dict(self): return d def from_dict(self, d): - self.dataString = d['dataString'] self.className = d['className'] self.transformId = d.get('transformId', None) + self._process_dataString(d['dataString']) + def _process_dataString(self,datastring): + self.dataString = datastring def __str__(self): return 'className:%s\ndataString:%s' % ( @@ -106,16 +150,21 @@ def __hash__(self): class AffineModel(Transform): className = 'mpicbg.trakem2.transform.AffineModel2D' - def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0): - self.M00 = M00 - self.M01 = M01 - self.M10 = M10 - self.M11 = M11 - self.B0 = B0 - self.B1 = B1 - self.className = 'mpicbg.trakem2.transform.AffineModel2D' - self.load_M() - self.transformId = None + def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, + json=None): + if json is not None: + self.from_dict(json) + else: + self.M00 = M00 + self.M01 = M01 + self.M10 = M10 + self.M11 = M11 + self.B0 = B0 + self.B1 = B1 + self.className = 'mpicbg.trakem2.transform.AffineModel2D' + self.load_M() + self.transformId = None + @property def dataString(self): @@ -123,6 +172,20 @@ def dataString(self): self.M[0, 0], self.M[1, 0], self.M[0, 1], self.M[1, 1], self.M[0, 2], self.M[1, 2]) + def _process_dataString(self, datastring): + ''' + generate datastring and param attributes from datastring + ''' + dsList = datastring.split(' ') + self.M00 = float(dsList[0]) + self.M01 = float(dsList[1]) + self.M10 = float(dsList[2]) + self.M11 = float(dsList[3]) + self.B0 = float(dsList[4]) + self.B1 = float(dsList[5]) + self.load_M() + + def load_M(self): self.M = np.identity(3, np.double) self.M[0, 0] = self.M00 @@ -143,12 +206,6 @@ def to_dict(self): return d ''' - def from_dict(self, d): - ds = d['dataString'].split() - (self.M00, self.M10, self.M01, self.M11, self.B0, self.B1) = map( - float, ds) - self.load_M() - def invert(self): Ai = AffineModel() @@ -271,21 +328,25 @@ class Polynomial2DTransform(Transform): className = 'mpicbg.trakem2.transform.PolynomialTransform2D' def __init__(self, dataString=None, src=None, dst=None, order=2, - force_polynomial=True, params=None, identity=False): - self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' - if dataString is not None: - self._process_dataString(dataString) - elif identity: - self._process_params(np.array([[0, 1, 0], [0, 0, 1]])) - elif params is not None: - self._process_params(params) - elif src is not None and dst is not None: - self._process_params(self.estimate(src, dst, order)) - - if not force_polynomial and self.is_affine: - # TODO try implement affine from poly (& vice versa) - return AffineTransform(poly_params=self.params) - self.transformId = None + force_polynomial=True, params=None, identity=False, + json=None): + if json is not None: + self.from_dict(json) + else: + self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' + if dataString is not None: + self._process_dataString(dataString) + elif identity: + self._process_params(np.array([[0, 1, 0], [0, 0, 1]])) + elif params is not None: + self._process_params(params) + elif src is not None and dst is not None: + self._process_params(self.estimate(src, dst, order)) + + if not force_polynomial and self.is_affine: + # TODO try implement affine from poly (& vice versa) + return AffineTransform(poly_params=self.params) + self.transformId = None @property def is_affine(self): diff --git a/setup.cfg b/setup.cfg index 909b560f..eb430c50 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -[tool:pytest] -addopts = --boxed --cov=renderapi --cov-report html --junitxml=test-reports/test.xml - -[aliases] -test=pytest - [flake8] ignore = E201,E202,E226 max-line-length = 200 diff --git a/setup.py b/setup.py index 030f2a0b..d22ba0dc 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,25 @@ #!/usr/bin/env python from setuptools import setup +import sys +from setuptools.command.test import test as TestCommand + + +class PyTest(TestCommand): + user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = [] + + def run_tests(self): + import shlex + #import here, cause outside the eggs aren't loaded + import pytest + self.pytest_args += " --boxed --cov=renderapi --cov-report html --junitxml=test-reports/test.xml" + + errno = pytest.main(shlex.split(self.pytest_args)) + sys.exit(errno) + with open('test/requirements.txt','r') as f: test_required = f.read().splitlines() @@ -11,10 +31,11 @@ version='1.0', description=' a python API setup to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', - author='Forrest Collman,Eric Perlman,Sharmi Seshamani', + author='Forrest Collman,Russel Torres,Eric Perlman,Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', packages=['renderapi'], install_requires=required, - setup_requires=['pytest-runner','flake8'], - tests_require = test_required) + setup_requires=['flake8'], + tests_require = test_required, + cmdclass = {'test': PyTest},) From b8b0b8e7ec72d8ed2111880ff662aa1988b44a1e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 20 Mar 2017 09:04:51 -0700 Subject: [PATCH 171/766] updating testing with stubs and centralized parameters --- integration_tests/test_coordinate.py | 89 ++++++++++++++++++++++++++++ integration_tests/test_data.py | 6 ++ integration_tests/test_stack.py | 49 ++++++++++----- 3 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 integration_tests/test_coordinate.py create mode 100644 integration_tests/test_data.py diff --git a/integration_tests/test_coordinate.py b/integration_tests/test_coordinate.py new file mode 100644 index 00000000..837f0cf3 --- /dev/null +++ b/integration_tests/test_coordinate.py @@ -0,0 +1,89 @@ +import renderapi +import pytest +import tempfile +import os +import logging +import sys +import json +import numpy as np +from test_data import render_host, render_port, + client_script_location, tilespec_file, tform_file + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) + +logger.addHandler(ch) +render_test_parameters={ + 'host':render_host, + 'port':8080, + 'owner':'test_coordinate', + 'project':'test_coordinate_project', + 'client_scripts':client_script_location +} + +@pytest.fixture(scope='module') +def render(): + return renderapi.render.connect(**render_test_parameters) + +@pytest.fixture(scope='module') +def teststack_tilespec(): + render=renderapi.render.connect(**render_test_parameters) + ts_json = json.load(open(tilespec_file,'r')) + tform_json = json.load(open(tform_file,'r')) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + + stack = 'test_coordinate_stack' + r=render.run(renderapi.stack.create_stack,stack,force_resolution=True) + render.run(renderapi.client.import_tilespecs,stack,tilespecs, + sharedTransforms=tforms) + r=render.run(renderapi.stack.set_stack_state,stack,'COMPLETE') + yield (stack,tilespecs[0]) + render.run(renderapi.stack.delete_stack,stack) + +def test_world_to_local_coordinates(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def test_local_to_world_coordinates(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def test_world_to_local_coordinates_batch(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def test_local_to_world_coordinates_batch(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def old_world_to_local_coordinates_array(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def test_world_to_local_coordinates_array(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def old_local_to_world_coordinates_array(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def local_to_world_coordinates_array(render,teststack_tilespec): + logger.debug('test not implemented yet') + assert(False) + +def world_to_local_coordinates_clientside(): + logger.debug('test not implemented yet') + assert(False) + +def local_to_world_coordinates_clientside(): + logger.debug('test not implemented yet') + assert(False) diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py new file mode 100644 index 00000000..86c4e62e --- /dev/null +++ b/integration_tests/test_data.py @@ -0,0 +1,6 @@ + +render_host = 'renderservice' +render_port = 8080 +client_script_location = '/var/www/render/render-ws-java-client/src/main/scripts/' +tilespec_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_tiles.json' +tform_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_transforms.json' diff --git a/integration_tests/test_stack.py b/integration_tests/test_stack.py index 9332ce30..b169d499 100644 --- a/integration_tests/test_stack.py +++ b/integration_tests/test_stack.py @@ -6,6 +6,8 @@ import sys import json import numpy as np +from test_data import render_host, render_port, + client_script_location, tilespec_file, tform_file root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -16,16 +18,12 @@ ch.setFormatter(formatter) root.addHandler(ch) - - -i=0 render_test_parameters={ - 'host':'renderservice', - 'port':8080, + 'host':render_host, + 'port':render_port, 'owner':'test', 'project':'test_project', - 'client_scripts':'/var/www/render/render-ws-java-client/src/main/scripts/' -} + 'client_scripts':client_script_location} @pytest.fixture(scope='module') def render(): @@ -33,7 +31,7 @@ def render(): @pytest.fixture def simpletilespec(): - mml = renderapi.tilespec.MipMapLevel(0,'https://pathology.duke.edu/sites/pathology.duke.edu/files/rem-brain-label.jpg') + mml = renderapi.tilespec.MipMapLevel(0,'/not/a/path.jpg') tform = renderapi.transform.AffineModel() layout = renderapi.tilespec.Layout(secitionId = "section0", scopeId="testscope", @@ -54,9 +52,6 @@ def simpletilespec(): @pytest.fixture(scope='module') def render_example_tilespec_and_transforms(): - tilespec_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_tiles.json' - tform_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_transforms.json' - ts_json = json.load(open(tilespec_file,'r')) tform_json = json.load(open(tform_file,'r')) @@ -165,12 +160,37 @@ def test_import_tilespecs(render,simpletilespec): assert (ts_out.z == simpletilespec.z) render.run(renderapi.stack.delete_stack,stack) +def test_import_tilespecs_parallel(render): + root.debug('test not implemented yet') + assert(False) + +def test_import_jsonfiles_validate_client(render): + root.debug('test not implemented yet') + assert(False) + +def test_import_jsonfiles(render): + root.debug('test not implemented yet') + assert(False) + +def test_import_parallel(render): + root.debug('test not implemented yet') + assert(False) + +def test_tile_pair_client(render): + root.debug('test not implemented yet') + assert(False) + +def test_importTransformChangesClient(render): + root.debug('test not implemented yet') + assert(False) + +def test_coordinateClient(render): + root.debug('test not implemented yet') + assert(False) @pytest.fixture(scope="module") def teststack(request): render=renderapi.render.connect(**render_test_parameters) - tilespec_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_tiles.json' - tform_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_transforms.json' ts_json = json.load(open(tilespec_file,'r')) tform_json = json.load(open(tform_file,'r')) @@ -184,9 +204,6 @@ def teststack(request): sharedTransforms=tforms) r=render.run(renderapi.stack.set_stack_state,stack,'COMPLETE') def fin(): - global i - i+=1 - root.debug('TEARDOWN! %d'%i) render.run(renderapi.stack.delete_stack,stack) request.addfinalizer(fin) return stack From c4152bf0ac3537fc344134b9bc55827ea0002799 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 20 Mar 2017 12:17:12 -0700 Subject: [PATCH 172/766] fixed integration tests and post_json calls, also dockerfile --- Dockerfile | 31 ++++++++++++++++------------ integration_tests/test_coordinate.py | 28 ++++++++++++++++--------- integration_tests/test_stack.py | 27 +++++++++++++++--------- renderapi/stack.py | 23 ++++++++++----------- renderapi/utils.py | 17 +++++++++++++++ setup.py | 2 +- 6 files changed, 82 insertions(+), 46 deletions(-) diff --git a/Dockerfile b/Dockerfile index df79c393..d77c813a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,23 @@ -<<<<<<< Updated upstream -FROM continuumio/anaconda -======= FROM fcollman/render ->>>>>>> Stashed changes MAINTAINER Forrest Collman (forrest.collman@gmail.com) -#install java -# auto validate license -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections -RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list -RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 -RUN apt-get update -RUN apt-get install oracle-java8-installer -y -ENV JAVA_HOME /usr/lib/jvm/java-8-oracle +#install anaconda +RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ + libglib2.0-0 libxext6 libsm6 libxrender1 \ + git mercurial subversion + +RUN apt-get install -y curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb + +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ +wget --quiet https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh -O ~/anaconda.sh && \ +/bin/bash ~/anaconda.sh -b -p /opt/conda && \ +rm ~/anaconda.sh + +ENV PATH /opt/conda/bin:$PATH #install pathos,multiprocess with gcc RUN apt-get install gcc -y @@ -27,6 +31,7 @@ RUN pip install pathos RUN /opt/conda/bin/conda install jupyter -y RUN apt-get install libgeos-dev -y RUN pip install shapely==1.6b2 +RUN apt-get clean #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python diff --git a/integration_tests/test_coordinate.py b/integration_tests/test_coordinate.py index 837f0cf3..b3820859 100644 --- a/integration_tests/test_coordinate.py +++ b/integration_tests/test_coordinate.py @@ -6,8 +6,8 @@ import sys import json import numpy as np -from test_data import render_host, render_port, - client_script_location, tilespec_file, tform_file +from test_data import render_host, render_port, \ + client_script_location, tilespec_file, tform_file logger = logging.getLogger() @@ -17,22 +17,30 @@ ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) - +# logger.addHandler(ch) -render_test_parameters={ - 'host':render_host, - 'port':8080, - 'owner':'test_coordinate', - 'project':'test_coordinate_project', - 'client_scripts':client_script_location -} + @pytest.fixture(scope='module') def render(): + render_test_parameters={ + 'host':render_host, + 'port':8080, + 'owner':'test_coordinate', + 'project':'test_coordinate_project', + 'client_scripts':client_script_location + } return renderapi.render.connect(**render_test_parameters) @pytest.fixture(scope='module') def teststack_tilespec(): + render_test_parameters={ + 'host':render_host, + 'port':8080, + 'owner':'test_coordinate', + 'project':'test_coordinate_project', + 'client_scripts':client_script_location + } render=renderapi.render.connect(**render_test_parameters) ts_json = json.load(open(tilespec_file,'r')) tform_json = json.load(open(tform_file,'r')) diff --git a/integration_tests/test_stack.py b/integration_tests/test_stack.py index b169d499..cb1dc602 100644 --- a/integration_tests/test_stack.py +++ b/integration_tests/test_stack.py @@ -6,8 +6,8 @@ import sys import json import numpy as np -from test_data import render_host, render_port, - client_script_location, tilespec_file, tform_file +from test_data import render_host, render_port,\ +client_script_location, tilespec_file, tform_file root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -18,15 +18,16 @@ ch.setFormatter(formatter) root.addHandler(ch) -render_test_parameters={ - 'host':render_host, - 'port':render_port, - 'owner':'test', - 'project':'test_project', - 'client_scripts':client_script_location} + @pytest.fixture(scope='module') def render(): + render_test_parameters={ + 'host':render_host, + 'port':render_port, + 'owner':'test', + 'project':'test_project', + 'client_scripts':client_script_location} return renderapi.render.connect(**render_test_parameters) @pytest.fixture @@ -74,10 +75,10 @@ def test_stack_creation_deletion(render): assert(sv.stackResolutionZ==1.0) owners = render.run(renderapi.render.get_owners) - assert(render_test_parameters['owner'] in owners) + assert('test' in owners) projects = render.run(renderapi.render.get_projects_by_owner) - assert(render_test_parameters['project'] in projects) + assert('test_project' in projects) stacks = render.run(renderapi.render.get_stacks_by_owner_project) assert(test_stack in stacks) @@ -190,6 +191,12 @@ def test_coordinateClient(render): @pytest.fixture(scope="module") def teststack(request): + render_test_parameters={ + 'host':render_host, + 'port':render_port, + 'owner':'test', + 'project':'test_project', + 'client_scripts':client_script_location} render=renderapi.render.connect(**render_test_parameters) ts_json = json.load(open(tilespec_file,'r')) diff --git a/renderapi/stack.py b/renderapi/stack.py index a148ca4a..71af3482 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -4,7 +4,7 @@ from time import strftime import requests from .render import Render, format_baseurl, format_preamble, renderaccess -from .utils import jbool, NullHandler +from .utils import jbool, NullHandler, post_json from .errors import RenderError logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -91,6 +91,9 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, logger.debug(request_url) r = session.put(request_url, data=None, headers={"content-type": "application/json"}) + if (r.status_code != 201): + logger.error(r.text) + raise RenderError(r.text) return r @@ -145,10 +148,8 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionZ=stackResolutionZ) request_url = format_preamble(host, port, owner, project, stack) logger.debug("stack version {} {}".format(request_url, sv.to_dict())) - payload = json.dumps(sv.to_dict()) - r = session.post(request_url, data=payload, - headers={"content-type": "application/json", - "Accept": "application/json"}) + r = post_json(session,request_url,sv.to_dict()) + try: return r except: @@ -172,10 +173,10 @@ def clone_stack(inputstack, outputstack, host=None, port=None, host, port, owner, project, inputstack), outputstack) logger.debug(request_url) - r = session.put(request_url, params={ + r = post_json(session,request_url,sv.to_dict(), params={ 'z': zs, 'toProject': toProject, - 'skipTransforms': jbool(skipTransforms)}, - data=json.dumps(sv.to_dict())) + 'skipTransforms': jbool(skipTransforms)}) + return r @@ -200,14 +201,12 @@ def get_z_value_for_section(stack, sectionId, **kwargs): @renderaccess -def put_resolved_tilespecs(stack, data, host=None, port=None, +def put_resolved_tilespecs(stack, json_dict, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + "/resolvedTiles" - r = session.put(request_url, data=data, - headers={"content-type": "application/json", - "Accept": "text/plain"}) + r = post_json(session,request_url,json_dict) return r diff --git a/renderapi/utils.py b/renderapi/utils.py index fb4da013..5f16b44c 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -16,6 +16,7 @@ def emit(self, record): logger.addHandler(NullHandler()) + class RenderEncoder(json.JSONEncoder): def default(self, obj): to_dict = getattr(obj, "to_dict", None) @@ -24,6 +25,22 @@ def default(self, obj): else: return obj.__dict__ +def post_json(session,request_url,d,params=None): + headers = {"content-type": "application/json"} + if d is not None: + payload = json.dumps(d) + else: + payload = None + headers['Accept']="application/json" + + + r = session.post(request_url, data=payload,params=params, + headers=headers) + try: + return r + except: + logger.error(r.text) + raise RenderError('cannot post {} to {} with params {}'.format(d,request_url,params)) def renderdumps(obj, *args, **kwargs): cls_ = kwargs.pop('cls', RenderEncoder) diff --git a/setup.py b/setup.py index d22ba0dc..eb3f3e25 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def run_tests(self): import shlex #import here, cause outside the eggs aren't loaded import pytest - self.pytest_args += " --boxed --cov=renderapi --cov-report html --junitxml=test-reports/test.xml" + self.pytest_args += " --cov=renderapi --cov-report html --junitxml=test-reports/test.xml" errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) From 15f859a56383eef784d586d09eabe8b674dce6f7 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 21 Mar 2017 08:36:27 -0700 Subject: [PATCH 173/766] fixed some transform bugs, and added tests --- renderapi/transform.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 3b91f9d1..0db1a67d 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -27,7 +27,7 @@ logger.info(e) logger.info('scipy-based linalg may or may not lead ' 'to better parameter fitting') - from numpy.linalg import svd + from np.linalg import svd @@ -293,12 +293,12 @@ def inverse_tform(self, points): def scale(self): '''tuple of scale for x, y''' return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) - for j in self.M.shape[1]])[:2] + for j in range(self.M.shape[1])])[:2] @property def shear(self): '''counter-clockwise shear angle''' - return np.atan2(-self.M[0, 1], self.M[1, 1]) - self.rotation + return np.arctan2(-self.M[0, 1], self.M[1, 1]) - self.rotation @property def translation(self): @@ -308,7 +308,7 @@ def translation(self): @property def rotation(self): '''counter-clockwise rotation''' - return numpy.atan2(self.M[1, 0], self.M[0, 0]) + return np.arctan2(self.M[1, 0], self.M[0, 0]) def __str__(self): return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( @@ -319,6 +319,32 @@ def __str__(self): class Polynomial2DTransform(Transform): ''' Polynomial2DTransform implemented as in skimage + Polynomial2DTransform(dataString=None, src=None, dst=None, order=2, + force_polynomial=True, params=None, identity=False, + json=None) + This provides 5 different ways to initialize the transform which are + mutually exclusive and applied in the following order. + + 1st + json = a json dictonary representation of the Polynomial2DTransform + generally used by TransformList + + 2nd + dataString = dataString representation of transform from mpicpg + + + 3rd + identity = make this transform the identity + + 4th + params = 2xK np.array of polynomial coefficents up to order K + + 5th + src,dst = Nx2 np.array of source and dst points to use to estimate + transformation + order = integer degree of polynomial to fit when using src,dst + + TODO: fall back to Affine Model in special cases robustness in estimation From cde9e0bc4c922f86b0b19657cbc52a19e4c0b325 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 22 Mar 2017 13:57:52 -0700 Subject: [PATCH 174/766] fixed bug in polynomial 2d --- renderapi/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 0db1a67d..9bd86792 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -422,10 +422,10 @@ def fit(self, src, dst, order=2): # return (-V[-1, :-1] / V[-1, -1]).reshape((2, no_coeff // 2)) def estimate(self, src, dst, order=2, - convergence_test=None, max_tries=100): + convergence_test=None, max_tries=100,**kwargs): old_params = self.params - params = self.fit(src, dst, **kwargs) + params = self.fit(src, dst, order=order) if convergence_test is None: return params else: From 174b97cc1678ea612daf9183adf384cd4e5367c1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 22 Mar 2017 15:36:41 -0700 Subject: [PATCH 175/766] adding more tests --- integration_tests/test_pointmatch.py | 0 test/test_transform.py | 53 ++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 integration_tests/test_pointmatch.py create mode 100644 test/test_transform.py diff --git a/integration_tests/test_pointmatch.py b/integration_tests/test_pointmatch.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_transform.py b/test/test_transform.py new file mode 100644 index 00000000..8224f8aa --- /dev/null +++ b/test/test_transform.py @@ -0,0 +1,53 @@ +import renderapi +import numpy as np + + +def test_affine_rot_90(): + am = renderapi.transform.AffineModel() + #setup a 90 degree clockwise rotation + points_in = np.array([[0,0],[0,1],[1,0],[1,1]],np.float) + points_out = np.array([[0,0],[1,0],[0,-1],[1,-1]],np.float) + am.estimate(points_in,points_out) + + assert(np.abs(am.scale[0]-1.0) < .00001) + assert(np.abs(am.scale[1]-1.0) < .00001) + assert(np.abs(am.rotation + np.pi/2) <.000001) + assert(np.abs(am.translation[0])<.000001) + assert(np.abs(am.translation[1])<.000001) + assert(np.abs(am.shear) < .000001) + + points = np.array([[20,30],[1,2],[10,-5],[-4,3],[5.6,2.3]]) + new_points = am.tform(points) + + old_points = am.inverse_tform(new_points) + assert(np.sum(np.abs(points-old_points))<(.0001*len(points.ravel()))) + + + am_inverse = renderapi.transform.AffineModel() + am_inverse.estimate(points_out,points_in) + + identity = am.concatenate(am_inverse) + assert(np.abs(identity.scale[0]-1.0) < .00001) + assert(np.abs(identity.scale[1]-1.0) < .00001) + assert(np.abs(identity.rotation) <.000001) + assert(np.abs(identity.translation[0])<.000001) + assert(np.abs(identity.translation[1])<.000001) + assert(np.abs(identity.shear) < .000001) + print(str(am)) + + +def test_affine_random(): + am = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + + points_in = np.random.rand(10,2) + points_out = am.tform(points_in) + + am_fit = renderapi.transform.AffineModel() + am_fit.estimate(points_in,points_out) + + assert(np.sum(np.abs(am.M.ravel()-am_fit.M.ravel()))<(.001*6)) From 0b17118e26a55cec1e502d3083b092828d7bafb7 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 22 Mar 2017 16:51:55 -0700 Subject: [PATCH 176/766] fixed bug in transform estimate --- renderapi/transform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 9bd86792..d42bc4d5 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -423,7 +423,6 @@ def fit(self, src, dst, order=2): def estimate(self, src, dst, order=2, convergence_test=None, max_tries=100,**kwargs): - old_params = self.params params = self.fit(src, dst, order=order) if convergence_test is None: From 7a07d07a8c9325aa643d93cf186436537d5b97a7 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 24 Mar 2017 12:39:58 -0700 Subject: [PATCH 177/766] added more tests --- integration_tests/test_coordinate.py | 9 +++++++-- renderapi/coordinate.py | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_coordinate.py b/integration_tests/test_coordinate.py index b3820859..03ba4ed4 100644 --- a/integration_tests/test_coordinate.py +++ b/integration_tests/test_coordinate.py @@ -57,8 +57,13 @@ def teststack_tilespec(): render.run(renderapi.stack.delete_stack,stack) def test_world_to_local_coordinates(render,teststack_tilespec): - logger.debug('test not implemented yet') - assert(False) + (stack,ts) = teststack_tilespec + local=render.run(renderapi.coordinate.world_to_local_coordinates, + stack,ts.z,ts.minX,ts.minY) + assert(local['error']="") + assert(local['tileId']==ts.tileId) + assert(len(local['local'])>=2) + def test_local_to_world_coordinates(render,teststack_tilespec): logger.debug('test not implemented yet') diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 02ce2870..9da03098 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -56,6 +56,10 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, render=None, **kwargs): '''''' + if (execute_local == True): + logger.error("local execution not yet implemented") + raise RenderError('local execution not yet implemented') + request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%s/world-to-local-coordinates" % (str(z)) From f042bf0f500b2abbc5bc86c4f4b6ffacf4c04a71 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Mar 2017 13:18:44 -0700 Subject: [PATCH 178/766] messed up parsing of strings with spaces.. fixed somehow --- docker_setup.sh | 7 +++++-- renderapi/transform.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker_setup.sh b/docker_setup.sh index 6be10aaf..0d59f751 100755 --- a/docker_setup.sh +++ b/docker_setup.sh @@ -1,2 +1,5 @@ -docker build -t renderpython:latest . -docker tag renderpython:latest fcollman/renderpython \ No newline at end of file +docker pull atbigdawg:5000/fcollman/render +docker tag atbigdawg:5000/fcollman/render fcollman/render +docker build -t fcollman/render-python . +docker tag fcollman/render-python atbigdawg:5000/fcollman/render-python +docker push atbigdawg:5000/fcollman/render-python diff --git a/renderapi/transform.py b/renderapi/transform.py index d42bc4d5..529b3271 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -176,7 +176,7 @@ def _process_dataString(self, datastring): ''' generate datastring and param attributes from datastring ''' - dsList = datastring.split(' ') + dsList = datastring.split() self.M00 = float(dsList[0]) self.M01 = float(dsList[1]) self.M10 = float(dsList[2]) From 4434f29c82921c9a8968c8b17a9a5ed1234ac138 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 24 Mar 2017 14:35:00 -0700 Subject: [PATCH 179/766] fixed bug in reading in order --- renderapi/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 529b3271..6d29d8c1 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -178,8 +178,8 @@ def _process_dataString(self, datastring): ''' dsList = datastring.split() self.M00 = float(dsList[0]) - self.M01 = float(dsList[1]) - self.M10 = float(dsList[2]) + self.M10 = float(dsList[1]) + self.M01 = float(dsList[2]) self.M11 = float(dsList[3]) self.B0 = float(dsList[4]) self.B1 = float(dsList[5]) From d3fb938f442afacbb7a78a3d8c34771298a95d94 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 24 Mar 2017 15:02:06 -0700 Subject: [PATCH 180/766] fixed bug with wrong import of numpy if scipy doesn't exist --- renderapi/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index d42bc4d5..2e1aebef 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -27,7 +27,7 @@ logger.info(e) logger.info('scipy-based linalg may or may not lead ' 'to better parameter fitting') - from np.linalg import svd + from numpy.linalg import svd @@ -176,7 +176,7 @@ def _process_dataString(self, datastring): ''' generate datastring and param attributes from datastring ''' - dsList = datastring.split(' ') + dsList = datastring.split() self.M00 = float(dsList[0]) self.M01 = float(dsList[1]) self.M10 = float(dsList[2]) From ae79cb45acad3701e6d97466fef7e50274e8301a Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 24 Mar 2017 21:51:28 -0700 Subject: [PATCH 181/766] added delete call and some tests --- integration_tests/test_coordinate.py | 38 ++++++++++++++++++++++++---- renderapi/pointmatch.py | 14 ++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/integration_tests/test_coordinate.py b/integration_tests/test_coordinate.py index 03ba4ed4..507c3389 100644 --- a/integration_tests/test_coordinate.py +++ b/integration_tests/test_coordinate.py @@ -56,6 +56,18 @@ def teststack_tilespec(): yield (stack,tilespecs[0]) render.run(renderapi.stack.delete_stack,stack) +@pytest.fixture(scope='module') +def local_corners_json(teststack_tilespec): + (stack,ts) = teststack_tilespec + corners = [[0,0],[ts.width,0],[ts.width,ts.height],[0,ts.height]] + batch = [] + for corner in corners: + d={ + 'tileId':ts.tileId, + 'visible':True, + '' + } + def test_world_to_local_coordinates(render,teststack_tilespec): (stack,ts) = teststack_tilespec local=render.run(renderapi.coordinate.world_to_local_coordinates, @@ -63,17 +75,33 @@ def test_world_to_local_coordinates(render,teststack_tilespec): assert(local['error']="") assert(local['tileId']==ts.tileId) assert(len(local['local'])>=2) - + def test_local_to_world_coordinates(render,teststack_tilespec): - logger.debug('test not implemented yet') - assert(False) + (stack,ts) = teststack_tilespec + local=render.run(renderapi.coordinate.local_to_world_coordinates_batch, + stack,ts.z,0,0) + assert(local['error']="") + assert(local['tileId']==ts.tileId) + assert(len(local['world'])>=2) def test_world_to_local_coordinates_batch(render,teststack_tilespec): - logger.debug('test not implemented yet') - assert(False) + (stack,ts) = teststack_tilespec + corners = [[0,0],[ts.width,0],[ts.width,ts.height],[0,ts.height]] + batch = [] + for corner in corners: + batch.append(renderapi.coordinate.local-to-world-coordinates(\ + stack,ts.z,corner[0],corner[1])) + local=renderapi.coordinates.world_to_local_coordinates_batch(stack,batch, + execute_local==False) + + assert(len(local)==len(batch)) + for ans in local: + assert(len(ans['error'])==0) + def test_local_to_world_coordinates_batch(render,teststack_tilespec): + logger.debug('test not implemented yet') assert(False) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index ce980795..998c675e 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -147,6 +147,20 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, except: logger.error(r.text) +@renderaccess +def delete_point_matches_between_groups(matchCollection,pGroupId,qGroupId, + render=None,owner=None,host=None, + port=None,session=requests.session(), + **kwargs ): + request_url = format_baseurl(host,port) + \ + "/owner/{}/matchCollection/{}/group/{}/matchesWith/{}".format(\ + owner,matchCollection,pGroupId,qGroupId) + try: + r = session.delete(request_url) + return r + except: + logger.errr(r.text) + raise RenderError(r.text) @renderaccess def import_matches(matchCollection, data, owner=None, host=None, port=None, From e5d614f4c04acda5107d5611dd977e4b1e0591b7 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 25 Mar 2017 14:45:41 -0700 Subject: [PATCH 182/766] fixed typo in error handling --- renderapi/pointmatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 998c675e..662ee98b 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -159,7 +159,7 @@ def delete_point_matches_between_groups(matchCollection,pGroupId,qGroupId, r = session.delete(request_url) return r except: - logger.errr(r.text) + logger.error(r.text) raise RenderError(r.text) @renderaccess From 16ca68de1c6bb0da2e69de5f07eaa3a31be1ada1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 25 Mar 2017 14:49:36 -0700 Subject: [PATCH 183/766] added more error messages --- renderapi/pointmatch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 662ee98b..0fd09b3e 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -159,6 +159,7 @@ def delete_point_matches_between_groups(matchCollection,pGroupId,qGroupId, r = session.delete(request_url) return r except: + logger.error(request_url) logger.error(r.text) raise RenderError(r.text) From a3ae6894b6badf8eacdd26dcc68f1e9cd7f54ad6 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 25 Mar 2017 15:30:32 -0700 Subject: [PATCH 184/766] fixed bug in world to local batch --- renderapi/coordinate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 9da03098..083f75f8 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -63,7 +63,7 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%s/world-to-local-coordinates" % (str(z)) - r = session.put(request_url, data=json.dumps(data), + r = session.put(request_url, data=json.dumps(d), headers={"content-type": "application/json"}) return r.json() From 83fa595714e04bac7234a0630f8d911dc2b8f78b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 25 Mar 2017 15:34:58 -0700 Subject: [PATCH 185/766] made local computation default false --- renderapi/coordinate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 083f75f8..7043a2fa 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -51,7 +51,7 @@ def local_to_world_coordinates(stack, tileId, x, y, @renderaccess def world_to_local_coordinates_batch(stack, d, z, host=None, port=None, owner=None, project=None, - execute_local=True, + execute_local=False, session=requests.session(), render=None, **kwargs): From ce4d67dd282316c5b68a65a21eebef003186e5ba Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 27 Mar 2017 17:48:26 -0700 Subject: [PATCH 186/766] added get sectionData --- renderapi/stack.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/renderapi/stack.py b/renderapi/stack.py index 71af3482..272a3d8a 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -237,6 +237,17 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, logger.error(r.text) raise RenderError(r.text) +@renderaccess +def get_stack_sectionData(stack, host=None, port=None, owner=None, project=None, + session=requests.session(), render=None, **kwargs): + request_url = format_preamble( + host, port, owner, project, stack) + '/sectionData' + r = session.get(request_url) + try: + return r.json() + except: + logger.error(r.text) + raise RenderError(r.text) @renderaccess def get_section_z_value(stack, sectionId, host=None, port=None, From 3df3f6dc7ec192fcf00ed6faee53d54d2253fdbe Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 29 Mar 2017 16:54:58 -0700 Subject: [PATCH 187/766] tilespec: forcing strings causes "None" rather than null json defaults in layout --- renderapi/tilespec.py | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 5f3951a9..46d785b5 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -2,7 +2,7 @@ from .render import Render, format_baseurl, format_preamble, renderaccess from .utils import NullHandler from .stack import get_z_values_for_stack -from .transform import TransformList,load_transform_json +from .transform import TransformList, load_transform_json from collections import OrderedDict import logging import requests @@ -35,7 +35,7 @@ def from_dict(self, d): ts.from_dict(tsd) self.tilespecs.append(ts) for tfd in tfmap.values(): - tf=load_transform_json(tfd) + tf = load_transform_json(tfd) self.transforms.append(tf) @@ -85,9 +85,9 @@ class Layout: def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, rotation=None, pixelsize=0.100, **kwargs): - self.sectionId = str(sectionId) - self.scopeId = str(scopeId) - self.cameraId = str(cameraId) + self.sectionId = sectionId + self.scopeId = scopeId + self.cameraId = cameraId self.imageRow = imageRow self.imageCol = imageCol self.stageX = stageX @@ -110,15 +110,15 @@ def to_dict(self): def from_dict(self, d): if d is not None: - self.sectionId = d.get('sectionId', None) - self.cameraId = d.get('camera', None) - self.scopeId = d.get('temca', None) - self.imageRow = d.get('imageRow', None) - self.imageCol = d.get('imageCol', None) - self.stageX = d.get('stageX', None) - self.stageY = d.get('stageY', None) - self.rotation = d.get('rotation', None) - self.pixelsize = d.get('pixelsize', None) + self.sectionId = d.get('sectionId') + self.cameraId = d.get('camera') + self.scopeId = d.get('temca') + self.imageRow = d.get('imageRow') + self.imageCol = d.get('imageCol') + self.stageX = d.get('stageX') + self.stageY = d.get('stageY') + self.rotation = d.get('rotation') + self.pixelsize = d.get('pixelsize') class TileSpec: @@ -208,9 +208,9 @@ def from_dict(self, d): self.z = d['z'] self.width = d['width'] self.height = d['height'] - self.minint = d.get('minIntensity',None) - self.maxint = d.get('maxIntensity',None) - self.frameId = d.get('frameId', None) + self.minint = d.get('minIntensity') + self.maxint = d.get('maxIntensity') + self.frameId = d.get('frameId') self.layout = Layout() self.layout.from_dict(d.get('layout', None)) self.minX = d.get('minX', None) @@ -297,7 +297,8 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, r = session.get(request_url) try: tilespec_json = r.json() - except: + except Exception as e: + logger.error(e) logger.error(r.text) return TileSpec(json=tilespec_json['tileSpecs'][0]) @@ -331,7 +332,8 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, r = session.get(request_url) try: tilespecs_json = r.json() - except: + except Exception as e: + logger.error(e) logger.error(r.text) return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json['tileSpecs']] @@ -347,7 +349,8 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, r = session.get(request_url) try: tilespecs_json = r.json() - except: + except Exception as e: + logger.error(e) logger.error(r.text) if len(tilespecs_json) == 0: From 6d9405622cd204a10e737b438a64c5661822fb1a Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 29 Mar 2017 17:04:56 -0700 Subject: [PATCH 188/766] transform: PEP8, avoid builtin variables --- renderapi/transform.py | 68 ++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 8be74c2f..6713b543 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -30,23 +30,22 @@ from numpy.linalg import svd - class TransformList: - def __init__(self,tforms=None,transformId=None,json=None): + def __init__(self, tforms=None, transformId=None, json=None): if json is not None: self.from_dict(json) else: if tforms is None: self.tforms = [] else: - assert(type(tforms)==list) + assert(type(tforms) == list) self.tforms = tforms self.transformId = transformId def to_dict(self): - d= {} + d = {} d['type'] = 'list' - d['specList'] =[tform.to_dict() for tform in self.tforms] + d['specList'] = [tform.to_dict() for tform in self.tforms] if self.transformId is not None: d['id'] = self.transformId return d @@ -54,36 +53,44 @@ def to_dict(self): def to_json(self): return json.dumps(self.to_dict()) - def from_dict(self,d): - self.tforms=[] + def from_dict(self, d): + self.tforms = [] if d is not None: - self.transformId = d.get('id',None) + self.transformId = d.get('id') for td in d['specList']: self.tforms.append(load_transform_json(td)) return self.tforms -def load_transform_json(d): - type = d.get('type','leaf') - if type=='leaf': - return load_leaf_json(d) - elif type == 'list': - return TransformList(json=d) - elif type == 'ref': - return ReferenceTransform(json=d) - raise RenderError("Unknown Transform Type %s"%type) + +def load_transform_json(d, default_type='leaf'): + handle_load_tform = {'leaf': load_leaf_json, + 'list': lambda x: TransformList(json=x), + 'ref': lambda x: ReferenceTransform(json=x)} + try: + return handle_load_tform[d.get('type', default_type)](d) + except KeyError as e: + raise RenderError('Unknown Transform Type {}'.format(e)) + def load_leaf_json(d): - type = d.get('type','leaf') - assert(type=='leaf') - cls = d['className'] - - if cls==AffineModel.className: - return AffineModel(json=d) - elif cls==Polynomial2DTransform.className: - return Polynomial2DTransform(json=d) - else: + handle_load_leaf = { + AffineModel.className: lambda x: AffineModel(json=d), + Polynomial2DTransform.className: + lambda x: Polynomial2DTransform(json=d)} + + tform_type = d.get('type', 'leaf') + if tform_type != 'leaf': + raise RenderError( + 'Unexpected or unknown Transform Type {}'.format(tform_type)) + tform_class = d['className'] + try: + return handle_load_leaf[tform_class](d) + except KeyError as e: + logger.info('Leaf transform class {} not defined in ' + 'transform module, using generic'.format(e)) return Transform(json=d) + class ReferenceTransform: def __init__(self, refId=None, json=None): if json is not None: @@ -130,7 +137,8 @@ def from_dict(self, d): self.className = d['className'] self.transformId = d.get('transformId', None) self._process_dataString(d['dataString']) - def _process_dataString(self,datastring): + + def _process_dataString(self, datastring): self.dataString = datastring def __str__(self): @@ -165,7 +173,6 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, self.load_M() self.transformId = None - @property def dataString(self): return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( @@ -185,7 +192,6 @@ def _process_dataString(self, datastring): self.B1 = float(dsList[5]) self.load_M() - def load_M(self): self.M = np.identity(3, np.double) self.M[0, 0] = self.M00 @@ -348,8 +354,6 @@ class Polynomial2DTransform(Transform): TODO: fall back to Affine Model in special cases robustness in estimation - - there is a hierarchy in the __init__ that should probably be defined ''' className = 'mpicbg.trakem2.transform.PolynomialTransform2D' @@ -422,7 +426,7 @@ def fit(self, src, dst, order=2): # return (-V[-1, :-1] / V[-1, -1]).reshape((2, no_coeff // 2)) def estimate(self, src, dst, order=2, - convergence_test=None, max_tries=100,**kwargs): + convergence_test=None, max_tries=100, **kwargs): params = self.fit(src, dst, order=order) if convergence_test is None: From 7d17ad47664db25f0121d953dd85371b7fd210e8 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 29 Mar 2017 18:20:20 -0700 Subject: [PATCH 189/766] transform: add support for InterpolatedTransform --- renderapi/transform.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 6713b543..f6d4085d 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -65,7 +65,9 @@ def from_dict(self, d): def load_transform_json(d, default_type='leaf'): handle_load_tform = {'leaf': load_leaf_json, 'list': lambda x: TransformList(json=x), - 'ref': lambda x: ReferenceTransform(json=x)} + 'ref': lambda x: ReferenceTransform(json=x), + 'interpolated': + lambda x: InterpolatedTransform(json=x)} try: return handle_load_tform[d.get('type', default_type)](d) except KeyError as e: @@ -91,6 +93,39 @@ def load_leaf_json(d): return Transform(json=d) +class InterpolatedTransform: + ''' + Transform spec defined by linear interpolation of two other transform specs + inputs: + a -- transform spec at minimum weight + b -- transform spec at maximum weight + lambda_ -- float value (0.-1.) which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + ''' + def __init__(self, a=None, b=None, lambda_=None, json=None): + if json is not None: + self.from_dict(json) + else: + self.a = a + self.b = b + self.lambda_ = lambda_ + + def to_dict(self): + return dict(self) + + def from_dict(self, d): + self.a = load_transform_json(d['a']) + self.b = load_transform_json(d['b']) + self.lambda_ = d['lambda'] + + def __iter__(self): + # TODO I think AffineModel requires to_dict() + return iter([('type', 'interpolated'), + ('a', self.a.to_dict()), + ('b', self.b.to_dict()), + ('lambda', self.lambda_)]) + + class ReferenceTransform: def __init__(self, refId=None, json=None): if json is not None: From cab09fe6a73dc774e08aa6b674af98ffad98ce9b Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 29 Mar 2017 19:02:30 -0700 Subject: [PATCH 190/766] replace asserts with custom errors, PEP8 --- renderapi/stack.py | 4 +++- renderapi/transform.py | 28 +++++++++++----------------- setup.py | 11 ++++++----- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index fe204439..8944a557 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -86,7 +86,9 @@ def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - assert state in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'] + if state not in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY']: + raise RenderError('state {} not in known states {}'.format( + state, ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'])) request_url = format_preamble( host, port, owner, project, stack) + "/state/%s" % state logger.debug(request_url) diff --git a/renderapi/transform.py b/renderapi/transform.py index f6d4085d..8f197f25 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -38,7 +38,10 @@ def __init__(self, tforms=None, transformId=None, json=None): if tforms is None: self.tforms = [] else: - assert(type(tforms) == list) + if not isinstance(tforms, list): + raise RenderError( + 'unexpected type {} for transforms!'.format( + type(tforms))) self.tforms = tforms self.transformId = transformId @@ -119,7 +122,6 @@ def from_dict(self, d): self.lambda_ = d['lambda'] def __iter__(self): - # TODO I think AffineModel requires to_dict() return iter([('type', 'interpolated'), ('a', self.a.to_dict()), ('b', self.b.to_dict()), @@ -236,24 +238,14 @@ def load_M(self): self.M[0, 2] = self.B0 self.M[1, 2] = self.B1 - ''' - def to_dict(self): - d = {} - d['type'] = 'leaf' - d['className'] = self.className - d['dataString'] = "%.10f %.10f %.10f %.10f %.10f %.10f" % ( - self.M[0, 0], self.M[1, 0], self.M[0, 1], - self.M[1, 1], self.M[0, 2], self.M[1, 2]) - return d - ''' - def invert(self): Ai = AffineModel() def fit(self, A, B): - assert A.shape[0] == B.shape[0] - assert A.shape[1] == 2 - assert B.shape[1] == 2 + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) N = A.shape[0] # total points @@ -287,7 +279,9 @@ def convert_to_point_vector(self, points): zerovec = np.zeros((Np, 1), np.double) onevec = np.ones((Np, 1), np.double) - assert(points.shape[1] == 2) + if points.shape[1] != 2: + raise ConversionError('Points must be of shape (:, 2) ' + '-- got {}'.format(points.shape)) Nd = 2 points = np.concatenate((points, onevec), axis=1) return points, Nd diff --git a/setup.py b/setup.py index eb3f3e25..c2ce8682 100644 --- a/setup.py +++ b/setup.py @@ -12,16 +12,17 @@ def initialize_options(self): self.pytest_args = [] def run_tests(self): + # import here, cause outside the eggs aren't loaded import shlex - #import here, cause outside the eggs aren't loaded import pytest - self.pytest_args += " --cov=renderapi --cov-report html --junitxml=test-reports/test.xml" + self.pytest_args += " --cov=renderapi --cov-report html "\ + "--junitxml=test-reports/test.xml" errno = pytest.main(shlex.split(self.pytest_args)) sys.exit(errno) -with open('test/requirements.txt','r') as f: +with open('test/requirements.txt', 'r') as f: test_required = f.read().splitlines() with open('requirements.txt', 'r') as f: @@ -37,5 +38,5 @@ def run_tests(self): packages=['renderapi'], install_requires=required, setup_requires=['flake8'], - tests_require = test_required, - cmdclass = {'test': PyTest},) + tests_require=test_required, + cmdclass={'test': PyTest},) From 02e5046c3a1dd7037800e766c5db1751eb5d0d0b Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 30 Mar 2017 14:31:44 -0700 Subject: [PATCH 191/766] render, stack: add docstrings --- renderapi/render.py | 60 ++++++++++++++++++++++++++++++++++++++++++--- renderapi/stack.py | 8 ++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index a29808a4..06e2ee7a 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -12,6 +12,9 @@ class Render(object): + ''' + Render object to store connection settings for render server + ''' def __init__(self, host=None, port=None, owner=None, project=None, client_scripts=None): self.DEFAULT_HOST = host @@ -64,7 +67,10 @@ def run(self, f, *args, **kwargs): class RenderClient(Render): - '''Draft object for run_ws_client.sh calls''' + ''' + Render object to run java client commands via a wrapped client script. + This is a work in progress. + ''' def __init__(self, client_script=None, memGB=None, *args, **kwargs): super(RenderClient, self).__init__(**kwargs) if client_script is None: @@ -96,8 +102,35 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, - json_dict=None, force_http=True, **kwargs): - '''helper function to connect to a render instance''' + force_http=True, **kwargs): + ''' + helper function to connect to a render instance + can default to using environment variables if not specified in call. + input: + host -- string hostname for target render server -- will prepend + "http://" if not found and force_http keyword evaluates True. + Can be set by environment variable RENDER_HOST. + port -- string, int, or None port for target render server. + Optional as in 'http://hostname[:port]'. + Can be set by environment variable RENDER_PORT. + owner -- string owner for render-ws. + Can be set by environment variable RENDER_OWNER. + project -- string project for render webservice. + Can be set by environment variable RENDER_PROJECT. + client_scripts -- string specifying directory path + for render-ws-java-client scripts. + Can be set by environment variable RENDER_CLIENT_SCRIPTS. + client_script -- string path to a wrapper for java client classes. + Used only in RenderClient. + Can be set by environment variable RENDER_CLIENT_SCRIPT. + memGB -- string specifying heap size in GB for java client scripts, + example for 1 GB: '1G'. Used only in RenderClient. + Can be set by environment variable RENDER_CLIENT_HEAP. + force_http -- boolean determining whether to prepend + 'http://' to render host + returns: + RenderClient or Render object + ''' if host is None: if 'RENDER_HOST' not in os.environ: host = str(raw_input("Enter Render Host: ")) @@ -177,6 +210,10 @@ def connect(host=None, port=None, owner=None, project=None, def renderaccess(f): + ''' + decorator allowing functions asking for host, port, owner, project + to default to a connection defined by a render object kwarg + ''' @wraps(f) def wrapper(*args, **kwargs): args, kwargs = fitargspec(f, args, kwargs) @@ -206,12 +243,17 @@ def post_json(session, request, jsondict): def format_baseurl(host, port): + '''format host and port to a standard template render-ws url''' # return 'http://%s:%d/render-ws/v1' % (host, port) server = '{}{}'.format(host, ('' if port is None else ':{}'.format(port))) return '{}/render-ws/v1'.format(server) def format_preamble(host, port, owner, project, stack): + ''' + format host, port, owner, project, and stack parameters + to the access point to stack-based apis + ''' preamble = "%s/owner/%s/project/%s/stack/%s" % ( format_baseurl(host, port), owner, project, stack) return preamble @@ -220,6 +262,9 @@ def format_preamble(host, port, owner, project, stack): @renderaccess def get_owners(host=None, port=None, session=requests.session(), render=None, **kwargs): + ''' + return list of owners across all Projects and Stacks for a render server + ''' request_url = "%s/owners/" % format_baseurl(host, port) r = session.get(request_url) try: @@ -233,6 +278,11 @@ def get_owners(host=None, port=None, session=requests.session(), def get_stack_metadata_by_owner(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + ''' + return metadata for all stacks belonging to particular + owner on render server + TODO example + ''' request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) logger.debug(request_url) @@ -247,6 +297,7 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, @renderaccess def get_projects_by_owner(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + '''return list of projects belonging to a single owner for render stack''' metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) projects = list(set([m['stackId']['project'] for m in metadata])) @@ -257,6 +308,9 @@ def get_projects_by_owner(owner=None, host=None, port=None, def get_stacks_by_owner_project(owner=None, project=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + ''' + return list of stacks belonging to an owner's project on render server + ''' metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) stacks = ([m['stackId']['stack'] for m in metadata diff --git a/renderapi/stack.py b/renderapi/stack.py index 8944a557..dc9a01f1 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -86,6 +86,14 @@ def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + ''' + set state of selected stack. Acceptable states are listed below: + LOADING: stack is accepting additional information. + COMPLETE: stack is finished loading. + OFFLINE: stack is not in use. + READ_ONLY: stack cannot be changed. + TODO there is a limited direction in which these stack changes can go + ''' if state not in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY']: raise RenderError('state {} not in known states {}'.format( state, ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'])) From ea8d5352f57d30b2146841084fafb00fc65abe19 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 30 Mar 2017 14:42:37 -0700 Subject: [PATCH 192/766] README: make more current, reference master of rpa --- README.md | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 3ed89fcc..193399bf 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,10 @@ # render-python -This is a python API setup to interact via python with render databases and facilitate python scripting of tilespec creation -see -https://github.com/saalfeldlab/render -it presently interacts with render via a web-api, though a couple functions are presently implemented with the java client scripts that are part of render, ideally those would be cutout. +This is a python API client to interact with [render-webservices](https://github.com/saalfeldlab/render) and facilitate python scripting of [tilespec](https://github.com/saalfeldlab/render/blob/master/docs/src/site/markdown/data-model.md) creation -tilespec.py is a set of python objects that facilitate serializing and deserializing objects to/from json. +it presently interacts with render via a web-api, though the [client module](renderapi/client.py) aims to interface by calling java client scripts to avoid server-side processing. -[add_downsample_to_render_project.py](docs/examples/add_downsample_to_render_project.py) is an example script that makes use of these files to read tilespecs from a render stack, -modify the tilespecs to include paths to downsampled images, and then write those tilespecs to disk, upload them back to render, and launch all the jobs to create those downsampled images in a parallel fashion. This program assumes a filestructure path that is unique to the Synapse Biology group at the Allen Institute, but nonetheless demonstrate the general utility of the above files. - -[create_mipmaps.py](docs/examples/create_mipmaps.py) is a simple python program for using Pillow to create downsampled images from a single image that is included simply for reference. - -Render connection objects created with `renderapi.connect()` can default to environment variables. Below is an example of the variables which can be sourced and added to ~/.bashrc or ~/.bash_profile. +Render connection objects created with `renderapi.connect()` can default to environment variables. Below is an example of the variables which can be sourced and added to, e.g., ~/.bashrc or ~/.bash_profile. ``` export RENDER_HOST="localhost" export RENDER_PORT="8080" @@ -23,4 +15,4 @@ Render connection objects created with `renderapi.connect()` can default to envi export RENDER_CLIENT_HEAP="1G" ``` -[Usage examples for an Array Tomography workflow](https://github.com/fcollman/render-python-apps/tree/newrender) are available. +[Usage examples for a development Array Tomography workflow](https://github.com/fcollman/render-python-apps) are available. From 74940c77da7106a3c7d9ebb66411ac78c073b28f Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 30 Mar 2017 15:09:47 -0700 Subject: [PATCH 193/766] tilespec.py: add some docstrings --- renderapi/tilespec.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 46d785b5..f314de33 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -234,6 +234,15 @@ def from_dict(self, d): class MipMapLevel: + ''' + MipMapLevel class to represent a level of an image pyramid. + Can be put in dictionary formatting using dict(mML) + + init: + level -- integer level of 2x downsampling represented by mipmaplevel + imageUrl (optional) -- url corresponding to image + maskUrl (optional) -- url corresponding to mask + ''' def __init__(self, level, imageUrl=None, maskUrl=None): self.level = level self.imageUrl = imageUrl @@ -255,6 +264,24 @@ def __iter__(self): class ImagePyramid: + ''' + Image Pyramid class representing a set of MipMapLevels which correspond + to mipmapped (continuously downsmapled by 2x) representations + of an image at level 0 + Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) + + init: + mipMapLevels -- list of MipMapLevel objects + append: + adds MipmapLevel without checking if it exists + input: MipMapLevel object + update: + adds MipMapLevel object replacing a corresponding level if it exists + input: MipMapLevel object + to_ordered_dict: + input: key(optional) -- key to sort ordered dictionary + default sort by level via lambda x: x[0] + ''' def __init__(self, mipMapLevels=[]): self.mipMapLevels = mipMapLevels @@ -343,6 +370,12 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, def get_tile_specs_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + ''' + input: + stack -- string render stack + z -- render z + output: list of tilespec objects + ''' request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) logger.debug(request_url) From c8cfbbcf38bc8ddeb9fa80528caa6264f8d40d5f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 3 Apr 2017 11:00:02 -0700 Subject: [PATCH 194/766] added minIntensity and maxIntensity calls --- renderapi/image.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/renderapi/image.py b/renderapi/image.py index 9d6ab3b1..833f3f28 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -25,6 +25,7 @@ @renderaccess def get_bb_image(stack, z, x, y, width, height, scale=1.0, + minIntensity=None,maxIntensity=None, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): @@ -43,8 +44,17 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, request_url = format_preamble( host, port, owner, project, stack) + \ - "/z/%d/box/%d,%d,%d,%d,%3.2f/%s" % ( + "/z/%d/box/%d,%d,%d,%d,%3.2f/%s?" % ( z, x, y, width, height, scale, image_ext) + args = [] + if minIntensity is not None: + args+=['minIntensity=%d'%minIntensity] + if maxIntensity is not None: + args+=['maxIntensity=%d'%maxIntensity] + if len(args)>0: + args = "&".join(args) + request_url+=args + r = session.get(request_url) try: image = np.asarray(Image.open(io.BytesIO(r.content))) From 28f7232f268cd8c76ffd4e6a247cad3be82a4e1a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 3 Apr 2017 12:09:11 -0700 Subject: [PATCH 195/766] trying a different setup.py string --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index eb3f3e25..f7672f93 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ class PyTest(TestCommand): def initialize_options(self): TestCommand.initialize_options(self) - self.pytest_args = [] + self.pytest_args = "" def run_tests(self): import shlex From 023e9891d2a37c651842a47716bb3de7749b1543 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 12:50:15 -0700 Subject: [PATCH 196/766] tests: PEP8 --- test/test_client.py | 15 ++++++++------- test/test_transform.py | 43 +++++++++++++++++++++--------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/test/test_client.py b/test/test_client.py index 65a58a2b..c96c0f4a 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -1,11 +1,12 @@ import renderapi + def test_render_client(): - args={ - 'host':'renderhost', - 'port':8080, - 'owner':'renderowner', - 'project':'renderproject', - 'client_scripts':'/path/to/client_scripts' - } + args = { + 'host': 'renderhost', + 'port': 8080, + 'owner': 'renderowner', + 'project': 'renderproject', + 'client_scripts': '/path/to/client_scripts' + } r = renderapi.render.connect(**args) diff --git a/test/test_transform.py b/test/test_transform.py index 8224f8aa..54211948 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -4,50 +4,49 @@ def test_affine_rot_90(): am = renderapi.transform.AffineModel() - #setup a 90 degree clockwise rotation - points_in = np.array([[0,0],[0,1],[1,0],[1,1]],np.float) - points_out = np.array([[0,0],[1,0],[0,-1],[1,-1]],np.float) - am.estimate(points_in,points_out) + # setup a 90 degree clockwise rotation + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float) + points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float) + am.estimate(points_in, points_out) assert(np.abs(am.scale[0]-1.0) < .00001) assert(np.abs(am.scale[1]-1.0) < .00001) - assert(np.abs(am.rotation + np.pi/2) <.000001) - assert(np.abs(am.translation[0])<.000001) - assert(np.abs(am.translation[1])<.000001) + assert(np.abs(am.rotation + np.pi/2) < .000001) + assert(np.abs(am.translation[0]) < .000001) + assert(np.abs(am.translation[1]) < .000001) assert(np.abs(am.shear) < .000001) - points = np.array([[20,30],[1,2],[10,-5],[-4,3],[5.6,2.3]]) + points = np.array([[20, 30], [1, 2], [10, -5], [-4, 3], [5.6, 2.3]]) new_points = am.tform(points) old_points = am.inverse_tform(new_points) - assert(np.sum(np.abs(points-old_points))<(.0001*len(points.ravel()))) - + assert(np.sum(np.abs(points-old_points)) < (.0001*len(points.ravel()))) am_inverse = renderapi.transform.AffineModel() - am_inverse.estimate(points_out,points_in) + am_inverse.estimate(points_out, points_in) identity = am.concatenate(am_inverse) assert(np.abs(identity.scale[0]-1.0) < .00001) assert(np.abs(identity.scale[1]-1.0) < .00001) - assert(np.abs(identity.rotation) <.000001) - assert(np.abs(identity.translation[0])<.000001) - assert(np.abs(identity.translation[1])<.000001) + assert(np.abs(identity.rotation) < .000001) + assert(np.abs(identity.translation[0]) < .000001) + assert(np.abs(identity.translation[1]) < .000001) assert(np.abs(identity.shear) < .000001) print(str(am)) def test_affine_random(): am = renderapi.transform.AffineModel(M00=.9, - M10=-0.2, - M01=0.3, - M11=.85, - B0=245.3, - B1=-234.1) + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) - points_in = np.random.rand(10,2) + points_in = np.random.rand(10, 2) points_out = am.tform(points_in) am_fit = renderapi.transform.AffineModel() - am_fit.estimate(points_in,points_out) + am_fit.estimate(points_in, points_out) - assert(np.sum(np.abs(am.M.ravel()-am_fit.M.ravel()))<(.001*6)) + assert(np.sum(np.abs(am.M.ravel()-am_fit.M.ravel())) < (.001*6)) From 6bd21c5e413844af1acd4af8595f44caeed7fee9 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 14:21:43 -0700 Subject: [PATCH 197/766] tests: PEP8 --- integration_tests/test_coordinate.py | 131 +++++++----- integration_tests/test_data.py | 9 +- integration_tests/test_stack.py | 309 +++++++++++++++------------ 3 files changed, 255 insertions(+), 194 deletions(-) diff --git a/integration_tests/test_coordinate.py b/integration_tests/test_coordinate.py index 507c3389..424aa6b5 100644 --- a/integration_tests/test_coordinate.py +++ b/integration_tests/test_coordinate.py @@ -15,7 +15,8 @@ ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) # logger.addHandler(ch) @@ -23,108 +24,120 @@ @pytest.fixture(scope='module') def render(): - render_test_parameters={ - 'host':render_host, - 'port':8080, - 'owner':'test_coordinate', - 'project':'test_coordinate_project', - 'client_scripts':client_script_location + render_test_parameters = { + 'host': render_host, + 'port': 8080, + 'owner': 'test_coordinate', + 'project': 'test_coordinate_project', + 'client_scripts': client_script_location } return renderapi.render.connect(**render_test_parameters) + @pytest.fixture(scope='module') def teststack_tilespec(): - render_test_parameters={ - 'host':render_host, - 'port':8080, - 'owner':'test_coordinate', - 'project':'test_coordinate_project', - 'client_scripts':client_script_location + render_test_parameters = { + 'host': render_host, + 'port': 8080, + 'owner': 'test_coordinate', + 'project': 'test_coordinate_project', + 'client_scripts': client_script_location } - render=renderapi.render.connect(**render_test_parameters) - ts_json = json.load(open(tilespec_file,'r')) - tform_json = json.load(open(tform_file,'r')) + render = renderapi.render.connect(**render_test_parameters) + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] stack = 'test_coordinate_stack' - r=render.run(renderapi.stack.create_stack,stack,force_resolution=True) - render.run(renderapi.client.import_tilespecs,stack,tilespecs, - sharedTransforms=tforms) - r=render.run(renderapi.stack.set_stack_state,stack,'COMPLETE') - yield (stack,tilespecs[0]) - render.run(renderapi.stack.delete_stack,stack) + r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run( + renderapi.client.import_tilespecs, stack, tilespecs, + sharedTransforms=tforms) + r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') + yield (stack, tilespecs[0]) + render.run(renderapi.stack.delete_stack, stack) + @pytest.fixture(scope='module') def local_corners_json(teststack_tilespec): - (stack,ts) = teststack_tilespec - corners = [[0,0],[ts.width,0],[ts.width,ts.height],[0,ts.height]] + (stack, ts) = teststack_tilespec + corners = [[0, 0], [ts.width, 0], [ts.width, ts.height], [0, ts.height]] batch = [] for corner in corners: - d={ - 'tileId':ts.tileId, - 'visible':True, + d = { + 'tileId': ts.tileId, + 'visible': True, '' } -def test_world_to_local_coordinates(render,teststack_tilespec): - (stack,ts) = teststack_tilespec - local=render.run(renderapi.coordinate.world_to_local_coordinates, - stack,ts.z,ts.minX,ts.minY) - assert(local['error']="") - assert(local['tileId']==ts.tileId) - assert(len(local['local'])>=2) - - -def test_local_to_world_coordinates(render,teststack_tilespec): - (stack,ts) = teststack_tilespec - local=render.run(renderapi.coordinate.local_to_world_coordinates_batch, - stack,ts.z,0,0) - assert(local['error']="") - assert(local['tileId']==ts.tileId) - assert(len(local['world'])>=2) - -def test_world_to_local_coordinates_batch(render,teststack_tilespec): - (stack,ts) = teststack_tilespec - corners = [[0,0],[ts.width,0],[ts.width,ts.height],[0,ts.height]] + +def test_world_to_local_coordinates(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local = render.run(renderapi.coordinate.world_to_local_coordinates, + stack, ts.z, ts.minX, ts.minY) + assert(local['error'] == "") + assert(local['tileId'] == ts.tileId) + assert(len(local['local']) >= 2) + + +def test_local_to_world_coordinates(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local = render.run(renderapi.coordinate.local_to_world_coordinates_batch, + stack, ts.z, 0, 0) + assert(local['error'] == "") + assert(local['tileId'] == ts.tileId) + assert(len(local['world']) >= 2) + + +def test_world_to_local_coordinates_batch(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + corners = [[0, 0], [ts.width, 0], [ts.width, ts.height], [0, ts.height]] batch = [] for corner in corners: - batch.append(renderapi.coordinate.local-to-world-coordinates(\ - stack,ts.z,corner[0],corner[1])) - local=renderapi.coordinates.world_to_local_coordinates_batch(stack,batch, - execute_local==False) + batch.append(renderapi.coordinate.local-to-world-coordinates( + stack, ts.z, corner[0], corner[1])) + local = renderapi.coordinates.world_to_local_coordinates_batch( + stack, batch, execute_local=False) - assert(len(local)==len(batch)) + assert(len(local) == len(batch)) for ans in local: - assert(len(ans['error'])==0) + assert(len(ans['error']) == 0) -def test_local_to_world_coordinates_batch(render,teststack_tilespec): - +def test_local_to_world_coordinates_batch(render, teststack_tilespec): logger.debug('test not implemented yet') assert(False) -def old_world_to_local_coordinates_array(render,teststack_tilespec): + +def old_world_to_local_coordinates_array(render, teststack_tilespec): logger.debug('test not implemented yet') assert(False) -def test_world_to_local_coordinates_array(render,teststack_tilespec): + +def test_world_to_local_coordinates_array(render, teststack_tilespec): logger.debug('test not implemented yet') assert(False) -def old_local_to_world_coordinates_array(render,teststack_tilespec): + +def old_local_to_world_coordinates_array(render, teststack_tilespec): logger.debug('test not implemented yet') assert(False) -def local_to_world_coordinates_array(render,teststack_tilespec): + +def local_to_world_coordinates_array(render, teststack_tilespec): logger.debug('test not implemented yet') assert(False) + def world_to_local_coordinates_clientside(): logger.debug('test not implemented yet') assert(False) + def local_to_world_coordinates_clientside(): logger.debug('test not implemented yet') assert(False) diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py index 86c4e62e..2e82500a 100644 --- a/integration_tests/test_data.py +++ b/integration_tests/test_data.py @@ -1,6 +1,9 @@ render_host = 'renderservice' render_port = 8080 -client_script_location = '/var/www/render/render-ws-java-client/src/main/scripts/' -tilespec_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_tiles.json' -tform_file = '/var/www/render/examples/example_1/cycle1_step1_acquire_transforms.json' +client_script_location = ('/var/www/render/render-ws-java-client/' + 'src/main/scripts/') +tilespec_file = ('/var/www/render/examples/' + 'example_1/cycle1_step1_acquire_tiles.json') +tform_file = ('/var/www/render/examples/' + 'example_1/cycle1_step1_acquire_transforms.json') diff --git a/integration_tests/test_stack.py b/integration_tests/test_stack.py index cb1dc602..6a0d74bc 100644 --- a/integration_tests/test_stack.py +++ b/integration_tests/test_stack.py @@ -6,73 +6,78 @@ import sys import json import numpy as np -from test_data import render_host, render_port,\ -client_script_location, tilespec_file, tform_file +from test_data import (render_host, render_port, + client_script_location, tilespec_file, tform_file) root = logging.getLogger() root.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) root.addHandler(ch) - @pytest.fixture(scope='module') def render(): - render_test_parameters={ - 'host':render_host, - 'port':render_port, - 'owner':'test', - 'project':'test_project', - 'client_scripts':client_script_location} + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location} return renderapi.render.connect(**render_test_parameters) + @pytest.fixture def simpletilespec(): - mml = renderapi.tilespec.MipMapLevel(0,'/not/a/path.jpg') + mml = renderapi.tilespec.MipMapLevel(0, '/not/a/path.jpg') tform = renderapi.transform.AffineModel() - layout = renderapi.tilespec.Layout(secitionId = "section0", - scopeId="testscope", - cameraId="testcamera", - imageRow=1, - imageCol=1, - stageX=40.0, - stageY=50.0, - rotation=0) - ts = renderapi.tilespec.TileSpec(tileId = "1000", - width=2048, - height=2048, - mipMapLevels=[mml], - z=0, - tforms=[tform], - layout=layout) + layout = renderapi.tilespec.Layout(sectionId="section0", + scopeId="testscope", + cameraId="testcamera", + imageRow=1, + imageCol=1, + stageX=40.0, + stageY=50.0, + rotation=0) + ts = renderapi.tilespec.TileSpec(tileId="1000", + width=2048, + height=2048, + mipMapLevels=[mml], + z=0, + tforms=[tform], + layout=layout) return ts + @pytest.fixture(scope='module') def render_example_tilespec_and_transforms(): - ts_json = json.load(open(tilespec_file,'r')) - tform_json = json.load(open(tform_file,'r')) + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] print tforms - return (tilespecs,tforms) + return (tilespecs, tforms) def test_stack_creation_deletion(render): test_stack = 'test_stack1' - r=render.run(renderapi.stack.create_stack,test_stack,force_resolution=True) + r = render.run(renderapi.stack.create_stack, + test_stack, force_resolution=True) assert (r.status_code == 201) - sv = render.run(renderapi.stack.get_stack_metadata,test_stack) + sv = render.run(renderapi.stack.get_stack_metadata, test_stack) assert (sv is not None) - assert(sv.stackResolutionX==1.0) - assert(sv.stackResolutionY==1.0) - assert(sv.stackResolutionZ==1.0) + assert(sv.stackResolutionX == 1.0) + assert(sv.stackResolutionY == 1.0) + assert(sv.stackResolutionZ == 1.0) owners = render.run(renderapi.render.get_owners) assert('test' in owners) @@ -83,197 +88,237 @@ def test_stack_creation_deletion(render): stacks = render.run(renderapi.render.get_stacks_by_owner_project) assert(test_stack in stacks) - r = render.run(renderapi.stack.delete_stack,test_stack) + r = render.run(renderapi.stack.delete_stack, test_stack) assert (r.status_code != 400) + def test_failed_metadata(render): with pytest.raises(renderapi.errors.RenderError): - render.run(renderapi.stack.get_stack_metadata,'NOTASTACKNAME') + render.run(renderapi.stack.get_stack_metadata, 'NOTASTACKNAME') + def test_set_stack_metadata(render): test_stack = 'test_stack2' - r=render.run(renderapi.stack.create_stack,test_stack,force_resolution=True) + r = render.run(renderapi.stack.create_stack, + test_stack, force_resolution=True) assert (r.status_code == 201) - sv = render.run(renderapi.stack.get_stack_metadata,test_stack) + sv = render.run(renderapi.stack.get_stack_metadata, test_stack) sv.stackResolutionX = 2.0 sv.stackResolutionY = 3.0 sv.stackResolutionZ = 4.0 - r = render.run(renderapi.stack.set_stack_metadata,test_stack,sv) + r = render.run(renderapi.stack.set_stack_metadata, test_stack, sv) assert (r.status_code == 201) - sv = render.run(renderapi.stack.get_stack_metadata,test_stack) + sv = render.run(renderapi.stack.get_stack_metadata, test_stack) assert (sv.stackResolutionX == 2.0) assert (sv.stackResolutionY == 3.0) assert (sv.stackResolutionZ == 4.0) -def test_simple_import(render,simpletilespec,tmpdir): - #open a temporary file +def test_simple_import(render, simpletilespec, tmpdir): + # open a temporary file tfile = tmpdir.join('testfile.json') fp = tfile.open('w') - #write the file to disk - renderapi.utils.renderdump([simpletilespec],fp) + # write the file to disk + renderapi.utils.renderdump([simpletilespec], fp) fp.close() - r=render.run(renderapi.stack.create_stack,'test_insert',force_resolution=True) - render.run(renderapi.client.import_single_json_file,'test_insert',str(tfile)) - r=render.run(renderapi.stack.set_stack_state,'test_insert','COMPLETE') + r = render.run(renderapi.stack.create_stack, + 'test_insert', force_resolution=True) + render.run(renderapi.client.import_single_json_file, + 'test_insert', str(tfile)) + r = render.run(renderapi.stack.set_stack_state, + 'test_insert', 'COMPLETE') assert (r.status_code == 201) - ts_out = render.run(renderapi.tilespec.get_tile_spec,'test_insert',simpletilespec.tileId) + ts_out = render.run(renderapi.tilespec.get_tile_spec, + 'test_insert', simpletilespec.tileId) assert (ts_out.z == simpletilespec.z) - render.run(renderapi.stack.delete_stack,'test_insert') + render.run(renderapi.stack.delete_stack, 'test_insert') + -def test_simple_import_with_transforms(render,render_example_tilespec_and_transforms,tmpdir): - (tilespecs,tforms)=render_example_tilespec_and_transforms +def test_simple_import_with_transforms( + render, render_example_tilespec_and_transforms, tmpdir): + (tilespecs, tforms) = render_example_tilespec_and_transforms - #open a temporary file + # open a temporary file tilespecfile = tmpdir.join('tilespecs.json') fp = tilespecfile.open('w') - #write the file to disk - renderapi.utils.renderdump(tilespecs,fp) + # write the file to disk + renderapi.utils.renderdump(tilespecs, fp) fp.close() transformfile = tmpdir.join('transforms.json') fp = transformfile.open('w') - #write the file to disk - renderapi.utils.renderdump(tforms,fp) + # write the file to disk + renderapi.utils.renderdump(tforms, fp) fp.close() - r=render.run(renderapi.stack.create_stack,'test_insert_tform',force_resolution=True) - render.run(renderapi.client.import_single_json_file,'test_insert_tform',str(tilespecfile),transformFile=str(transformfile)) - r=render.run(renderapi.stack.set_stack_state,'test_insert_tform','COMPLETE') + r = render.run(renderapi.stack.create_stack, + 'test_insert_tform', force_resolution=True) + render.run(renderapi.client.import_single_json_file, 'test_insert_tform', + str(tilespecfile), transformFile=str(transformfile)) + r = render.run(renderapi.stack.set_stack_state, + 'test_insert_tform', 'COMPLETE') assert (r.status_code == 201) - ts_out = render.run(renderapi.tilespec.get_tile_spec,'test_insert_tform',tilespecs[0].tileId) + ts_out = render.run(renderapi.tilespec.get_tile_spec, + 'test_insert_tform', tilespecs[0].tileId) assert (ts_out.z == tilespecs[0].z) - render.run(renderapi.stack.delete_stack,'test_insert_tform') + render.run(renderapi.stack.delete_stack, 'test_insert_tform') + + # os.remove(tfile) - #os.remove(tfile) -def test_import_tilespecs(render,simpletilespec): + +def test_import_tilespecs(render, simpletilespec): stack = 'test_insert2' - r=render.run(renderapi.stack.create_stack,stack,force_resolution=True) - render.run(renderapi.client.import_tilespecs,stack,[simpletilespec]) - r=render.run(renderapi.stack.set_stack_state,stack,'COMPLETE') + r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.client.import_tilespecs, stack, [simpletilespec]) + r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') assert (r.status_code == 201) - ts_out = render.run(renderapi.tilespec.get_tile_spec,stack,simpletilespec.tileId) + ts_out = render.run(renderapi.tilespec.get_tile_spec, + stack, simpletilespec.tileId) assert (ts_out.z == simpletilespec.z) - render.run(renderapi.stack.delete_stack,stack) + render.run(renderapi.stack.delete_stack, stack) + def test_import_tilespecs_parallel(render): root.debug('test not implemented yet') assert(False) + def test_import_jsonfiles_validate_client(render): root.debug('test not implemented yet') assert(False) + def test_import_jsonfiles(render): root.debug('test not implemented yet') assert(False) + def test_import_parallel(render): root.debug('test not implemented yet') assert(False) + def test_tile_pair_client(render): root.debug('test not implemented yet') assert(False) + def test_importTransformChangesClient(render): root.debug('test not implemented yet') assert(False) + def test_coordinateClient(render): root.debug('test not implemented yet') assert(False) + @pytest.fixture(scope="module") def teststack(request): - render_test_parameters={ - 'host':render_host, - 'port':render_port, - 'owner':'test', - 'project':'test_project', - 'client_scripts':client_script_location} - render=renderapi.render.connect(**render_test_parameters) - - ts_json = json.load(open(tilespec_file,'r')) - tform_json = json.load(open(tform_file,'r')) + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location} + render = renderapi.render.connect(**render_test_parameters) + + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] stack = 'test_insert3' - r=render.run(renderapi.stack.create_stack,stack,force_resolution=True) - render.run(renderapi.client.import_tilespecs,stack,tilespecs, - sharedTransforms=tforms) - r=render.run(renderapi.stack.set_stack_state,stack,'COMPLETE') + r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.client.import_tilespecs, stack, tilespecs, + sharedTransforms=tforms) + r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') + def fin(): - render.run(renderapi.stack.delete_stack,stack) + render.run(renderapi.stack.delete_stack, stack) request.addfinalizer(fin) return stack -def test_stack_bounds(render,teststack): - #check the stack bounds - stack_bounds = render.run(renderapi.stack.get_stack_bounds,teststack) - expected_bounds={u'maxZ': 3408.0, u'maxX': 5176.0, u'maxY': 5319.0, u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} - assert(stack_bounds==expected_bounds) - -def test_z_bounds(render,teststack,render_example_tilespec_and_transforms): - (tilespecs,tforms)=render_example_tilespec_and_transforms - #check a single z stack bounds - zbounds = render.run(renderapi.stack.get_bounds_from_z,teststack,tilespecs[0].z) - expected_bounds = {u'maxZ': 3407.0, u'maxX': 5009.0, u'maxY': 4395.0, u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} - assert(zbounds==expected_bounds) - -def test_get_section_z(render,teststack): - #check getting section Z - z = render.run(renderapi.stack.get_section_z_value,teststack,"3407.0") - assert(z==3407) - z = render.run(renderapi.stack.get_z_value_for_section,teststack,"3407.0") - assert(z==3407) - -def test_get_z_values(render,teststack): - #check get z values - zvalues = render.run(renderapi.stack.get_z_values_for_stack,teststack) + +def test_stack_bounds(render, teststack): + # check the stack bounds + stack_bounds = render.run(renderapi.stack.get_stack_bounds, teststack) + expected_bounds = {u'maxZ': 3408.0, u'maxX': 5176.0, u'maxY': 5319.0, + u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} + assert(stack_bounds == expected_bounds) + + +def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + # check a single z stack bounds + zbounds = render.run(renderapi.stack.get_bounds_from_z, + teststack, tilespecs[0].z) + expected_bounds = {u'maxZ': 3407.0, u'maxX': 5009.0, u'maxY': 4395.0, + u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} + assert(zbounds == expected_bounds) + + +def test_get_section_z(render, teststack): + # check getting section Z + z = render.run(renderapi.stack.get_section_z_value, + teststack, "3407.0") + assert(z == 3407) + z = render.run(renderapi.stack.get_z_value_for_section, + teststack, "3407.0") + assert(z == 3407) + + +def test_get_z_values(render, teststack): + # check get z values + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) assert(zvalues == [3407.0, 3408.0]) + def test_uniq_value(render): - #check likelyUniqueId + # check likelyUniqueId uniq = render.run(renderapi.stack.likelyUniqueId) - assert(len(uniq)>=len('58ceebb7a7b11b0001dc4e32')) + assert(len(uniq) >= len('58ceebb7a7b11b0001dc4e32')) -def test_bb_image(render,teststack): +def test_bb_image(render, teststack): formats = renderapi.image.IMAGE_FORMATS.keys() - zvalues = render.run(renderapi.stack.get_z_values_for_stack,teststack) + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) z = zvalues[0] - bounds = render.run(renderapi.stack.get_bounds_from_z,teststack,z) - width = (bounds['maxX']-bounds['minX'])/2 - height = (bounds['maxY']-bounds['minY'])/2 - x= bounds['minX']+width/4 - y = bounds['minY']+width/4 - - for format in formats: - data = render.run(renderapi.image.get_bb_image,teststack, - z,x,y,width,height, - scale=.25, - img_format = format) + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + width = (bounds['maxX'] - bounds['minX']) / 2 + height = (bounds['maxY'] - bounds['minY']) / 2 + x = bounds['minX'] + width / 4 + y = bounds['minY'] + width / 4 + + for fmt in formats: + data = render.run(renderapi.image.get_bb_image, + teststack, z, x, y, width, height, + scale=.25, img_format=fmt) dr = data.ravel() - assert (data.shape[0]==(np.floor(height*.25))) - assert (data.shape[1]==(np.floor(width*.25))) - assert (data.shape[2]>=3) + assert (data.shape[0] == (np.floor(height*.25))) + assert (data.shape[1] == (np.floor(width*.25))) + assert (data.shape[2] >= 3) + -def test_tile_image(render,teststack,render_example_tilespec_and_transforms): - (tilespecs,tforms)=render_example_tilespec_and_transforms +def test_tile_image(render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms format = 'png' - data=render.run(renderapi.image.get_tile_image_data,teststack,tilespecs[0].tileId) - assert len(data.shape)==3 - assert data.shape[0]>=tilespecs[0].height - assert data.shape[1]>=tilespecs[0].width + data = render.run(renderapi.image.get_tile_image_data, + teststack, tilespecs[0].tileId) + assert len(data.shape) == 3 + assert data.shape[0] >= tilespecs[0].height + assert data.shape[1] >= tilespecs[0].width + -def fail_image_get(render,teststack,render_example_tilespec_and_transforms): +def fail_image_get(render, teststack, render_example_tilespec_and_transforms): with pytest.raises(KeyError): - render.run(renderapi.image.get_tile_image_data,teststack,'test',img_format='JUNK') + render.run(renderapi.image.get_tile_image_data, teststack, + 'test', img_format='JUNK') From eb7d9c4c5683b06b47e5cc919d4fd1d675e71d08 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 14:39:32 -0700 Subject: [PATCH 198/766] tests: fix bad dictionary --- integration_tests/test_coordinate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/test_coordinate.py b/integration_tests/test_coordinate.py index 424aa6b5..c0a6b4c5 100644 --- a/integration_tests/test_coordinate.py +++ b/integration_tests/test_coordinate.py @@ -71,7 +71,6 @@ def local_corners_json(teststack_tilespec): d = { 'tileId': ts.tileId, 'visible': True, - '' } From f50a6c77b08604a680ec68abf6ff12091d69c5dd Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 15:12:16 -0700 Subject: [PATCH 199/766] flake8 --- renderapi/coordinate.py | 4 ++-- renderapi/pointmatch.py | 3 ++- renderapi/stack.py | 6 ++---- renderapi/tilespec.py | 3 +-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 2b8815f8..1de445ab 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -2,9 +2,9 @@ ''' coordinate mapping functions for render api ''' -from .render import Render, format_preamble, renderaccess +from .render import format_preamble, renderaccess from .utils import NullHandler -from .client import call_run_ws_client, coordinateClient +from .client import coordinateClient import requests import json import numpy as np diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 513cd242..614fefe4 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -4,7 +4,8 @@ ''' import requests import logging -from .render import Render, format_baseurl, renderaccess +from .render import format_baseurl, renderaccess +from .errors import RenderError from .utils import NullHandler logger = logging.getLogger(__name__) diff --git a/renderapi/stack.py b/renderapi/stack.py index dc9a01f1..b63e5657 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -import json import logging from time import strftime import requests from .errors import RenderError from .utils import jbool, NullHandler -from .render import (Render, format_baseurl, format_preamble, +from .render import (format_baseurl, format_preamble, renderaccess, post_json) logger = logging.getLogger(__name__) @@ -85,7 +84,7 @@ def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, @renderaccess def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): + session=requests.session(), render=None, **kwargs): ''' set state of selected stack. Acceptable states are listed below: LOADING: stack is accepting additional information. @@ -278,4 +277,3 @@ def get_section_z_value(stack, sectionId, host=None, port=None, logger.error(e) logger.error(r.text) raise RenderError(r.text) - return float(process_simple_url_request(request_url, session)) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index f314de33..0ab0d0ad 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,12 +1,11 @@ #!/usr/bin/env python -from .render import Render, format_baseurl, format_preamble, renderaccess +from .render import format_preamble, renderaccess from .utils import NullHandler from .stack import get_z_values_for_stack from .transform import TransformList, load_transform_json from collections import OrderedDict import logging import requests -import numpy as np logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) From f6206b9b82617e6043ec035b343d9314906ae89b Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 17:06:39 -0700 Subject: [PATCH 200/766] test: adding client and tilespec coverage --- test/rendersettings.py | 32 ++++++++ test/test_client.py | 40 +++++++++ test/test_files/tilespecs.json | 146 +++++++++++++++++++++++++++++++++ test/test_tilespec.py | 14 ++++ 4 files changed, 232 insertions(+) create mode 100644 test/rendersettings.py create mode 100644 test/test_files/tilespecs.json create mode 100644 test/test_tilespec.py diff --git a/test/rendersettings.py b/test/rendersettings.py new file mode 100644 index 00000000..66726963 --- /dev/null +++ b/test/rendersettings.py @@ -0,0 +1,32 @@ +''' +default settings for render tests +''' +import os + +DEFAULT_RENDER = { + 'host': 'http://renderhost', + 'port': 8080, + 'owner': 'renderowner', + 'project': 'renderproject', + 'client_scripts': '/path/to/client_scripts' + } + +DEFAULT_RENDER_CLIENT = dict(DEFAULT_RENDER, **{ + 'client_script': '/path/to/client_scripts/run_ws_client.sh', + 'memGB': '2G'}) + +DEFAULT_RENDER_ENVIRONMENT_VARIABLES = { + 'RENDER_HOST': DEFAULT_RENDER['host'], + 'RENDER_PORT': DEFAULT_RENDER['port'], + 'RENDER_OWNER': DEFAULT_RENDER['owner'], + 'RENDER_PROJECT': DEFAULT_RENDER['project'], + 'RENDER_CLIENT_SCRIPTS': DEFAULT_RENDER['client_scripts'] + } + +DEFAULT_RENDER_CLIENT_ENVIRONMENT_VARIABLES = dict( + DEFAULT_RENDER_ENVIRONMENT_VARIABLES, **{ + 'RENDER_CLIENT_SCRIPT': DEFAULT_RENDER_CLIENT['client_script'], + 'RENDER_CLIENT_HEAP': DEFAULT_RENDER_CLIENT['memGB']}) + +TEST_TILESPECS_FILE = os.path.join(os.path.dirname( + __file__), 'test_files', 'tilespecs.json') diff --git a/test/test_client.py b/test/test_client.py index c96c0f4a..a657b264 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -1,4 +1,6 @@ +import os import renderapi +import rendersettings def test_render_client(): @@ -10,3 +12,41 @@ def test_render_client(): 'client_scripts': '/path/to/client_scripts' } r = renderapi.render.connect(**args) + + +def test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER): + r = renderapi.connect(**rkwargs) + new_r = renderapi.connect(**r.DEFAULT_KWARGS) + assert(new_r.DEFAULT_KWARGS == r.DEFAULT_KWARGS == rkwargs) + + +''' +def test_default_kwargs_client(): + test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER_CLIENT) +''' + + +def test_environment_variables( + rkwargs=rendersettings.DEFAULT_RENDER, + renvkwargs=rendersettings.DEFAULT_RENDER_ENVIRONMENT_VARIABLES): + def valstostring(d): + return {k: str(v) for k, v in d.items()} + old_env = os.environ.copy() + os.environ.update(valstostring(renvkwargs)) + + env_render = renderapi.connect() + + # restore environment + os.environ.clear() + os.environ.update(old_env) + + kwarg_render = renderapi.connect(**valstostring(rkwargs)) + assert(valstostring(kwarg_render.DEFAULT_KWARGS) == + valstostring(env_render.DEFAULT_KWARGS) == + valstostring(rkwargs)) + + +''' +def test_environment_variables_client(): + test_environment_variables(rkwargs=rendersettings.DEFAULT_RENDER_CLIENT) +''' diff --git a/test/test_files/tilespecs.json b/test/test_files/tilespecs.json new file mode 100644 index 00000000..51d6f74d --- /dev/null +++ b/test/test_files/tilespecs.json @@ -0,0 +1,146 @@ +[ + { + "tileId": "20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.2266.0", + "layout": { + "sectionId": "2266.0", + "temca": null, + "camera": null + }, + "z": 2266, + "minX": 1857, + "minY": -32318, + "maxX": 7275, + "maxY": -27014, + "width": 3840, + "height": 3840, + "minIntensity": 0, + "maxIntensity": 65535, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.PolynomialTransform2D", + "dataString": "65508.72594194093 -0.99868791634 -0.008854442595 -7.96199E-7 1.286356E-6 -7.21762E-7 70663.99671294086 -0.039478147906 -0.954683561039 6.052858E-6 9.11723E-7 -6.219026E-6" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000" + } + ] + }, + "meshCellSize": 64 + }, + { + "tileId": "20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8.2266.0", + "layout": { + "sectionId": "2266.0", + "temca": null, + "camera": null + }, + "z": 2266, + "minX": 3915, + "minY": -34958, + "maxX": 9339, + "maxY": -29519, + "width": 3840, + "height": 3840, + "minIntensity": 0, + "maxIntensity": 65535, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8.tif" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.PolynomialTransform2D", + "dataString": "62218.570504642266 -0.995583671407 2.06111858E-4 -1.782885E-6 6.1908E-8 -1.6858E-6 70741.89265480703 0.009744971533 -1.006717263266 3.12237E-7 -1.701837E-6 2.245153E-6" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000" + } + ] + }, + "meshCellSize": 64 + }, + { + "tileId": "20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8.2266.0", + "layout": { + "sectionId": "2266.0", + "temca": null, + "camera": null + }, + "z": 2266, + "minX": 5957, + "minY": -37482, + "maxX": 11373, + "maxY": -32062, + "width": 3840, + "height": 3840, + "minIntensity": 0, + "maxIntensity": 65535, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8.tif" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.PolynomialTransform2D", + "dataString": "58956.67195722279 -1.009281426241 -0.002631989396 2.226803E-6 3.96506E-7 -9.49469E-7 70815.8432399658 0.004676678124 -0.99930426995 3.8057E-8 7.7865E-8 1.52374E-7" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961993" + }, + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000" + } + ] + }, + "meshCellSize": 64 + } +] diff --git a/test/test_tilespec.py b/test/test_tilespec.py new file mode 100644 index 00000000..9d866a8e --- /dev/null +++ b/test/test_tilespec.py @@ -0,0 +1,14 @@ +import json +from operator import eq +import renderapi +import rendersettings + + +def test_load_tilespecs(): + with open(rendersettings.TEST_TILESPECS_FILE, 'r') as f: + ts_json = json.load(f) + ts_jsons = json.dumps(ts_json) + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ts_json] + assert(map( + lambda x: eq(*x), + zip(json.loads(renderapi.utils.renderdumps(tilespecs)), ts_json))) From 7d5c445623868649e8a6fdace78bbc3705c7dee3 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 17:08:46 -0700 Subject: [PATCH 201/766] client: adding draft rendersectionclient --- renderapi/client.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 9116ef12..0417673d 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,9 +8,9 @@ import logging import subprocess import tempfile -from .utils import jbool, renderdump, NullHandler +from .utils import renderdump, NullHandler from .errors import ClientScriptError -from .render import Render, RenderClient, renderaccess +from .render import RenderClient, renderaccess from .stack import set_stack_state, make_stack_params # setup logger @@ -203,7 +203,6 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, trjson = tempjson.name with open(trjson, 'w') as f: renderdump(sharedTransforms, f) - importJsonClient(stack, tileFiles=[tsjson], transformFile=( trjson if sharedTransforms is not None else None), subprocess_mode=subprocess_mode, host=host, port=port, @@ -226,9 +225,9 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, pool = Pool(poolsize) partial_import = partial( import_tilespecs, stack, sharedTransforms=sharedTransforms, - subprocess_mode=subprocess_mode, **render.make_kwargs( - host=host, port=port, owner=owner, project=project, - client_script=client_script, memGB=memGB, **kwargs)) + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, client_script=client_script, + memGB=memGB, **kwargs) # TODO this is a weird way to do splits.... is that okay? tilespec_groups = [tilespecs[i::poolsize] for i in xrange(poolsize)] @@ -397,7 +396,7 @@ def importTransformChangesClient(stack, targetStack, transformFile, 'changeMode {} is not valid!'.format(changeMode)) argvs = (make_stack_params(host, port, owner, project, stack) + - ['--stack', stack, '--targetStack', targetStack] + + ['--targetStack', targetStack] + ['--transformFile', transformFile] + get_param(targetOwner, '--targetOwner') + get_param(targetProject, '--targetProject') + @@ -405,7 +404,7 @@ def importTransformChangesClient(stack, targetStack, transformFile, call_run_ws_client( 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, - add_args=argv) + add_args=argvs) @renderaccess @@ -436,3 +435,22 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, jsondata = json.load(f) return jsondata + + +@renderaccess +def renderSectionClient(stack, rootDirectory, zs, scale=None, format=None, + doFilter=None, fillWithNoise=None, + subprocess_mode=None, host=None, port=None, owner=None, + project=None, client_script=None, memGB=None, + render=None, **kwargs): + ''' + run RenderSectionClient.java + ''' + argvs = (make_stack_params(host, port, owner, project, stack) + + ['--rootDirectory', rootDirectory] + + get_param(scale, '--scale') + get_param(format, '--format') + + get_param(doFilter, '--doFilter') + + get_param(fillWithNoise, '--fillWithNoise') + zs) + call_run_ws_client('org.janelia.render.client.RenderSectionClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs) From 749e06c6e690245b35a07c88768695df4e0d84fc Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 17:09:55 -0700 Subject: [PATCH 202/766] tilespec: do not explicitly set null parameters --- renderapi/tilespec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 0ab0d0ad..fe065171 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -105,6 +105,7 @@ def to_dict(self): d['stageY'] = self.stageY d['rotation'] = self.rotation d['pixelsize'] = self.pixelsize + d = {k: v for k, v in d.items() if v is not None} return d def from_dict(self, d): @@ -199,6 +200,7 @@ def to_dict(self): thedict['inputfilters']['type'] = 'list' thedict['inputfilters']['specList'] = [f.to_dict() for f in self.inputfilters] + thedict = {k: v for k, v in thedict.items() if v is not None} return thedict def from_dict(self, d): From 703f7880a82092cbbc6efa72d061ea9a542d1969 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 17:11:13 -0700 Subject: [PATCH 203/766] utils: add documentation, maybe break RenderEncoder in later attempts --- renderapi/utils.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 33ab83db..5e339314 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -4,11 +4,13 @@ ''' import logging import inspect +import tempfile import copy import json class NullHandler(logging.Handler): + '''handler to avoid logging errors for, e.g., missing logger setup''' def emit(self, record): pass @@ -18,12 +20,30 @@ def emit(self, record): class RenderEncoder(json.JSONEncoder): + ''' + json Encoder in the following hierarchy for serialization: + obj.to_dict() + dict(obj) + JsonEncoder.default(obj) + obj.__dict__ + ''' def default(self, obj): to_dict = getattr(obj, "to_dict", None) if callable(to_dict): return obj.to_dict() else: - return obj.__dict__ + try: + return dict(obj) + except TypeError as e: + logger.debug("{} object is not recognized dictionary".format( + type(obj))) + try: + return super(RenderEncoder, self).default(obj) + except TypeError as e: + logger.warning( + "cannot json serialize {}. " + "Defaulting to __dict__".format(type(obj))) + return obj.__dict__ def post_json(session, request_url, d, params=None): @@ -46,11 +66,13 @@ def post_json(session, request_url, d, params=None): def renderdumps(obj, *args, **kwargs): + '''json.dumps using the RenderEncoder''' cls_ = kwargs.pop('cls', RenderEncoder) return json.dumps(obj, *args, cls=cls_, **kwargs) def renderdump(obj, *args, **kwargs): + '''json.dump using the RenderEncoder''' cls_ = kwargs.pop('cls', RenderEncoder) return json.dump(obj, *args, cls=cls_, **kwargs) From 341ec65bbc58e315fe92dc46eff81e58e5609483 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 19:12:20 -0700 Subject: [PATCH 204/766] tests: add coverage to client, stack, tilespec, utils --- test/test_client.py | 11 ++++---- test/test_files/tilespecs.json | 12 ++------ test/test_stack.py | 9 ++++++ test/test_tilespec.py | 50 +++++++++++++++++++++++++++++++--- test/test_utils.py | 8 ++++++ 5 files changed, 71 insertions(+), 19 deletions(-) create mode 100644 test/test_stack.py create mode 100644 test/test_utils.py diff --git a/test/test_client.py b/test/test_client.py index a657b264..3c5c50d8 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -14,16 +14,15 @@ def test_render_client(): r = renderapi.render.connect(**args) -def test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER): - r = renderapi.connect(**rkwargs) - new_r = renderapi.connect(**r.DEFAULT_KWARGS) +def test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER, **kwargs): + r = renderapi.connect(**dict(rkwargs, **kwargs)) + new_r = renderapi.connect(**dict(r.DEFAULT_KWARGS, **kwargs)) assert(new_r.DEFAULT_KWARGS == r.DEFAULT_KWARGS == rkwargs) -''' def test_default_kwargs_client(): - test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER_CLIENT) -''' + test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER_CLIENT, + validate_client=False) def test_environment_variables( diff --git a/test/test_files/tilespecs.json b/test/test_files/tilespecs.json index 51d6f74d..ef01e560 100644 --- a/test/test_files/tilespecs.json +++ b/test/test_files/tilespecs.json @@ -2,9 +2,7 @@ { "tileId": "20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.2266.0", "layout": { - "sectionId": "2266.0", - "temca": null, - "camera": null + "sectionId": "2266.0" }, "z": 2266, "minX": 1857, @@ -50,9 +48,7 @@ { "tileId": "20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8.2266.0", "layout": { - "sectionId": "2266.0", - "temca": null, - "camera": null + "sectionId": "2266.0" }, "z": 2266, "minX": 3915, @@ -98,9 +94,7 @@ { "tileId": "20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8.2266.0", "layout": { - "sectionId": "2266.0", - "temca": null, - "camera": null + "sectionId": "2266.0" }, "z": 2266, "minX": 5957, diff --git a/test/test_stack.py b/test/test_stack.py new file mode 100644 index 00000000..a95a5bb2 --- /dev/null +++ b/test/test_stack.py @@ -0,0 +1,9 @@ +import renderapi + + +def test_blank_stackversion(): + sv = renderapi.stack.StackVersion() + der_sv = renderapi.stack.StackVersion(**sv.to_dict()) + fd_sv = renderapi.stack.StackVersion() + fd_sv.from_dict(sv.to_dict()) + assert(sv.to_dict() == der_sv.to_dict() == fd_sv.to_dict()) diff --git a/test/test_tilespec.py b/test/test_tilespec.py index 9d866a8e..1e5fa696 100644 --- a/test/test_tilespec.py +++ b/test/test_tilespec.py @@ -4,11 +4,53 @@ import rendersettings -def test_load_tilespecs(): +def test_load_tilespecs_json(): with open(rendersettings.TEST_TILESPECS_FILE, 'r') as f: ts_json = json.load(f) - ts_jsons = json.dumps(ts_json) + # current TileSpec objects do not put, e.g. min/max X/Y values in in dict + ts_json_expected_fields = ['layout', 'width', 'height', 'tileId', + 'minIntensity', 'maxIntensity', 'mipmapLevels', + 'z', 'transforms'] + ts_json_expected = [{k: v for k, v in ts.items() + if k in ts_json_expected_fields} + for ts in ts_json] tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ts_json] - assert(map( + assert(all(map( lambda x: eq(*x), - zip(json.loads(renderapi.utils.renderdumps(tilespecs)), ts_json))) + zip(json.loads( + renderapi.utils.renderdumps(tilespecs)), ts_json_expected)))) + + +def test_load_tilespecs_args(): + with open(rendersettings.TEST_TILESPECS_FILE, 'r') as f: + ts_json = json.load(f) + # current TileSpec objects do not put, e.g. min/max X/Y values in in dict + ts_json_expected_fields = ['layout', 'width', 'height', 'tileId', + 'minIntensity', 'maxIntensity', 'mipmapLevels', + 'z', 'transforms'] + ts_json_expected = [{k: v for k, v in ts.items() + if k in ts_json_expected_fields} + for ts in ts_json] + + tilespecs = [renderapi.tilespec.TileSpec( + tileId=ts['tileId'], z=ts['z'], width=ts['width'], + height=ts['height'], mipMapLevels=[ + renderapi.tilespec.MipMapLevel( + l, d.get('imageUrl'), d.get('maskUrl')) + for l, d in ts['mipmapLevels'].items()], + layout=renderapi.tilespec.Layout( + force_pixelsize=False, **ts['layout']), + minint=ts['minIntensity'], maxint=ts['maxIntensity'], + tforms=renderapi.transform.TransformList( + json=ts['transforms']).tforms) + for ts in ts_json_expected] + assert(all(map( + lambda x: eq(*x), + zip([ts.to_dict() for ts in tilespecs], ts_json_expected)))) + + +def test_bbox_shape(): + with open(rendersettings.TEST_TILESPECS_FILE, 'r') as f: + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in json.load(f)] + + assert(all([len(ts.bbox) == 4 for ts in tilespecs])) diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 00000000..ea3733fc --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,8 @@ +import renderapi + + +def test_jbool(): + assert(renderapi.utils.jbool(True) == 'true') + assert(renderapi.utils.jbool(False) == 'false') + assert(renderapi.utils.jbool(0) == 'false') + assert(renderapi.utils.jbool(1) == 'true') From fe80414c568c4b93853b01573d06e0fe2faa8bca Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 19:13:23 -0700 Subject: [PATCH 205/766] tilespec: make pixelsize and inputfilter kind of optional --- renderapi/tilespec.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index fe065171..b9fda24a 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -83,7 +83,8 @@ def from_dict(self, d): class Layout: def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, - rotation=None, pixelsize=0.100, **kwargs): + rotation=None, pixelsize=None, + force_pixelsize=True, **kwargs): self.sectionId = sectionId self.scopeId = scopeId self.cameraId = cameraId @@ -92,6 +93,8 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, self.stageX = stageX self.stageY = stageY self.rotation = rotation + if force_pixelsize: + pixelsize = 0.100 if pixelsize is None else pixelsize self.pixelsize = pixelsize def to_dict(self): @@ -195,11 +198,12 @@ def to_dict(self): thedict['transforms']['specList'].append(strlist) else: thedict['transforms']['specList'].append(t.to_dict()) + if len(self.inputfilters): + thedict['inputfilters'] = {} + thedict['inputfilters']['type'] = 'list' + thedict['inputfilters']['specList'] = [f.to_dict() for f + in self.inputfilters] - thedict['inputfilters'] = {} - thedict['inputfilters']['type'] = 'list' - thedict['inputfilters']['specList'] = [f.to_dict() for f - in self.inputfilters] thedict = {k: v for k, v in thedict.items() if v is not None} return thedict From e6cb706220f796e22ca6182a0ac96150910fd426 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 3 Apr 2017 19:14:13 -0700 Subject: [PATCH 206/766] render: make RenderClient validation optional --- renderapi/render.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 06e2ee7a..c91ad1a7 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -71,13 +71,15 @@ class RenderClient(Render): Render object to run java client commands via a wrapped client script. This is a work in progress. ''' - def __init__(self, client_script=None, memGB=None, *args, **kwargs): + def __init__(self, client_script=None, memGB=None, validate_client=True, + *args, **kwargs): super(RenderClient, self).__init__(**kwargs) - if client_script is None: - raise ClientScriptError('No RenderClient script specified!') - elif not os.path.isfile(client_script): - raise ClientScriptError('Client script {} not found!'.format( - client_script)) + if validate_client: + if client_script is None: + raise ClientScriptError('No RenderClient script specified!') + elif not os.path.isfile(client_script): + raise ClientScriptError('Client script {} not found!'.format( + client_script)) if 'run_ws_client.sh' not in os.path.basename(client_script): logger.warning( 'Unrecognized client script {}!'.format(client_script)) @@ -102,7 +104,7 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, - force_http=True, **kwargs): + force_http=True, validate_client=True, **kwargs): ''' helper function to connect to a render instance can default to using environment variables if not specified in call. @@ -200,7 +202,8 @@ def connect(host=None, port=None, owner=None, project=None, return RenderClient(client_script=client_script, memGB=memGB, host=host, port=port, owner=owner, project=project, - client_scripts=client_scripts) + client_scripts=client_scripts, + validate_client=validate_client) except ClientScriptError as e: logger.info(e) logger.warning( From 60f14cf871470bc8b96fcb10d065c51d302fd523 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 3 Apr 2017 21:25:44 -0700 Subject: [PATCH 207/766] renaming tests to avoid conflicts in running both integration and regular tests --- .../{test_coordinate.py => test_coordinate_integrated.py} | 0 .../{test_pointmatch.py => test_pointmatch_integrated.py} | 0 integration_tests/{test_stack.py => test_stack_integrated.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename integration_tests/{test_coordinate.py => test_coordinate_integrated.py} (100%) rename integration_tests/{test_pointmatch.py => test_pointmatch_integrated.py} (100%) rename integration_tests/{test_stack.py => test_stack_integrated.py} (100%) diff --git a/integration_tests/test_coordinate.py b/integration_tests/test_coordinate_integrated.py similarity index 100% rename from integration_tests/test_coordinate.py rename to integration_tests/test_coordinate_integrated.py diff --git a/integration_tests/test_pointmatch.py b/integration_tests/test_pointmatch_integrated.py similarity index 100% rename from integration_tests/test_pointmatch.py rename to integration_tests/test_pointmatch_integrated.py diff --git a/integration_tests/test_stack.py b/integration_tests/test_stack_integrated.py similarity index 100% rename from integration_tests/test_stack.py rename to integration_tests/test_stack_integrated.py From 4a6d57acfbd49a7f3e7b8baff4e55bd315b19f2b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 3 Apr 2017 21:27:09 -0700 Subject: [PATCH 208/766] added more gitignore items --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 850de96e..4487c7e5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ *.pyc *source_me.sh +build/ +.eggs/ +*.egg-info +*.egg \ No newline at end of file From b458d6a0b869da9438df6a21a8662ae0d13c680d Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 07:35:25 -0700 Subject: [PATCH 209/766] changed starting image tag --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3902a3ea..1233174a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM atbigdawg:5000/fcollman/render:forrest +FROM fcollman/render:latest MAINTAINER Forrest Collman (forrest.collman@gmail.com) ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 From 35e231f3253a51f9a7b9c048adda6973e88a4881 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 4 Apr 2017 09:55:50 -0700 Subject: [PATCH 210/766] image, utils: add get_section_image, do flake8 --- renderapi/image.py | 49 ++++++++++++++++++++++++++++++++++++---------- renderapi/utils.py | 3 ++- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index 833f3f28..1baf956b 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -5,8 +5,8 @@ from PIL import Image import numpy as np import logging -from .render import Render, format_baseurl, format_preamble, renderaccess -from .utils import NullHandler +from .render import format_preamble, renderaccess +from .utils import NullHandler, jbool logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -25,7 +25,7 @@ @renderaccess def get_bb_image(stack, z, x, y, width, height, scale=1.0, - minIntensity=None,maxIntensity=None, + minIntensity=None, maxIntensity=None, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): @@ -48,18 +48,19 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, z, x, y, width, height, scale, image_ext) args = [] if minIntensity is not None: - args+=['minIntensity=%d'%minIntensity] + args += ['minIntensity=%d' % minIntensity] if maxIntensity is not None: - args+=['maxIntensity=%d'%maxIntensity] - if len(args)>0: + args += ['maxIntensity=%d' % maxIntensity] + if len(args) > 0: args = "&".join(args) - request_url+=args + request_url += args r = session.get(request_url) try: image = np.asarray(Image.open(io.BytesIO(r.content))) return image - except: + except Exception as e: + logger.error(e) logger.error(r.text) @@ -78,7 +79,7 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, request_url = format_preamble( host, port, owner, project, stack) + \ - "/tile/%s/png-image" % (tileId) + "/tile/%s/%s" % (tileId, image_ext) if normalizeForMatching: request_url += "?normalizeForMatching=true" logger.debug(request_url) @@ -87,5 +88,33 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, img = Image.open(io.BytesIO(r.content)) array = np.asarray(img) return array - except: + except Exception as e: + logger.error(e) logger.error(r.text) + + +@renderaccess +def get_section_image(stack, z, scale=1.0, filter=False, + maxTileSpecsToRender=None, img_format=None, + host=None, port=None, owner=None, project=None, + session=requests.session(), + render=None, **kwargs): + ''' + z: layer Z + scale: float -- linear scale at which to render image (e.g. 0.5) + filter: boolean -- whether or not to apply Khaled's preferred filter + maxTileSpecsToRender: int -- maximum number of tile specs in rendering + img_format: string -- format defined by IMAGE_FORMATS + ''' + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = format_preamble( + host, port, owner, project, stack) + '/z/{}/{}'.format(z, image_ext) + qparams = {'scale': scale, 'filter': jbool(filter)} + if maxTileSpecsToRender is not None: + qparams.update({'maxTileSpecsToRender': maxTileSpecsToRender}) + r = session.get(request_url, params=qparams) + return np.asarray(Image.open(io.BytesIO(r.content))) diff --git a/renderapi/utils.py b/renderapi/utils.py index 5e339314..acd001fc 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -4,9 +4,9 @@ ''' import logging import inspect -import tempfile import copy import json +from .errors import RenderError class NullHandler(logging.Handler): @@ -40,6 +40,7 @@ def default(self, obj): try: return super(RenderEncoder, self).default(obj) except TypeError as e: + logger.info(e) logger.warning( "cannot json serialize {}. " "Defaulting to __dict__".format(type(obj))) From 3524885ca8eb9d05a71589273e9df00dce5a8f9a Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 10:30:04 -0700 Subject: [PATCH 211/766] fixed bug in test --- integration_tests/test_coordinate_integrated.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index c0a6b4c5..56f0bb9b 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -97,10 +97,10 @@ def test_world_to_local_coordinates_batch(render, teststack_tilespec): corners = [[0, 0], [ts.width, 0], [ts.width, ts.height], [0, ts.height]] batch = [] for corner in corners: - batch.append(renderapi.coordinate.local-to-world-coordinates( + batch.append(renderapi.coordinate.local_to_world_coordinates( stack, ts.z, corner[0], corner[1])) - local = renderapi.coordinates.world_to_local_coordinates_batch( - stack, batch, execute_local=False) + local = renderapi.coordinate.world_to_local_coordinates_batch( + stack, batch, ts.z,execute_local=False) assert(len(local) == len(batch)) for ans in local: From 6e16b6adba45744efa7b9d5626aeb1cbedb06ab8 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 10:35:41 -0700 Subject: [PATCH 212/766] fixed bug in test --- integration_tests/test_coordinate_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 56f0bb9b..172d930f 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -98,9 +98,9 @@ def test_world_to_local_coordinates_batch(render, teststack_tilespec): batch = [] for corner in corners: batch.append(renderapi.coordinate.local_to_world_coordinates( - stack, ts.z, corner[0], corner[1])) + stack, ts.z, corner[0], corner[1],render=render)) local = renderapi.coordinate.world_to_local_coordinates_batch( - stack, batch, ts.z,execute_local=False) + stack, batch, ts.z,execute_local=False,render=render) assert(len(local) == len(batch)) for ans in local: From 275b1abea2e83f922fd020cec6d4339cfd824968 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 10:38:05 -0700 Subject: [PATCH 213/766] more test bugs --- integration_tests/test_coordinate_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 172d930f..3495458c 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -86,7 +86,7 @@ def test_world_to_local_coordinates(render, teststack_tilespec): def test_local_to_world_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec local = render.run(renderapi.coordinate.local_to_world_coordinates_batch, - stack, ts.z, 0, 0) + stack, ts.z, 0) assert(local['error'] == "") assert(local['tileId'] == ts.tileId) assert(len(local['world']) >= 2) From 210e88e30606cbc059121a5c93a27fbc06516ee0 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 10:47:51 -0700 Subject: [PATCH 214/766] fixed more tests --- integration_tests/test_coordinate_integrated.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 3495458c..c11d2821 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -85,8 +85,8 @@ def test_world_to_local_coordinates(render, teststack_tilespec): def test_local_to_world_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec - local = render.run(renderapi.coordinate.local_to_world_coordinates_batch, - stack, ts.z, 0) + local = render.run(renderapi.coordinate.local_to_world_coordinates, + stack, ts.z, 0,0) assert(local['error'] == "") assert(local['tileId'] == ts.tileId) assert(len(local['world']) >= 2) @@ -104,7 +104,8 @@ def test_world_to_local_coordinates_batch(render, teststack_tilespec): assert(len(local) == len(batch)) for ans in local: - assert(len(ans['error']) == 0) + for tile in ans: + assert(len(tile['error']) == 0) def test_local_to_world_coordinates_batch(render, teststack_tilespec): From 352340843fce59eb7a5c0959004b13b8285fe6ca Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 4 Apr 2017 11:07:06 -0700 Subject: [PATCH 215/766] transform: make invert work and add test --- renderapi/transform.py | 5 ++++- test/test_transform.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 8f197f25..9be54db2 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -239,7 +239,10 @@ def load_M(self): self.M[1, 2] = self.B1 def invert(self): - Ai = AffineModel() + inv_M = np.linalg.inv(self.M) + Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], + inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) + return Ai def fit(self, A, B): if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): diff --git a/test/test_transform.py b/test/test_transform.py index 54211948..f91904e3 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -1,5 +1,6 @@ import renderapi import numpy as np +import scipy.linalg def test_affine_rot_90(): @@ -50,3 +51,49 @@ def test_affine_random(): am_fit.estimate(points_in, points_out) assert(np.sum(np.abs(am.M.ravel()-am_fit.M.ravel())) < (.001*6)) + + +def test_invert_Affine(): + am = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + Iam = am.invert() + assert(np.allclose(Iam.concatenate(am).M, np.eye(3))) + assert(np.allclose(am.concatenate(Iam).M, np.eye(3))) + + +def test_Polynomial_estimation(use_numpy=False): + if use_numpy: + try: + import builtins + except ImportError: + import __builtin__ as builtins + realimport = builtins.__import__ + + def noscipy_import(name, globals=None, locals=None, + fromlist=(), level=0): + if 'scipy' in name: + raise ImportError + return realimport(name, globals, locals, fromlist, level) + builtins.__import__ = noscipy_import + reload(renderapi.transform) + assert(renderapi.transform.svd is np.linalg.svd + if use_numpy else renderapi.transform.svd is scipy.linalg.svd) + + # TODO now that import framework is good, do actual test + + if use_numpy: + builtins.__import__ = realimport + reload(renderapi.transform) + assert(renderapi.transform.svd is scipy.linalg.svd) + + +def test_Polynomial_estimation_numpy(): + test_Polynomial_estimation(use_numpy=True) + + +def test_transformsum(): + pass From 9d94c6737f9d860d385b06fd7ceaa9aca34addd7 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 11:25:40 -0700 Subject: [PATCH 216/766] fixed bug in test --- integration_tests/test_coordinate_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index c11d2821..ea809c00 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -59,7 +59,7 @@ def teststack_tilespec(): sharedTransforms=tforms) r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') yield (stack, tilespecs[0]) - render.run(renderapi.stack.delete_stack, stack) + #render.run(renderapi.stack.delete_stack, stack) @pytest.fixture(scope='module') @@ -86,7 +86,7 @@ def test_world_to_local_coordinates(render, teststack_tilespec): def test_local_to_world_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec local = render.run(renderapi.coordinate.local_to_world_coordinates, - stack, ts.z, 0,0) + stack, ts.tileId, 0,0) assert(local['error'] == "") assert(local['tileId'] == ts.tileId) assert(len(local['world']) >= 2) From d7086694b8cebdd867b4d27f1128c2f522746207 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 13:35:22 -0700 Subject: [PATCH 217/766] fixing tests --- .../test_coordinate_integrated.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index ea809c00..ba307a67 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -72,40 +72,48 @@ def local_corners_json(teststack_tilespec): 'tileId': ts.tileId, 'visible': True, } + d['local']=corner + return corners +@pytest.fixture(scope='module') +def world_corners_json(render,teststack_tilespec): + (stack, ts) = teststack_tilespec + corners = [[0, 0], [ts.width-100, 0], [ts.width-100, ts.height-100], [0, ts.height-100]] + world_corners = [] + for corner in corners: + world_corners.append(renderapi.coordinate.local_to_world_coordinates(\ + stack, ts.tileId, corner[0], corner[1],render=render)) + return world_corners def test_world_to_local_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec local = render.run(renderapi.coordinate.world_to_local_coordinates, stack, ts.z, ts.minX, ts.minY) - assert(local['error'] == "") + logger.debug(local) + assert('error' not in local.keys()) assert(local['tileId'] == ts.tileId) assert(len(local['local']) >= 2) def test_local_to_world_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec - local = render.run(renderapi.coordinate.local_to_world_coordinates, + world = render.run(renderapi.coordinate.local_to_world_coordinates, stack, ts.tileId, 0,0) - assert(local['error'] == "") - assert(local['tileId'] == ts.tileId) - assert(len(local['world']) >= 2) + logger.debug(world) + assert('error' not in world.keys()) + assert(world['tileId'] == ts.tileId) + assert(len(world['world']) >= 2) -def test_world_to_local_coordinates_batch(render, teststack_tilespec): +def test_world_to_local_coordinates_batch(render, teststack_tilespec,world_corners_json): (stack, ts) = teststack_tilespec - corners = [[0, 0], [ts.width, 0], [ts.width, ts.height], [0, ts.height]] - batch = [] - for corner in corners: - batch.append(renderapi.coordinate.local_to_world_coordinates( - stack, ts.z, corner[0], corner[1],render=render)) local = renderapi.coordinate.world_to_local_coordinates_batch( - stack, batch, ts.z,execute_local=False,render=render) - - assert(len(local) == len(batch)) + stack, world_corners_json, ts.z,execute_local=False,render=render) + logger.debug(local) + assert(len(local) == len(world_corners_json)) for ans in local: for tile in ans: - assert(len(tile['error']) == 0) + assert('error' not in tile.keys()) def test_local_to_world_coordinates_batch(render, teststack_tilespec): From 90b89188adb3fb4aa02e4c72496b5314e9aaeb1e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 13:42:28 -0700 Subject: [PATCH 218/766] finally actually fixed!!! --- integration_tests/test_coordinate_integrated.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index ba307a67..d1ce6534 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -53,11 +53,11 @@ def teststack_tilespec(): tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] stack = 'test_coordinate_stack' - r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.stack.create_stack, stack, force_resolution=True) render.run( renderapi.client.import_tilespecs, stack, tilespecs, sharedTransforms=tforms) - r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') + render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') yield (stack, tilespecs[0]) #render.run(renderapi.stack.delete_stack, stack) @@ -65,7 +65,7 @@ def teststack_tilespec(): @pytest.fixture(scope='module') def local_corners_json(teststack_tilespec): (stack, ts) = teststack_tilespec - corners = [[0, 0], [ts.width, 0], [ts.width, ts.height], [0, ts.height]] + corners = [[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]] batch = [] for corner in corners: d = { @@ -78,7 +78,7 @@ def local_corners_json(teststack_tilespec): @pytest.fixture(scope='module') def world_corners_json(render,teststack_tilespec): (stack, ts) = teststack_tilespec - corners = [[0, 0], [ts.width-100, 0], [ts.width-100, ts.height-100], [0, ts.height-100]] + corners = [[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]] world_corners = [] for corner in corners: world_corners.append(renderapi.coordinate.local_to_world_coordinates(\ From 6341bd373038ee433168545bb0f68a3628713e79 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 13:55:32 -0700 Subject: [PATCH 219/766] attempting to fix more tests --- .../test_coordinate_integrated.py | 2 +- integration_tests/test_stack_integrated.py | 38 ++++++------------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index d1ce6534..29be68f3 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -59,7 +59,7 @@ def teststack_tilespec(): sharedTransforms=tforms) render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') yield (stack, tilespecs[0]) - #render.run(renderapi.stack.delete_stack, stack) + render.run(renderapi.stack.delete_stack, stack) @pytest.fixture(scope='module') diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 6a0d74bc..553b3802 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -219,42 +219,26 @@ def test_coordinateClient(render): @pytest.fixture(scope="module") -def teststack(request): - render_test_parameters = { - 'host': render_host, - 'port': render_port, - 'owner': 'test', - 'project': 'test_project', - 'client_scripts': client_script_location} - render = renderapi.render.connect(**render_test_parameters) - - with open(tilespec_file, 'r') as f: - ts_json = json.load(f) - with open(tform_file, 'r') as f: - tform_json = json.load(f) - - tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] - tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] +def teststack(request,render,render_example_tilespec_and_transforms): + (tilespecs,tforms)=render_example_tilespec_and_transforms stack = 'test_insert3' r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) render.run(renderapi.client.import_tilespecs, stack, tilespecs, sharedTransforms=tforms) r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') - - def fin(): - render.run(renderapi.stack.delete_stack, stack) - request.addfinalizer(fin) - return stack - + yield stack + render.run(renderapi.stack.delete_stack, stack) def test_stack_bounds(render, teststack): # check the stack bounds stack_bounds = render.run(renderapi.stack.get_stack_bounds, teststack) - expected_bounds = {u'maxZ': 3408.0, u'maxX': 5176.0, u'maxY': 5319.0, - u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} - assert(stack_bounds == expected_bounds) + expected_bounds = {u'maxZ': 3408.0, u'maxX': 5102.0, u'maxY': 5385.0, + u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} + for key in stack_bounds.keys(): + assert(np.abs(stack_bounds[key]-expected_bounds[key])<1.0) + def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -263,7 +247,9 @@ def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): teststack, tilespecs[0].z) expected_bounds = {u'maxZ': 3407.0, u'maxX': 5009.0, u'maxY': 4395.0, u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} - assert(zbounds == expected_bounds) + for key in zbounds.keys(): + assert(np.abs(zbounds[key]-expected_bounds[key])<1.0) + def test_get_section_z(render, teststack): From 2551e71af029da24e2b4524b0681efcea3999141 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 13:59:33 -0700 Subject: [PATCH 220/766] formatting --- integration_tests/test_stack_integrated.py | 81 +++++++++------------- 1 file changed, 31 insertions(+), 50 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 553b3802..b4378c54 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -23,11 +23,11 @@ @pytest.fixture(scope='module') def render(): render_test_parameters = { - 'host': render_host, - 'port': render_port, - 'owner': 'test', - 'project': 'test_project', - 'client_scripts': client_script_location} + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location} return renderapi.render.connect(**render_test_parameters) @@ -110,12 +110,11 @@ def test_set_stack_metadata(render): sv.stackResolutionZ = 4.0 r = render.run(renderapi.stack.set_stack_metadata, test_stack, sv) - assert (r.status_code == 201) + assert r.status_code == 201 sv = render.run(renderapi.stack.get_stack_metadata, test_stack) - assert (sv.stackResolutionX == 2.0) - assert (sv.stackResolutionY == 3.0) - assert (sv.stackResolutionZ == 4.0) - + assert sv.stackResolutionX == 2.0 + assert sv.stackResolutionY == 3.0 + assert sv.stackResolutionZ == 4.0 def test_simple_import(render, simpletilespec, tmpdir): # open a temporary file @@ -138,7 +137,6 @@ def test_simple_import(render, simpletilespec, tmpdir): assert (ts_out.z == simpletilespec.z) render.run(renderapi.stack.delete_stack, 'test_insert') - def test_simple_import_with_transforms( render, render_example_tilespec_and_transforms, tmpdir): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -162,61 +160,52 @@ def test_simple_import_with_transforms( str(tilespecfile), transformFile=str(transformfile)) r = render.run(renderapi.stack.set_stack_state, 'test_insert_tform', 'COMPLETE') - assert (r.status_code == 201) + assert r.status_code == 201 ts_out = render.run(renderapi.tilespec.get_tile_spec, 'test_insert_tform', tilespecs[0].tileId) - assert (ts_out.z == tilespecs[0].z) + assert ts_out.z == tilespecs[0].z render.run(renderapi.stack.delete_stack, 'test_insert_tform') - # os.remove(tfile) - - def test_import_tilespecs(render, simpletilespec): stack = 'test_insert2' - r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.stack.create_stack, stack, force_resolution=True) render.run(renderapi.client.import_tilespecs, stack, [simpletilespec]) - r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') - assert (r.status_code == 201) + response = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') + assert response.status_code == 201 ts_out = render.run(renderapi.tilespec.get_tile_spec, stack, simpletilespec.tileId) - assert (ts_out.z == simpletilespec.z) + assert ts_out.z == simpletilespec.z render.run(renderapi.stack.delete_stack, stack) def test_import_tilespecs_parallel(render): root.debug('test not implemented yet') - assert(False) - + assert False def test_import_jsonfiles_validate_client(render): root.debug('test not implemented yet') - assert(False) - + assert False def test_import_jsonfiles(render): root.debug('test not implemented yet') - assert(False) - + assert False def test_import_parallel(render): root.debug('test not implemented yet') - assert(False) - + assert False def test_tile_pair_client(render): root.debug('test not implemented yet') - assert(False) + assert False def test_importTransformChangesClient(render): root.debug('test not implemented yet') - assert(False) - + assert False def test_coordinateClient(render): root.debug('test not implemented yet') - assert(False) - + assert False @pytest.fixture(scope="module") def teststack(request,render,render_example_tilespec_and_transforms): @@ -237,8 +226,7 @@ def test_stack_bounds(render, teststack): u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} for key in stack_bounds.keys(): - assert(np.abs(stack_bounds[key]-expected_bounds[key])<1.0) - + assert np.abs(stack_bounds[key]-expected_bounds[key]) < 1.0 def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -248,31 +236,26 @@ def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): expected_bounds = {u'maxZ': 3407.0, u'maxX': 5009.0, u'maxY': 4395.0, u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} for key in zbounds.keys(): - assert(np.abs(zbounds[key]-expected_bounds[key])<1.0) - - + assert np.abs(zbounds[key]-expected_bounds[key]) < 1.0 def test_get_section_z(render, teststack): # check getting section Z z = render.run(renderapi.stack.get_section_z_value, teststack, "3407.0") - assert(z == 3407) + assert z == 3407 z = render.run(renderapi.stack.get_z_value_for_section, teststack, "3407.0") - assert(z == 3407) - + assert z == 3407 def test_get_z_values(render, teststack): # check get z values zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) - assert(zvalues == [3407.0, 3408.0]) - + assert zvalues == [3407.0, 3408.0] def test_uniq_value(render): # check likelyUniqueId uniq = render.run(renderapi.stack.likelyUniqueId) - assert(len(uniq) >= len('58ceebb7a7b11b0001dc4e32')) - + assert len(uniq) >= len('58ceebb7a7b11b0001dc4e32') def test_bb_image(render, teststack): formats = renderapi.image.IMAGE_FORMATS.keys() @@ -289,10 +272,9 @@ def test_bb_image(render, teststack): teststack, z, x, y, width, height, scale=.25, img_format=fmt) dr = data.ravel() - assert (data.shape[0] == (np.floor(height*.25))) - assert (data.shape[1] == (np.floor(width*.25))) - assert (data.shape[2] >= 3) - + assert data.shape[0] == (np.floor(height*.25)) + assert data.shape[1] == (np.floor(width*.25)) + assert data.shape[2] >= 3 def test_tile_image(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -303,7 +285,6 @@ def test_tile_image(render, teststack, render_example_tilespec_and_transforms): assert data.shape[0] >= tilespecs[0].height assert data.shape[1] >= tilespecs[0].width - def fail_image_get(render, teststack, render_example_tilespec_and_transforms): with pytest.raises(KeyError): render.run(renderapi.image.get_tile_image_data, teststack, From fa1c5c336726542883b342d0ee65aef67cb5d2c5 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 14:37:02 -0700 Subject: [PATCH 221/766] fixed more tests --- integration_tests/test_coordinate_integrated.py | 9 +++++---- integration_tests/test_stack_integrated.py | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 29be68f3..5f1026b4 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -88,11 +88,12 @@ def world_corners_json(render,teststack_tilespec): def test_world_to_local_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec local = render.run(renderapi.coordinate.world_to_local_coordinates, - stack, ts.z, ts.minX, ts.minY) + stack, ts.z, 1500, 1500) logger.debug(local) - assert('error' not in local.keys()) - assert(local['tileId'] == ts.tileId) - assert(len(local['local']) >= 2) + for tile in local: + assert('error' not in tile.keys()) + assert(tile['tileId'] == ts.tileId) + assert(len(tile['local']) >= 2) def test_local_to_world_coordinates(render, teststack_tilespec): diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index b4378c54..372dc589 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -233,8 +233,9 @@ def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): # check a single z stack bounds zbounds = render.run(renderapi.stack.get_bounds_from_z, teststack, tilespecs[0].z) - expected_bounds = {u'maxZ': 3407.0, u'maxX': 5009.0, u'maxY': 4395.0, - u'minX': 232.0, u'minY': 17.0, u'minZ': 3407.0} + + expected_bounds = {u'maxZ': 3407.0, u'maxX': 4917.0, u'maxY': 4506.0, + u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} for key in zbounds.keys(): assert np.abs(zbounds[key]-expected_bounds[key]) < 1.0 From 4816d47b3e83d7d0171b0abb6ee4ab9c00d352e9 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 16:21:13 -0700 Subject: [PATCH 222/766] added more tests and debugging logs --- .../test_coordinate_integrated.py | 28 +++++++++++++------ renderapi/coordinate.py | 4 +-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 5f1026b4..a54e5560 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -73,7 +73,8 @@ def local_corners_json(teststack_tilespec): 'visible': True, } d['local']=corner - return corners + batch.append(d) + return batch @pytest.fixture(scope='module') def world_corners_json(render,teststack_tilespec): @@ -117,10 +118,13 @@ def test_world_to_local_coordinates_batch(render, teststack_tilespec,world_corne assert('error' not in tile.keys()) -def test_local_to_world_coordinates_batch(render, teststack_tilespec): - logger.debug('test not implemented yet') - assert(False) - +def test_local_to_world_coordinates_batch(render, teststack_tilespec,local_corners_json): + (stack, ts) = teststack_tilespec + world = renderapi.coordinate.local_to_world_coordinates_batch(stack,local_corners_json, + ts.z,execute_local=False,render=render) + assert(len(local_corners_json)==len(world)) + for ans in world: + assert('error' not in ans.keys()) def old_world_to_local_coordinates_array(render, teststack_tilespec): logger.debug('test not implemented yet') @@ -133,14 +137,22 @@ def test_world_to_local_coordinates_array(render, teststack_tilespec): def old_local_to_world_coordinates_array(render, teststack_tilespec): + + logger.debug('test not implemented yet') assert(False) def local_to_world_coordinates_array(render, teststack_tilespec): - logger.debug('test not implemented yet') - assert(False) - + (stack, ts) = teststack_tilespec + local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, + local_corners, + ts.tileId, + ts.z, + render=render) + logger.debug('world corners:{}'.format(world_corners)) + assert world_corners.shape[0]==local_corners.shape[0] def world_to_local_coordinates_clientside(): logger.debug('test not implemented yet') diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 1de445ab..c5b08a10 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -193,8 +193,8 @@ def old_local_to_world_coordinates_array(stack, dataarray, tileId, z=0, json_answer = r.json() try: answer = np.zeros(dataarray.shape) - print dataarray.shape - print len(json_answer) + logger.debug('shape {}'.format(dataarray.shape)) + logger.debug('length of json_answer {}'.format(len(json_answer))) for i, coord in enumerate(json_answer): c = coord['world'] answer[i, 0] = c[0] From 9e0b08041502cede0d480d91e546d6e2a619e529 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 16:48:02 -0700 Subject: [PATCH 223/766] implemented world to local array test --- .../test_coordinate_integrated.py | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index a54e5560..aa0c7f9f 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -126,22 +126,20 @@ def test_local_to_world_coordinates_batch(render, teststack_tilespec,local_corne for ans in world: assert('error' not in ans.keys()) -def old_world_to_local_coordinates_array(render, teststack_tilespec): - logger.debug('test not implemented yet') - assert(False) - - def test_world_to_local_coordinates_array(render, teststack_tilespec): - logger.debug('test not implemented yet') - assert(False) - - -def old_local_to_world_coordinates_array(render, teststack_tilespec): - - - logger.debug('test not implemented yet') - assert(False) - + local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, + local_corners, + ts.tileId, + ts.z, + render=render) + local_corners2 = renderapi.coordinate.world_to_local_coordinates_array(stack, + world_corners, + ts.tileId, + ts.z, + render=render) + for pt,ptafter in zip(local_corners,local_corners2): + assert np.sum(np.abs(pt-ptafter))<.1 def local_to_world_coordinates_array(render, teststack_tilespec): (stack, ts) = teststack_tilespec @@ -155,6 +153,7 @@ def local_to_world_coordinates_array(render, teststack_tilespec): assert world_corners.shape[0]==local_corners.shape[0] def world_to_local_coordinates_clientside(): + logger.debug('test not implemented yet') assert(False) From df71d3d994434457f27ae316c3b5048243a6f697 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 16:52:47 -0700 Subject: [PATCH 224/766] implemented clientside tests --- .../test_coordinate_integrated.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index aa0c7f9f..9fbab454 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -138,6 +138,7 @@ def test_world_to_local_coordinates_array(render, teststack_tilespec): ts.tileId, ts.z, render=render) + logger.debug('local corners2: {}'.format(local_corners2)) for pt,ptafter in zip(local_corners,local_corners2): assert np.sum(np.abs(pt-ptafter))<.1 @@ -152,12 +153,33 @@ def local_to_world_coordinates_array(render, teststack_tilespec): logger.debug('world corners:{}'.format(world_corners)) assert world_corners.shape[0]==local_corners.shape[0] -def world_to_local_coordinates_clientside(): - - logger.debug('test not implemented yet') - assert(False) +def world_to_local_coordinates_clientside(render, teststack_tilespec): + local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, + local_corners, + ts.tileId, + ts.z, + render=render, + doClientSide=True) + local_corners2 = renderapi.coordinate.world_to_local_coordinates_array(stack, + world_corners, + ts.tileId, + ts.z, + render=render, + doClientSide=True) + logger.debug('local corners2: {}'.format(local_corners2)) + for pt,ptafter in zip(local_corners,local_corners2): + assert np.sum(np.abs(pt-ptafter))<.1 -def local_to_world_coordinates_clientside(): - logger.debug('test not implemented yet') - assert(False) +def local_to_world_coordinates_clientside(render, teststack_tilespec): + (stack, ts) = teststack_tilespec + local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, + local_corners, + ts.tileId, + ts.z, + doClientSide=True, + render=render) + logger.debug('world corners:{}'.format(world_corners)) + assert world_corners.shape[0]==local_corners.shape[0] From 0c4f56d99ee669bb2ca6089d9d47b9ba79fc0b7b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 17:30:57 -0700 Subject: [PATCH 225/766] fixed bugs --- integration_tests/test_coordinate_integrated.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 9fbab454..078a0ce4 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -127,6 +127,7 @@ def test_local_to_world_coordinates_batch(render, teststack_tilespec,local_corne assert('error' not in ans.keys()) def test_world_to_local_coordinates_array(render, teststack_tilespec): + (stack, ts) = teststack_tilespec local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, local_corners, @@ -154,6 +155,7 @@ def local_to_world_coordinates_array(render, teststack_tilespec): assert world_corners.shape[0]==local_corners.shape[0] def world_to_local_coordinates_clientside(render, teststack_tilespec): + (stack, ts) = teststack_tilespec local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, local_corners, From b3d3f41c7ebefe1c5b3f08672889cf31093fba21 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 4 Apr 2017 17:37:05 -0700 Subject: [PATCH 226/766] more tests! --- integration_tests/test_coordinate_integrated.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 078a0ce4..283d9394 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -143,7 +143,7 @@ def test_world_to_local_coordinates_array(render, teststack_tilespec): for pt,ptafter in zip(local_corners,local_corners2): assert np.sum(np.abs(pt-ptafter))<.1 -def local_to_world_coordinates_array(render, teststack_tilespec): +def test_local_to_world_coordinates_array(render, teststack_tilespec): (stack, ts) = teststack_tilespec local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, @@ -154,7 +154,7 @@ def local_to_world_coordinates_array(render, teststack_tilespec): logger.debug('world corners:{}'.format(world_corners)) assert world_corners.shape[0]==local_corners.shape[0] -def world_to_local_coordinates_clientside(render, teststack_tilespec): +def test_world_to_local_coordinates_clientside(render, teststack_tilespec): (stack, ts) = teststack_tilespec local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, @@ -174,7 +174,7 @@ def world_to_local_coordinates_clientside(render, teststack_tilespec): assert np.sum(np.abs(pt-ptafter))<.1 -def local_to_world_coordinates_clientside(render, teststack_tilespec): +def test_local_to_world_coordinates_clientside(render, teststack_tilespec): (stack, ts) = teststack_tilespec local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, From 7a8bfc860967da6ef4ff2d746bd958f9617b940e Mon Sep 17 00:00:00 2001 From: Eric Perlman Date: Tue, 4 Apr 2017 21:52:39 -0400 Subject: [PATCH 227/766] Don't limit scale precision %3.2f causes havoc. e.g, 1/32 = .03125 !+ 0.03 --- renderapi/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/image.py b/renderapi/image.py index 9d6ab3b1..24619364 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -43,7 +43,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, request_url = format_preamble( host, port, owner, project, stack) + \ - "/z/%d/box/%d,%d,%d,%d,%3.2f/%s" % ( + "/z/%d/box/%d,%d,%d,%d,%f/%s" % ( z, x, y, width, height, scale, image_ext) r = session.get(request_url) try: From 689a38ba408719323193daeb038e25a69c2326be Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 08:15:49 -0700 Subject: [PATCH 228/766] adding more to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 850de96e..4264034b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ *.pyc *source_me.sh +build/ +.eggs* +*.egg-info +*.egg \ No newline at end of file From 03a906e40163bd5508d9e03068f2e32ae6241529 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 08:16:17 -0700 Subject: [PATCH 229/766] adding dist to gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4264034b..5f80c9e2 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ build/ .eggs* *.egg-info -*.egg \ No newline at end of file +*.egg +dist/ From 3c3deb1ee543f76caf9db5acef9173014567b858 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 10:14:55 -0700 Subject: [PATCH 230/766] transform: refactor Polynomial2D, add tests dynamic Polynomial2D datastring brought floating point rounding errors when reading from fresh tilespecs, so removed Polynomial from tilespec reading test --- renderapi/transform.py | 135 ++++++++++++++++++++++----------- test/test_files/tilespecs.json | 15 ---- test/test_transform.py | 12 ++- 3 files changed, 100 insertions(+), 62 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 9be54db2..f874f65e 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -22,12 +22,13 @@ # TODO preference for svd version? try: - from scipy.linalg import svd + from scipy.linalg import svd, LinAlgError except ImportError as e: logger.info(e) logger.info('scipy-based linalg may or may not lead ' 'to better parameter fitting') from numpy.linalg import svd + from numpy.linalg.linalg import LinAlgError class TransformList: @@ -278,8 +279,6 @@ def estimate(self, A, B): def convert_to_point_vector(self, points): Np = points.shape[0] - - zerovec = np.zeros((Np, 1), np.double) onevec = np.ones((Np, 1), np.double) if points.shape[1] != 2: @@ -391,7 +390,7 @@ class Polynomial2DTransform(Transform): def __init__(self, dataString=None, src=None, dst=None, order=2, force_polynomial=True, params=None, identity=False, - json=None): + json=None, **kwargs): if json is not None: self.from_dict(json) else: @@ -399,15 +398,15 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, if dataString is not None: self._process_dataString(dataString) elif identity: - self._process_params(np.array([[0, 1, 0], [0, 0, 1]])) + self.params = np.array([[0, 1, 0], [0, 0, 1]]) elif params is not None: - self._process_params(params) + self.params = params elif src is not None and dst is not None: - self._process_params(self.estimate(src, dst, order)) + self.params = self.estimate(src, dst, order, **kwargs) if not force_polynomial and self.is_affine: - # TODO try implement affine from poly (& vice versa) - return AffineTransform(poly_params=self.params) + raise NotImplementedError('Falling back to Affine model is ' + 'not supported {}') self.transformId = None @property @@ -421,7 +420,12 @@ def order(self): no_coeffs = len(self.params.ravel()) return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) - def fit(self, src, dst, order=2): + @property + def dataString(self): + return Polynomial2DTransform._dataStringfromParams(self.params) + + @staticmethod + def fit(src, dst, order=2): '''This is unreliable -- add tests to ensure repeatability''' xs = src[:, 0] ys = src[:, 1] @@ -439,7 +443,7 @@ def fit(self, src, dst, order=2): 'order {} is too large to fit {} points!'.format( order, len(src))) - A = np.empty([rows * 2, no_coeff + 1]) + A = np.zeros([rows * 2, no_coeff + 1]) pidx = 0 for j in range(order + 1): for i in range(j + 1): @@ -459,39 +463,35 @@ def fit(self, src, dst, order=2): def estimate(self, src, dst, order=2, convergence_test=None, max_tries=100, **kwargs): - - params = self.fit(src, dst, order=order) - if convergence_test is None: - return params - else: - # FIXME discuss plan for estimate vs fit & stability handling - raise NotImplementedError( - 'Stability checking is unavailable') - self._process_params(params) - if convergence_test is not None: - if(convergence_test(self)): - return params - - for i in range(max_tries): - params = fit(src, dst, **kwargs) - self._process_params(params) - if convergence_test(self): - return params - - raise EstimationError( - 'could not find a converged estimate in %d tries' % max_tries) - else: - return params - - def _process_params(self, params): - ''' - generate datastring and param attributes from params - ''' + def fitgood(src, dst, params): + result = Polynomial2DTransform(params=params).tform(src) + t = np.allclose( + result, dst, + atol=1e-3, rtol=0) + return t + + estimated = False + tries = 0 + while (tries < max_tries and not estimated): + tries += 1 + try: + params = Polynomial2DTransform.fit(src, dst, order=2) + except (LinAlgError, ValueError) as e: + logger.debug('Encountered error {}'.format(e)) + continue + estimated = fitgood(src, dst, params) + + if tries == max_tries and not estimated: + raise EstimationError('Could not fit Polynomial ' + 'in {} attempts!'.format(tries)) + logger.debug('fit parameters in {} attempts'.format(tries)) self.params = params - self.dataString = self._dataStringfromParams(params) + return self.params - def _dataStringfromParams(self, params=None): - return ' '.join([str(i) for i in params.flatten()]).replace('e', 'E') + @staticmethod + def _dataStringfromParams(params=None): + return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in params.flatten()]).replace('e', 'E') def _process_dataString(self, datastring): ''' @@ -501,9 +501,9 @@ def _process_dataString(self, datastring): self.params = np.array( [[float(d) for d in dsList[:len(dsList)/2]], [float(d) for d in dsList[len(dsList)/2:]]]) - self.dataString = datastring - def _format_raveled_params(self, raveled_params): + @staticmethod + def _format_raveled_params(raveled_params): return np.array( [[float(d) for d in raveled_params[:len(raveled_params)/2]], [float(d) for d in raveled_params[len(raveled_params)/2:]]]) @@ -523,12 +523,21 @@ def tform(self, points): return dst def coefficients(self, order=None): + ''' + determine number of coefficient terms in transform for a given order + input: order of polynomial -- defaults to self.order + output: integer number of coefficient terms expected in transform + ''' if order is None: order = self.order return (order + 1) * (order + 2) def asorder(self, order): - '''''' + ''' + input: order > current order + output: new Transform object of selected order with coefficients + from self + ''' if self.order > order: raise ConversionError( 'transformation {} is order {} -- conversion to ' @@ -538,7 +547,12 @@ def asorder(self, order): new_params[:self.params.shape[0], :self.params.shape[1]] = self.params return Polynomial2DTransform(params=new_params) - def _fromAffine(self, aff): + @staticmethod + def fromAffine(aff): + ''' + input: AffineModel + output: Polynomial2DTransform defined by Affine model + ''' if not isinstance(aff, AffineModel): raise ConversionError('attempting to convert a nonaffine model!') return Polynomial2DTransform(params=np.array([ @@ -570,6 +584,8 @@ def transformsum(transformlist, src=None): Polynomial2DTransform representing the sum of the input list ''' + logger.warning('This implementation of transformsum is not recommended! ' + 'Please consider using estimate_transformsum') sumtform = Polynomial2DTransform(identity=True) for tform in transformlist: if isinstance(tform, list): @@ -579,3 +595,30 @@ def transformsum(transformlist, src=None): else: sumtform = sumtform.concatenate(tform, srcpts=src) return sumtform + + +def estimate_dstpts(transformlist, src=None): + ''' + estimate destination points for list of transforms + input: + transformlist -- list of transform classes with tform method + src -- Nx2 numpy array of source points + output: Nx2 numpt array of destination points + ''' + dstpts = src + for tform in transformlist: + if isinstance(tform, list): + dstpts = estimate_dstpts(tform, dstpts) + else: + dstpts = tform.tform(dstpts) + return dstpts + + +def estimate_transformsum(transformlist, src=None, order=2): + ''' + pseudo-composition of transforms in list of transforms using source point + transformation and a single estimation. + input: trans + ''' + dstpts = estimate_dstpts(transformlist, src) + return Polynomial2DTransform(src=src, dst=dstpts) diff --git a/test/test_files/tilespecs.json b/test/test_files/tilespecs.json index ef01e560..bd6eaea7 100644 --- a/test/test_files/tilespecs.json +++ b/test/test_files/tilespecs.json @@ -26,11 +26,6 @@ "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " }, - { - "type": "leaf", - "className": "mpicbg.trakem2.transform.PolynomialTransform2D", - "dataString": "65508.72594194093 -0.99868791634 -0.008854442595 -7.96199E-7 1.286356E-6 -7.21762E-7 70663.99671294086 -0.039478147906 -0.954683561039 6.052858E-6 9.11723E-7 -6.219026E-6" - }, { "type": "leaf", "className": "mpicbg.trakem2.transform.AffineModel2D", @@ -72,11 +67,6 @@ "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " }, - { - "type": "leaf", - "className": "mpicbg.trakem2.transform.PolynomialTransform2D", - "dataString": "62218.570504642266 -0.995583671407 2.06111858E-4 -1.782885E-6 6.1908E-8 -1.6858E-6 70741.89265480703 0.009744971533 -1.006717263266 3.12237E-7 -1.701837E-6 2.245153E-6" - }, { "type": "leaf", "className": "mpicbg.trakem2.transform.AffineModel2D", @@ -118,11 +108,6 @@ "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " }, - { - "type": "leaf", - "className": "mpicbg.trakem2.transform.PolynomialTransform2D", - "dataString": "58956.67195722279 -1.009281426241 -0.002631989396 2.226803E-6 3.96506E-7 -9.49469E-7 70815.8432399658 0.004676678124 -0.99930426995 3.8057E-8 7.7865E-8 1.52374E-7" - }, { "type": "leaf", "className": "mpicbg.trakem2.transform.AffineModel2D", diff --git a/test/test_transform.py b/test/test_transform.py index f91904e3..e838c172 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -83,7 +83,17 @@ def noscipy_import(name, globals=None, locals=None, assert(renderapi.transform.svd is np.linalg.svd if use_numpy else renderapi.transform.svd is scipy.linalg.svd) - # TODO now that import framework is good, do actual test + datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') + default_pt = renderapi.transform.Polynomial2DTransform( + dataString=datastring) + srcpts = np.random.rand(30, 2) + dstpts = default_pt.tform(srcpts) + derived_pt = renderapi.transform.Polynomial2DTransform( + src=srcpts, dst=dstpts) + assert(np.allclose(derived_pt.params, default_pt.params)) if use_numpy: builtins.__import__ = realimport From d47e907349fba2d38fe0d94e6ed008b4e3d1b144 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 10:23:31 -0700 Subject: [PATCH 231/766] added point match tests --- .../test_pointmatch_integrated.py | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index e69de29b..fd76ccc7 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -0,0 +1,179 @@ +import renderapi +import pytest +import tempfile +import os +import logging +import sys +import json +import numpy as np +from test_data import render_host, render_port, \ + client_script_location, tilespec_file, tform_file + + +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +# +logger.addHandler(ch) + +test_matches = [ + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "1", + "qId": "1-1", + "matches": { + "p": [ + [0,0], + [100,100], + [0,100], + [100,0] + ], + "q": [ + [0,0], + [100,100], + [0,100], + [100,0] + ], + "w": [ + 1,1,1,1 + ] + } + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "2", + "qId": "2-1", + "matches": { + "p": [ + [0,0], + [100,100], + [0,100], + [100,0] + ], + "q": [ + [0,0], + [100,100], + [0,100], + [100,0] + ], + "w": [ + 1,1,1,1 + ] + } + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "0", + "qId": "0-2", + "matches": { + "p": [ + [100,100], + [100,0] + ], + "q": [ + [0,100], + [0,0] + ], + "w": [ + 1,1 + ] + } + } +] + + +@pytest.fixture(scope='module') +def render(): + render_test_parameters = { + 'host': render_host, + 'port': 8080, + 'owner': 'test_coordinate', + 'project': 'test_coordinate_project', + 'client_scripts': client_script_location + } + return renderapi.render.connect(**render_test_parameters) + +@pytest.fixture(scope='module') +def test_pm_collection_owner(render): + owner='test' + collection = 'test_collection' + renderapi.pointmatch.import_matches(owner,collection,test_matches,render=render) + return (owner,collection) + +def test_get_matchcollection_owners(render,test_pm_collection_owner): + (owner, collection) = test_pm_collection_owner + owners = renderapi.pointmatch.get_matchcollection_owners(render=render) + assert (owner in owners) + +def test_get_matchcollections(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + collections = renderapi.pointmatch.get_matchcollections(render=render) + assert (collection in collections) + +def test_get_match_groupIds(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) + assert len(groups)==3 + +def test_get_matches_outside_group(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) + matches = renderapi.pointmatch.get_matches_outside_group(collection,"0") + assert test_matches[0] in matches + assert test_matches[1] in matches + +def test_get_matches_within_group(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) + matches = renderapi.pointmatch.get_matches_outside_group(collection,"0") + assert matches[0]==test_matches[2] + +def test_get_matches_from_group_to_group(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + group1="0" + group2="1" + matches = renderapi.pointmatch.get_matches_outside_group(collection,group1,group2,render=render) + assert matches[0] == test_matches[0] + +def test_get_matches_from_tile_to_tile(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + group1="0" + group2="1" + tile1="0-1" + tile2="1-1" + matches = renderapi.pointmatch.get_matches_outside_group(collection,group1,tile1,group2,tile2,render=render) + assert matches[0]==test_matches[0] + +def test_get_matches_with_group(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + group1="0" + matches = renderapi.pointmatch.get_matches_outside_group(collection,group1,render=render) + assert len(matches)==3 + +def test_get_match_groupIds_from_only(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + groups = renderapi.pointmatch.get_match_groupIds_from_only(collection,render=render) + assert len(groups)==2 + +def test_get_match_groupIds_to_only(render,test_pm_collection_owner): + (owner,collection)=test_pm_collection_owner + groups = renderapi.pointmatch.get_match_groupIds_to_only(collection,render=render) + assert len(groups)==2 + +def test_delete_point_matches_between_groups(render): + collection = 'test_delete' + owner = 'test' + renderapi.pointmatch.import_matches(owner,collection,test_matches,render=render) + group1 = '0' + group2 = '1' + renderapi.pointmatch.delete_point_matches_between_groups(collection,'0','1',render=render) + groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) + assert len(groups)==2 From cc12620eb4795804ca56e20b69ab95a5337695ca Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 10:30:03 -0700 Subject: [PATCH 232/766] fixed bug in tests --- integration_tests/test_pointmatch_integrated.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index fd76ccc7..bcec1fb2 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -95,17 +95,16 @@ def render(): render_test_parameters = { 'host': render_host, 'port': 8080, - 'owner': 'test_coordinate', - 'project': 'test_coordinate_project', + 'owner': 'test', + 'project': 'test_pointmatch_project', 'client_scripts': client_script_location } return renderapi.render.connect(**render_test_parameters) @pytest.fixture(scope='module') def test_pm_collection_owner(render): - owner='test' collection = 'test_collection' - renderapi.pointmatch.import_matches(owner,collection,test_matches,render=render) + renderapi.pointmatch.import_matches(collection,test_matches,render=render) return (owner,collection) def test_get_matchcollection_owners(render,test_pm_collection_owner): @@ -171,7 +170,7 @@ def test_get_match_groupIds_to_only(render,test_pm_collection_owner): def test_delete_point_matches_between_groups(render): collection = 'test_delete' owner = 'test' - renderapi.pointmatch.import_matches(owner,collection,test_matches,render=render) + renderapi.pointmatch.import_matches(collection,test_matches,render=render) group1 = '0' group2 = '1' renderapi.pointmatch.delete_point_matches_between_groups(collection,'0','1',render=render) From 254a421a39b2a580bb046b45b67916b6ae19319e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 10:31:39 -0700 Subject: [PATCH 233/766] making more flexible import_matches to serialize dictionary --- renderapi/pointmatch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 614fefe4..42f6eeea 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -7,7 +7,7 @@ from .render import format_baseurl, renderaccess from .errors import RenderError from .utils import NullHandler - +import json logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -183,6 +183,8 @@ def import_matches(matchCollection, data, owner=None, host=None, port=None, request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/matches" % (owner, matchCollection) logger.debug(request_url) + if isinstance(data, dict): + data = json.dumps(data) r = session.put(request_url, data=data, headers={ "content-type": "application/json", "Accept": "application/json"}) return r From de81215b7c709ecfc3dbb291452e6262eed15f86 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 10:42:07 -0700 Subject: [PATCH 234/766] fixed bug in flexible data passing --- integration_tests/test_pointmatch_integrated.py | 12 +++--------- renderapi/pointmatch.py | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index bcec1fb2..83828a03 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -40,9 +40,7 @@ [0,100], [100,0] ], - "w": [ - 1,1,1,1 - ] + "w": [1,1,1,1] } }, { @@ -63,9 +61,7 @@ [0,100], [100,0] ], - "w": [ - 1,1,1,1 - ] + "w": [1,1,1,1] } }, { @@ -82,9 +78,7 @@ [0,100], [0,0] ], - "w": [ - 1,1 - ] + "w": [1,1] } } ] diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 42f6eeea..873b0633 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -183,7 +183,7 @@ def import_matches(matchCollection, data, owner=None, host=None, port=None, request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/matches" % (owner, matchCollection) logger.debug(request_url) - if isinstance(data, dict): + if not isinstance(data, str): data = json.dumps(data) r = session.put(request_url, data=data, headers={ "content-type": "application/json", "Accept": "application/json"}) From 2bca5c1b287ce8793ef540efd5f986a5d5c7f726 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 10:54:10 -0700 Subject: [PATCH 235/766] fixedpoint match tests --- .../test_pointmatch_integrated.py | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 83828a03..3716bea7 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -96,69 +96,59 @@ def render(): return renderapi.render.connect(**render_test_parameters) @pytest.fixture(scope='module') -def test_pm_collection_owner(render): +def test_pm_collection(render): collection = 'test_collection' renderapi.pointmatch.import_matches(collection,test_matches,render=render) - return (owner,collection) + return collection -def test_get_matchcollection_owners(render,test_pm_collection_owner): - (owner, collection) = test_pm_collection_owner +def test_get_matchcollection_owners(render): owners = renderapi.pointmatch.get_matchcollection_owners(render=render) assert (owner in owners) -def test_get_matchcollections(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner +def test_get_matchcollections(render,test_pm_collection): collections = renderapi.pointmatch.get_matchcollections(render=render) - assert (collection in collections) + assert (test_pm_collection in collections) -def test_get_match_groupIds(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner - groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) +def test_get_match_groupIds(render,test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) assert len(groups)==3 -def test_get_matches_outside_group(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner - groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) - matches = renderapi.pointmatch.get_matches_outside_group(collection,"0") +def test_get_matches_outside_group(render,test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) + matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0") assert test_matches[0] in matches assert test_matches[1] in matches -def test_get_matches_within_group(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner - groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) - matches = renderapi.pointmatch.get_matches_outside_group(collection,"0") +def test_get_matches_within_group(render,test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) + matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0") assert matches[0]==test_matches[2] -def test_get_matches_from_group_to_group(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner +def test_get_matches_from_group_to_group(render,test_pm_collection): group1="0" group2="1" - matches = renderapi.pointmatch.get_matches_outside_group(collection,group1,group2,render=render) + matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,group1,group2,render=render) assert matches[0] == test_matches[0] -def test_get_matches_from_tile_to_tile(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner +def test_get_matches_from_tile_to_tile(render,test_pm_collection): group1="0" group2="1" tile1="0-1" tile2="1-1" - matches = renderapi.pointmatch.get_matches_outside_group(collection,group1,tile1,group2,tile2,render=render) + matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,group1,tile1,group2,tile2,render=render) assert matches[0]==test_matches[0] -def test_get_matches_with_group(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner +def test_get_matches_with_group(render,test_pm_collection): group1="0" - matches = renderapi.pointmatch.get_matches_outside_group(collection,group1,render=render) + matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,group1,render=render) assert len(matches)==3 -def test_get_match_groupIds_from_only(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner - groups = renderapi.pointmatch.get_match_groupIds_from_only(collection,render=render) +def test_get_match_groupIds_from_only(render,test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds_from_only(test_pm_collection,render=render) assert len(groups)==2 -def test_get_match_groupIds_to_only(render,test_pm_collection_owner): - (owner,collection)=test_pm_collection_owner - groups = renderapi.pointmatch.get_match_groupIds_to_only(collection,render=render) +def test_get_match_groupIds_to_only(render,test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds_to_only(test_pm_collection,render=render) assert len(groups)==2 def test_delete_point_matches_between_groups(render): From e338127f5c2b54b7fa637f10c04e66ce89e28c59 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 11:00:44 -0700 Subject: [PATCH 236/766] forgot to include render in a call --- integration_tests/test_pointmatch_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 3716bea7..171c9789 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -115,13 +115,13 @@ def test_get_match_groupIds(render,test_pm_collection): def test_get_matches_outside_group(render,test_pm_collection): groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) - matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0") + matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0",render=render) assert test_matches[0] in matches assert test_matches[1] in matches def test_get_matches_within_group(render,test_pm_collection): groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) - matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0") + matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0",render=render) assert matches[0]==test_matches[2] def test_get_matches_from_group_to_group(render,test_pm_collection): From 31524e37b86f780c66a6b4e0c4e1cabe0e8e4167 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 11:01:31 -0700 Subject: [PATCH 237/766] got owner from test parameters --- integration_tests/test_pointmatch_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 171c9789..249571b2 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -103,7 +103,7 @@ def test_pm_collection(render): def test_get_matchcollection_owners(render): owners = renderapi.pointmatch.get_matchcollection_owners(render=render) - assert (owner in owners) + assert (render_test_parameters['owner'] in owners) def test_get_matchcollections(render,test_pm_collection): collections = renderapi.pointmatch.get_matchcollections(render=render) From b9572260fa1f39f5783185075807db5d48faa6ed Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 11:02:30 -0700 Subject: [PATCH 238/766] mislabelled call --- integration_tests/test_pointmatch_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 249571b2..588f2c3e 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -135,7 +135,7 @@ def test_get_matches_from_tile_to_tile(render,test_pm_collection): group2="1" tile1="0-1" tile2="1-1" - matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,group1,tile1,group2,tile2,render=render) + matches = renderapi.pointmatch.get_matches_from_tile_to_tile(test_pm_collection,group1,tile1,group2,tile2,render=render) assert matches[0]==test_matches[0] def test_get_matches_with_group(render,test_pm_collection): From 60013fb76ab9062b7dfcae298c5e41439b936314 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 11:03:28 -0700 Subject: [PATCH 239/766] fixed outside group match --- integration_tests/test_pointmatch_integrated.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 588f2c3e..2bc35fdf 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -114,13 +114,11 @@ def test_get_match_groupIds(render,test_pm_collection): assert len(groups)==3 def test_get_matches_outside_group(render,test_pm_collection): - groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0",render=render) assert test_matches[0] in matches assert test_matches[1] in matches def test_get_matches_within_group(render,test_pm_collection): - groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0",render=render) assert matches[0]==test_matches[2] From 202945e60343a884359c4730317ea2dc48420bea Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 11:07:17 -0700 Subject: [PATCH 240/766] fixed typos in tests --- integration_tests/test_pointmatch_integrated.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 2bc35fdf..824566b3 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -138,16 +138,16 @@ def test_get_matches_from_tile_to_tile(render,test_pm_collection): def test_get_matches_with_group(render,test_pm_collection): group1="0" - matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,group1,render=render) + matches = renderapi.pointmatch.get_matches_with_group(test_pm_collection,group1,render=render) assert len(matches)==3 def test_get_match_groupIds_from_only(render,test_pm_collection): groups = renderapi.pointmatch.get_match_groupIds_from_only(test_pm_collection,render=render) - assert len(groups)==2 + assert len(groups)==1 def test_get_match_groupIds_to_only(render,test_pm_collection): groups = renderapi.pointmatch.get_match_groupIds_to_only(test_pm_collection,render=render) - assert len(groups)==2 + assert len(groups)==3 def test_delete_point_matches_between_groups(render): collection = 'test_delete' From 26db0d2e51ea50a47f9da40699ff56443efc9ae4 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 11:17:51 -0700 Subject: [PATCH 241/766] fixed test bugs --- integration_tests/test_pointmatch_integrated.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 824566b3..567a7dc6 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -103,11 +103,12 @@ def test_pm_collection(render): def test_get_matchcollection_owners(render): owners = renderapi.pointmatch.get_matchcollection_owners(render=render) - assert (render_test_parameters['owner'] in owners) + assert 'test' in owners def test_get_matchcollections(render,test_pm_collection): collections = renderapi.pointmatch.get_matchcollections(render=render) - assert (test_pm_collection in collections) + matched_collection = next(coljson for coljson in collections if coljson['collectionId']['name']==test_pm_collection) + assert matched_collection is not None def test_get_match_groupIds(render,test_pm_collection): groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) @@ -119,13 +120,13 @@ def test_get_matches_outside_group(render,test_pm_collection): assert test_matches[1] in matches def test_get_matches_within_group(render,test_pm_collection): - matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0",render=render) + matches = renderapi.pointmatch.get_matches_within_group(test_pm_collection,"0",render=render) assert matches[0]==test_matches[2] def test_get_matches_from_group_to_group(render,test_pm_collection): group1="0" group2="1" - matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,group1,group2,render=render) + matches = renderapi.pointmatch.get_matches_from_group_to_group(test_pm_collection,group1,group2,render=render) assert matches[0] == test_matches[0] def test_get_matches_from_tile_to_tile(render,test_pm_collection): From ce03bec5be8548e8432472188cab3c2d0b20760c Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 11:38:18 -0700 Subject: [PATCH 242/766] setup: ignore non-PEP8 E126 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index eb430c50..30bc29fe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ [flake8] -ignore = E201,E202,E226 +ignore = E226,E126 max-line-length = 200 From 8411572abb21832c323b5e532688da2df68ae6f8 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 11:39:39 -0700 Subject: [PATCH 243/766] test_coordinate: PEP8 --- .../test_coordinate_integrated.py | 109 +++++++++--------- 1 file changed, 52 insertions(+), 57 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 283d9394..2beedfc3 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -65,27 +65,31 @@ def teststack_tilespec(): @pytest.fixture(scope='module') def local_corners_json(teststack_tilespec): (stack, ts) = teststack_tilespec - corners = [[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]] + corners = [[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]] batch = [] for corner in corners: d = { 'tileId': ts.tileId, - 'visible': True, + 'visible': True } - d['local']=corner + d['local'] = corner batch.append(d) return batch + @pytest.fixture(scope='module') -def world_corners_json(render,teststack_tilespec): +def world_corners_json(render, teststack_tilespec): (stack, ts) = teststack_tilespec - corners = [[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]] + corners = [[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]] world_corners = [] for corner in corners: - world_corners.append(renderapi.coordinate.local_to_world_coordinates(\ - stack, ts.tileId, corner[0], corner[1],render=render)) + world_corners.append(renderapi.coordinate.local_to_world_coordinates( + stack, ts.tileId, corner[0], corner[1], render=render)) return world_corners + def test_world_to_local_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec local = render.run(renderapi.coordinate.world_to_local_coordinates, @@ -100,17 +104,18 @@ def test_world_to_local_coordinates(render, teststack_tilespec): def test_local_to_world_coordinates(render, teststack_tilespec): (stack, ts) = teststack_tilespec world = render.run(renderapi.coordinate.local_to_world_coordinates, - stack, ts.tileId, 0,0) + stack, ts.tileId, 0, 0) logger.debug(world) assert('error' not in world.keys()) assert(world['tileId'] == ts.tileId) assert(len(world['world']) >= 2) -def test_world_to_local_coordinates_batch(render, teststack_tilespec,world_corners_json): +def test_world_to_local_coordinates_batch(render, teststack_tilespec, + world_corners_json): (stack, ts) = teststack_tilespec local = renderapi.coordinate.world_to_local_coordinates_batch( - stack, world_corners_json, ts.z,execute_local=False,render=render) + stack, world_corners_json, ts.z, execute_local=False, render=render) logger.debug(local) assert(len(local) == len(world_corners_json)) for ans in local: @@ -118,70 +123,60 @@ def test_world_to_local_coordinates_batch(render, teststack_tilespec,world_corne assert('error' not in tile.keys()) -def test_local_to_world_coordinates_batch(render, teststack_tilespec,local_corners_json): +def test_local_to_world_coordinates_batch(render, teststack_tilespec, + local_corners_json): (stack, ts) = teststack_tilespec - world = renderapi.coordinate.local_to_world_coordinates_batch(stack,local_corners_json, - ts.z,execute_local=False,render=render) - assert(len(local_corners_json)==len(world)) + world = renderapi.coordinate.local_to_world_coordinates_batch( + stack, local_corners_json, ts.z, execute_local=False, render=render) + assert(len(local_corners_json) == len(world)) for ans in world: assert('error' not in ans.keys()) + def test_world_to_local_coordinates_array(render, teststack_tilespec): (stack, ts) = teststack_tilespec - local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) - world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, - local_corners, - ts.tileId, - ts.z, - render=render) - local_corners2 = renderapi.coordinate.world_to_local_coordinates_array(stack, - world_corners, - ts.tileId, - ts.z, - render=render) + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, render=render) + local_corners2 = renderapi.coordinate.world_to_local_coordinates_array( + stack, world_corners, ts.tileId, ts.z, render=render) logger.debug('local corners2: {}'.format(local_corners2)) - for pt,ptafter in zip(local_corners,local_corners2): - assert np.sum(np.abs(pt-ptafter))<.1 + for pt, ptafter in zip(local_corners, local_corners2): + assert(np.sum(np.abs(pt-ptafter)) < .1) + def test_local_to_world_coordinates_array(render, teststack_tilespec): (stack, ts) = teststack_tilespec - local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) - world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, - local_corners, - ts.tileId, - ts.z, - render=render) + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, render=render) logger.debug('world corners:{}'.format(world_corners)) - assert world_corners.shape[0]==local_corners.shape[0] + assert(world_corners.shape[0] == local_corners.shape[0]) + def test_world_to_local_coordinates_clientside(render, teststack_tilespec): (stack, ts) = teststack_tilespec - local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) - world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, - local_corners, - ts.tileId, - ts.z, - render=render, - doClientSide=True) - local_corners2 = renderapi.coordinate.world_to_local_coordinates_array(stack, - world_corners, - ts.tileId, - ts.z, - render=render, - doClientSide=True) + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, + render=render, doClientSide=True) + local_corners2 = renderapi.coordinate.world_to_local_coordinates_array( + stack, world_corners, ts.tileId, ts.z, + render=render, doClientSide=True) logger.debug('local corners2: {}'.format(local_corners2)) - for pt,ptafter in zip(local_corners,local_corners2): - assert np.sum(np.abs(pt-ptafter))<.1 + for pt, ptafter in zip(local_corners, local_corners2): + assert(np.sum(np.abs(pt-ptafter)) < .1) def test_local_to_world_coordinates_clientside(render, teststack_tilespec): (stack, ts) = teststack_tilespec - local_corners = np.array([[10, 10], [ts.width-10, 10], [ts.width-10, ts.height-10], [10, ts.height-10]]) - world_corners = renderapi.coordinate.local_to_world_coordinates_array(stack, - local_corners, - ts.tileId, - ts.z, - doClientSide=True, - render=render) + local_corners = np.array([[10, 10], [ts.width-10, 10], + [ts.width-10, ts.height-10], [10, ts.height-10]]) + world_corners = renderapi.coordinate.local_to_world_coordinates_array( + stack, local_corners, ts.tileId, ts.z, + doClientSide=True, render=render) logger.debug('world corners:{}'.format(world_corners)) - assert world_corners.shape[0]==local_corners.shape[0] + assert(world_corners.shape[0] == local_corners.shape[0]) From 205aa956acebfea172cda1da95e2a411e70fa895 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 15:00:03 -0700 Subject: [PATCH 244/766] added some more integration tests --- integration_tests/test_stack_integrated.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 372dc589..4c9223ea 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -248,6 +248,26 @@ def test_get_section_z(render, teststack): teststack, "3407.0") assert z == 3407 +def test_clone_stack(render,teststack): + stack2 = 'cloned_stack' + zvalues = renderapi.stack.get_z_values_for_stack(test_stack,render=render) + renderapi.stack.clone_stack(teststack, cloned_stack, render=render) + zvalues2 = renderapi.stack.get_z_values_for_stack(stack2, render=render) + renderapi.stack.delete_stack(stack2,render=render) + assert zvalues == zvalues2 + +def test_clone_stack_subset(render, teststack): + stack2 = 'cloned_stack_subset' + zvalues = renderapi.stack.get_z_values_for_stack(test_stack,render=render) + renderapi.stack.clone_stack(teststack,cloned_stack,z=zvalues[0:1],render=render) + zvalues2 = renderapi.stack.get_z_values_for_stack(stack2,render=render) + renderapi.stack.delete_stack(stack2,render=render) + assert zvalues[0:1] == zvalues2 + +def test_get_stack_sectionData(render,teststack): + sectionData = renderapi.stack.get_stack_sectionData(teststack,render=render) + assert len(sectionData)==2 + def test_get_z_values(render, teststack): # check get z values zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) From 85ee77a4b655e6200839eb5a26af9171b26fb4ff Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 15:13:10 -0700 Subject: [PATCH 245/766] bug in tests --- integration_tests/test_stack_integrated.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 4c9223ea..4ff7de2c 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -250,7 +250,7 @@ def test_get_section_z(render, teststack): def test_clone_stack(render,teststack): stack2 = 'cloned_stack' - zvalues = renderapi.stack.get_z_values_for_stack(test_stack,render=render) + zvalues = renderapi.stack.get_z_values_for_stack(teststack,render=render) renderapi.stack.clone_stack(teststack, cloned_stack, render=render) zvalues2 = renderapi.stack.get_z_values_for_stack(stack2, render=render) renderapi.stack.delete_stack(stack2,render=render) @@ -258,7 +258,7 @@ def test_clone_stack(render,teststack): def test_clone_stack_subset(render, teststack): stack2 = 'cloned_stack_subset' - zvalues = renderapi.stack.get_z_values_for_stack(test_stack,render=render) + zvalues = renderapi.stack.get_z_values_for_stack(teststack,render=render) renderapi.stack.clone_stack(teststack,cloned_stack,z=zvalues[0:1],render=render) zvalues2 = renderapi.stack.get_z_values_for_stack(stack2,render=render) renderapi.stack.delete_stack(stack2,render=render) @@ -267,7 +267,7 @@ def test_clone_stack_subset(render, teststack): def test_get_stack_sectionData(render,teststack): sectionData = renderapi.stack.get_stack_sectionData(teststack,render=render) assert len(sectionData)==2 - + def test_get_z_values(render, teststack): # check get z values zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) From df8c1025b3749b639576dfa04051ed6a9c850764 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 15:21:04 -0700 Subject: [PATCH 246/766] image: add functionality to get_tile_image_data, test --- integration_tests/test_stack_integrated.py | 72 ++++++++++++++++++---- renderapi/image.py | 30 +++++---- 2 files changed, 78 insertions(+), 24 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 372dc589..025328d9 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -116,6 +116,7 @@ def test_set_stack_metadata(render): assert sv.stackResolutionY == 3.0 assert sv.stackResolutionZ == 4.0 + def test_simple_import(render, simpletilespec, tmpdir): # open a temporary file tfile = tmpdir.join('testfile.json') @@ -137,6 +138,7 @@ def test_simple_import(render, simpletilespec, tmpdir): assert (ts_out.z == simpletilespec.z) render.run(renderapi.stack.delete_stack, 'test_insert') + def test_simple_import_with_transforms( render, render_example_tilespec_and_transforms, tmpdir): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -166,6 +168,7 @@ def test_simple_import_with_transforms( assert ts_out.z == tilespecs[0].z render.run(renderapi.stack.delete_stack, 'test_insert_tform') + def test_import_tilespecs(render, simpletilespec): stack = 'test_insert2' render.run(renderapi.stack.create_stack, stack, force_resolution=True) @@ -182,18 +185,22 @@ def test_import_tilespecs_parallel(render): root.debug('test not implemented yet') assert False + def test_import_jsonfiles_validate_client(render): root.debug('test not implemented yet') assert False + def test_import_jsonfiles(render): root.debug('test not implemented yet') assert False + def test_import_parallel(render): root.debug('test not implemented yet') assert False + def test_tile_pair_client(render): root.debug('test not implemented yet') assert False @@ -203,14 +210,15 @@ def test_importTransformChangesClient(render): root.debug('test not implemented yet') assert False + def test_coordinateClient(render): root.debug('test not implemented yet') assert False -@pytest.fixture(scope="module") -def teststack(request,render,render_example_tilespec_and_transforms): - (tilespecs,tforms)=render_example_tilespec_and_transforms +@pytest.fixture(scope="module") +def teststack(request, render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms stack = 'test_insert3' r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) render.run(renderapi.client.import_tilespecs, stack, tilespecs, @@ -219,6 +227,7 @@ def teststack(request,render,render_example_tilespec_and_transforms): yield stack render.run(renderapi.stack.delete_stack, stack) + def test_stack_bounds(render, teststack): # check the stack bounds stack_bounds = render.run(renderapi.stack.get_stack_bounds, teststack) @@ -228,6 +237,7 @@ def test_stack_bounds(render, teststack): for key in stack_bounds.keys(): assert np.abs(stack_bounds[key]-expected_bounds[key]) < 1.0 + def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms # check a single z stack bounds @@ -237,7 +247,8 @@ def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): expected_bounds = {u'maxZ': 3407.0, u'maxX': 4917.0, u'maxY': 4506.0, u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} for key in zbounds.keys(): - assert np.abs(zbounds[key]-expected_bounds[key]) < 1.0 + assert np.abs(zbounds[key]-expected_bounds[key]) < 1.0 + def test_get_section_z(render, teststack): # check getting section Z @@ -248,16 +259,19 @@ def test_get_section_z(render, teststack): teststack, "3407.0") assert z == 3407 + def test_get_z_values(render, teststack): # check get z values zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) assert zvalues == [3407.0, 3408.0] + def test_uniq_value(render): # check likelyUniqueId uniq = render.run(renderapi.stack.likelyUniqueId) assert len(uniq) >= len('58ceebb7a7b11b0001dc4e32') + def test_bb_image(render, teststack): formats = renderapi.image.IMAGE_FORMATS.keys() zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) @@ -277,14 +291,50 @@ def test_bb_image(render, teststack): assert data.shape[1] == (np.floor(width*.25)) assert data.shape[2] >= 3 -def test_tile_image(render, teststack, render_example_tilespec_and_transforms): + +def test_tile_image(render, teststack, render_example_tilespec_and_transforms, + **kwargs): (tilespecs, tforms) = render_example_tilespec_and_transforms - format = 'png' - data = render.run(renderapi.image.get_tile_image_data, - teststack, tilespecs[0].tileId) - assert len(data.shape) == 3 - assert data.shape[0] >= tilespecs[0].height - assert data.shape[1] >= tilespecs[0].width + for fmt in renderapi.image.IMAGE_FORMATS: + data = render.run(renderapi.image.get_tile_image_data, + teststack, tilespecs[0].tileId, **kwargs) + if kwargs.get('scale') is None: + testscale = kwargs['scale'] + else: + testscale = 1. + assert len(data.shape) == 3 + assert data.shape[0] >= np.floor(tilespecs[0].height * testscale) + assert data.shape[1] >= np.floor(tilespecs[0].width * testscale) + + +def test_tile_image_options(render, teststack, + render_example_tilespec_and_transforms): + testscale = 0.5 + test_tile_image_options( + render, teststack, render_example_tilespec_and_transforms, + scale=testscale, filter=True, normalizeForMatching=False) + + +def test_section_image(render, teststack, **kwargs): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + + width = bounds['maxX'] - bounds['minX'] + height = bounds['maxY'] - bounds['minY'] + fmt = 'png' + scalefactor = 0.05 + data = renderapi.image.get_section_image(teststack, z, scale=scalefactor, + img_format=fmt, **kwargs) + assert data.shape[0] == (np.floor(height * scalefactor)) + assert data.shape[1] == (np.floor(width * scalefactor)) + assert data.shape[2] >= 3 + + +def test_section_image_options(render, teststack): + test_section_image(render, teststack, filter=True, + maxTileSpecsToRender=50) + def fail_image_get(render, teststack, render_example_tilespec_and_transforms): with pytest.raises(KeyError): diff --git a/renderapi/image.py b/renderapi/image.py index 0ff9568f..62ba04b8 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -46,16 +46,13 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%f/%s" % ( z, x, y, width, height, scale, image_ext) - args = [] + qparams = {} if minIntensity is not None: - args += ['minIntensity=%d' % minIntensity] + qparams['minIntensity'] = minIntensity if maxIntensity is not None: - args += ['maxIntensity=%d' % maxIntensity] - if len(args) > 0: - args = "&".join(args) - request_url += args + qparams['maxIntensity'] = maxIntensity - r = session.get(request_url) + r = session.get(request_url, params=qparams) try: image = np.asarray(Image.open(io.BytesIO(r.content))) return image @@ -65,10 +62,10 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, @renderaccess -def get_tile_image_data(stack, tileId, normalizeForMatching=True, - host=None, port=None, owner=None, project=None, - img_format=None, session=requests.session(), - render=None, **kwargs): +def get_tile_image_data(stack, tileId, normalizeForMatching=True, scale=None, + filter=None, host=None, port=None, owner=None, + project=None, img_format=None, + session=requests.session(), render=None, **kwargs): ''' render image from a tile with all transforms and return numpy array ''' @@ -80,10 +77,17 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/%s" % (tileId, image_ext) + + qparams = {} if normalizeForMatching: - request_url += "?normalizeForMatching=true" + qparams['normalizeForMatching'] = jbool(normalizeForMatching) + if scale is not None: + qparams['scale'] = scale + if filter is not None: + qparams['filter'] = jbool(filter) logger.debug(request_url) - r = session.get(request_url) + + r = session.get(request_url, params=qparams) try: img = Image.open(io.BytesIO(r.content)) array = np.asarray(img) From d43fa62f00e48c416cc9094a9542ccf1616a0efb Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 15:35:34 -0700 Subject: [PATCH 247/766] integration: fix image tests... --- integration_tests/test_stack_integrated.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 7e2c2ba9..dc7d3c00 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -324,9 +324,10 @@ def test_tile_image(render, teststack, render_example_tilespec_and_transforms, data = render.run(renderapi.image.get_tile_image_data, teststack, tilespecs[0].tileId, **kwargs) if kwargs.get('scale') is None: - testscale = kwargs['scale'] - else: testscale = 1. + else: + testscale = kwargs['scale'] + assert len(data.shape) == 3 assert data.shape[0] >= np.floor(tilespecs[0].height * testscale) assert data.shape[1] >= np.floor(tilespecs[0].width * testscale) @@ -335,7 +336,7 @@ def test_tile_image(render, teststack, render_example_tilespec_and_transforms, def test_tile_image_options(render, teststack, render_example_tilespec_and_transforms): testscale = 0.5 - test_tile_image_options( + test_tile_image( render, teststack, render_example_tilespec_and_transforms, scale=testscale, filter=True, normalizeForMatching=False) @@ -349,8 +350,8 @@ def test_section_image(render, teststack, **kwargs): height = bounds['maxY'] - bounds['minY'] fmt = 'png' scalefactor = 0.05 - data = renderapi.image.get_section_image(teststack, z, scale=scalefactor, - img_format=fmt, **kwargs) + data = render.run(renderapi.image.get_section_image, teststack, z, + scale=scalefactor, img_format=fmt, **kwargs) assert data.shape[0] == (np.floor(height * scalefactor)) assert data.shape[1] == (np.floor(width * scalefactor)) assert data.shape[2] >= 3 From d17fcfd70ad79ed75e9052c14c158c7bf4e90d73 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 15:49:56 -0700 Subject: [PATCH 248/766] client: pathos is official requirement, do not need import checking --- renderapi/client.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 0417673d..2f7c4b8a 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -12,19 +12,12 @@ from .errors import ClientScriptError from .render import RenderClient, renderaccess from .stack import set_stack_state, make_stack_params +from pathos.multiprocessing import ProcessingPool as Pool # setup logger logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) -try: - from pathos.multiprocessing import ProcessingPool as Pool - has_pathos = True -except ImportError as e: - logger.warning(e) - has_pathos = False - from multiprocessing import Pool - @renderaccess def import_single_json_file(stack, jsonfile, transformFile=None, @@ -94,7 +87,7 @@ def import_jsonfiles_parallel( host=host, port=port, owner=owner, project=project) - rs = pool.map(partial_import, jsonfiles) + pool.map(partial_import, jsonfiles) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -231,7 +224,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, # TODO this is a weird way to do splits.... is that okay? tilespec_groups = [tilespecs[i::poolsize] for i in xrange(poolsize)] - rs = pool.map(partial_import, tilespec_groups) + pool.map(partial_import, tilespec_groups) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From e58aaf8c29dd5c07351b16b8973cc67ac279e2a9 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 16:00:22 -0700 Subject: [PATCH 249/766] image: bb_image options and tests --- integration_tests/test_stack_integrated.py | 31 +++++++++++++--------- renderapi/image.py | 12 ++++++++- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index dc7d3c00..5b0619c8 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -297,7 +297,7 @@ def test_uniq_value(render): assert len(uniq) >= len('58ceebb7a7b11b0001dc4e32') -def test_bb_image(render, teststack): +def test_bb_image(render, teststack, **kwargs): formats = renderapi.image.IMAGE_FORMATS.keys() zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) z = zvalues[0] @@ -310,27 +310,32 @@ def test_bb_image(render, teststack): for fmt in formats: data = render.run(renderapi.image.get_bb_image, teststack, z, x, y, width, height, - scale=.25, img_format=fmt) + scale=.25, img_format=fmt, **kwargs) dr = data.ravel() assert data.shape[0] == (np.floor(height*.25)) assert data.shape[1] == (np.floor(width*.25)) assert data.shape[2] >= 3 +def test_bb_image_options(render, teststack): + test_bb_image(render, teststack, filter=True, + binaryMask=True, maxTileSpecsToRender=20) + + def test_tile_image(render, teststack, render_example_tilespec_and_transforms, **kwargs): (tilespecs, tforms) = render_example_tilespec_and_transforms - for fmt in renderapi.image.IMAGE_FORMATS: - data = render.run(renderapi.image.get_tile_image_data, - teststack, tilespecs[0].tileId, **kwargs) - if kwargs.get('scale') is None: - testscale = 1. - else: - testscale = kwargs['scale'] - - assert len(data.shape) == 3 - assert data.shape[0] >= np.floor(tilespecs[0].height * testscale) - assert data.shape[1] >= np.floor(tilespecs[0].width * testscale) + fmt = 'png' + data = render.run(renderapi.image.get_tile_image_data, + teststack, tilespecs[0].tileId, **kwargs) + if kwargs.get('scale') is None: + testscale = 1. + else: + testscale = kwargs['scale'] + + assert len(data.shape) == 3 + assert data.shape[0] >= np.floor(tilespecs[0].height * testscale) + assert data.shape[1] >= np.floor(tilespecs[0].width * testscale) def test_tile_image_options(render, teststack, diff --git a/renderapi/image.py b/renderapi/image.py index 62ba04b8..a60919d9 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -25,7 +25,8 @@ @renderaccess def get_bb_image(stack, z, x, y, width, height, scale=1.0, - minIntensity=None, maxIntensity=None, + minIntensity=None, maxIntensity=None, binaryMask=None, + filter=None, maxTileSpecsToRender=None, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): @@ -36,6 +37,9 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, y: topmost pont of bounding rectangle width: extent to right in x height: extent down in y + binaryMask: optional, boolean whether to treat maskimage as binary + maxTileSpecsToRender: optional, int number of tilespecs to render + filter: optional, boolean whether to use Khaled's preferred filter ''' try: image_ext = IMAGE_FORMATS[img_format] @@ -51,6 +55,12 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, qparams['minIntensity'] = minIntensity if maxIntensity is not None: qparams['maxIntensity'] = maxIntensity + if binaryMask is not None: + qparams['binaryMask'] = jbool(binaryMask) + if filter is not None: + qparams['filter'] = jbool(filter) + if maxTileSpecsToRender is not None: + qparams['maxTileSpecsToRender'] = maxTileSpecsToRender r = session.get(request_url, params=qparams) try: From 1c7d4dbc0e532dc2ca7823b49f36a8f281a2e232 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 16:27:52 -0700 Subject: [PATCH 250/766] removed unused post_json --- renderapi/utils.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index acd001fc..2417dbb8 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -47,24 +47,6 @@ def default(self, obj): return obj.__dict__ -def post_json(session, request_url, d, params=None): - headers = {"content-type": "application/json"} - if d is not None: - payload = json.dumps(d) - else: - payload = None - headers['Accept'] = "application/json" - r = session.post(request_url, data=payload, params=params, - headers=headers) - try: - return r - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError( - 'cannot post {} to {} with params {}'.format( - d, request_url, params)) - def renderdumps(obj, *args, **kwargs): '''json.dumps using the RenderEncoder''' From 956e43dd2c217a616d790ba7372fd74462971395 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 16:31:21 -0700 Subject: [PATCH 251/766] putting utils post_json back --- renderapi/utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/renderapi/utils.py b/renderapi/utils.py index 2417dbb8..acd001fc 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -47,6 +47,24 @@ def default(self, obj): return obj.__dict__ +def post_json(session, request_url, d, params=None): + headers = {"content-type": "application/json"} + if d is not None: + payload = json.dumps(d) + else: + payload = None + headers['Accept'] = "application/json" + r = session.post(request_url, data=payload, params=params, + headers=headers) + try: + return r + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError( + 'cannot post {} to {} with params {}'.format( + d, request_url, params)) + def renderdumps(obj, *args, **kwargs): '''json.dumps using the RenderEncoder''' From 81fabf912aef97d26226c2661fa2680a59ca54e5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Apr 2017 17:09:14 -0700 Subject: [PATCH 252/766] stack, image: update and document clone_stack, tests --- integration_tests/test_stack_integrated.py | 5 +-- renderapi/stack.py | 39 ++++++++++++++-------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 5b0619c8..9d8b018a 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -273,7 +273,7 @@ def test_clone_stack_subset(render, teststack): stack2 = 'cloned_stack_subset' zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) renderapi.stack.clone_stack( - teststack, cloned_stack, z=zvalues[0:1], render=render) + teststack, cloned_stack, zs=zvalues[0:1], render=render) zvalues2 = renderapi.stack.get_z_values_for_stack(stack2, render=render) renderapi.stack.delete_stack(stack2, render=render) assert zvalues[0:1] == zvalues2 @@ -319,7 +319,8 @@ def test_bb_image(render, teststack, **kwargs): def test_bb_image_options(render, teststack): test_bb_image(render, teststack, filter=True, - binaryMask=True, maxTileSpecsToRender=20) + binaryMask=True, maxTileSpecsToRender=20, minIntensity=0, + maxIntensity=255) def test_tile_image(render, teststack, render_example_tilespec_and_transforms, diff --git a/renderapi/stack.py b/renderapi/stack.py index b63e5657..811f8445 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,9 +3,9 @@ from time import strftime import requests from .errors import RenderError -from .utils import jbool, NullHandler +from .utils import jbool, NullHandler, post_json from .render import (format_baseurl, format_preamble, - renderaccess, post_json) + renderaccess) logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -166,26 +166,39 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, logger.error(r.text) -# FIXME multiple z indices require multiple params? @renderaccess -def clone_stack(inputstack, outputstack, host=None, port=None, - owner=None, project=None, skipTransforms=False, toProject=None, - z=None, session=None, render=None, **kwargs): +def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, + zs=None, close_stack=True, host=None, port=None, + owner=None, project=None, session=None, render=None, **kwargs): ''' - result: - cloned stack in LOADING state with tiles in layers specified by z' + input: + inputstack: string name of input stack to clone + outputstack: string name of destination stack. + if exists, must be LOADING + close_stack: boolean, whether to set stack to COMPLETE when finished + toProject: optional, string name of project + skipTransforms: optional, boolean whether to strip transformations + in new stack + zs: optional, list of selected z values to clone into stack ''' - if z is not None: - zs = [float(i) for i in z] # TODO test me session = requests.session() if session is None else session sv = StackVersion(**kwargs) + qparams = {} + if zs is not None: + qparams['z'] = [float(i) for i in zs] + if skipTransforms is not None: + qparams['skipTransforms'] = jbool(skipTransforms) + if toProject is not None: + qparams['toProject'] = toProject + request_url = '{}/{}'.format(format_preamble( host, port, owner, project, inputstack), outputstack) logger.debug(request_url) - r = post_json(session, request_url, sv.to_dict(), params={ - 'z': zs, 'toProject': toProject, - 'skipTransforms': jbool(skipTransforms)}) + r = post_json(session, request_url, sv.to_dict(), params=qparams) + + if close_stack: + set_stack_state(outputstack, 'COMPLETE', host, port, owner, project) return r From c56265af3e1a3b85ba5c278bf0e45ebf3f575bf5 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 20:34:26 -0700 Subject: [PATCH 253/766] added more tests --- integration_tests/test_stack_integrated.py | 34 +++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 9d8b018a..09ab267d 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -362,9 +362,40 @@ def test_section_image(render, teststack, **kwargs): assert data.shape[1] == (np.floor(width * scalefactor)) assert data.shape[2] >= 3 +def test_get_tilespecs_from_z(render, teststack,render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + tiles = renderapi.tilespec.get_tile_specs_from_z(teststack, tilespecs[0].z, render=render) + tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z ] + assert len(ts)==len(tsz) + +def test_get_tile_specs_from_minmax_box(render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + z = tilespecs[0].z + tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] + zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) + ts = renderapi.tilespec.get_tile_specs_from_minmax_box(teststack, z, zbounds['minX'], + zbounds['maxX'], zbounds['minY'], zbounds['maxY'], render=render) + assert len(ts) == len(tsz) + +def test_get_tile_specs_from_box(render,teststack,render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + z = tilespecs[0].z + tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] + zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) + width = zbounds['maxX']-zbounds['minX'] + height = zbounds['maxY']-zbounds['minY'] + + ts = renderapi.tilespec.get_tile_specs_from_box(teststack, z, zbounds['minX'], + zbounds['minY'], width, height, render=render) + assert len(ts) == len(tsz) + +def test_get_tile_specs_from_stack(render,teststack,render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + ts = renderapi.tilespec.get_tile_specs_from_stack(teststack, render=render) + assert len(ts) == len(tsz) def test_section_image_options(render, teststack): - test_section_image(render, teststack, filter=True, + img=test_section_image(render, teststack, filter=True, maxTileSpecsToRender=50) @@ -372,3 +403,4 @@ def fail_image_get(render, teststack, render_example_tilespec_and_transforms): with pytest.raises(KeyError): render.run(renderapi.image.get_tile_image_data, teststack, 'test', img_format='JUNK') + From 11fc7e83c9b77f80502c36a53f5d147e31764f05 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 5 Apr 2017 20:34:41 -0700 Subject: [PATCH 254/766] commented out unsupported functions --- renderapi/stack.py | 19 +++--- renderapi/tilespec.py | 137 +++++++++++++++++++++--------------------- 2 files changed, 79 insertions(+), 77 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 811f8445..dee42887 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -55,7 +55,7 @@ def from_dict(self, d): self.__dict__.update({k: v for k, v in d.items()}) -@renderaccess +@renderaccess def set_stack_metadata(stack, sv, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): @@ -222,14 +222,15 @@ def get_z_value_for_section(stack, sectionId, **kwargs): return get_section_z_value(stack, sectionId, **kwargs) -@renderaccess -def put_resolved_tilespecs(stack, json_dict, host=None, port=None, - owner=None, project=None, - session=requests.session(), render=None, **kwargs): - request_url = format_preamble( - host, port, owner, project, stack) + "/resolvedTiles" - r = post_json(session, request_url, json_dict) - return r +# haven't fully supported this yet +# @renderaccess +# def put_resolved_tilespecs(stack, json_dict, host=None, port=None, +# owner=None, project=None, +# session=requests.session(), render=None, **kwargs): +# request_url = format_preamble( +# host, port, owner, project, stack) + "/resolvedTiles" +# r = post_json(session, request_url, json_dict) +# return r @renderaccess diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index b9fda24a..ef0d365f 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -11,74 +11,6 @@ logger.addHandler(NullHandler()) -class ResolvedTileSpecMap: - def __init__(self, tilespecs=[], transforms=[]): - self.tilespecs = tilespecs - self.transforms = transforms - - def to_dict(self): - d = {} - d['tileIdToSpecMap'] = {} - for ts in self.tilespecs: - d['tileIdToSpecMap'][ts.tileId] = ts.to_dict() - d['transformIdToSpecMap'] = {} - for tf in self.transforms: - d['transformIdToSpecMap'][tf.transformId] = tf.to_dict() - return d - - def from_dict(self, d): - tsmap = d['tileIdToSpecMap'] - tfmap = d['transformIdToSpecMap'] - for tsd in tsmap.values(): - ts = TileSpec() - ts.from_dict(tsd) - self.tilespecs.append(ts) - for tfd in tfmap.values(): - tf = load_transform_json(tfd) - self.transforms.append(tf) - - -class ResolvedTileSpecCollection: - def __init__(self, tilespecs=[], transforms=[]): - self.tilespecs = tilespecs - self.transforms = transforms - - def to_dict(self): - d = {} - d['tileCount'] = len(self.tilespecs) - d['tileSpecs'] = [ts.to_dict() for ts in self.tilespecs] - d['transformCount'] = len(self.transforms) - d['transformSpecs'] = [tf.to_dict() for tf in self.transforms] - return d - - def from_dict(self, d): - self.tilespecs = [] - self.transforms = [] - for i in range(d['tileCount']): - ts = TileSpec() - ts.from_dict(d['tileSpecs'][i]) - self.tilespecs.append(ts) - for i in range(d['tranformCount']): - tfd = d['transformSpecs'][i] - tf = load_transform_json(tfd) - self.transforms.append(tf) - - -class Filter: - def __init__(self, classname, params={}): - self.classname = classname - self.params = params - - def to_dict(self): - d = {} - d['className'] = self.classname - d['params'] = self.params - return d - - def from_dict(self, d): - self.classname = d['className'] - self.params = d['params'] - class Layout: def __init__(self, sectionId=None, scopeId=None, cameraId=None, @@ -410,3 +342,72 @@ def get_tile_specs_from_stack(stack, host=None, port=None, for z in get_z_values_for_stack(stack, host=host, port=port, owner=owner, project=project, session=session)] for i in sl] + +#TODO: ADD FEATURES THAT REQUIRED THESE TO SUPPORT.. NOT YET FULLY IMPLEMENTED +# class ResolvedTileSpecMap: +# def __init__(self, tilespecs=[], transforms=[]): +# self.tilespecs = tilespecs +# self.transforms = transforms + +# def to_dict(self): +# d = {} +# d['tileIdToSpecMap'] = {} +# for ts in self.tilespecs: +# d['tileIdToSpecMap'][ts.tileId] = ts.to_dict() +# d['transformIdToSpecMap'] = {} +# for tf in self.transforms: +# d['transformIdToSpecMap'][tf.transformId] = tf.to_dict() +# return d + +# def from_dict(self, d): +# tsmap = d['tileIdToSpecMap'] +# tfmap = d['transformIdToSpecMap'] +# for tsd in tsmap.values(): +# ts = TileSpec() +# ts.from_dict(tsd) +# self.tilespecs.append(ts) +# for tfd in tfmap.values(): +# tf = load_transform_json(tfd) +# self.transforms.append(tf) + + +# class ResolvedTileSpecCollection: +# def __init__(self, tilespecs=[], transforms=[]): +# self.tilespecs = tilespecs +# self.transforms = transforms + +# def to_dict(self): +# d = {} +# d['tileCount'] = len(self.tilespecs) +# d['tileSpecs'] = [ts.to_dict() for ts in self.tilespecs] +# d['transformCount'] = len(self.transforms) +# d['transformSpecs'] = [tf.to_dict() for tf in self.transforms] +# return d + +# def from_dict(self, d): +# self.tilespecs = [] +# self.transforms = [] +# for i in range(d['tileCount']): +# ts = TileSpec() +# ts.from_dict(d['tileSpecs'][i]) +# self.tilespecs.append(ts) +# for i in range(d['tranformCount']): +# tfd = d['transformSpecs'][i] +# tf = load_transform_json(tfd) +# self.transforms.append(tf) + + +# class Filter: +# def __init__(self, classname, params={}): +# self.classname = classname +# self.params = params + +# def to_dict(self): +# d = {} +# d['className'] = self.classname +# d['params'] = self.params +# return d + +# def from_dict(self, d): +# self.classname = d['className'] +# self.params = d['params'] From 5d31d79eabadcd19e6ea2a6acb7a72667af7fe2a Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 01:37:35 -0700 Subject: [PATCH 255/766] integration tests: misnamed variable in clone, PEP8 --- integration_tests/test_stack_integrated.py | 57 +++++++++++++--------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 09ab267d..63df701b 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -263,7 +263,7 @@ def test_get_section_z(render, teststack): def test_clone_stack(render, teststack): stack2 = 'cloned_stack' zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) - renderapi.stack.clone_stack(teststack, cloned_stack, render=render) + renderapi.stack.clone_stack(teststack, stack2, render=render) zvalues2 = renderapi.stack.get_z_values_for_stack(stack2, render=render) renderapi.stack.delete_stack(stack2, render=render) assert zvalues == zvalues2 @@ -362,22 +362,41 @@ def test_section_image(render, teststack, **kwargs): assert data.shape[1] == (np.floor(width * scalefactor)) assert data.shape[2] >= 3 -def test_get_tilespecs_from_z(render, teststack,render_example_tilespec_and_transforms): + +def test_section_image_options(render, teststack): + img = test_section_image(render, teststack, filter=True, + maxTileSpecsToRender=50) + + +def fail_image_get(render, teststack, render_example_tilespec_and_transforms): + with pytest.raises(KeyError): + render.run(renderapi.image.get_tile_image_data, teststack, + 'test', img_format='JUNK') + + +def test_get_tilespecs_from_z(render, teststack, + render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms - tiles = renderapi.tilespec.get_tile_specs_from_z(teststack, tilespecs[0].z, render=render) - tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z ] - assert len(ts)==len(tsz) - -def test_get_tile_specs_from_minmax_box(render, teststack, render_example_tilespec_and_transforms): + tiles = renderapi.tilespec.get_tile_specs_from_z( + teststack, tilespecs[0].z, render=render) + tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] + assert len(ts) == len(tsz) + + +def test_get_tile_specs_from_minmax_box( + render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms z = tilespecs[0].z tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) - ts = renderapi.tilespec.get_tile_specs_from_minmax_box(teststack, z, zbounds['minX'], - zbounds['maxX'], zbounds['minY'], zbounds['maxY'], render=render) + ts = renderapi.tilespec.get_tile_specs_from_minmax_box( + teststack, z, zbounds['minX'], zbounds['maxX'], + zbounds['minY'], zbounds['maxY'], render=render) assert len(ts) == len(tsz) -def test_get_tile_specs_from_box(render,teststack,render_example_tilespec_and_transforms): + +def test_get_tile_specs_from_box(render, teststack, + render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms z = tilespecs[0].z tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] @@ -385,22 +404,14 @@ def test_get_tile_specs_from_box(render,teststack,render_example_tilespec_and_tr width = zbounds['maxX']-zbounds['minX'] height = zbounds['maxY']-zbounds['minY'] - ts = renderapi.tilespec.get_tile_specs_from_box(teststack, z, zbounds['minX'], + ts = renderapi.tilespec.get_tile_specs_from_box( + teststack, z, zbounds['minX'], zbounds['minY'], width, height, render=render) assert len(ts) == len(tsz) -def test_get_tile_specs_from_stack(render,teststack,render_example_tilespec_and_transforms): + +def test_get_tile_specs_from_stack(render, teststack, + render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms ts = renderapi.tilespec.get_tile_specs_from_stack(teststack, render=render) assert len(ts) == len(tsz) - -def test_section_image_options(render, teststack): - img=test_section_image(render, teststack, filter=True, - maxTileSpecsToRender=50) - - -def fail_image_get(render, teststack, render_example_tilespec_and_transforms): - with pytest.raises(KeyError): - render.run(renderapi.image.get_tile_image_data, teststack, - 'test', img_format='JUNK') - From 0939abd0aaab0e67e93b488c86da61d9332dd1a7 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 02:02:34 -0700 Subject: [PATCH 256/766] clone stack bugs and tests, PEP8 --- integration_tests/test_stack_integrated.py | 2 +- renderapi/stack.py | 7 +++++-- renderapi/tilespec.py | 8 +++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 63df701b..f13f044f 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -273,7 +273,7 @@ def test_clone_stack_subset(render, teststack): stack2 = 'cloned_stack_subset' zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) renderapi.stack.clone_stack( - teststack, cloned_stack, zs=zvalues[0:1], render=render) + teststack, stack2, zs=zvalues[0:1], render=render) zvalues2 = renderapi.stack.get_z_values_for_stack(stack2, render=render) renderapi.stack.delete_stack(stack2, render=render) assert zvalues[0:1] == zvalues2 diff --git a/renderapi/stack.py b/renderapi/stack.py index dee42887..62d57a7b 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -183,6 +183,7 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, ''' session = requests.session() if session is None else session sv = StackVersion(**kwargs) + newstack_project = project qparams = {} if zs is not None: qparams['z'] = [float(i) for i in zs] @@ -190,15 +191,17 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, qparams['skipTransforms'] = jbool(skipTransforms) if toProject is not None: qparams['toProject'] = toProject + newstack_project = toProject - request_url = '{}/{}'.format(format_preamble( + request_url = '{}/cloneTo/{}'.format(format_preamble( host, port, owner, project, inputstack), outputstack) logger.debug(request_url) r = post_json(session, request_url, sv.to_dict(), params=qparams) if close_stack: - set_stack_state(outputstack, 'COMPLETE', host, port, owner, project) + set_stack_state(outputstack, 'COMPLETE', host, port, owner, + newstack_project) return r diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index ef0d365f..f45f38a4 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -2,7 +2,7 @@ from .render import format_preamble, renderaccess from .utils import NullHandler from .stack import get_z_values_for_stack -from .transform import TransformList, load_transform_json +from .transform import TransformList from collections import OrderedDict import logging import requests @@ -11,7 +11,6 @@ logger.addHandler(NullHandler()) - class Layout: def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, @@ -162,12 +161,15 @@ def from_dict(self, d): tfl = TransformList(json=d['transforms']) self.tforms = tfl.tforms + # TODO filters not implemented -- should skip + ''' self.inputfilters = [] if d.get('inputfilters', None) is not None: for f in d['inputfilters']['specList']: f = Filter() f.from_dict(f) self.inputfilters.append(f) + ''' class MipMapLevel: @@ -343,7 +345,7 @@ def get_tile_specs_from_stack(stack, host=None, port=None, owner=owner, project=project, session=session)] for i in sl] -#TODO: ADD FEATURES THAT REQUIRED THESE TO SUPPORT.. NOT YET FULLY IMPLEMENTED +# TODO: ADD FEATURES THAT REQUIRED THESE TO SUPPORT.. NOT YET FULLY IMPLEMENTED # class ResolvedTileSpecMap: # def __init__(self, tilespecs=[], transforms=[]): # self.tilespecs = tilespecs From 6279331c3fe75699b482761c7b3434cfc658c644 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 02:10:21 -0700 Subject: [PATCH 257/766] fixing filter error --- renderapi/tilespec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index f45f38a4..22cb7435 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -129,11 +129,15 @@ def to_dict(self): thedict['transforms']['specList'].append(strlist) else: thedict['transforms']['specList'].append(t.to_dict()) + + # TODO filters not implemented + ''' if len(self.inputfilters): thedict['inputfilters'] = {} thedict['inputfilters']['type'] = 'list' thedict['inputfilters']['specList'] = [f.to_dict() for f in self.inputfilters] + ''' thedict = {k: v for k, v in thedict.items() if v is not None} return thedict From df885825fd649218d54711b30fb4973b922b9293 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 02:27:31 -0700 Subject: [PATCH 258/766] stack, utils: add put_json to facilitate PUT requests and fix clone_stack --- renderapi/stack.py | 4 ++-- renderapi/utils.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 62d57a7b..31855092 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,7 +3,7 @@ from time import strftime import requests from .errors import RenderError -from .utils import jbool, NullHandler, post_json +from .utils import jbool, NullHandler, post_json, put_json from .render import (format_baseurl, format_preamble, renderaccess) @@ -197,7 +197,7 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, host, port, owner, project, inputstack), outputstack) logger.debug(request_url) - r = post_json(session, request_url, sv.to_dict(), params=qparams) + r = put_json(session, request_url, sv.to_dict(), params=qparams) if close_stack: set_stack_state(outputstack, 'COMPLETE', host, port, owner, diff --git a/renderapi/utils.py b/renderapi/utils.py index acd001fc..5c6df9f2 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -66,6 +66,25 @@ def post_json(session, request_url, d, params=None): d, request_url, params)) +def put_json(session, request_url, d, params=None): + headers = {"content-type": "application/json"} + if d is not None: + payload = json.dumps(d) + else: + payload = None + headers['Accept'] = "application/json" + r = session.put(request_url, data=payload, params=params, + headers=headers) + try: + return r + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError( + 'cannot put {} to {} with params {}'.format( + d, request_url, params)) + + def renderdumps(obj, *args, **kwargs): '''json.dumps using the RenderEncoder''' cls_ = kwargs.pop('cls', RenderEncoder) From 5c0fe69b26aee5d5a51288a486ea80e47dfe7188 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 03:10:12 -0700 Subject: [PATCH 259/766] coverage: add coveragerc to exclude some things not being tested --- .coveragerc | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..7225285f --- /dev/null +++ b/.coveragerc @@ -0,0 +1,12 @@ +[report] +exclude_lines = + pragma: no cover + + # bad form, but annoying when reported + except Exception + + # raw input is low priority and may be removed + raw_input + + # __repr__ is usually only important in debugging + def __repr__ From a4daee2015dd07381e8492f859c7a50c9e366945 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 07:33:36 -0700 Subject: [PATCH 260/766] fixed owner test as it wasn't using the setup fixture --- integration_tests/test_pointmatch_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 567a7dc6..9bc4d9a4 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -101,7 +101,7 @@ def test_pm_collection(render): renderapi.pointmatch.import_matches(collection,test_matches,render=render) return collection -def test_get_matchcollection_owners(render): +def test_get_matchcollection_owners(render,test_pm_collection): owners = renderapi.pointmatch.get_matchcollection_owners(render=render) assert 'test' in owners From 4ba4a975790cd35ce7e15e127ace82750da91297 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 07:34:47 -0700 Subject: [PATCH 261/766] fixed typos --- integration_tests/test_stack_integrated.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 09ab267d..b2ee35b0 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -365,13 +365,13 @@ def test_section_image(render, teststack, **kwargs): def test_get_tilespecs_from_z(render, teststack,render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms tiles = renderapi.tilespec.get_tile_specs_from_z(teststack, tilespecs[0].z, render=render) - tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z ] + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z ] assert len(ts)==len(tsz) def test_get_tile_specs_from_minmax_box(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms z = tilespecs[0].z - tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z ] zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) ts = renderapi.tilespec.get_tile_specs_from_minmax_box(teststack, z, zbounds['minX'], zbounds['maxX'], zbounds['minY'], zbounds['maxY'], render=render) @@ -380,7 +380,7 @@ def test_get_tile_specs_from_minmax_box(render, teststack, render_example_tilesp def test_get_tile_specs_from_box(render,teststack,render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms z = tilespecs[0].z - tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z ] zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) width = zbounds['maxX']-zbounds['minX'] height = zbounds['maxY']-zbounds['minY'] From 3e8abe7f9c95fc0e9edaf8609250928054f7ee23 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 07:36:57 -0700 Subject: [PATCH 262/766] commenting out deprecated calls --- renderapi/coordinate.py | 146 ++++++++++++++++++++-------------------- 1 file changed, 73 insertions(+), 73 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index c5b08a10..fe5e4aab 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -107,47 +107,47 @@ def unpackage_world_to_local_point_match_from_json(json_answer, tileId): return answer -@renderaccess -def old_world_to_local_coordinates_array(stack, dataarray, tileId, z=0, - host=None, port=None, - owner=None, project=None, - session=requests.session(), - render=None, **kwargs): - '''''' - - request_url = format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/world-to-local-coordinates" % (z) - dlist = [] - for i in range(dataarray.shape[0]): - d = {} - d['tileId'] = tileId - d['world'] = [dataarray[i, 0], dataarray[i, 1]] - dlist.append(d) - jsondata = json.dumps(dlist) - r = session.put(request_url, data=jsondata, - headers={"content-type": "application/json"}) - json_answer = r.json() - try: - answer = np.zeros(dataarray.shape) - for i, coord in enumerate(json_answer): - c = coord['local'] - answer[i, 0] = c[0] - answer[i, 1] = c[1] - return answer - except Exception as e: - logger.error(e) - logger.error(json_answer) - - -def unpackage_local_to_world_point_match_from_json(json_answer): - logger.debug("json_answer_length %d" % len(json_answer)) - answer = np.zeros((len(json_answer), 2)) - for i, coord in enumerate(json_answer): - c = coord['world'] - answer[i, 0] = c[0] - answer[i, 1] = c[1] - return answer +# @renderaccess +# def old_world_to_local_coordinates_array(stack, dataarray, tileId, z=0, +# host=None, port=None, +# owner=None, project=None, +# session=requests.session(), +# render=None, **kwargs): +# '''''' + +# request_url = format_preamble( +# host, port, owner, project, stack) + \ +# "/z/%d/world-to-local-coordinates" % (z) +# dlist = [] +# for i in range(dataarray.shape[0]): +# d = {} +# d['tileId'] = tileId +# d['world'] = [dataarray[i, 0], dataarray[i, 1]] +# dlist.append(d) +# jsondata = json.dumps(dlist) +# r = session.put(request_url, data=jsondata, +# headers={"content-type": "application/json"}) +# json_answer = r.json() +# try: +# answer = np.zeros(dataarray.shape) +# for i, coord in enumerate(json_answer): +# c = coord['local'] +# answer[i, 0] = c[0] +# answer[i, 1] = c[1] +# return answer +# except Exception as e: +# logger.error(e) +# logger.error(json_answer) + + +# def unpackage_local_to_world_point_match_from_json(json_answer): +# logger.debug("json_answer_length %d" % len(json_answer)) +# answer = np.zeros((len(json_answer), 2)) +# for i, coord in enumerate(json_answer): +# c = coord['world'] +# answer[i, 0] = c[0] +# answer[i, 1] = c[1] +# return answer @renderaccess @@ -171,38 +171,38 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, return unpackage_world_to_local_point_match_from_json(json_answer, tileId) -@renderaccess -def old_local_to_world_coordinates_array(stack, dataarray, tileId, z=0, - host=None, port=None, - owner=None, project=None, - session=requests.session(), - render=None, **kwargs): - '''''' - request_url = format_preamble( - host, port, owner, project, stack) + \ - "/z/%d/local-to-world-coordinates" % (z) - dlist = [] - for i in range(dataarray.shape[0]): - d = {} - d['tileId'] = tileId - d['local'] = [dataarray[i, 0], dataarray[i, 1]] - dlist.append(d) - jsondata = json.dumps(dlist) - r = session.put(request_url, data=jsondata, - headers={"content-type": "application/json"}) - json_answer = r.json() - try: - answer = np.zeros(dataarray.shape) - logger.debug('shape {}'.format(dataarray.shape)) - logger.debug('length of json_answer {}'.format(len(json_answer))) - for i, coord in enumerate(json_answer): - c = coord['world'] - answer[i, 0] = c[0] - answer[i, 1] = c[1] - return answer - except Exception as e: - logger.error(e) - logger.error(json_answer) +# @renderaccess +# def old_local_to_world_coordinates_array(stack, dataarray, tileId, z=0, +# host=None, port=None, +# owner=None, project=None, +# session=requests.session(), +# render=None, **kwargs): +# '''''' +# request_url = format_preamble( +# host, port, owner, project, stack) + \ +# "/z/%d/local-to-world-coordinates" % (z) +# dlist = [] +# for i in range(dataarray.shape[0]): +# d = {} +# d['tileId'] = tileId +# d['local'] = [dataarray[i, 0], dataarray[i, 1]] +# dlist.append(d) +# jsondata = json.dumps(dlist) +# r = session.put(request_url, data=jsondata, +# headers={"content-type": "application/json"}) +# json_answer = r.json() +# try: +# answer = np.zeros(dataarray.shape) +# logger.debug('shape {}'.format(dataarray.shape)) +# logger.debug('length of json_answer {}'.format(len(json_answer))) +# for i, coord in enumerate(json_answer): +# c = coord['world'] +# answer[i, 0] = c[0] +# answer[i, 1] = c[1] +# return answer +# except Exception as e: +# logger.error(e) +# logger.error(json_answer) @renderaccess From 3886822313e1ab949ecf74f8707a4ca3e2bfeff3 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 07:53:09 -0700 Subject: [PATCH 263/766] uncommented accidentally commented function --- renderapi/coordinate.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index fe5e4aab..73d2b2d7 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -140,14 +140,14 @@ def unpackage_world_to_local_point_match_from_json(json_answer, tileId): # logger.error(json_answer) -# def unpackage_local_to_world_point_match_from_json(json_answer): -# logger.debug("json_answer_length %d" % len(json_answer)) -# answer = np.zeros((len(json_answer), 2)) -# for i, coord in enumerate(json_answer): -# c = coord['world'] -# answer[i, 0] = c[0] -# answer[i, 1] = c[1] -# return answer +def unpackage_local_to_world_point_match_from_json(json_answer): + logger.debug("json_answer_length %d" % len(json_answer)) + answer = np.zeros((len(json_answer), 2)) + for i, coord in enumerate(json_answer): + c = coord['world'] + answer[i, 0] = c[0] + answer[i, 1] = c[1] + return answer @renderaccess From ee85d2d7a4bb02e8622527f27670a53d3cea5000 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 07:53:48 -0700 Subject: [PATCH 264/766] fixed test --- integration_tests/test_stack_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index d2a573aa..83c9911e 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -414,4 +414,4 @@ def test_get_tile_specs_from_stack(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms ts = renderapi.tilespec.get_tile_specs_from_stack(teststack, render=render) - assert len(ts) == len(tsz) + assert len(ts) == len(tilespecs) From 88f128a0032a81d145d47e63a45e7b652ec8a80c Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 07:54:40 -0700 Subject: [PATCH 265/766] fixed typo --- integration_tests/test_stack_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 83c9911e..632a23fd 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -379,7 +379,7 @@ def test_get_tilespecs_from_z(render, teststack, (tilespecs, tforms) = render_example_tilespec_and_transforms tiles = renderapi.tilespec.get_tile_specs_from_z( teststack, tilespecs[0].z, render=render) - tsz = [ts for ts in tilespces if ts.z == tilespecs[0].z] + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] assert len(ts) == len(tsz) From b81dc94221937b59e8d80e494ab9d7bbdfc1873b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 07:59:27 -0700 Subject: [PATCH 266/766] fixed error in test --- integration_tests/test_stack_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 632a23fd..38eace0f 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -380,7 +380,7 @@ def test_get_tilespecs_from_z(render, teststack, tiles = renderapi.tilespec.get_tile_specs_from_z( teststack, tilespecs[0].z, render=render) tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] - assert len(ts) == len(tsz) + assert len(tiles) == len(tsz) def test_get_tile_specs_from_minmax_box( From 56be7d419ff1bc70b6f6470e1a9977f2c42e1928 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 08:01:58 -0700 Subject: [PATCH 267/766] minor changes to formatting of setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da534646..af99635d 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,9 @@ def run_tests(self): setup(name='render-python', version='1.0', - description=' a python API setup to interact via python with render ' + description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', - author='Forrest Collman,Russel Torres,Eric Perlman,Sharmi Seshamani', + author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', packages=['renderapi'], From 1336ab0a56a5d34befa0cd2895ff52e88ec30c85 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 08:48:47 -0700 Subject: [PATCH 268/766] adding first versions of some client tests --- integration_tests/test_client_integrated.py | 115 ++++++++++++++++++++ integration_tests/test_stack_integrated.py | 33 ------ 2 files changed, 115 insertions(+), 33 deletions(-) create mode 100644 integration_tests/test_client_integrated.py diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py new file mode 100644 index 00000000..bc237451 --- /dev/null +++ b/integration_tests/test_client_integrated.py @@ -0,0 +1,115 @@ +import renderapi +import pytest +import tempfile +import os +import logging +import sys +import json +import numpy as np +from test_data import (render_host, render_port, + client_script_location, tilespec_file, tform_file) + +root = logging.getLogger() +root.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +root.addHandler(ch) + + +@pytest.fixture(scope='module') +def render(): + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location} + return renderapi.render.connect(**render_test_parameters) + + +@pytest.fixture(scope='module') +def render_example_tilespec_and_transforms(): + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + print tforms + return (tilespecs, tforms) + + +def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack = 'test_import_tilespecs_parallel'): + + renderapi.stack.create_stack(stack,render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, + render=render) + stacks = renderapi.render.get_stacks_by_owner_project(render=render) + assert stack in stacks + ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) + assert len(ts) == len(tilespecs) + +def test_import_jsonfiles_validate_client(render): + root.debug('test not implemented yet') + assert False + + +def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=None): + if stack is None: + stack = 'test_import_jsonfiles' + renderapi.stack.create_stack(stack,render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + tfiles=[] + for ts in tilespecs: + tfile = tempfile.NamedTemporaryFile(mode = 'w', suffix = '.json', delete=False) + tfile.write(renderapi.utils.renderdumps(ts)) + tfile.close() + tfiles.append(tfile) + transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) + transformFile.write(renderapi.utils.renderdumps(tforms)) + + renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile) + + stacks = renderapi.render.get_stacks_by_owner_project(render=render) + assert stack in stacks + ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) + assert len(ts) == len(tilespecs) + + +@pytest.fixture(scope = "moudle") +def teststack(render,render_example_tilespec_and_transforms): + + stack = 'teststack' + test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) + yield stack + renderapi.stack.delete_stack(stack, render=render) + + +def test_tile_pair_client(render,teststack,**kwargs): + zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack)) + outjson = kwargs.pop('outjson',None) + if outjson is None: + outjson = 'test_tile_pair_client.json' + + renderapi.client.tilePairClient(teststack, np.min(zvalues), + np.max(zvalues), outjson=outjson, + **kwargs) + + tilepairjson = json.load(open(outjson,'r')) + assert isinstance(tilepairjson,dict) + assert len(tilepairjson['neighborPairs'])>3 + +def test_importTransformChangesClient(render): + root.debug('test not implemented yet') + assert False + + +def test_coordinateClient(render): + root.debug('test not implemented yet') + assert False \ No newline at end of file diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 38eace0f..070f3474 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -181,39 +181,6 @@ def test_import_tilespecs(render, simpletilespec): render.run(renderapi.stack.delete_stack, stack) -def test_import_tilespecs_parallel(render): - root.debug('test not implemented yet') - assert False - - -def test_import_jsonfiles_validate_client(render): - root.debug('test not implemented yet') - assert False - - -def test_import_jsonfiles(render): - root.debug('test not implemented yet') - assert False - - -def test_import_parallel(render): - root.debug('test not implemented yet') - assert False - - -def test_tile_pair_client(render): - root.debug('test not implemented yet') - assert False - - -def test_importTransformChangesClient(render): - root.debug('test not implemented yet') - assert False - - -def test_coordinateClient(render): - root.debug('test not implemented yet') - assert False @pytest.fixture(scope="module") From b6d2f6fc5bae87105a168050c202e574a9d34976 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 09:01:35 -0700 Subject: [PATCH 269/766] typo --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index bc237451..52b3b7da 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -82,7 +82,7 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=No assert len(ts) == len(tilespecs) -@pytest.fixture(scope = "moudle") +@pytest.fixture(scope = "module") def teststack(render,render_example_tilespec_and_transforms): stack = 'teststack' From 3d3ab2cb6dd437ad5a2a41f94f2b04d18b5e9dc1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 09:46:52 -0700 Subject: [PATCH 270/766] missing render --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 52b3b7da..4b3a9e96 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -74,7 +74,7 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=No transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) transformFile.write(renderapi.utils.renderdumps(tforms)) - renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile) + renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile, render=render) stacks = renderapi.render.get_stacks_by_owner_project(render=render) assert stack in stacks @@ -92,7 +92,7 @@ def teststack(render,render_example_tilespec_and_transforms): def test_tile_pair_client(render,teststack,**kwargs): - zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack)) + zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) outjson = kwargs.pop('outjson',None) if outjson is None: outjson = 'test_tile_pair_client.json' From 24012a19db215719f4c98aef1f25d908e7e509d5 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 09:49:48 -0700 Subject: [PATCH 271/766] adding dill as requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index e316b911..08f7c8ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ requests numpy pillow pathos +dill \ No newline at end of file From 31a3482108967cd05850b1d671de23082a0ba99f Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 09:53:20 -0700 Subject: [PATCH 272/766] fixed missing render --- integration_tests/test_client_integrated.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 4b3a9e96..2f01136d 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -99,6 +99,7 @@ def test_tile_pair_client(render,teststack,**kwargs): renderapi.client.tilePairClient(teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, + render = render, **kwargs) tilepairjson = json.load(open(outjson,'r')) From 333f58551239539e8110d5348e7281c17a121052 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 09:53:52 -0700 Subject: [PATCH 273/766] missing render --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 2f01136d..6cc02dd4 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -86,7 +86,7 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=No def teststack(render,render_example_tilespec_and_transforms): stack = 'teststack' - test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) + test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack, render=render) yield stack renderapi.stack.delete_stack(stack, render=render) From 00c7caf463af942c84775c05280e8e6d72c21f4b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 09:56:56 -0700 Subject: [PATCH 274/766] fixed bug in tempfile usage --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 6cc02dd4..773a15fc 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -70,11 +70,11 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=No tfile = tempfile.NamedTemporaryFile(mode = 'w', suffix = '.json', delete=False) tfile.write(renderapi.utils.renderdumps(ts)) tfile.close() - tfiles.append(tfile) + tfiles.append(tfile.name) transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) transformFile.write(renderapi.utils.renderdumps(tforms)) - renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile, render=render) + renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) stacks = renderapi.render.get_stacks_by_owner_project(render=render) assert stack in stacks From dc2dc190f2f2c6f02e9884c870791b7e7ea8badb Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 10:03:01 -0700 Subject: [PATCH 275/766] closed transform file --- integration_tests/test_client_integrated.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 773a15fc..8374b01b 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -73,7 +73,8 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=No tfiles.append(tfile.name) transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) transformFile.write(renderapi.utils.renderdumps(tforms)) - + transformFile.close() + renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) stacks = renderapi.render.get_stacks_by_owner_project(render=render) From a34a956b289d0bf7c574a011fb19784c237a30f1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 10:09:42 -0700 Subject: [PATCH 276/766] removed bug --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 8374b01b..94e9f170 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -74,7 +74,7 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=No transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) transformFile.write(renderapi.utils.renderdumps(tforms)) transformFile.close() - + renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) stacks = renderapi.render.get_stacks_by_owner_project(render=render) @@ -87,7 +87,7 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=No def teststack(render,render_example_tilespec_and_transforms): stack = 'teststack' - test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack, render=render) + test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) yield stack renderapi.stack.delete_stack(stack, render=render) From 4cf690504b657855d87cee78099b4c0d46259da9 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 10:11:12 -0700 Subject: [PATCH 277/766] adding dill to pip install in dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1233174a..06c05806 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get install build-essential -y RUN apt-get clean RUN pip install multiprocess RUN pip install pathos +RUN pip install dill #install components for common render-python apps #jupyter notebook, shapely with geos From 6bf25c4a6b9ddb407d86919adb4a31cc45419db8 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 10:11:43 -0700 Subject: [PATCH 278/766] changing install process to avoid using multiprocess --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 06c05806..65b37f8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,9 +35,9 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-oracle RUN apt-get install gcc -y RUN apt-get install build-essential -y RUN apt-get clean -RUN pip install multiprocess -RUN pip install pathos RUN pip install dill +RUN pip install pathos + #install components for common render-python apps #jupyter notebook, shapely with geos From cd4447c4ec03e121b14894f445d1ce2eac2cbcee Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 11:00:47 -0700 Subject: [PATCH 279/766] transform: some pruning , fixes, and tests --- renderapi/transform.py | 50 ++++---------------------------- test/test_transform.py | 66 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 47 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index f874f65e..5f495814 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -20,7 +20,6 @@ logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) -# TODO preference for svd version? try: from scipy.linalg import svd, LinAlgError except ImportError as e: @@ -475,7 +474,7 @@ def fitgood(src, dst, params): while (tries < max_tries and not estimated): tries += 1 try: - params = Polynomial2DTransform.fit(src, dst, order=2) + params = Polynomial2DTransform.fit(src, dst, order=order) except (LinAlgError, ValueError) as e: logger.debug('Encountered error {}'.format(e)) continue @@ -498,9 +497,7 @@ def _process_dataString(self, datastring): generate datastring and param attributes from datastring ''' dsList = datastring.split(' ') - self.params = np.array( - [[float(d) for d in dsList[:len(dsList)/2]], - [float(d) for d in dsList[len(dsList)/2:]]]) + self.params = Polynomial2DTransform._format_raveled_params(dsList) @staticmethod def _format_raveled_params(raveled_params): @@ -543,7 +540,7 @@ def asorder(self, order): 'transformation {} is order {} -- conversion to ' 'order {} not supported'.format( self.dataString, self.order, order)) - new_params = np.zeros([2, self.coefficients(order)]) + new_params = np.zeros([2, self.coefficients(order) // 2]) new_params[:self.params.shape[0], :self.params.shape[1]] = self.params return Polynomial2DTransform(params=new_params) @@ -555,47 +552,10 @@ def fromAffine(aff): ''' if not isinstance(aff, AffineModel): raise ConversionError('attempting to convert a nonaffine model!') - return Polynomial2DTransform(params=np.array([ + return Polynomial2DTransform(order=1, params=np.array([ [aff.M[0, 2], aff.M[0, 0], aff.M[0, 1]], [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) - def concatenate(self, othertform, order=None, srcpts=None): - ''' - currently uses an estimation to represent composition of transforms - ''' - if isinstance(othertform, AffineModel): - othertform = self._fromAffine(othertform) - if order is None: - order = max([self.order, othertform.order]) - # TODO define srcpts and dstpts - if srcpts is None: - raise NotImplementedError('default source points unavailable!') - dstpts = othertform.tform(self.tform(srcpts)) - return Polynomial2DTransform(src=srcpts, dst=dstpts, order=order) - - -def transformsum(transformlist, src=None): - ''' - summation of all transforms in a list of transforms. - Will force affines as polynomials. Does not support LC. - input: - src -- test points representing the - Returns: - Polynomial2DTransform representing the sum of the - input list - ''' - logger.warning('This implementation of transformsum is not recommended! ' - 'Please consider using estimate_transformsum') - sumtform = Polynomial2DTransform(identity=True) - for tform in transformlist: - if isinstance(tform, list): - logger.debug('found transformlist!') - sumtform = sumtform.concatenate( - transformsum(tform, src=src), srcpts=src) - else: - sumtform = sumtform.concatenate(tform, srcpts=src) - return sumtform - def estimate_dstpts(transformlist, src=None): ''' @@ -621,4 +581,4 @@ def estimate_transformsum(transformlist, src=None, order=2): input: trans ''' dstpts = estimate_dstpts(transformlist, src) - return Polynomial2DTransform(src=src, dst=dstpts) + return Polynomial2DTransform(src=src, dst=dstpts, order=order) diff --git a/test/test_transform.py b/test/test_transform.py index e838c172..bf1fcd22 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -105,5 +105,67 @@ def test_Polynomial_estimation_numpy(): test_Polynomial_estimation(use_numpy=True) -def test_transformsum(): - pass +def test_transformsum_polynomial_identity(): + srcpts = np.random.rand(50, 2) + am = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + invam = am.invert() + + datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') + pt = renderapi.transform.Polynomial2DTransform( + dataString=datastring) + ptest_dstpts = pt.tform(srcpts) + invpt = renderapi.transform.Polynomial2DTransform( + src=ptest_dstpts, dst=srcpts) + + tformlist = [am, pt, invpt, invam] + new_tform = renderapi.transform.estimate_transformsum( + tformlist, src=srcpts) + + poly_identity = renderapi.transform.Polynomial2DTransform( + identity=True).asorder(new_tform.order) + assert all([i < 1e-3 for i in + (new_tform.params[:, 0] - poly_identity.params[:, 0]).ravel()]) + + assert np.allclose( + new_tform.params[:, 1:-1], + poly_identity.params[:, 1:-1], atol=1e-5) + + +def test_load_polynomial(): + datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') + pt = renderapi.transform.Polynomial2DTransform( + dataString=datastring) + pt_dict = renderapi.transform.Polynomial2DTransform(json=pt.to_dict()) + pt_dataString = renderapi.transform.Polynomial2DTransform( + dataString=pt.dataString) + pt_params = renderapi.transform.Polynomial2DTransform(params=pt.params) + assert (pt_dict.to_dict() == pt_dataString.to_dict() == + pt_params.to_dict() == pt.to_dict()) + + +def test_Polynomial_from_affine(): + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + pt = renderapi.transform.Polynomial2DTransform.fromAffine(am1) + pt_params_raveled = pt.params.ravel() + assert pt_params_raveled[0] == am1.M[0, 2] + assert pt_params_raveled[1] == am1.M[0, 0] + assert pt_params_raveled[2] == am1.M[0, 1] + assert pt_params_raveled[3] == am1.M[1, 2] + assert pt_params_raveled[4] == am1.M[1, 0] + assert pt_params_raveled[5] == am1.M[1, 1] From 9c7cd833049cd8bb2e0571e069453f449dc21caa Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 11:02:06 -0700 Subject: [PATCH 280/766] coverage: no cover comments and modifications to .coveragerc --- .coveragerc | 9 +++++++++ renderapi/image.py | 6 +++--- renderapi/render.py | 10 +++++----- renderapi/utils.py | 4 ++-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.coveragerc b/.coveragerc index 7225285f..2660be86 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,8 @@ exclude_lines = pragma: no cover + raise NotImplementedError + # bad form, but annoying when reported except Exception @@ -10,3 +12,10 @@ exclude_lines = # __repr__ is usually only important in debugging def __repr__ + + # ignore logger messages + logger.debug + logger.info + logger.warning + logger.error + logger.critical diff --git a/renderapi/image.py b/renderapi/image.py index a60919d9..f9060689 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -43,7 +43,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, ''' try: image_ext = IMAGE_FORMATS[img_format] - except KeyError as e: + except KeyError as e: # pragma no cover raise ValueError('{} is not a valid render image format!'.format(e)) request_url = format_preamble( @@ -81,7 +81,7 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, scale=None, ''' try: image_ext = IMAGE_FORMATS[img_format] - except KeyError as e: + except KeyError as e: # pragma no cover raise ValueError('{} is not a valid render image format!'.format(e)) request_url = format_preamble( @@ -122,7 +122,7 @@ def get_section_image(stack, z, scale=1.0, filter=False, ''' try: image_ext = IMAGE_FORMATS[img_format] - except KeyError as e: + except KeyError as e: # pragma no cover raise ValueError('{} is not a valid render image format!'.format(e)) request_url = format_preamble( diff --git a/renderapi/render.py b/renderapi/render.py index c91ad1a7..aebb2ea5 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -136,7 +136,7 @@ def connect(host=None, port=None, owner=None, project=None, if host is None: if 'RENDER_HOST' not in os.environ: host = str(raw_input("Enter Render Host: ")) - if host == '': + if host == '': # pragma no cover logger.critical('Render Host must not be empty!') raise ValueError('Render Host must not be empty!') else: @@ -148,7 +148,7 @@ def connect(host=None, port=None, owner=None, project=None, if port is None: if 'RENDER_PORT' not in os.environ: port = str(int(raw_input("Enter Render Port: "))) - if port == '': + if port == '': # pragma no cover # TODO better (no) port handling logger.critical('Render Port must not be empty!') raise ValueError('Render Port must not be empty!') @@ -160,7 +160,7 @@ def connect(host=None, port=None, owner=None, project=None, project = str(raw_input("Enter Render Project: ")) else: project = str(os.environ['RENDER_PROJECT']) - if project == '': + if project == '': # pragma no cover logger.critical('Render Project must not be empty!') raise ValueError('Render Project must not be empty!') @@ -169,7 +169,7 @@ def connect(host=None, port=None, owner=None, project=None, owner = str(raw_input("Enter Render Owner: ")) else: owner = str(os.environ['RENDER_OWNER']) - if owner == '': + if owner == '': # pragma no cover logger.critical('Render Owner must not be empty!') raise ValueError('Render Owner must not be empty!') @@ -180,7 +180,7 @@ def connect(host=None, port=None, owner=None, project=None, "Enter Render Client Scripts location: ")) else: client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) - if client_scripts == '': + if client_scripts == '': # pragma no cover logger.critical('Render Client Scripts must ' 'not be empty!') raise ValueError('Render Client Scripts must ' diff --git a/renderapi/utils.py b/renderapi/utils.py index 5c6df9f2..4e2699ad 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -39,7 +39,7 @@ def default(self, obj): type(obj))) try: return super(RenderEncoder, self).default(obj) - except TypeError as e: + except TypeError as e: # pragma no cover logger.info(e) logger.warning( "cannot json serialize {}. " @@ -105,7 +105,7 @@ def jbool(val): return 'true' if val else 'false' -def stripLogger(logger_tostrip): +def stripLogger(logger_tostrip): # pragma no cover ''' remove all handlers from a logger -- useful for redefining input: From ea812f32aaf66371c44cf6f43173071522d7ccf1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 11:38:50 -0700 Subject: [PATCH 281/766] trying to isolate build problem --- Dockerfile | 3 +- integration_tests/test_client_integrated.py | 208 ++++++++++---------- requirements.txt | 2 +- 3 files changed, 106 insertions(+), 107 deletions(-) diff --git a/Dockerfile b/Dockerfile index 65b37f8a..228e92b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,7 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-oracle RUN apt-get install gcc -y RUN apt-get install build-essential -y RUN apt-get clean -RUN pip install dill -RUN pip install pathos + #install components for common render-python apps diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 94e9f170..61c4a860 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -1,117 +1,117 @@ -import renderapi -import pytest -import tempfile -import os -import logging -import sys -import json -import numpy as np -from test_data import (render_host, render_port, - client_script_location, tilespec_file, tform_file) - -root = logging.getLogger() -root.setLevel(logging.DEBUG) - -ch = logging.StreamHandler(sys.stdout) -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') -ch.setFormatter(formatter) -root.addHandler(ch) - - -@pytest.fixture(scope='module') -def render(): - render_test_parameters = { - 'host': render_host, - 'port': render_port, - 'owner': 'test', - 'project': 'test_project', - 'client_scripts': client_script_location} - return renderapi.render.connect(**render_test_parameters) - - -@pytest.fixture(scope='module') -def render_example_tilespec_and_transforms(): - with open(tilespec_file, 'r') as f: - ts_json = json.load(f) - with open(tform_file, 'r') as f: - tform_json = json.load(f) - - tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] - tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] - print tforms - return (tilespecs, tforms) - - -def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack = 'test_import_tilespecs_parallel'): +# import renderapi +# import pytest +# import tempfile +# import os +# import logging +# import sys +# import json +# import numpy as np +# from test_data import (render_host, render_port, +# client_script_location, tilespec_file, tform_file) + +# root = logging.getLogger() +# root.setLevel(logging.DEBUG) + +# ch = logging.StreamHandler(sys.stdout) +# ch.setLevel(logging.DEBUG) +# formatter = logging.Formatter( +# '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# ch.setFormatter(formatter) +# root.addHandler(ch) + + +# @pytest.fixture(scope='module') +# def render(): +# render_test_parameters = { +# 'host': render_host, +# 'port': render_port, +# 'owner': 'test', +# 'project': 'test_project', +# 'client_scripts': client_script_location} +# return renderapi.render.connect(**render_test_parameters) + + +# @pytest.fixture(scope='module') +# def render_example_tilespec_and_transforms(): +# with open(tilespec_file, 'r') as f: +# ts_json = json.load(f) +# with open(tform_file, 'r') as f: +# tform_json = json.load(f) + +# tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] +# tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] +# print tforms +# return (tilespecs, tforms) + + +# def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack = 'test_import_tilespecs_parallel'): - renderapi.stack.create_stack(stack,render=render) - (tilespecs, tforms) = render_example_tilespec_and_transforms - renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, - render=render) - stacks = renderapi.render.get_stacks_by_owner_project(render=render) - assert stack in stacks - ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) - assert len(ts) == len(tilespecs) +# renderapi.stack.create_stack(stack,render=render) +# (tilespecs, tforms) = render_example_tilespec_and_transforms +# renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, +# render=render) +# stacks = renderapi.render.get_stacks_by_owner_project(render=render) +# assert stack in stacks +# ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) +# assert len(ts) == len(tilespecs) -def test_import_jsonfiles_validate_client(render): - root.debug('test not implemented yet') - assert False - - -def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=None): - if stack is None: - stack = 'test_import_jsonfiles' - renderapi.stack.create_stack(stack,render=render) - (tilespecs, tforms) = render_example_tilespec_and_transforms - tfiles=[] - for ts in tilespecs: - tfile = tempfile.NamedTemporaryFile(mode = 'w', suffix = '.json', delete=False) - tfile.write(renderapi.utils.renderdumps(ts)) - tfile.close() - tfiles.append(tfile.name) - transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) - transformFile.write(renderapi.utils.renderdumps(tforms)) - transformFile.close() - - renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) +# def test_import_jsonfiles_validate_client(render): +# root.debug('test not implemented yet') +# assert False + + +# def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=None): +# if stack is None: +# stack = 'test_import_jsonfiles' +# renderapi.stack.create_stack(stack,render=render) +# (tilespecs, tforms) = render_example_tilespec_and_transforms +# tfiles=[] +# for ts in tilespecs: +# tfile = tempfile.NamedTemporaryFile(mode = 'w', suffix = '.json', delete=False) +# tfile.write(renderapi.utils.renderdumps(ts)) +# tfile.close() +# tfiles.append(tfile.name) +# transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) +# transformFile.write(renderapi.utils.renderdumps(tforms)) +# transformFile.close() + +# renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) - stacks = renderapi.render.get_stacks_by_owner_project(render=render) - assert stack in stacks - ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) - assert len(ts) == len(tilespecs) +# stacks = renderapi.render.get_stacks_by_owner_project(render=render) +# assert stack in stacks +# ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) +# assert len(ts) == len(tilespecs) -@pytest.fixture(scope = "module") -def teststack(render,render_example_tilespec_and_transforms): +# @pytest.fixture(scope = "module") +# def teststack(render,render_example_tilespec_and_transforms): - stack = 'teststack' - test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) - yield stack - renderapi.stack.delete_stack(stack, render=render) +# stack = 'teststack' +# test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) +# yield stack +# renderapi.stack.delete_stack(stack, render=render) -def test_tile_pair_client(render,teststack,**kwargs): - zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) - outjson = kwargs.pop('outjson',None) - if outjson is None: - outjson = 'test_tile_pair_client.json' +# def test_tile_pair_client(render,teststack,**kwargs): +# zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) +# outjson = kwargs.pop('outjson',None) +# if outjson is None: +# outjson = 'test_tile_pair_client.json' - renderapi.client.tilePairClient(teststack, np.min(zvalues), - np.max(zvalues), outjson=outjson, - render = render, - **kwargs) +# renderapi.client.tilePairClient(teststack, np.min(zvalues), +# np.max(zvalues), outjson=outjson, +# render = render, +# **kwargs) - tilepairjson = json.load(open(outjson,'r')) - assert isinstance(tilepairjson,dict) - assert len(tilepairjson['neighborPairs'])>3 +# tilepairjson = json.load(open(outjson,'r')) +# assert isinstance(tilepairjson,dict) +# assert len(tilepairjson['neighborPairs'])>3 -def test_importTransformChangesClient(render): - root.debug('test not implemented yet') - assert False +# def test_importTransformChangesClient(render): +# root.debug('test not implemented yet') +# assert False -def test_coordinateClient(render): - root.debug('test not implemented yet') - assert False \ No newline at end of file +# def test_coordinateClient(render): +# root.debug('test not implemented yet') +# assert False \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 08f7c8ec..83589c97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ requests numpy pillow +dill>=0.2.6 pathos -dill \ No newline at end of file From 3f146342dd6a35db738215fbd18fd048bdecea37 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 11:46:58 -0700 Subject: [PATCH 282/766] isolating single test --- integration_tests/test_client_integrated.py | 112 ++++++++++---------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 61c4a860..5f6ac09e 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -1,59 +1,59 @@ -# import renderapi -# import pytest -# import tempfile -# import os -# import logging -# import sys -# import json -# import numpy as np -# from test_data import (render_host, render_port, -# client_script_location, tilespec_file, tform_file) - -# root = logging.getLogger() -# root.setLevel(logging.DEBUG) - -# ch = logging.StreamHandler(sys.stdout) -# ch.setLevel(logging.DEBUG) -# formatter = logging.Formatter( -# '%(asctime)s - %(name)s - %(levelname)s - %(message)s') -# ch.setFormatter(formatter) -# root.addHandler(ch) - - -# @pytest.fixture(scope='module') -# def render(): -# render_test_parameters = { -# 'host': render_host, -# 'port': render_port, -# 'owner': 'test', -# 'project': 'test_project', -# 'client_scripts': client_script_location} -# return renderapi.render.connect(**render_test_parameters) - - -# @pytest.fixture(scope='module') -# def render_example_tilespec_and_transforms(): -# with open(tilespec_file, 'r') as f: -# ts_json = json.load(f) -# with open(tform_file, 'r') as f: -# tform_json = json.load(f) - -# tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] -# tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] -# print tforms -# return (tilespecs, tforms) - - -# def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack = 'test_import_tilespecs_parallel'): - -# renderapi.stack.create_stack(stack,render=render) -# (tilespecs, tforms) = render_example_tilespec_and_transforms -# renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, -# render=render) -# stacks = renderapi.render.get_stacks_by_owner_project(render=render) -# assert stack in stacks -# ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) -# assert len(ts) == len(tilespecs) +import renderapi +import pytest +import tempfile +import os +import logging +import sys +import json +import numpy as np +import dill +from test_data import (render_host, render_port, + client_script_location, tilespec_file, tform_file) + +root = logging.getLogger() +root.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +root.addHandler(ch) + + +@pytest.fixture(scope='module') +def render(): + render_test_parameters = { + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location} + return renderapi.render.connect(**render_test_parameters) + + +@pytest.fixture(scope='module') +def render_example_tilespec_and_transforms(): + with open(tilespec_file, 'r') as f: + ts_json = json.load(f) + with open(tform_file, 'r') as f: + tform_json = json.load(f) + + tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] + tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] + print tforms + return (tilespecs, tforms) + + +def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack = 'test_import_tilespecs_parallel'): + renderapi.stack.create_stack(stack,render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, + render=render) + stacks = renderapi.render.get_stacks_by_owner_project(render=render) + assert stack in stacks + ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) + assert len(ts) == len(tilespecs) # def test_import_jsonfiles_validate_client(render): # root.debug('test not implemented yet') From 0f116541c5a6d5a4cb8d6f09761d769592dd6f17 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 11:58:04 -0700 Subject: [PATCH 283/766] changed install for speedier testing --- Dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 228e92b1..9d45836b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,10 +42,14 @@ RUN apt-get clean #jupyter notebook, shapely with geos RUN /opt/conda/bin/conda install jupyter -y RUN apt-get install libgeos-dev -y +RUN apt-get install imagemagick -y + RUN pip install shapely==1.6b2 RUN pip install opencv-python - -RUN apt-get install imagemagick -y +RUN pip install dill==0.2.6 +RUN pip install multiprocess==0.70.5 +RUN pip install pathos==0.2.0 +RUN pip install pillow #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python From 266f023ff6288976c964ba0f9ba5dd247e725b49 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 12:04:02 -0700 Subject: [PATCH 284/766] replace mistpyed coverage ignore comment --- renderapi/image.py | 6 +++--- renderapi/render.py | 10 +++++----- renderapi/utils.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index f9060689..1d41071e 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -43,7 +43,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, ''' try: image_ext = IMAGE_FORMATS[img_format] - except KeyError as e: # pragma no cover + except KeyError as e: # pragma: no cover raise ValueError('{} is not a valid render image format!'.format(e)) request_url = format_preamble( @@ -81,7 +81,7 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, scale=None, ''' try: image_ext = IMAGE_FORMATS[img_format] - except KeyError as e: # pragma no cover + except KeyError as e: # pragma: no cover raise ValueError('{} is not a valid render image format!'.format(e)) request_url = format_preamble( @@ -122,7 +122,7 @@ def get_section_image(stack, z, scale=1.0, filter=False, ''' try: image_ext = IMAGE_FORMATS[img_format] - except KeyError as e: # pragma no cover + except KeyError as e: # pragma: no cover raise ValueError('{} is not a valid render image format!'.format(e)) request_url = format_preamble( diff --git a/renderapi/render.py b/renderapi/render.py index aebb2ea5..7dbcd863 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -136,7 +136,7 @@ def connect(host=None, port=None, owner=None, project=None, if host is None: if 'RENDER_HOST' not in os.environ: host = str(raw_input("Enter Render Host: ")) - if host == '': # pragma no cover + if host == '': # pragma: no cover logger.critical('Render Host must not be empty!') raise ValueError('Render Host must not be empty!') else: @@ -148,7 +148,7 @@ def connect(host=None, port=None, owner=None, project=None, if port is None: if 'RENDER_PORT' not in os.environ: port = str(int(raw_input("Enter Render Port: "))) - if port == '': # pragma no cover + if port == '': # pragma: no cover # TODO better (no) port handling logger.critical('Render Port must not be empty!') raise ValueError('Render Port must not be empty!') @@ -160,7 +160,7 @@ def connect(host=None, port=None, owner=None, project=None, project = str(raw_input("Enter Render Project: ")) else: project = str(os.environ['RENDER_PROJECT']) - if project == '': # pragma no cover + if project == '': # pragma: no cover logger.critical('Render Project must not be empty!') raise ValueError('Render Project must not be empty!') @@ -169,7 +169,7 @@ def connect(host=None, port=None, owner=None, project=None, owner = str(raw_input("Enter Render Owner: ")) else: owner = str(os.environ['RENDER_OWNER']) - if owner == '': # pragma no cover + if owner == '': # pragma: no cover logger.critical('Render Owner must not be empty!') raise ValueError('Render Owner must not be empty!') @@ -180,7 +180,7 @@ def connect(host=None, port=None, owner=None, project=None, "Enter Render Client Scripts location: ")) else: client_scripts = str(os.environ['RENDER_CLIENT_SCRIPTS']) - if client_scripts == '': # pragma no cover + if client_scripts == '': # pragma: no cover logger.critical('Render Client Scripts must ' 'not be empty!') raise ValueError('Render Client Scripts must ' diff --git a/renderapi/utils.py b/renderapi/utils.py index 4e2699ad..b27e1950 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -39,7 +39,7 @@ def default(self, obj): type(obj))) try: return super(RenderEncoder, self).default(obj) - except TypeError as e: # pragma no cover + except TypeError as e: # pragma: no cover logger.info(e) logger.warning( "cannot json serialize {}. " @@ -105,7 +105,7 @@ def jbool(val): return 'true' if val else 'false' -def stripLogger(logger_tostrip): # pragma no cover +def stripLogger(logger_tostrip): # pragma: no cover ''' remove all handlers from a logger -- useful for redefining input: From 6561b64589da2ccd1b638aa6686a6ef92d085887 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 12:08:32 -0700 Subject: [PATCH 285/766] trying to close pool to fix problem --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 2f7c4b8a..2772108f 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -225,7 +225,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, # TODO this is a weird way to do splits.... is that okay? tilespec_groups = [tilespecs[i::poolsize] for i in xrange(poolsize)] pool.map(partial_import, tilespec_groups) - + pool.close() if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From b831fd6a03bb870586e41793fb6e467312891510 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 12:16:07 -0700 Subject: [PATCH 286/766] adding back tests --- integration_tests/test_client_integrated.py | 74 ++++++++++----------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 5f6ac09e..6e49f829 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -60,52 +60,50 @@ def test_import_tilespecs_parallel(render, render_example_tilespec_and_transform # assert False -# def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=None): -# if stack is None: -# stack = 'test_import_jsonfiles' -# renderapi.stack.create_stack(stack,render=render) -# (tilespecs, tforms) = render_example_tilespec_and_transforms -# tfiles=[] -# for ts in tilespecs: -# tfile = tempfile.NamedTemporaryFile(mode = 'w', suffix = '.json', delete=False) -# tfile.write(renderapi.utils.renderdumps(ts)) -# tfile.close() -# tfiles.append(tfile.name) -# transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) -# transformFile.write(renderapi.utils.renderdumps(tforms)) -# transformFile.close() - -# renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) +def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack='test_import_jsonfiles'): + renderapi.stack.create_stack(stack,render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + tfiles=[] + for ts in tilespecs: + tfile = tempfile.NamedTemporaryFile(mode = 'w', suffix = '.json', delete=False) + tfile.write(renderapi.utils.renderdumps(ts)) + tfile.close() + tfiles.append(tfile.name) + transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) + transformFile.write(renderapi.utils.renderdumps(tforms)) + transformFile.close() + + renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) -# stacks = renderapi.render.get_stacks_by_owner_project(render=render) -# assert stack in stacks -# ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) -# assert len(ts) == len(tilespecs) + stacks = renderapi.render.get_stacks_by_owner_project(render=render) + assert stack in stacks + ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) + assert len(ts) == len(tilespecs) -# @pytest.fixture(scope = "module") -# def teststack(render,render_example_tilespec_and_transforms): +@pytest.fixture(scope = "module") +def teststack(render,render_example_tilespec_and_transforms): -# stack = 'teststack' -# test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) -# yield stack -# renderapi.stack.delete_stack(stack, render=render) + stack = 'teststack' + test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) + yield stack + renderapi.stack.delete_stack(stack, render=render) -# def test_tile_pair_client(render,teststack,**kwargs): -# zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) -# outjson = kwargs.pop('outjson',None) -# if outjson is None: -# outjson = 'test_tile_pair_client.json' +def test_tile_pair_client(render,teststack,**kwargs): + zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) + outjson = kwargs.pop('outjson',None) + if outjson is None: + outjson = 'test_tile_pair_client.json' -# renderapi.client.tilePairClient(teststack, np.min(zvalues), -# np.max(zvalues), outjson=outjson, -# render = render, -# **kwargs) + renderapi.client.tilePairClient(teststack, np.min(zvalues), + np.max(zvalues), outjson=outjson, + render = render, + **kwargs) -# tilepairjson = json.load(open(outjson,'r')) -# assert isinstance(tilepairjson,dict) -# assert len(tilepairjson['neighborPairs'])>3 + tilepairjson = json.load(open(outjson,'r')) + assert isinstance(tilepairjson,dict) + assert len(tilepairjson['neighborPairs'])>3 # def test_importTransformChangesClient(render): # root.debug('test not implemented yet') From 726391fa83f8911f06da47cea57c33f7bb0b111d Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 12:18:44 -0700 Subject: [PATCH 287/766] adding join calls to pool in hopes of cleaning up errors --- renderapi/client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 2772108f..91938e3a 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -63,6 +63,8 @@ def import_jsonfiles_and_transforms_parallel_by_z( port=port, owner=owner, project=project) rs = pool.amap(partial_import, jsonfiles, transformfiles) rs.wait() + pool.close() + pool.join() if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -88,7 +90,8 @@ def import_jsonfiles_parallel( project=project) pool.map(partial_import, jsonfiles) - + pool.close() + pool.join() if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -226,6 +229,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, tilespec_groups = [tilespecs[i::poolsize] for i in xrange(poolsize)] pool.map(partial_import, tilespec_groups) pool.close() + pool.join() if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From 5cbb8a39eb583cecd5bb66056a5b5ceff2e4a632 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 12:59:50 -0700 Subject: [PATCH 288/766] transform test: better test --- test/test_transform.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/test_transform.py b/test/test_transform.py index bf1fcd22..0efde0a8 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -125,7 +125,7 @@ def test_transformsum_polynomial_identity(): invpt = renderapi.transform.Polynomial2DTransform( src=ptest_dstpts, dst=srcpts) - tformlist = [am, pt, invpt, invam] + tformlist = [am, [[pt, invpt]], invam] new_tform = renderapi.transform.estimate_transformsum( tformlist, src=srcpts) @@ -163,9 +163,10 @@ def test_Polynomial_from_affine(): B1=-234.1) pt = renderapi.transform.Polynomial2DTransform.fromAffine(am1) pt_params_raveled = pt.params.ravel() - assert pt_params_raveled[0] == am1.M[0, 2] - assert pt_params_raveled[1] == am1.M[0, 0] - assert pt_params_raveled[2] == am1.M[0, 1] - assert pt_params_raveled[3] == am1.M[1, 2] - assert pt_params_raveled[4] == am1.M[1, 0] - assert pt_params_raveled[5] == am1.M[1, 1] + assert pt.order == 1 + assert pt_params_raveled[0] == am1.B0 + assert pt_params_raveled[1] == am1.M00 + assert pt_params_raveled[2] == am1.M01 + assert pt_params_raveled[3] == am1.B1 + assert pt_params_raveled[4] == am1.M10 + assert pt_params_raveled[5] == am1.M11 From 2183a123b2a578283853abdc5c75242dc867b5a2 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 13:24:39 -0700 Subject: [PATCH 289/766] fixing temp files --- integration_tests/test_client_integrated.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 6e49f829..fe180c89 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -65,15 +65,22 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack='t (tilespecs, tforms) = render_example_tilespec_and_transforms tfiles=[] for ts in tilespecs: - tfile = tempfile.NamedTemporaryFile(mode = 'w', suffix = '.json', delete=False) - tfile.write(renderapi.utils.renderdumps(ts)) - tfile.close() - tfiles.append(tfile.name) - transformFile = tempfile.NamedTemporaryFile(mode = 'w',suffix = '.json', delete=False) - transformFile.write(renderapi.utils.renderdumps(tforms)) + tempjson = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) + tempjson.close() + tsjson = tempjson.name + with open(tsjson, 'w') as f: + renderapi.utils.renderdump(tilespecs, f) + tfiles.append(tsjson) + + transformFile = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) transformFile.close() + tfjson = transformFile.name + with open(tfjson, 'w') as f: + renderapi.utils.renderdump(tforms, f) - renderapi.client.import_jsonfiles(stack, tfiles, transformFile = transformFile.name, render=render) + renderapi.client.import_jsonfiles(stack, tfiles, transformFile = tfjson, render=render) stacks = renderapi.render.get_stacks_by_owner_project(render=render) assert stack in stacks From 0212b6f2a495d4dbda6da1226a56a0f5d11218a9 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 6 Apr 2017 15:10:31 -0700 Subject: [PATCH 290/766] transform: more tests, make dict(ReferenceTransform) viable --- renderapi/transform.py | 3 + test/rendersettings.py | 38 ++++++++++++ test/test_files/tilespec_interpolated.json | 70 ++++++++++++++++++++++ test/test_files/tilespec_ref.json | 29 +++++++++ test/test_transform.py | 59 ++++++++++++++++++ 5 files changed, 199 insertions(+) create mode 100644 test/test_files/tilespec_interpolated.json create mode 100644 test/test_files/tilespec_ref.json diff --git a/renderapi/transform.py b/renderapi/transform.py index 5f495814..3facf755 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -150,6 +150,9 @@ def __str__(self): def __repr__(self): return self.__str__() + def __iter__(self): + return iter([('type', 'ref'), ('refId', self.refId)]) + class Transform(object): def __init__(self, className=None, dataString=None, diff --git a/test/rendersettings.py b/test/rendersettings.py index 66726963..92409a08 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -30,3 +30,41 @@ TEST_TILESPECS_FILE = os.path.join(os.path.dirname( __file__), 'test_files', 'tilespecs.json') + +INTERPOLATED_TRANSFORM_TILESPEC = os.path.join(os.path.dirname( + __file__), 'test_files', 'tilespec_interpolated.json') + +REFERENCE_TRANSFORM_TILESPEC = os.path.join(os.path.dirname( + __file__), 'test_files', 'tilespec_ref.json') + +NONLINEAR_TRANSFORM_KWARGS = { + 'className': "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + 'dataString': ( + "5 21 1103.117269905359 -5.606757285805906 " + "5.321142212984271 1041.3480932107918 -10.43426929509782 " + "7.428382306562735 -14.137269334106124 7.299563138046944 " + "-15.904206050362355 18.284978789358718 3.4106705605908587 " + "-23.24346650864392 -22.18066749731861 -3.583245180840507 " + "3.2706525226745384 27.117126387399466 20.96651792381158 " + "-1.2219610254066424 -10.531856407305256 0.3306037423046746 " + "2.054811912746283 28.849184526610635 34.799030853758985 " + "-7.4227129043141495 -8.211918715339516 -27.434469512200955 " + "-3.1066643785881 3.8503170392714026 1.7124046587349628 " + "1.8506627747180158 4.886666182237065 -2.949889582798017 " + "-4.5150121503501515 -11.363818954372583 -9.8507389567363 " + "8.672520277073502 5.89027776049669 4.924363838689751 " + "-1.4446630732848895 -2.3876262067533727 19.9189118558668 " + "21.72003155506043 1991.9406329917294 2171.99091870329 " + "5136325.014398676 4300827.048910938 5862798.046430213 " + "1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 " + "1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 " + "2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 " + "1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 " + "8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 " + "100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 " + "3334088.839525756 4464532.472685248 1.6068544862341637E10 " + "1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 " + "5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 " + "4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 " + "1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 " + "1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 ")} diff --git a/test/test_files/tilespec_interpolated.json b/test/test_files/tilespec_interpolated.json new file mode 100644 index 00000000..8109351b --- /dev/null +++ b/test/test_files/tilespec_interpolated.json @@ -0,0 +1,70 @@ +{ + "maxX": 7275, + "maxY": -27014, + "layout": { + "sectionId": "2266.0" + }, + "width": 3840, + "height": 3840, + "minX": 1857, + "minY": -32318, + "minIntensity": 0, + "tileId": "20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.2266.0", + "meshCellSize": 64, + "maxIntensity": 65535, + "z": 2266, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + } + }, + "transforms": { + "specList": [ + { + "a": { + "specList": [ + { + "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 ", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000", + "type": "leaf" + } + ], + "type": "list" + }, + "b": { + "specList": [ + { + "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", + "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 ", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "-0.6433267575 0.7655917209 -0.7655917209 -0.6433267575 115071.0014383696 23073.7167961992", + "type": "leaf" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.0000000000 0.0000000000 0.0000000000 1.0000000000 -16981.0000000000 -57152.0000000000", + "type": "leaf" + } + ], + "type": "list" + }, + "type": "interpolated", + "lambda": 0.2 + } + ], + "type": "list" + } +} diff --git a/test/test_files/tilespec_ref.json b/test/test_files/tilespec_ref.json new file mode 100644 index 00000000..bc4e1278 --- /dev/null +++ b/test/test_files/tilespec_ref.json @@ -0,0 +1,29 @@ +{ + "maxX": 7275, + "maxY": -27014, + "layout": { + "sectionId": "2266.0" + }, + "width": 3840, + "height": 3840, + "minX": 1857, + "minY": -32318, + "minIntensity": 0, + "tileId": "20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.2266.0", + "meshCellSize": 64, + "maxIntensity": 65535, + "z": 2266, + "mipmapLevels": { + "0": { + "imageUrl": "file:/data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + } + }, + "transforms": { + "specList": [ + { "type": "ref", + "refId": "lc_em_test_2266.2266.0" + } + ], + "type": "list" + } +} diff --git a/test/test_transform.py b/test/test_transform.py index 0efde0a8..3e743399 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -1,6 +1,8 @@ +import json import renderapi import numpy as np import scipy.linalg +import rendersettings def test_affine_rot_90(): @@ -170,3 +172,60 @@ def test_Polynomial_from_affine(): assert pt_params_raveled[3] == am1.B1 assert pt_params_raveled[4] == am1.M10 assert pt_params_raveled[5] == am1.M11 + + +def test_interpolated_transform(): + with open(rendersettings.INTERPOLATED_TRANSFORM_TILESPEC, 'r') as f: + j = json.load(f) + ts = renderapi.tilespec.TileSpec(json=j) + it_ts = [tform for tform in ts.tforms + if isinstance( + tform, renderapi.transform.InterpolatedTransform)][0] + it_args = renderapi.transform.InterpolatedTransform( + it_ts.a, it_ts.b, it_ts.lambda_) + it_dd = renderapi.transform.InterpolatedTransform( + json=it_ts.to_dict()) + + assert (dict(it_args) == it_args.to_dict() == dict(it_ts) == + it_ts.to_dict() == dict(it_dd) == it_dd.to_dict()) + + +def test_reference_transform(): + with open(rendersettings.REFERENCE_TRANSFORM_TILESPEC, 'r') as f: + j = json.load(f) + ts = renderapi.tilespec.TileSpec(json=j) + ref_ts = [tform for tform in ts.tforms + if isinstance( + tform, renderapi.transform.ReferenceTransform)][0] + ref_args = renderapi.transform.ReferenceTransform(ref_ts.refId) + ref_dd = renderapi.transform.ReferenceTransform(json=ref_ts.to_dict()) + + assert (dict(ref_args) == ref_args.to_dict() == dict(ref_ts) == + ref_ts.to_dict() == dict(ref_dd) == ref_dd.to_dict()) + + +def test_transform_hash_eq(): + t1 = renderapi.transform.Transform( + **rendersettings.NONLINEAR_TRANSFORM_KWARGS) + t2 = renderapi.transform.Transform( + **rendersettings.NONLINEAR_TRANSFORM_KWARGS) + + assert t1 is not t2 + assert t1 == t2 + assert len({t1, t2}) == 1 + + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + am2 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + assert am1 is not am2 + assert am1 == am2 + assert len({am1, am2}) == 1 From 28019b81778fa7831fd9f4a4626e4bb05f0b3718 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 6 Apr 2017 17:39:57 -0700 Subject: [PATCH 291/766] simplifiying tile pair testing --- integration_tests/test_client_integrated.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index fe180c89..53d6f68b 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -103,12 +103,11 @@ def test_tile_pair_client(render,teststack,**kwargs): if outjson is None: outjson = 'test_tile_pair_client.json' - renderapi.client.tilePairClient(teststack, np.min(zvalues), + tilepairjson=renderapi.client.tilePairClient(teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, render = render, **kwargs) - tilepairjson = json.load(open(outjson,'r')) assert isinstance(tilepairjson,dict) assert len(tilepairjson['neighborPairs'])>3 From d53fbac1bd5705724b34ea34a942c718c584086d Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 07:32:53 -0700 Subject: [PATCH 292/766] added rendersectionclient test and cleaned up other test formats --- integration_tests/test_client_integrated.py | 87 +++++++++++++++------ 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 53d6f68b..5baa00b3 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -45,23 +45,7 @@ def render_example_tilespec_and_transforms(): return (tilespecs, tforms) -def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack = 'test_import_tilespecs_parallel'): - renderapi.stack.create_stack(stack,render=render) - (tilespecs, tforms) = render_example_tilespec_and_transforms - renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, - render=render) - stacks = renderapi.render.get_stacks_by_owner_project(render=render) - assert stack in stacks - ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) - assert len(ts) == len(tilespecs) - -# def test_import_jsonfiles_validate_client(render): -# root.debug('test not implemented yet') -# assert False - - -def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack='test_import_jsonfiles'): - renderapi.stack.create_stack(stack,render=render) +def render_example_json_files(render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms tfiles=[] for ts in tilespecs: @@ -79,30 +63,67 @@ def test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack='t tfjson = transformFile.name with open(tfjson, 'w') as f: renderapi.utils.renderdump(tforms, f) + yield (tfiles,transformFile) + os.remove(transformFile) + for tf in tfiles: + os.remove(tfiles) - renderapi.client.import_jsonfiles(stack, tfiles, transformFile = tfjson, render=render) - +def validate_stack_import(render,stack,tilespecs): stacks = renderapi.render.get_stacks_by_owner_project(render=render) assert stack in stacks ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) +def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms, + render_example_json_files): + stack = 'test_import_jsonfiles_validate_client' + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files + renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) + validate_stack_import(render, stack, tilespecs) + renderapi.stack.delete_stack(stack,render=render) + +def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, + render_example_json_files, + stack='test_import_jsonfiles_parallel'): + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files + renderapi.client.import_jsonfiles_parallel(stack, tfiles, transformFile=transformFile, render=render) + validate_stack_import(render, stack, tilespecs) + renderapi.stack.delete_stack(stack, render=render) -@pytest.fixture(scope = "module") -def teststack(render,render_example_tilespec_and_transforms): +def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, + stack='test_import_tilespecs_parallel'): + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, + render=render) + validate_stack_import(render, stack, tilespecs) + + +def test_import_jsonfiles(render, render_example_tilespec_and_transforms, + render_example_json_files, stack='test_import_jsonfiles'): + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files + + renderapi.client.import_jsonfiles(stack, tfiles, transformFile=tfjson, render=render) + validate_stack_import(render, stack, tilespecs) + +@pytest.fixture(scope = "module") +def teststack(render, render_example_tilespec_and_transforms): stack = 'teststack' - test_import_jsonfiles(render,render_example_tilespec_and_transforms,stack=stack) + test_import_jsonfiles(render, render_example_tilespec_and_transforms, stack=stack) yield stack renderapi.stack.delete_stack(stack, render=render) -def test_tile_pair_client(render,teststack,**kwargs): +def test_tile_pair_client(render, teststack, **kwargs): zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) outjson = kwargs.pop('outjson',None) - if outjson is None: - outjson = 'test_tile_pair_client.json' - tilepairjson=renderapi.client.tilePairClient(teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, render = render, @@ -111,6 +132,20 @@ def test_tile_pair_client(render,teststack,**kwargs): assert isinstance(tilepairjson,dict) assert len(tilepairjson['neighborPairs'])>3 +def test_renderSectionClient(render, teststack): + zvalues = renderapi.stack.get_z_values_for_stack(stack, render=render) + section_directory = tempfile.mkdtemp() + renderapi.client.renderSectionClient(stack, + section_directory, + zvalues, + scale=.05, + render=render, + format='png') + (dirpath, dirnames, filenames) = os.walk(section_directory) + pngfiles = [f for f in filenames if f.endswith('png')] + assert len(pngfiles) == len(zvalues) + + # def test_importTransformChangesClient(render): # root.debug('test not implemented yet') # assert False From 60ef3d33dbbb4c70685ed96a417df929a6663898 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 07:33:11 -0700 Subject: [PATCH 293/766] bug fix --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 5baa00b3..89fe072e 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -109,7 +109,7 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files - renderapi.client.import_jsonfiles(stack, tfiles, transformFile=tfjson, render=render) + renderapi.client.import_jsonfiles(stack, tfiles, transformFile=transformFile, render=render) validate_stack_import(render, stack, tilespecs) @@ -144,7 +144,7 @@ def test_renderSectionClient(render, teststack): (dirpath, dirnames, filenames) = os.walk(section_directory) pngfiles = [f for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) - + # def test_importTransformChangesClient(render): # root.debug('test not implemented yet') From 49f5bf7b4252220eb899c3218f2c9e24e059f689 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 07:34:28 -0700 Subject: [PATCH 294/766] more bug fix --- integration_tests/test_client_integrated.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 89fe072e..d30df168 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -114,23 +114,25 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, @pytest.fixture(scope = "module") -def teststack(render, render_example_tilespec_and_transforms): +def teststack(render, render_example_tilespec_and_transforms, + render_example_json_files): stack = 'teststack' - test_import_jsonfiles(render, render_example_tilespec_and_transforms, stack=stack) + test_import_jsonfiles(render, render_example_tilespec_and_transforms, + render_example_json_files, stack=stack) yield stack renderapi.stack.delete_stack(stack, render=render) def test_tile_pair_client(render, teststack, **kwargs): zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) - outjson = kwargs.pop('outjson',None) + outjson = kwargs.pop('outjson', None) tilepairjson=renderapi.client.tilePairClient(teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, render = render, **kwargs) - assert isinstance(tilepairjson,dict) - assert len(tilepairjson['neighborPairs'])>3 + assert isinstance(tilepairjson, dict) + assert len(tilepairjson['neighborPairs']) > 3 def test_renderSectionClient(render, teststack): zvalues = renderapi.stack.get_z_values_for_stack(stack, render=render) From 318a9c3778dd47a8d271d6b9dbb6acea87d2ab29 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 07:37:37 -0700 Subject: [PATCH 295/766] forgot to make json files fixtures --- integration_tests/test_client_integrated.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index d30df168..b3288b0a 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -44,7 +44,7 @@ def render_example_tilespec_and_transforms(): print tforms return (tilespecs, tforms) - +@pytest.fixture(scope='module') def render_example_json_files(render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms tfiles=[] @@ -67,6 +67,7 @@ def render_example_json_files(render_example_tilespec_and_transforms): os.remove(transformFile) for tf in tfiles: os.remove(tfiles) + return (tfiles,transformFile) def validate_stack_import(render,stack,tilespecs): stacks = renderapi.render.get_stacks_by_owner_project(render=render) @@ -112,7 +113,6 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, renderapi.client.import_jsonfiles(stack, tfiles, transformFile=transformFile, render=render) validate_stack_import(render, stack, tilespecs) - @pytest.fixture(scope = "module") def teststack(render, render_example_tilespec_and_transforms, render_example_json_files): @@ -122,7 +122,6 @@ def teststack(render, render_example_tilespec_and_transforms, yield stack renderapi.stack.delete_stack(stack, render=render) - def test_tile_pair_client(render, teststack, **kwargs): zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) outjson = kwargs.pop('outjson', None) @@ -130,7 +129,6 @@ def test_tile_pair_client(render, teststack, **kwargs): np.max(zvalues), outjson=outjson, render = render, **kwargs) - assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 From 781687f81618d9bf40b650be778c80d067b0ef0e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 07:41:40 -0700 Subject: [PATCH 296/766] nevermind --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index b3288b0a..97a90e59 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -67,7 +67,7 @@ def render_example_json_files(render_example_tilespec_and_transforms): os.remove(transformFile) for tf in tfiles: os.remove(tfiles) - return (tfiles,transformFile) + def validate_stack_import(render,stack,tilespecs): stacks = renderapi.render.get_stacks_by_owner_project(render=render) From 99db45448815880aa6fb5c6ad3b9fbc014809349 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 07:45:07 -0700 Subject: [PATCH 297/766] fixed renderSectionClient test --- integration_tests/test_client_integrated.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 97a90e59..9d5f89e2 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -83,7 +83,7 @@ def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_tr (tfiles, transformFile) = render_example_json_files renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) validate_stack_import(render, stack, tilespecs) - renderapi.stack.delete_stack(stack,render=render) + renderapi.stack.delete_stack(stack, render=render) def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, render_example_json_files, @@ -115,7 +115,7 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, @pytest.fixture(scope = "module") def teststack(render, render_example_tilespec_and_transforms, - render_example_json_files): + render_example_json_files): stack = 'teststack' test_import_jsonfiles(render, render_example_tilespec_and_transforms, render_example_json_files, stack=stack) @@ -133,9 +133,9 @@ def test_tile_pair_client(render, teststack, **kwargs): assert len(tilepairjson['neighborPairs']) > 3 def test_renderSectionClient(render, teststack): - zvalues = renderapi.stack.get_z_values_for_stack(stack, render=render) + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) section_directory = tempfile.mkdtemp() - renderapi.client.renderSectionClient(stack, + renderapi.client.renderSectionClient(teststack, section_directory, zvalues, scale=.05, From 275364dc4c94ce1eefd2038121979dbf5320f5c0 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 08:47:29 -0700 Subject: [PATCH 298/766] trying to solve test issue --- integration_tests/test_client_integrated.py | 51 ++++++++++----------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 9d5f89e2..15fc3528 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -44,7 +44,6 @@ def render_example_tilespec_and_transforms(): print tforms return (tilespecs, tforms) -@pytest.fixture(scope='module') def render_example_json_files(render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms tfiles=[] @@ -54,7 +53,8 @@ def render_example_json_files(render_example_tilespec_and_transforms): tempjson.close() tsjson = tempjson.name with open(tsjson, 'w') as f: - renderapi.utils.renderdump(tilespecs, f) + renderapi.utils.renderdump(tilespecs, f) + f.close() tfiles.append(tsjson) transformFile = tempfile.NamedTemporaryFile( @@ -62,12 +62,9 @@ def render_example_json_files(render_example_tilespec_and_transforms): transformFile.close() tfjson = transformFile.name with open(tfjson, 'w') as f: - renderapi.utils.renderdump(tforms, f) - yield (tfiles,transformFile) - os.remove(transformFile) - for tf in tfiles: - os.remove(tfiles) - + renderapi.utils.renderdump(tforms, f) + f.close() + return (tfiles,transformFile) def validate_stack_import(render,stack,tilespecs): stacks = renderapi.render.get_stacks_by_owner_project(render=render) @@ -75,22 +72,20 @@ def validate_stack_import(render,stack,tilespecs): ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) -def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms, - render_example_json_files): +def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms): stack = 'test_import_jsonfiles_validate_client' renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files + (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, - render_example_json_files, stack='test_import_jsonfiles_parallel'): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files + (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles_parallel(stack, tfiles, transformFile=transformFile, render=render) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) @@ -105,10 +100,10 @@ def test_import_tilespecs_parallel(render, render_example_tilespec_and_transform def test_import_jsonfiles(render, render_example_tilespec_and_transforms, - render_example_json_files, stack='test_import_jsonfiles'): + stack='test_import_jsonfiles'): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files + (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles(stack, tfiles, transformFile=transformFile, render=render) validate_stack_import(render, stack, tilespecs) @@ -118,7 +113,7 @@ def teststack(render, render_example_tilespec_and_transforms, render_example_json_files): stack = 'teststack' test_import_jsonfiles(render, render_example_tilespec_and_transforms, - render_example_json_files, stack=stack) + stack=stack) yield stack renderapi.stack.delete_stack(stack, render=render) @@ -132,18 +127,18 @@ def test_tile_pair_client(render, teststack, **kwargs): assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 -def test_renderSectionClient(render, teststack): - zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) - section_directory = tempfile.mkdtemp() - renderapi.client.renderSectionClient(teststack, - section_directory, - zvalues, - scale=.05, - render=render, - format='png') - (dirpath, dirnames, filenames) = os.walk(section_directory) - pngfiles = [f for f in filenames if f.endswith('png')] - assert len(pngfiles) == len(zvalues) +# def test_renderSectionClient(render, teststack): +# zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) +# section_directory = tempfile.mkdtemp() +# renderapi.client.renderSectionClient(teststack, +# section_directory, +# zvalues, +# scale=.05, +# render=render, +# format='png') +# (dirpath, dirnames, filenames) = os.walk(section_directory) +# pngfiles = [f for f in filenames if f.endswith('png')] +# assert len(pngfiles) == len(zvalues) # def test_importTransformChangesClient(render): From befea8fd67aac231b52017762946799a3e8965a5 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 09:05:41 -0700 Subject: [PATCH 299/766] stil trying to fix tests --- integration_tests/test_client_integrated.py | 35 ++++++++++----------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 15fc3528..16178647 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -72,23 +72,23 @@ def validate_stack_import(render,stack,tilespecs): ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) -def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms): - stack = 'test_import_jsonfiles_validate_client' - renderapi.stack.create_stack(stack, render=render) - (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) - renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) - validate_stack_import(render, stack, tilespecs) - renderapi.stack.delete_stack(stack, render=render) - -def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, - stack='test_import_jsonfiles_parallel'): - renderapi.stack.create_stack(stack, render=render) - (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) - renderapi.client.import_jsonfiles_parallel(stack, tfiles, transformFile=transformFile, render=render) - validate_stack_import(render, stack, tilespecs) - renderapi.stack.delete_stack(stack, render=render) +# def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms): +# stack = 'test_import_jsonfiles_validate_client' +# renderapi.stack.create_stack(stack, render=render) +# (tilespecs, tforms) = render_example_tilespec_and_transforms +# (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) +# renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) +# validate_stack_import(render, stack, tilespecs) +# renderapi.stack.delete_stack(stack, render=render) + +# def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, +# stack='test_import_jsonfiles_parallel'): +# renderapi.stack.create_stack(stack, render=render) +# (tilespecs, tforms) = render_example_tilespec_and_transforms +# (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) +# renderapi.client.import_jsonfiles_parallel(stack, tfiles, transformFile=transformFile, render=render) +# validate_stack_import(render, stack, tilespecs) +# renderapi.stack.delete_stack(stack, render=render) def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack='test_import_tilespecs_parallel'): @@ -97,7 +97,6 @@ def test_import_tilespecs_parallel(render, render_example_tilespec_and_transform renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, render=render) validate_stack_import(render, stack, tilespecs) - def test_import_jsonfiles(render, render_example_tilespec_and_transforms, stack='test_import_jsonfiles'): From d01bfd0080bf05186f6f03f3e8a37e9df0757b77 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 09:21:35 -0700 Subject: [PATCH 300/766] fix test stack --- integration_tests/test_client_integrated.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 16178647..6b0c273e 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -108,8 +108,7 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, validate_stack_import(render, stack, tilespecs) @pytest.fixture(scope = "module") -def teststack(render, render_example_tilespec_and_transforms, - render_example_json_files): +def teststack(render, render_example_tilespec_and_transforms): stack = 'teststack' test_import_jsonfiles(render, render_example_tilespec_and_transforms, stack=stack) From dfd043291a4b2b1683d27da97b323026bd126012 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 09:29:26 -0700 Subject: [PATCH 301/766] adding debug statement to help figure out test breakage --- renderapi/client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/renderapi/client.py b/renderapi/client.py index 91938e3a..34178bea 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -279,6 +279,9 @@ def call_run_ws_client(className, add_args=[], renderclient=None, ''' simple call for run_ws_client.sh -- all arguments set in add_args ''' + logger.debug('call_run_ws_client -- classname:{} add_args:{} client_script:{} memGB:{}'\ + .format(className, add_args, client_script, memGB)) + if renderclient is not None: if isinstance(renderclient, RenderClient): return call_run_ws_client(className, add_args=add_args, From 0f8f99103e158c8380998ea49c3fe654729a04e1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 09:41:20 -0700 Subject: [PATCH 302/766] fixed bug --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 6b0c273e..47b501ab 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -64,7 +64,7 @@ def render_example_json_files(render_example_tilespec_and_transforms): with open(tfjson, 'w') as f: renderapi.utils.renderdump(tforms, f) f.close() - return (tfiles,transformFile) + return (tfiles,tfjson) def validate_stack_import(render,stack,tilespecs): stacks = renderapi.render.get_stacks_by_owner_project(render=render) From 61944d5ebd1516e563976294eaa5969724cb50bb Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 09:56:26 -0700 Subject: [PATCH 303/766] adding back tests one by one --- integration_tests/test_client_integrated.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 47b501ab..4a500802 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -81,14 +81,14 @@ def validate_stack_import(render,stack,tilespecs): # validate_stack_import(render, stack, tilespecs) # renderapi.stack.delete_stack(stack, render=render) -# def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, -# stack='test_import_jsonfiles_parallel'): -# renderapi.stack.create_stack(stack, render=render) -# (tilespecs, tforms) = render_example_tilespec_and_transforms -# (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) -# renderapi.client.import_jsonfiles_parallel(stack, tfiles, transformFile=transformFile, render=render) -# validate_stack_import(render, stack, tilespecs) -# renderapi.stack.delete_stack(stack, render=render) +def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, + stack='test_import_jsonfiles_parallel'): + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) + renderapi.client.import_jsonfiles_parallel(stack, tfiles, transformFile=transformFile, render=render) + validate_stack_import(render, stack, tilespecs) + renderapi.stack.delete_stack(stack, render=render) def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack='test_import_tilespecs_parallel'): From bab1a68bfdb81e15cc1925c7294c54a29e297a50 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 10:16:23 -0700 Subject: [PATCH 304/766] activating sectionrender test --- integration_tests/test_client_integrated.py | 26 +++++++++++---------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 4a500802..7e9314b4 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -125,18 +125,20 @@ def test_tile_pair_client(render, teststack, **kwargs): assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 -# def test_renderSectionClient(render, teststack): -# zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) -# section_directory = tempfile.mkdtemp() -# renderapi.client.renderSectionClient(teststack, -# section_directory, -# zvalues, -# scale=.05, -# render=render, -# format='png') -# (dirpath, dirnames, filenames) = os.walk(section_directory) -# pngfiles = [f for f in filenames if f.endswith('png')] -# assert len(pngfiles) == len(zvalues) +def test_renderSectionClient(render, teststack): + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + section_directory = tempfile.mkdtemp() + renderapi.client.renderSectionClient(teststack, + section_directory, + zvalues, + scale=.05, + render=render, + format='png') + (dirpath, dirnames, filenames) = os.walk(section_directory) + pngfiles = [f for f in filenames if f.endswith('png')] + assert len(pngfiles) == len(zvalues) + + # def test_importTransformChangesClient(render): From e3b3c64608984313d75add0177627b80853cba41 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 10:23:15 -0700 Subject: [PATCH 305/766] closed temp file after writing --- renderapi/coordinate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 73d2b2d7..afe0f272 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -235,7 +235,7 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, with open(json_inpath, 'w') as fp: # d = json.loads(jsondata) json.dump(jsondata, fp) - # fp.close() + fp.close() # fp.write(jsondata) # json.dump(jsondata,open(json_inpath,'w')) From 43b13f87e0efa1690c4213350dbf65da5f1eb215 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 10:43:01 -0700 Subject: [PATCH 306/766] trying smaller pool sizes --- integration_tests/test_client_integrated.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 7e9314b4..af8afa85 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -95,7 +95,7 @@ def test_import_tilespecs_parallel(render, render_example_tilespec_and_transform renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, - render=render) + poolsize=3, render=render) validate_stack_import(render, stack, tilespecs) def test_import_jsonfiles(render, render_example_tilespec_and_transforms, @@ -104,7 +104,7 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) - renderapi.client.import_jsonfiles(stack, tfiles, transformFile=transformFile, render=render) + renderapi.client.import_jsonfiles(stack, tfiles, transformFile=transformFile, poolsize=3, render=render) validate_stack_import(render, stack, tilespecs) @pytest.fixture(scope = "module") @@ -128,6 +128,7 @@ def test_tile_pair_client(render, teststack, **kwargs): def test_renderSectionClient(render, teststack): zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) section_directory = tempfile.mkdtemp() + root.debug('section_directory:{}'.format(section_directory)) renderapi.client.renderSectionClient(teststack, section_directory, zvalues, From 26829d2b82e159daab1b5a75bd8d5c8ea4ee64ed Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 7 Apr 2017 10:43:06 -0700 Subject: [PATCH 307/766] added debugging --- renderapi/coordinate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index afe0f272..a4f1568e 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -233,6 +233,7 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, json_infile, json_inpath = tempfile.mkstemp( prefix='render_coordinates_in_', suffix='.json') with open(json_inpath, 'w') as fp: + logger.debug('jsondata:{}'.format(jsondata)) # d = json.loads(jsondata) json.dump(jsondata, fp) fp.close() From 24d33fb03faee704bc93cde3780ab5ff494e2fb8 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 7 Apr 2017 13:52:51 -0700 Subject: [PATCH 308/766] transform: make estimate_transformsum produce Affine if inputs are affine and remove rare broken test of transformsum using polynomial identity --- renderapi/transform.py | 24 +++++++++- test/test_transform.py | 103 +++++++++-------------------------------- 2 files changed, 46 insertions(+), 81 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 3facf755..31764a94 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -13,6 +13,7 @@ ''' import json import logging +from collections import Iterable import numpy as np from .errors import ConversionError, EstimationError, RenderError from .utils import NullHandler @@ -581,7 +582,28 @@ def estimate_transformsum(transformlist, src=None, order=2): ''' pseudo-composition of transforms in list of transforms using source point transformation and a single estimation. - input: trans + Will produce an Affine Model if all input transforms are Affine, + otherwise will produce a Polynomial of specified order + inputs: + transformlist: recursive list of transform objects + src: Nx2 numpy array of source points for estimation + order: optional, order of Polynomial output if transformlist + inputs are non-Affine ''' + def flatten(l): + for i in l: + if (isinstance(i, Iterable) and not + isinstance(i, basestring)): + for sub in flatten(i): + yield sub + else: + yield i + dstpts = estimate_dstpts(transformlist, src) + tforms = flatten(transformlist) + if all([tform.className == AffineModel.className + for tform in tforms]): + am = AffineModel() + _ = am.estimate(A=src, B=dstpts) + return am return Polynomial2DTransform(src=src, dst=dstpts, order=order) diff --git a/test/test_transform.py b/test/test_transform.py index 3e743399..ba32f3d7 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -1,8 +1,6 @@ -import json import renderapi import numpy as np import scipy.linalg -import rendersettings def test_affine_rot_90(): @@ -107,7 +105,8 @@ def test_Polynomial_estimation_numpy(): test_Polynomial_estimation(use_numpy=True) -def test_transformsum_polynomial_identity(): +def notatest_transformsum_polynomial_identity(): + # test not used currently in favor of more reproducible affine srcpts = np.random.rand(50, 2) am = renderapi.transform.AffineModel(M00=.9, M10=-0.2, @@ -127,7 +126,7 @@ def test_transformsum_polynomial_identity(): invpt = renderapi.transform.Polynomial2DTransform( src=ptest_dstpts, dst=srcpts) - tformlist = [am, [[pt, invpt]], invam] + tformlist = [am, [pt, invpt], invam] new_tform = renderapi.transform.estimate_transformsum( tformlist, src=srcpts) @@ -141,91 +140,35 @@ def test_transformsum_polynomial_identity(): poly_identity.params[:, 1:-1], atol=1e-5) -def test_load_polynomial(): - datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' - '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' - '5446.85340052 0.0224047626583 0.961202608454 ' - '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') - pt = renderapi.transform.Polynomial2DTransform( - dataString=datastring) - pt_dict = renderapi.transform.Polynomial2DTransform(json=pt.to_dict()) - pt_dataString = renderapi.transform.Polynomial2DTransform( - dataString=pt.dataString) - pt_params = renderapi.transform.Polynomial2DTransform(params=pt.params) - assert (pt_dict.to_dict() == pt_dataString.to_dict() == - pt_params.to_dict() == pt.to_dict()) - - -def test_Polynomial_from_affine(): +def test_transformsum_affine_concatenate(): + srcpts = np.random.rand(50, 2) am1 = renderapi.transform.AffineModel(M00=.9, M10=-0.2, M01=0.3, M11=.85, B0=245.3, B1=-234.1) - pt = renderapi.transform.Polynomial2DTransform.fromAffine(am1) - pt_params_raveled = pt.params.ravel() - assert pt.order == 1 - assert pt_params_raveled[0] == am1.B0 - assert pt_params_raveled[1] == am1.M00 - assert pt_params_raveled[2] == am1.M01 - assert pt_params_raveled[3] == am1.B1 - assert pt_params_raveled[4] == am1.M10 - assert pt_params_raveled[5] == am1.M11 - - -def test_interpolated_transform(): - with open(rendersettings.INTERPOLATED_TRANSFORM_TILESPEC, 'r') as f: - j = json.load(f) - ts = renderapi.tilespec.TileSpec(json=j) - it_ts = [tform for tform in ts.tforms - if isinstance( - tform, renderapi.transform.InterpolatedTransform)][0] - it_args = renderapi.transform.InterpolatedTransform( - it_ts.a, it_ts.b, it_ts.lambda_) - it_dd = renderapi.transform.InterpolatedTransform( - json=it_ts.to_dict()) - - assert (dict(it_args) == it_args.to_dict() == dict(it_ts) == - it_ts.to_dict() == dict(it_dd) == it_dd.to_dict()) - - -def test_reference_transform(): - with open(rendersettings.REFERENCE_TRANSFORM_TILESPEC, 'r') as f: - j = json.load(f) - ts = renderapi.tilespec.TileSpec(json=j) - ref_ts = [tform for tform in ts.tforms - if isinstance( - tform, renderapi.transform.ReferenceTransform)][0] - ref_args = renderapi.transform.ReferenceTransform(ref_ts.refId) - ref_dd = renderapi.transform.ReferenceTransform(json=ref_ts.to_dict()) - - assert (dict(ref_args) == ref_args.to_dict() == dict(ref_ts) == - ref_ts.to_dict() == dict(ref_dd) == ref_dd.to_dict()) - - -def test_transform_hash_eq(): - t1 = renderapi.transform.Transform( - **rendersettings.NONLINEAR_TRANSFORM_KWARGS) - t2 = renderapi.transform.Transform( - **rendersettings.NONLINEAR_TRANSFORM_KWARGS) - - assert t1 is not t2 - assert t1 == t2 - assert len({t1, t2}) == 1 - - am1 = renderapi.transform.AffineModel(M00=.9, + am2 = renderapi.transform.AffineModel(M00=.9, M10=-0.2, M01=0.3, M11=.85, - B0=245.3, - B1=-234.1) - am2 = renderapi.transform.AffineModel(M00=.9, + B0=-100, + B1=3) + am3 = renderapi.transform.AffineModel(M00=1.9, + M10=-0.2, + M01=0.3, + M11=1.85, + B0=-25.3, + B1=60.1) + am4 = renderapi.transform.AffineModel(M00=.9, M10=-0.2, M01=0.3, M11=.85, - B0=245.3, - B1=-234.1) - assert am1 is not am2 - assert am1 == am2 - assert len({am1, am2}) == 1 + B0=2.3, + B1=100.1) + + tformlist = [am1, [[am2, am3], am4]] + new_tform = renderapi.transform.estimate_transformsum( + tformlist, src=srcpts) + concat_tform = am4.concatenate(am3.concatenate(am2)).concatenate(am1) + assert np.allclose(new_tform.M, concat_tform.M) From 84ad0c78694b9bf90c1f0cc2ac18ab3ff39f5171 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 7 Apr 2017 14:03:41 -0700 Subject: [PATCH 309/766] transform: reverting and re-commiting changes lost due to bad sync --- test/test_transform.py | 92 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/test/test_transform.py b/test/test_transform.py index ba32f3d7..f7ecb61e 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -1,6 +1,8 @@ +import json import renderapi import numpy as np import scipy.linalg +import rendersettings def test_affine_rot_90(): @@ -172,3 +174,93 @@ def test_transformsum_affine_concatenate(): tformlist, src=srcpts) concat_tform = am4.concatenate(am3.concatenate(am2)).concatenate(am1) assert np.allclose(new_tform.M, concat_tform.M) + + +def test_load_polynomial(): + datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06') + pt = renderapi.transform.Polynomial2DTransform( + dataString=datastring) + pt_dict = renderapi.transform.Polynomial2DTransform(json=pt.to_dict()) + pt_dataString = renderapi.transform.Polynomial2DTransform( + dataString=pt.dataString) + pt_params = renderapi.transform.Polynomial2DTransform(params=pt.params) + assert (pt_dict.to_dict() == pt_dataString.to_dict() == + pt_params.to_dict() == pt.to_dict()) + + +def test_Polynomial_from_affine(): + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + pt = renderapi.transform.Polynomial2DTransform.fromAffine(am1) + pt_params_raveled = pt.params.ravel() + assert pt.order == 1 + assert pt_params_raveled[0] == am1.B0 + assert pt_params_raveled[1] == am1.M00 + assert pt_params_raveled[2] == am1.M01 + assert pt_params_raveled[3] == am1.B1 + assert pt_params_raveled[4] == am1.M10 + assert pt_params_raveled[5] == am1.M11 + + +def test_interpolated_transform(): + with open(rendersettings.INTERPOLATED_TRANSFORM_TILESPEC, 'r') as f: + j = json.load(f) + ts = renderapi.tilespec.TileSpec(json=j) + it_ts = [tform for tform in ts.tforms + if isinstance( + tform, renderapi.transform.InterpolatedTransform)][0] + it_args = renderapi.transform.InterpolatedTransform( + it_ts.a, it_ts.b, it_ts.lambda_) + it_dd = renderapi.transform.InterpolatedTransform( + json=it_ts.to_dict()) + + assert (dict(it_args) == it_args.to_dict() == dict(it_ts) == + it_ts.to_dict() == dict(it_dd) == it_dd.to_dict()) + + +def test_reference_transform(): + with open(rendersettings.REFERENCE_TRANSFORM_TILESPEC, 'r') as f: + j = json.load(f) + ts = renderapi.tilespec.TileSpec(json=j) + ref_ts = [tform for tform in ts.tforms + if isinstance( + tform, renderapi.transform.ReferenceTransform)][0] + ref_args = renderapi.transform.ReferenceTransform(ref_ts.refId) + ref_dd = renderapi.transform.ReferenceTransform(json=ref_ts.to_dict()) + + assert (dict(ref_args) == ref_args.to_dict() == dict(ref_ts) == + ref_ts.to_dict() == dict(ref_dd) == ref_dd.to_dict()) + + +def test_transform_hash_eq(): + t1 = renderapi.transform.Transform( + **rendersettings.NONLINEAR_TRANSFORM_KWARGS) + t2 = renderapi.transform.Transform( + **rendersettings.NONLINEAR_TRANSFORM_KWARGS) + + assert t1 is not t2 + assert t1 == t2 + assert len({t1, t2}) == 1 + + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + am2 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + assert am1 is not am2 + assert am1 == am2 + assert len({am1, am2}) == 1 From e10e6f1dc1799481fa69775117904cf69501ae75 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 07:27:51 +0100 Subject: [PATCH 310/766] fixed test for section client and added new test for point match feature --- integration_tests/test_client_integrated.py | 11 +++++++---- integration_tests/test_pointmatch_integrated.py | 4 ++++ renderapi/pointmatch.py | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index af8afa85..1264a482 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -127,16 +127,19 @@ def test_tile_pair_client(render, teststack, **kwargs): def test_renderSectionClient(render, teststack): zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) - section_directory = tempfile.mkdtemp() + root_directory = tempfile.mkdtemp() root.debug('section_directory:{}'.format(section_directory)) renderapi.client.renderSectionClient(teststack, - section_directory, + root_directory, zvalues, scale=.05, render=render, format='png') - (dirpath, dirnames, filenames) = os.walk(section_directory) - pngfiles = [f for f in filenames if f.endswith('png')] + + section_directory=os.path.join(section_directory,teststack,'sections_at_.05') + pngfiles = [] + for (dirpath, dirname, filenames) in os.walk(section_directory): + pngfiles += [f for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 9bc4d9a4..64d123c2 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -110,6 +110,10 @@ def test_get_matchcollections(render,test_pm_collection): matched_collection = next(coljson for coljson in collections if coljson['collectionId']['name']==test_pm_collection) assert matched_collection is not None +def test_get_matches_involving_tile(render,test_pm_collection): + matches = renderapi.pointmatch.get_matches_involving_tile(test_pm_collection, "0", "0-1", render=render) + assert len(matches) == 3 + def test_get_match_groupIds(render,test_pm_collection): groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) assert len(groups)==3 diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 873b0633..4037a5f1 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -158,7 +158,21 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, logger.error(e) logger.error(r.text) +@renderaccess +def get_matches_involving_tile(matchCollection, pGroupId, pTileId, + owner=None,host=None, port=None, + session=requests.session(), **kwargs): + request_url = format_baseurl(host, port) + \ + "/owner/{}/matchCollection/{}/pGroup/{}/pTileId/{}/matchesWith/".format(\ + owner, matchCollection, pGroupId, pTileId) + r = session.get(request_url) + try: + return r.json() + except Exception as e: + logger.error(e) + logger.error(r.text) + @renderaccess def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, render=None, owner=None, host=None, From 8eff4f4cdf5a44aeb28d7eefb6635cd91b426825 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 07:36:32 +0100 Subject: [PATCH 311/766] fixed typo and added dstore to gitignore --- .gitignore | 1 + integration_tests/test_client_integrated.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5f80c9e2..81b560a2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ build/ *.egg-info *.egg dist/ +.DS_Store \ No newline at end of file diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 1264a482..1f1e33ec 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -128,7 +128,7 @@ def test_tile_pair_client(render, teststack, **kwargs): def test_renderSectionClient(render, teststack): zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) root_directory = tempfile.mkdtemp() - root.debug('section_directory:{}'.format(section_directory)) + root.debug('section_directory:{}'.format(root_directory)) renderapi.client.renderSectionClient(teststack, root_directory, zvalues, From 9aebaa40cb9ad3e8cba6ab3e6ba2a02ad133b947 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 07:39:33 +0100 Subject: [PATCH 312/766] changed involving tile call --- renderapi/pointmatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 4037a5f1..3a890f3c 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -163,7 +163,7 @@ def get_matches_involving_tile(matchCollection, pGroupId, pTileId, owner=None,host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ - "/owner/{}/matchCollection/{}/pGroup/{}/pTileId/{}/matchesWith/".format(\ + "/owner/{}/matchCollection/{}/group/{}/id/{}/".format(\ owner, matchCollection, pGroupId, pTileId) r = session.get(request_url) try: From fe4bfecdad29924a72571503f74067fe4590e038 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 07:49:45 +0100 Subject: [PATCH 313/766] fixing bug --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 1f1e33ec..ec46c0a2 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -136,7 +136,7 @@ def test_renderSectionClient(render, teststack): render=render, format='png') - section_directory=os.path.join(section_directory,teststack,'sections_at_.05') + section_directory=os.path.join(root_directory,teststack,'sections_at_.05') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(section_directory): pngfiles += [f for f in filenames if f.endswith('png')] From 91686faf1ce08476a508ab08e0bef7abd8cb6a45 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 14:59:53 +0200 Subject: [PATCH 314/766] reconfigured dockerfile to be lighter weight --- Dockerfile | 57 +++++++++++++++++++++---------------------- test/requirements.txt | 3 ++- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9d45836b..b9ae4ed5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,47 +3,46 @@ MAINTAINER Forrest Collman (forrest.collman@gmail.com) ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ - libglib2.0-0 libxext6 libsm6 libxrender1 \ - git mercurial subversion +# RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ +# libglib2.0-0 libxext6 libsm6 libxrender1 \ +# git mercurial subversion -RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ - wget --quiet https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh -O ~/anaconda.sh && \ - /bin/bash ~/anaconda.sh -b -p /opt/conda && \ - rm ~/anaconda.sh +# RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ +# wget --quiet https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh -O ~/anaconda.sh && \ +# /bin/bash ~/anaconda.sh -b -p /opt/conda && \ +# rm ~/anaconda.sh -RUN apt-get install -y curl grep sed dpkg && \ - TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ - curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ - dpkg -i tini.deb && \ - rm tini.deb && \ - apt-get clean +# RUN apt-get install -y curl grep sed dpkg && \ +# TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ +# curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ +# dpkg -i tini.deb && \ +# rm tini.deb && \ +# apt-get clean -ENV PATH /opt/conda/bin:$PATH +# ENV PATH /opt/conda/bin:$PATH #install java # auto validate license -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections -RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list -RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 -RUN apt-get update -RUN apt-get install oracle-java8-installer -y -ENV JAVA_HOME /usr/lib/jvm/java-8-oracle +#RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections +#RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list +#RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list +#RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 +#RUN apt-get update +#RUN apt-get install oracle-java8-installer -y +#ENV JAVA_HOME /www/var/render/deploy/jdkd #install pathos,multiprocess with gcc -RUN apt-get install gcc -y -RUN apt-get install build-essential -y +RUN apt-get update +RUN apt-get install gcc build-essential libgeos-dev imagemagick -y +RUN apt-get install python-setuptools python-dev -y RUN apt-get clean - - #install components for common render-python apps #jupyter notebook, shapely with geos -RUN /opt/conda/bin/conda install jupyter -y -RUN apt-get install libgeos-dev -y -RUN apt-get install imagemagick -y +# RUN /opt/conda/bin/conda install jupyter -y + +RUN easy_install pip RUN pip install shapely==1.6b2 RUN pip install opencv-python RUN pip install dill==0.2.6 @@ -59,5 +58,5 @@ COPY . /usr/local/render-python WORKDIR /usr/local/render-python RUN python setup.py install -ENTRYPOINT [ "/usr/bin/tini", "--" ] +# ENTRYPOINT [ "/usr/bin/tini", "--" ] CMD [ "/bin/bash" ] diff --git a/test/requirements.txt b/test/requirements.txt index 9fbb4148..8a5a5ccc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -6,4 +6,5 @@ pytest-cov==2.2.1 pytest-pep8==1.0.6 pytest-xdist==1.14 flake8>=3.0.4 -pylint>=1.5.4 \ No newline at end of file +pylint>=1.5.4 +scipy \ No newline at end of file From 84ec0fc295dc7f0ffe8e0ab9f5da0d9950b3d96a Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 15:10:20 +0200 Subject: [PATCH 315/766] working on getting lighter dockerfile functional --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b9ae4ed5..e82882be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 RUN apt-get update RUN apt-get install gcc build-essential libgeos-dev imagemagick -y RUN apt-get install python-setuptools python-dev -y +RUN apt-get install libblas-dev liblapack-dev RUN apt-get clean #install components for common render-python apps From 8125cd1b00ad478fdae4e461f1e719fc3d742c1e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 15:13:58 +0200 Subject: [PATCH 316/766] forgot auto yes --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e82882be..7463b7e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 RUN apt-get update RUN apt-get install gcc build-essential libgeos-dev imagemagick -y RUN apt-get install python-setuptools python-dev -y -RUN apt-get install libblas-dev liblapack-dev +RUN apt-get install libblas-dev liblapack-dev -y RUN apt-get clean #install components for common render-python apps From 315fb51b1460ad9debfb98fb4588f1f109cc0bb4 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 15:19:35 +0200 Subject: [PATCH 317/766] changing docker file --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7463b7e0..3d579cb0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ RUN apt-get install python-setuptools python-dev -y RUN apt-get install libblas-dev liblapack-dev -y RUN apt-get clean + #install components for common render-python apps #jupyter notebook, shapely with geos # RUN /opt/conda/bin/conda install jupyter -y From ab6de1a8a3eccf75131cc95cc8f6bf23bc1f9cf8 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 15:37:27 +0200 Subject: [PATCH 318/766] upgrading setuptools --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 3d579cb0..a9fadd49 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ RUN apt-get clean RUN easy_install pip +RUN pip install -U pip setuptools RUN pip install shapely==1.6b2 RUN pip install opencv-python RUN pip install dill==0.2.6 From 2055e4c3823df76feef8bd9e67f07efb96244f82 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 15:51:29 +0200 Subject: [PATCH 319/766] changed to miniconda --- Dockerfile | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a9fadd49..b42d493d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,22 @@ MAINTAINER Forrest Collman (forrest.collman@gmail.com) ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ + libglib2.0-0 libxext6 libsm6 libxrender1 \ + git mercurial subversion + +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ + wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh + +RUN apt-get install -y curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb && \ + apt-get clean + # RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ # libglib2.0-0 libxext6 libsm6 libxrender1 \ # git mercurial subversion @@ -19,7 +35,7 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 # rm tini.deb && \ # apt-get clean -# ENV PATH /opt/conda/bin:$PATH +ENV PATH /opt/conda/bin:$PATH #install java # auto validate license @@ -35,7 +51,7 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 RUN apt-get update RUN apt-get install gcc build-essential libgeos-dev imagemagick -y RUN apt-get install python-setuptools python-dev -y -RUN apt-get install libblas-dev liblapack-dev -y +#RUN apt-get install libblas-dev liblapack-dev -y RUN apt-get clean @@ -44,15 +60,15 @@ RUN apt-get clean # RUN /opt/conda/bin/conda install jupyter -y -RUN easy_install pip -RUN pip install -U pip setuptools +#RUN easy_install pip +#RUN pip install -U pip setuptools RUN pip install shapely==1.6b2 RUN pip install opencv-python RUN pip install dill==0.2.6 RUN pip install multiprocess==0.70.5 RUN pip install pathos==0.2.0 RUN pip install pillow - +RUN conda install scipy #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python @@ -61,5 +77,5 @@ COPY . /usr/local/render-python WORKDIR /usr/local/render-python RUN python setup.py install -# ENTRYPOINT [ "/usr/bin/tini", "--" ] +ENTRYPOINT [ "/usr/bin/tini", "--" ] CMD [ "/bin/bash" ] From 5f419569cbe92e012ec021479612881edb740cca Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 15:56:30 +0200 Subject: [PATCH 320/766] fix test --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index ec46c0a2..ac606b6a 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -136,7 +136,7 @@ def test_renderSectionClient(render, teststack): render=render, format='png') - section_directory=os.path.join(root_directory,teststack,'sections_at_.05') + section_directory=os.path.join(root_directory,test_project,teststack,'sections_at_.05') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(section_directory): pngfiles += [f for f in filenames if f.endswith('png')] From 6d43579b4cfebb567a67d03b09991847ac91207b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 16:04:10 +0200 Subject: [PATCH 321/766] test typo --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index ac606b6a..4eb0ee47 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -136,7 +136,7 @@ def test_renderSectionClient(render, teststack): render=render, format='png') - section_directory=os.path.join(root_directory,test_project,teststack,'sections_at_.05') + section_directory=os.path.join(root_directory,'test_project',teststack,'sections_at_.05') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(section_directory): pngfiles += [f for f in filenames if f.endswith('png')] From b0a63a168edb4253a6784bcc4b2a3ceb67f858b1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 16:16:30 +0200 Subject: [PATCH 322/766] another typo --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 4eb0ee47..53202f16 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -136,7 +136,7 @@ def test_renderSectionClient(render, teststack): render=render, format='png') - section_directory=os.path.join(root_directory,'test_project',teststack,'sections_at_.05') + section_directory=os.path.join(root_directory,'test_project',teststack,'sections_at_0.05') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(section_directory): pngfiles += [f for f in filenames if f.endswith('png')] From 9107ca5332f1c76485d66b18cd83e4defbca4a0b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 17:08:11 +0200 Subject: [PATCH 323/766] adding another test back --- integration_tests/test_client_integrated.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 53202f16..efe6641d 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -72,14 +72,14 @@ def validate_stack_import(render,stack,tilespecs): ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) -# def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms): -# stack = 'test_import_jsonfiles_validate_client' -# renderapi.stack.create_stack(stack, render=render) -# (tilespecs, tforms) = render_example_tilespec_and_transforms -# (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) -# renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) -# validate_stack_import(render, stack, tilespecs) -# renderapi.stack.delete_stack(stack, render=render) +def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms): + stack = 'test_import_jsonfiles_validate_client' + renderapi.stack.create_stack(stack, render=render) + (tilespecs, tforms) = render_example_tilespec_and_transforms + (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) + renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) + validate_stack_import(render, stack, tilespecs) + renderapi.stack.delete_stack(stack, render=render) def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, stack='test_import_jsonfiles_parallel'): From 407b380224d3bc8573858d810fabea72e3f3802c Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 17:18:53 +0200 Subject: [PATCH 324/766] missed render call --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index efe6641d..0d4b2a86 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -77,7 +77,7 @@ def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_tr renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) - renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile) + renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile, render=render) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) From 1a603fa8025aa6e60ca5ee47685f77f791daf5c7 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 8 Apr 2017 17:50:34 +0200 Subject: [PATCH 325/766] fixed coordinate tests --- renderapi/coordinate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index a4f1568e..ad8cba50 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -216,7 +216,7 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, jsondata = package_point_match_data_into_json(dataarray, tileId, 'local') if doClientSide: json_answer = local_to_world_coordinates_clientside( - stack, jsondata, z, host=host, port=port, owner=owner, + stack, [[lp] for lp in jsondata], z, host=host, port=port, owner=owner, project=project, client_script=client_script, number_of_threads=number_of_threads) else: From 5fed9926e1c5d3506ddf30de007f8f64d70ca427 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Mon, 10 Apr 2017 01:45:32 -0400 Subject: [PATCH 326/766] tests: add tests for client environment variables, polynomial transformsum, polynomial asorder --- test/test_client.py | 14 ++++++++------ test/test_transform.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/test/test_client.py b/test/test_client.py index 3c5c50d8..94646281 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -27,25 +27,27 @@ def test_default_kwargs_client(): def test_environment_variables( rkwargs=rendersettings.DEFAULT_RENDER, - renvkwargs=rendersettings.DEFAULT_RENDER_ENVIRONMENT_VARIABLES): + renvkwargs=rendersettings.DEFAULT_RENDER_ENVIRONMENT_VARIABLES, + **kwargs): def valstostring(d): return {k: str(v) for k, v in d.items()} old_env = os.environ.copy() os.environ.update(valstostring(renvkwargs)) - env_render = renderapi.connect() + env_render = renderapi.connect(**kwargs) # restore environment os.environ.clear() os.environ.update(old_env) - kwarg_render = renderapi.connect(**valstostring(rkwargs)) + kwarg_render = renderapi.connect(**dict(valstostring(rkwargs), **kwargs)) assert(valstostring(kwarg_render.DEFAULT_KWARGS) == valstostring(env_render.DEFAULT_KWARGS) == valstostring(rkwargs)) -''' def test_environment_variables_client(): - test_environment_variables(rkwargs=rendersettings.DEFAULT_RENDER_CLIENT) -''' + test_environment_variables( + rkwargs=rendersettings.DEFAULT_RENDER_CLIENT, + renvkwargs=rendersettings.DEFAULT_RENDER_CLIENT_ENVIRONMENT_VARIABLES, + validate_client=False) diff --git a/test/test_transform.py b/test/test_transform.py index f7ecb61e..b1c6fb78 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -264,3 +264,41 @@ def test_transform_hash_eq(): assert am1 is not am2 assert am1 == am2 assert len({am1, am2}) == 1 + + +def test_polynomial_transform_asorder_identity(): + srcpts = np.random.rand(50, 2) + pt1 = renderapi.transform.Polynomial2DTransform(identity=True) + pt2 = renderapi.transform.Polynomial2DTransform(identity=True).asorder(2) + + dstpts1 = pt1.tform(srcpts) + dstpts2 = pt2.tform(srcpts) + + assert pt1.order != pt2.order + assert np.allclose(dstpts1, dstpts2) + + + +def test_transformsum_identity_polynomial(): + srcpts = np.random.rand(50, 2) + pt = renderapi.transform.Polynomial2DTransform(identity=True) + am1 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) + am2 = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=-100, + B1=3) + tformlist = [pt, am1, am2] + new_tform = renderapi.transform.estimate_transformsum( + tformlist, src=srcpts, order=1) + + new_srcpts = np.random.rand(50, 2) + new_dstpts_comp = new_tform.tform(new_srcpts) + new_dstpts = am2.concatenate(am1).tform(new_srcpts) + assert np.allclose(new_dstpts_comp, new_dstpts) From d697aae300a124928335b51a6f54e581d9228dae Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 11 Apr 2017 12:20:12 -0700 Subject: [PATCH 327/766] client, render: post_json from utils is better, PEP8 --- renderapi/client.py | 7 +++---- renderapi/render.py | 14 +------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 34178bea..0719ec1b 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -251,7 +251,6 @@ def local_to_world_array(stack, points, tileId, subprocess_mode=None, list of points in world coordinates corresponding to local points ''' raise NotImplementedError('Whoops') - pass @renderaccess @@ -270,7 +269,6 @@ def world_to_local_array(stack, points, subprocess_mode=None, and tileIds corresponding to world point ''' raise NotImplementedError('Whoops.') - pass def call_run_ws_client(className, add_args=[], renderclient=None, @@ -279,8 +277,9 @@ def call_run_ws_client(className, add_args=[], renderclient=None, ''' simple call for run_ws_client.sh -- all arguments set in add_args ''' - logger.debug('call_run_ws_client -- classname:{} add_args:{} client_script:{} memGB:{}'\ - .format(className, add_args, client_script, memGB)) + logger.debug('call_run_ws_client -- classname:{} add_args:{} ' + 'client_script:{} memGB:{}'.format( + className, add_args, client_script, memGB)) if renderclient is not None: if isinstance(renderclient, RenderClient): diff --git a/renderapi/render.py b/renderapi/render.py index 7dbcd863..af276a0e 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -4,7 +4,7 @@ from functools import wraps import requests import json -from .utils import defaultifNone, NullHandler, fitargspec +from .utils import defaultifNone, NullHandler, fitargspec, post_json, put_json from .errors import ClientScriptError logger = logging.getLogger(__name__) @@ -233,18 +233,6 @@ def wrapper(*args, **kwargs): return wrapper -def post_json(session, request, jsondict): - payload = json.dumps(jsondict) - r = session.post(request, data=payload, - headers={"content-type": "application/json", - "Accept": "application/json"}) - try: - return r - except Exception as e: - logger.error(e) - logger.error(r.text) - - def format_baseurl(host, port): '''format host and port to a standard template render-ws url''' # return 'http://%s:%d/render-ws/v1' % (host, port) From f0184a0a6509577f2f6b2487279448e5d087955c Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 12 Apr 2017 10:42:23 -0700 Subject: [PATCH 328/766] test: weird sync issue --- renderapi/stack.py | 11 +++++++++++ test/test_transform.py | 1 - 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 31855092..52f45efd 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -139,6 +139,17 @@ def delete_stack(stack, host=None, port=None, owner=None, return r +@renderaccess +def delete_section(stack, z, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + request_url = '{}/z/{}'.format( + format_preamble(host, port, owner, project, stack), z) + r = session.delete(request_url) + logger.debug(r.text) + return r + + @renderaccess def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, diff --git a/test/test_transform.py b/test/test_transform.py index b1c6fb78..42c98c2b 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -278,7 +278,6 @@ def test_polynomial_transform_asorder_identity(): assert np.allclose(dstpts1, dstpts2) - def test_transformsum_identity_polynomial(): srcpts = np.random.rand(50, 2) pt = renderapi.transform.Polynomial2DTransform(identity=True) From 91e46c12f996cc04fc85dac3150b810d5361984e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 12 Apr 2017 10:56:26 -0700 Subject: [PATCH 329/766] tests: add delete_section integration test --- integration_tests/test_stack_integrated.py | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 070f3474..556dd771 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -181,6 +181,33 @@ def test_import_tilespecs(render, simpletilespec): render.run(renderapi.stack.delete_stack, stack) +def test_remove_section(render, simpletilespec, tmpdir): + # open a temporary file + tfile = tmpdir.join('testfile.json') + fp = tfile.open('w') + + # write the file to disk + renderapi.utils.renderdump([simpletilespec], fp) + fp.close() + + r = render.run(renderapi.stack.create_stack, + 'test_insert', force_resolution=True) + render.run(renderapi.client.import_single_json_file, + 'test_insert', str(tfile)) + r = render.run(renderapi.stack.set_stack_state, + 'test_insert', 'COMPLETE') + stack_zs_before = render.run(renderapi.stack.get_z_values_for_stack, + 'test_insert') + assert simpletilespec.z in stack_zs_before + r = render.run(renderapi.stack.set_stack_state, + 'test_insert', 'LOADING') + r = renderapi.stack.delete_section('test_insert', + simpletilespec.z, render=render) + stack_zs_after = render.run(renderapi.stack.get_z_values_for_stack, + 'test_insert') + assert len(stack_zs_after) == (len(stack_zs_before) - 1) + assert simpletilespec.z not in stack_zs_after + render.run(renderapi.stack.delete_stack, 'test_insert') @pytest.fixture(scope="module") From 5c36d5f39e5a67db7a13698b060fbd70e76f52ba Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 13 Apr 2017 13:07:17 +0200 Subject: [PATCH 330/766] adding transformId to init method --- renderapi/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 31764a94..15f94783 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -200,7 +200,7 @@ class AffineModel(Transform): className = 'mpicbg.trakem2.transform.AffineModel2D' def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, - json=None): + transformId = None, json=None): if json is not None: self.from_dict(json) else: @@ -212,7 +212,7 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, self.B1 = B1 self.className = 'mpicbg.trakem2.transform.AffineModel2D' self.load_M() - self.transformId = None + self.transformId = transformId @property def dataString(self): From 915b9bd5ff7d83df496266625caf0895f6a5e6e4 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 13 Apr 2017 13:07:31 +0200 Subject: [PATCH 331/766] adding more tests --- integration_tests/test_client_integrated.py | 47 ++++++++++++--------- integration_tests/test_stack_integrated.py | 3 -- renderapi/utils.py | 16 ++++++- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 0d4b2a86..e0197038 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -48,22 +48,10 @@ def render_example_json_files(render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms tfiles=[] for ts in tilespecs: - tempjson = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - tempjson.close() - tsjson = tempjson.name - with open(tsjson, 'w') as f: - renderapi.utils.renderdump(tilespecs, f) - f.close() - tfiles.append(tsjson) - - transformFile = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - transformFile.close() - tfjson = transformFile.name - with open(tfjson, 'w') as f: - renderapi.utils.renderdump(tforms, f) - f.close() + tfile = renderapi.utils.renderdump_temp(ts) + tfiles.append(tfile) + tfjson = renderapi.utils.renderdump_temp(tforms) + return (tfiles,tfjson) def validate_stack_import(render,stack,tilespecs): @@ -150,6 +138,27 @@ def test_renderSectionClient(render, teststack): # assert False -# def test_coordinateClient(render): -# root.debug('test not implemented yet') -# assert False \ No newline at end of file +def test_importTransformChangesClient(render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + new_tforms = {} + for tform in tforms: + tformid = tform.transformId + new_tforms[tformid] = AffineModel(B0=np.random.rand(), B1=np.random.rand()) + transformFile = renderapi.utils.renderdump_temp(new_tforms.values()) + + renderapi.client.importTransformChangesClient(teststack, transformFile, render=render) + + tilespecs = renderapi.tilespec.get_tile_specs_from_stack(stack) + for ts in tilespecs: + for tf in ts.tforms: + if tf.transformId in new_tforms.keys(): + assert tf == new_tforms[tf.transformId] + + +# @renderaccess +# def importTransformChangesClient(stack, targetStack, transformFile, +# targetOwner=None, targetProject=None, +# changeMode=None, subprocess_mode=None, +# host=None, port=None, owner=None, +# project=None, client_script=None, memGB=None, +# render=None, **kwargs): diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 070f3474..ca13b3b0 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -180,9 +180,6 @@ def test_import_tilespecs(render, simpletilespec): assert ts_out.z == simpletilespec.z render.run(renderapi.stack.delete_stack, stack) - - - @pytest.fixture(scope="module") def teststack(request, render, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms diff --git a/renderapi/utils.py b/renderapi/utils.py index b27e1950..04f5ff7b 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -7,7 +7,7 @@ import copy import json from .errors import RenderError - +import tempfile class NullHandler(logging.Handler): '''handler to avoid logging errors for, e.g., missing logger setup''' @@ -85,6 +85,20 @@ def put_json(session, request_url, d, params=None): d, request_url, params)) +def renderdump_temp(obj): + '''json.dump into a temporary file + renderdump_temp(obj), obj will be dumped through renderdump + into a temporary file + returns tempfilename, path to file it was dumped into''' + tempfile = tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) + tempfile.close() + tempfilename = tempfile.name + with open(tempfilename, 'w') as f: + renderdump(obj, f) + f.close() + return tempfilename + def renderdumps(obj, *args, **kwargs): '''json.dumps using the RenderEncoder''' cls_ = kwargs.pop('cls', RenderEncoder) From c14c0b0bd5908190310c0e8363fe9ca7555b2a52 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 13 Apr 2017 14:26:05 -0700 Subject: [PATCH 332/766] fixing left in head --- integration_tests/test_stack_integrated.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 079e6825..2a598c36 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -180,9 +180,6 @@ def test_import_tilespecs(render, simpletilespec): assert ts_out.z == simpletilespec.z render.run(renderapi.stack.delete_stack, stack) -<<<<<<< HEAD -======= - def test_remove_section(render, simpletilespec, tmpdir): # open a temporary file tfile = tmpdir.join('testfile.json') @@ -212,7 +209,6 @@ def test_remove_section(render, simpletilespec, tmpdir): render.run(renderapi.stack.delete_stack, 'test_insert') ->>>>>>> 91e46c12f996cc04fc85dac3150b810d5361984e @pytest.fixture(scope="module") def teststack(request, render, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms From 78e147c6b262985fe91524ac2ef7e09a00885c26 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 13 Apr 2017 14:28:32 -0700 Subject: [PATCH 333/766] converting --- renderapi/utils.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 04f5ff7b..535e1046 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -90,13 +90,11 @@ def renderdump_temp(obj): renderdump_temp(obj), obj will be dumped through renderdump into a temporary file returns tempfilename, path to file it was dumped into''' - tempfile = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - tempfile.close() - tempfilename = tempfile.name - with open(tempfilename, 'w') as f: - renderdump(obj, f) - f.close() + tfile = tempfile.NamedTemporaryFile(suffix=".json", mode='r', delete=False) + tfile.close() + tempfilename = tfile.name + with open(tempfilename, 'w') as filepointer: + renderdump(obj, filepointer) return tempfilename def renderdumps(obj, *args, **kwargs): From 783553e6ebe6bbae18de1f182f5c7ecf7d503fee Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 13 Apr 2017 14:47:10 -0700 Subject: [PATCH 334/766] fixing utils --- integration_tests/test_stack_integrated.py | 1 - renderapi/utils.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 2a598c36..36e17281 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -208,7 +208,6 @@ def test_remove_section(render, simpletilespec, tmpdir): assert simpletilespec.z not in stack_zs_after render.run(renderapi.stack.delete_stack, 'test_insert') - @pytest.fixture(scope="module") def teststack(request, render, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms diff --git a/renderapi/utils.py b/renderapi/utils.py index 535e1046..42006ad5 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -2,12 +2,12 @@ ''' utilities to make render/java/web/life interfacing easier ''' +import tempfile import logging import inspect import copy import json from .errors import RenderError -import tempfile class NullHandler(logging.Handler): '''handler to avoid logging errors for, e.g., missing logger setup''' From 371cbb141dff8393c07d9ab5faf868c2a27091eb Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 13 Apr 2017 15:13:22 -0700 Subject: [PATCH 335/766] making renderapi utils temp dump integrate --- renderapi/client.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 0719ec1b..08efafe0 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,7 +8,7 @@ import logging import subprocess import tempfile -from .utils import renderdump, NullHandler +from .utils import renderdump, NullHandler, renderdump_temp from .errors import ClientScriptError from .render import RenderClient, renderaccess from .stack import set_stack_state, make_stack_params @@ -185,20 +185,11 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, sharedTransforms -- list of shared referenced transforms to be ingested ''' - tempjson = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - tempjson.close() - tsjson = tempjson.name - with open(tsjson, 'w') as f: - renderdump(tilespecs, f) + tsjson = renderdump_temp(tilespecs) if sharedTransforms is not None: - tempjson = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - tempjson.close() - trjson = tempjson.name - with open(trjson, 'w') as f: - renderdump(sharedTransforms, f) + trjson = renderdump_temp(tilespecs) + importJsonClient(stack, tileFiles=[tsjson], transformFile=( trjson if sharedTransforms is not None else None), subprocess_mode=subprocess_mode, host=host, port=port, From b8d1fd92338d4ab8d30c02f4c87fd8f9f3f76253 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 13 Apr 2017 15:15:38 -0700 Subject: [PATCH 336/766] fixing broken test --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index e0197038..0d1b6fbb 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -48,10 +48,10 @@ def render_example_json_files(render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms tfiles=[] for ts in tilespecs: - tfile = renderapi.utils.renderdump_temp(ts) + tfile = renderapi.utils.renderdump_temp([ts]) tfiles.append(tfile) tfjson = renderapi.utils.renderdump_temp(tforms) - + return (tfiles,tfjson) def validate_stack_import(render,stack,tilespecs): From fb6ddf089dab8acece514e17c6da5ac292c5f418 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 13 Apr 2017 16:13:18 -0700 Subject: [PATCH 337/766] PEP8 --- renderapi/client.py | 2 +- renderapi/transform.py | 2 +- renderapi/utils.py | 25 +++++++++++++------------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 08efafe0..55b0f696 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -189,7 +189,7 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, if sharedTransforms is not None: trjson = renderdump_temp(tilespecs) - + importJsonClient(stack, tileFiles=[tsjson], transformFile=( trjson if sharedTransforms is not None else None), subprocess_mode=subprocess_mode, host=host, port=port, diff --git a/renderapi/transform.py b/renderapi/transform.py index 15f94783..a40efd2b 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -200,7 +200,7 @@ class AffineModel(Transform): className = 'mpicbg.trakem2.transform.AffineModel2D' def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, - transformId = None, json=None): + transformId=None, json=None): if json is not None: self.from_dict(json) else: diff --git a/renderapi/utils.py b/renderapi/utils.py index 42006ad5..e860ed71 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -9,6 +9,7 @@ import json from .errors import RenderError + class NullHandler(logging.Handler): '''handler to avoid logging errors for, e.g., missing logger setup''' def emit(self, record): @@ -85,18 +86,6 @@ def put_json(session, request_url, d, params=None): d, request_url, params)) -def renderdump_temp(obj): - '''json.dump into a temporary file - renderdump_temp(obj), obj will be dumped through renderdump - into a temporary file - returns tempfilename, path to file it was dumped into''' - tfile = tempfile.NamedTemporaryFile(suffix=".json", mode='r', delete=False) - tfile.close() - tempfilename = tfile.name - with open(tempfilename, 'w') as filepointer: - renderdump(obj, filepointer) - return tempfilename - def renderdumps(obj, *args, **kwargs): '''json.dumps using the RenderEncoder''' cls_ = kwargs.pop('cls', RenderEncoder) @@ -109,6 +98,18 @@ def renderdump(obj, *args, **kwargs): return json.dump(obj, *args, cls=cls_, **kwargs) +def renderdump_temp(obj): + '''json.dump into a temporary file + renderdump_temp(obj), obj will be dumped through renderdump + into a temporary file + returns tempfilename, path to file it was dumped into''' + with tempfile.NamedTemporaryFile( + suffix=".json", mode='w', delete=False) as tf: + tempfilename = tf.name + renderdump(obj, tf) + return tempfilename + + def jbool(val): '''return string representing java string values of py booleans''' if not isinstance(val, bool): From bf2a4eae2e34e0fe46d27bd29725e9c7d02e4ce5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 13 Apr 2017 16:32:57 -0700 Subject: [PATCH 338/766] test: use render ficture --- integration_tests/test_coordinate_integrated.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 2beedfc3..16745686 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -35,7 +35,8 @@ def render(): @pytest.fixture(scope='module') -def teststack_tilespec(): +def teststack_tilespec(render): + ''' render_test_parameters = { 'host': render_host, 'port': 8080, @@ -44,6 +45,7 @@ def teststack_tilespec(): 'client_scripts': client_script_location } render = renderapi.render.connect(**render_test_parameters) + ''' with open(tilespec_file, 'r') as f: ts_json = json.load(f) with open(tform_file, 'r') as f: From 67990c8276a980a9f45894e7d73ca1d572ba07d5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 13 Apr 2017 16:43:31 -0700 Subject: [PATCH 339/766] client: importJson transform file should come from transforms --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 55b0f696..5f80c1bc 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -188,7 +188,7 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, tsjson = renderdump_temp(tilespecs) if sharedTransforms is not None: - trjson = renderdump_temp(tilespecs) + trjson = renderdump_temp(sharedTransforms) importJsonClient(stack, tileFiles=[tsjson], transformFile=( trjson if sharedTransforms is not None else None), From d0ca7a808ac33d569a2282d393f09e85dd315804 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 13 Apr 2017 17:04:52 -0700 Subject: [PATCH 340/766] tests: remove unfinished importTransformClient, PEP8 --- integration_tests/test_client_integrated.py | 113 +++++++------ .../test_pointmatch_integrated.py | 157 ++++++++++-------- integration_tests/test_stack_integrated.py | 6 +- renderapi/client.py | 50 +++--- 4 files changed, 181 insertions(+), 145 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 0d1b6fbb..4598f15a 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -44,58 +44,75 @@ def render_example_tilespec_and_transforms(): print tforms return (tilespecs, tforms) + def render_example_json_files(render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms - tfiles=[] + tfiles = [] for ts in tilespecs: tfile = renderapi.utils.renderdump_temp([ts]) tfiles.append(tfile) tfjson = renderapi.utils.renderdump_temp(tforms) - return (tfiles,tfjson) + return (tfiles, tfjson) + -def validate_stack_import(render,stack,tilespecs): +def validate_stack_import(render, stack, tilespecs): stacks = renderapi.render.get_stacks_by_owner_project(render=render) assert stack in stacks ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) -def test_import_jsonfiles_validate_client(render, render_example_tilespec_and_transforms): + +def test_import_jsonfiles_validate_client( + render, render_example_tilespec_and_transforms): stack = 'test_import_jsonfiles_validate_client' renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) - renderapi.client.import_jsonfiles_validate_client(stack, tfiles, transformFile=transformFile, render=render) + (tfiles, transformFile) = render_example_json_files( + render_example_tilespec_and_transforms) + renderapi.client.import_jsonfiles_validate_client( + stack, tfiles, transformFile=transformFile, render=render) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) -def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, + +def test_import_jsonfiles_parallel(render, + render_example_tilespec_and_transforms, stack='test_import_jsonfiles_parallel'): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) - renderapi.client.import_jsonfiles_parallel(stack, tfiles, transformFile=transformFile, render=render) + (tfiles, transformFile) = render_example_json_files( + render_example_tilespec_and_transforms) + renderapi.client.import_jsonfiles_parallel( + stack, tfiles, transformFile=transformFile, render=render) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) -def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, + +def test_import_tilespecs_parallel(render, + render_example_tilespec_and_transforms, stack='test_import_tilespecs_parallel'): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms - renderapi.client.import_tilespecs_parallel(stack, tilespecs, sharedTransforms=tforms, - poolsize=3, render=render) + renderapi.client.import_tilespecs_parallel( + stack, tilespecs, sharedTransforms=tforms, + poolsize=3, render=render) validate_stack_import(render, stack, tilespecs) + def test_import_jsonfiles(render, render_example_tilespec_and_transforms, stack='test_import_jsonfiles'): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms - (tfiles, transformFile) = render_example_json_files(render_example_tilespec_and_transforms) + (tfiles, transformFile) = render_example_json_files( + render_example_tilespec_and_transforms) - renderapi.client.import_jsonfiles(stack, tfiles, transformFile=transformFile, poolsize=3, render=render) + renderapi.client.import_jsonfiles( + stack, tfiles, transformFile=transformFile, poolsize=3, render=render) validate_stack_import(render, stack, tilespecs) -@pytest.fixture(scope = "module") + +@pytest.fixture(scope="module") def teststack(render, render_example_tilespec_and_transforms): stack = 'teststack' test_import_jsonfiles(render, render_example_tilespec_and_transforms, @@ -103,16 +120,18 @@ def teststack(render, render_example_tilespec_and_transforms): yield stack renderapi.stack.delete_stack(stack, render=render) + def test_tile_pair_client(render, teststack, **kwargs): - zvalues = np.array(renderapi.stack.get_z_values_for_stack(teststack, render=render)) + zvalues = np.array(renderapi.stack.get_z_values_for_stack( + teststack, render=render)) outjson = kwargs.pop('outjson', None) - tilepairjson=renderapi.client.tilePairClient(teststack, np.min(zvalues), - np.max(zvalues), outjson=outjson, - render = render, - **kwargs) + tilepairjson = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, + render=render, **kwargs) assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 + def test_renderSectionClient(render, teststack): zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) root_directory = tempfile.mkdtemp() @@ -124,41 +143,29 @@ def test_renderSectionClient(render, teststack): render=render, format='png') - section_directory=os.path.join(root_directory,'test_project',teststack,'sections_at_0.05') + section_directory = os.path.join( + root_directory, 'test_project', teststack, 'sections_at_0.05') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(section_directory): pngfiles += [f for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) - - - -# def test_importTransformChangesClient(render): -# root.debug('test not implemented yet') -# assert False - - -def test_importTransformChangesClient(render, teststack, render_example_tilespec_and_transforms): - (tilespecs, tforms) = render_example_tilespec_and_transforms - new_tforms = {} - for tform in tforms: - tformid = tform.transformId - new_tforms[tformid] = AffineModel(B0=np.random.rand(), B1=np.random.rand()) - transformFile = renderapi.utils.renderdump_temp(new_tforms.values()) - - renderapi.client.importTransformChangesClient(teststack, transformFile, render=render) - - tilespecs = renderapi.tilespec.get_tile_specs_from_stack(stack) - for ts in tilespecs: - for tf in ts.tforms: - if tf.transformId in new_tforms.keys(): - assert tf == new_tforms[tf.transformId] - - -# @renderaccess -# def importTransformChangesClient(stack, targetStack, transformFile, -# targetOwner=None, targetProject=None, -# changeMode=None, subprocess_mode=None, -# host=None, port=None, owner=None, -# project=None, client_script=None, memGB=None, -# render=None, **kwargs): +# TODO importTransformChangesClient is not implemented +# def test_importTransformChangesClient(render, teststack, +# render_example_tilespec_and_transforms): +# (tilespecs, tforms) = render_example_tilespec_and_transforms +# new_tforms = {} +# for tform in tforms: +# tformid = tform.transformId +# new_tforms[tformid] = AffineModel( +# B0=np.random.rand(), B1=np.random.rand()) +# transformFile = renderapi.utils.renderdump_temp(new_tforms.values()) +# +# renderapi.client.importTransformChangesClient(teststack, transformFile, +# render=render) +# +# tilespecs = renderapi.tilespec.get_tile_specs_from_stack(stack) +# for ts in tilespecs: +# for tf in ts.tforms: +# if tf.transformId in new_tforms.keys(): +# assert tf == new_tforms[tf.transformId] diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 64d123c2..4f8cd910 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -29,18 +29,18 @@ "qId": "1-1", "matches": { "p": [ - [0,0], - [100,100], - [0,100], - [100,0] + [0, 0], + [100, 100], + [0, 100], + [100, 0] ], "q": [ - [0,0], - [100,100], - [0,100], - [100,0] + [0, 0], + [100, 100], + [0, 100], + [100, 0] ], - "w": [1,1,1,1] + "w": [1, 1, 1, 1] } }, { @@ -50,18 +50,18 @@ "qId": "2-1", "matches": { "p": [ - [0,0], - [100,100], - [0,100], - [100,0] + [0, 0], + [100, 100], + [0, 100], + [100, 0] ], "q": [ - [0,0], - [100,100], - [0,100], - [100,0] + [0, 0], + [100, 100], + [0, 100], + [100, 0] ], - "w": [1,1,1,1] + "w": [1, 1, 1, 1] } }, { @@ -71,14 +71,14 @@ "qId": "0-2", "matches": { "p": [ - [100,100], - [100,0] + [100, 100], + [100, 0] ], "q": [ - [0,100], - [0,0] + [0, 100], + [0, 0] ], - "w": [1,1] + "w": [1, 1] } } ] @@ -95,71 +95,98 @@ def render(): } return renderapi.render.connect(**render_test_parameters) + @pytest.fixture(scope='module') def test_pm_collection(render): collection = 'test_collection' - renderapi.pointmatch.import_matches(collection,test_matches,render=render) + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) return collection -def test_get_matchcollection_owners(render,test_pm_collection): + +def test_get_matchcollection_owners(render, test_pm_collection): owners = renderapi.pointmatch.get_matchcollection_owners(render=render) assert 'test' in owners -def test_get_matchcollections(render,test_pm_collection): + +def test_get_matchcollections(render, test_pm_collection): collections = renderapi.pointmatch.get_matchcollections(render=render) - matched_collection = next(coljson for coljson in collections if coljson['collectionId']['name']==test_pm_collection) + matched_collection = next( + coljson for coljson in collections + if coljson['collectionId']['name'] == test_pm_collection) assert matched_collection is not None -def test_get_matches_involving_tile(render,test_pm_collection): - matches = renderapi.pointmatch.get_matches_involving_tile(test_pm_collection, "0", "0-1", render=render) + +def test_get_matches_involving_tile(render, test_pm_collection): + matches = renderapi.pointmatch.get_matches_involving_tile( + test_pm_collection, "0", "0-1", render=render) assert len(matches) == 3 -def test_get_match_groupIds(render,test_pm_collection): - groups = renderapi.pointmatch.get_match_groupIds(test_pm_collection,render=render) - assert len(groups)==3 -def test_get_matches_outside_group(render,test_pm_collection): - matches = renderapi.pointmatch.get_matches_outside_group(test_pm_collection,"0",render=render) +def test_get_match_groupIds(render, test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds( + test_pm_collection, render=render) + assert len(groups) == 3 + + +def test_get_matches_outside_group(render, test_pm_collection): + matches = renderapi.pointmatch.get_matches_outside_group( + test_pm_collection, "0", render=render) assert test_matches[0] in matches assert test_matches[1] in matches -def test_get_matches_within_group(render,test_pm_collection): - matches = renderapi.pointmatch.get_matches_within_group(test_pm_collection,"0",render=render) - assert matches[0]==test_matches[2] -def test_get_matches_from_group_to_group(render,test_pm_collection): - group1="0" - group2="1" - matches = renderapi.pointmatch.get_matches_from_group_to_group(test_pm_collection,group1,group2,render=render) +def test_get_matches_within_group(render, test_pm_collection): + matches = renderapi.pointmatch.get_matches_within_group( + test_pm_collection, "0", render=render) + assert matches[0] == test_matches[2] + + +def test_get_matches_from_group_to_group(render, test_pm_collection): + group1 = "0" + group2 = "1" + matches = renderapi.pointmatch.get_matches_from_group_to_group( + test_pm_collection, group1, group2, render=render) + assert matches[0] == test_matches[0] + + +def test_get_matches_from_tile_to_tile(render, test_pm_collection): + group1 = "0" + group2 = "1" + tile1 = "0-1" + tile2 = "1-1" + matches = renderapi.pointmatch.get_matches_from_tile_to_tile( + test_pm_collection, group1, tile1, group2, tile2, render=render) assert matches[0] == test_matches[0] -def test_get_matches_from_tile_to_tile(render,test_pm_collection): - group1="0" - group2="1" - tile1="0-1" - tile2="1-1" - matches = renderapi.pointmatch.get_matches_from_tile_to_tile(test_pm_collection,group1,tile1,group2,tile2,render=render) - assert matches[0]==test_matches[0] - -def test_get_matches_with_group(render,test_pm_collection): - group1="0" - matches = renderapi.pointmatch.get_matches_with_group(test_pm_collection,group1,render=render) - assert len(matches)==3 - -def test_get_match_groupIds_from_only(render,test_pm_collection): - groups = renderapi.pointmatch.get_match_groupIds_from_only(test_pm_collection,render=render) - assert len(groups)==1 - -def test_get_match_groupIds_to_only(render,test_pm_collection): - groups = renderapi.pointmatch.get_match_groupIds_to_only(test_pm_collection,render=render) - assert len(groups)==3 + +def test_get_matches_with_group(render, test_pm_collection): + group1 = "0" + matches = renderapi.pointmatch.get_matches_with_group( + test_pm_collection, group1, render=render) + assert len(matches) == 3 + + +def test_get_match_groupIds_from_only(render, test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds_from_only( + test_pm_collection, render=render) + assert len(groups) == 1 + + +def test_get_match_groupIds_to_only(render, test_pm_collection): + groups = renderapi.pointmatch.get_match_groupIds_to_only( + test_pm_collection, render=render) + assert len(groups) == 3 + def test_delete_point_matches_between_groups(render): collection = 'test_delete' owner = 'test' - renderapi.pointmatch.import_matches(collection,test_matches,render=render) + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) group1 = '0' group2 = '1' - renderapi.pointmatch.delete_point_matches_between_groups(collection,'0','1',render=render) - groups = renderapi.pointmatch.get_match_groupIds(collection,render=render) - assert len(groups)==2 + renderapi.pointmatch.delete_point_matches_between_groups( + collection, '0', '1', render=render) + groups = renderapi.pointmatch.get_match_groupIds(collection, render=render) + assert len(groups) == 2 diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 36e17281..4aae2eb1 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -180,6 +180,7 @@ def test_import_tilespecs(render, simpletilespec): assert ts_out.z == simpletilespec.z render.run(renderapi.stack.delete_stack, stack) + def test_remove_section(render, simpletilespec, tmpdir): # open a temporary file tfile = tmpdir.join('testfile.json') @@ -208,6 +209,7 @@ def test_remove_section(render, simpletilespec, tmpdir): assert simpletilespec.z not in stack_zs_after render.run(renderapi.stack.delete_stack, 'test_insert') + @pytest.fixture(scope="module") def teststack(request, render, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -379,7 +381,7 @@ def test_get_tile_specs_from_minmax_box( render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms z = tilespecs[0].z - tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z ] + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) ts = renderapi.tilespec.get_tile_specs_from_minmax_box( teststack, z, zbounds['minX'], zbounds['maxX'], @@ -391,7 +393,7 @@ def test_get_tile_specs_from_box(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms z = tilespecs[0].z - tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z ] + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] zbounds = renderapi.stack.get_bounds_from_z(teststack, z, render=render) width = zbounds['maxX']-zbounds['minX'] height = zbounds['maxY']-zbounds['minY'] diff --git a/renderapi/client.py b/renderapi/client.py index 5f80c1bc..e6b7ccde 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -370,31 +370,31 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, os.remove(outjson) return jsondata - -@renderaccess -def importTransformChangesClient(stack, targetStack, transformFile, - targetOwner=None, targetProject=None, - changeMode=None, subprocess_mode=None, - host=None, port=None, owner=None, - project=None, client_script=None, memGB=None, - render=None, **kwargs): - ''' - run ImportTransformChangesClient.java - ''' - if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: - raise ClientScriptError( - 'changeMode {} is not valid!'.format(changeMode)) - - argvs = (make_stack_params(host, port, owner, project, stack) + - ['--targetStack', targetStack] + - ['--transformFile', transformFile] + - get_param(targetOwner, '--targetOwner') + - get_param(targetProject, '--targetProject') + - get_param(changeMode, '--changeMode')) - call_run_ws_client( - 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, - client_script=client_script, subprocess_mode=subprocess_mode, - add_args=argvs) +# FIXME I do not know what input file this client takes +# @renderaccess +# def importTransformChangesClient(stack, targetStack, transformFile, +# targetOwner=None, targetProject=None, +# changeMode=None, subprocess_mode=None, +# host=None, port=None, owner=None, +# project=None, client_script=None, memGB=None, +# render=None, **kwargs): +# ''' +# run ImportTransformChangesClient.java +# ''' +# if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: +# raise ClientScriptError( +# 'changeMode {} is not valid!'.format(changeMode)) +# +# argvs = (make_stack_params(host, port, owner, project, stack) + +# ['--targetStack', targetStack] + +# ['--transformFile', transformFile] + +# get_param(targetOwner, '--targetOwner') + +# get_param(targetProject, '--targetProject') + +# get_param(changeMode, '--changeMode')) +# call_run_ws_client( +# 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, +# client_script=client_script, subprocess_mode=subprocess_mode, +# add_args=argvs) @renderaccess From 6679b0089da1f3fadce0dafec24d7e3026a3a0b9 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 14 Apr 2017 14:42:00 -0700 Subject: [PATCH 341/766] render, client, pointmatch: flake8 --- renderapi/client.py | 3 +-- renderapi/pointmatch.py | 7 ++++--- renderapi/render.py | 1 - 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index e6b7ccde..b202c873 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,8 +8,7 @@ import logging import subprocess import tempfile -from .utils import renderdump, NullHandler, renderdump_temp -from .errors import ClientScriptError +from .utils import NullHandler, renderdump_temp from .render import RenderClient, renderaccess from .stack import set_stack_state, make_stack_params from pathos.multiprocessing import ProcessingPool as Pool diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 3a890f3c..f3add735 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -158,12 +158,13 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, logger.error(e) logger.error(r.text) + @renderaccess def get_matches_involving_tile(matchCollection, pGroupId, pTileId, - owner=None,host=None, port=None, + owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ - "/owner/{}/matchCollection/{}/group/{}/id/{}/".format(\ + "/owner/{}/matchCollection/{}/group/{}/id/{}/".format( owner, matchCollection, pGroupId, pTileId) r = session.get(request_url) try: @@ -172,7 +173,7 @@ def get_matches_involving_tile(matchCollection, pGroupId, pTileId, logger.error(e) logger.error(r.text) - + @renderaccess def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, render=None, owner=None, host=None, diff --git a/renderapi/render.py b/renderapi/render.py index af276a0e..774473fd 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -3,7 +3,6 @@ import os from functools import wraps import requests -import json from .utils import defaultifNone, NullHandler, fitargspec, post_json, put_json from .errors import ClientScriptError From 421ad9436da4ec279283384d6a6c2d8645a4949e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 15 Apr 2017 05:49:38 -0700 Subject: [PATCH 342/766] client: make pathos pool easier to use and fix #14 --- renderapi/client.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index b202c873..3097f723 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -18,6 +18,15 @@ logger.addHandler(NullHandler()) +class WithPool(Pool): + '''pathos ProcessingPool with functioning __exit__ call''' + def __init__(self, *args, **kwargs): + super(WithPool, self).__init__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + super(WithPool, self)._clear() + + @renderaccess def import_single_json_file(stack, jsonfile, transformFile=None, client_scripts=None, host=None, port=None, @@ -208,7 +217,6 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, client_script=None, memGB=None, render=None, **kwargs): set_stack_state(stack, 'LOADING', host, port, owner, project) - pool = Pool(poolsize) partial_import = partial( import_tilespecs, stack, sharedTransforms=sharedTransforms, subprocess_mode=subprocess_mode, host=host, port=port, @@ -217,9 +225,8 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, # TODO this is a weird way to do splits.... is that okay? tilespec_groups = [tilespecs[i::poolsize] for i in xrange(poolsize)] - pool.map(partial_import, tilespec_groups) - pool.close() - pool.join() + with WithPool(poolsize) as pool: + pool.map(partial_import, tilespec_groups) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From 24916a4242f0b3088d94559afdc5f292a767ec59 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 15 Apr 2017 08:21:04 -0700 Subject: [PATCH 343/766] adding a test for running multiple instances of import_jsonfiles_parallel with an external use of pathos pool in a loop of 3 attempts --- integration_tests/test_client_integrated.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 4598f15a..5e02a4f1 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -9,6 +9,7 @@ import dill from test_data import (render_host, render_port, client_script_location, tilespec_file, tform_file) +from pathos.multiprocessing import ProcessingPool as Pool root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -78,17 +79,28 @@ def test_import_jsonfiles_validate_client( def test_import_jsonfiles_parallel(render, render_example_tilespec_and_transforms, - stack='test_import_jsonfiles_parallel'): + stack='test_import_jsonfiles_parallel', + poolsize = 5): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files( render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles_parallel( - stack, tfiles, transformFile=transformFile, render=render) + stack, tfiles, transformFile=transformFile, render=render, poolsize=poolsize) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) - +def test_import_jsonfiles_parallel_multiple(render, + render_example_tilespec_and_transforms, + poolsize=5): + stacks = ['testmultiple1','testmultiple2','testmultiple3'] + mylist = range(10) + for stack in stacks: + pool = Pool(poolsize) + results=pool.map(lambda x:x**2,mylist) + test_import_jsonfiles_parallel(render,render_example_tilespec_and_transforms,stack,poolsize) + + def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack='test_import_tilespecs_parallel'): From 9ee9752a6eeaf41be03f7e8a5a286dc0149d6f51 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 15 Apr 2017 08:36:04 -0700 Subject: [PATCH 344/766] switched to using WithPool helper for multiple imports --- integration_tests/test_client_integrated.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 5e02a4f1..bf29d174 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -96,11 +96,10 @@ def test_import_jsonfiles_parallel_multiple(render, stacks = ['testmultiple1','testmultiple2','testmultiple3'] mylist = range(10) for stack in stacks: - pool = Pool(poolsize) - results=pool.map(lambda x:x**2,mylist) - test_import_jsonfiles_parallel(render,render_example_tilespec_and_transforms,stack,poolsize) + with renderapi.client.WithPool(poolsize) as pool: + results=pool.map(lambda x:x**2,mylist) + test_import_jsonfiles_parallel(render,render_example_tilespec_and_transforms,stack,poolsize) - def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, stack='test_import_tilespecs_parallel'): From 12b2b73b66870e2d375e7dd72d1bcb55f602b974 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 15 Apr 2017 08:36:19 -0700 Subject: [PATCH 345/766] extended use of WithPool helper for all parallel import calls --- renderapi/client.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 3097f723..a15d08ca 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -65,14 +65,12 @@ def import_jsonfiles_and_transforms_parallel_by_z( close_stack: mark render stack as COMPLETE after successful import ''' set_stack_state(stack, 'LOADING', host, port, owner, project) - pool = Pool(poolsize) partial_import = partial(import_single_json_file, stack, render=render, client_scripts=client_scripts, host=host, port=port, owner=owner, project=project) - rs = pool.amap(partial_import, jsonfiles, transformfiles) - rs.wait() - pool.close() - pool.join() + with WithPool(poolsize) as pool: + rs = pool.map(partial_import, jsonfiles, transformfiles) + if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -90,16 +88,15 @@ def import_jsonfiles_parallel( in the jsonfiles ''' set_stack_state(stack, 'LOADING', host, port, owner, project) - pool = Pool(poolsize) + partial_import = partial(import_single_json_file, stack, render=render, transformFile=transformFile, client_scripts=client_scripts, host=host, port=port, owner=owner, project=project) + with WithPool(poolsize) as pool: + pool.map(partial_import, jsonfiles) - pool.map(partial_import, jsonfiles) - pool.close() - pool.join() if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From 7726fb7157aef89c35678672d67c03fe2248f65c Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 15 Apr 2017 09:08:44 -0700 Subject: [PATCH 346/766] chipping away at documentation --- renderapi/client.py | 35 +++++++++++++++++++++++++++++------ renderapi/stack.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index a15d08ca..14f0903a 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -33,7 +33,9 @@ def import_single_json_file(stack, jsonfile, transformFile=None, owner=None, project=None, render=None, **kwargs): ''' calls client script to import given jsonfile: - transformFile: ? + stack: stack to import into + jsonfile: path to jsonfile to import + transformFile: path to a file that contains shared transform references if necessary ''' if transformFile is None: transform_params = [] @@ -59,8 +61,12 @@ def import_jsonfiles_and_transforms_parallel_by_z( project=None, close_stack=True, render=None, **kwargs): ''' imports json files and transform files in parallel: + stack: the stack to import within jsonfiles: "list of tilespec" jsons to import - transformfiles: ? + transformfiles: "list of transform files" which matches in a 1-1 way with + jsonfiles, so referenced transforms are shared only within a single element + of these matched lists. Useful cases where there is as single z transforms shared + by all tiles within a single z, but not across z's poolsize: number of processes for multiprocessing pool close_stack: mark render stack as COMPLETE after successful import ''' @@ -82,10 +88,12 @@ def import_jsonfiles_parallel( project=None, close_stack=True, render=None, **kwargs): ''' import jsons using client script in parallel + stack: the stack to upload into jsonfiles: list of jsonfiles to upload poolsize: number of upload processes spawned by multiprocessing pool transformFile: a single json file containing transforms referenced in the jsonfiles + close_stack: mark render stack as COMPLETE after successful import ''' set_stack_state(stack, 'LOADING', host, port, owner, project) @@ -109,8 +117,8 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, ''' import jsons using client script serially jsonfiles: iterator of filenames to be uploaded - transformFile: ? - close_stack: ? + transformFile: path to a jsonfile that contains shared transform references (if necessary) + close_stack: mark render stack as COMPLETE after successful import ''' set_stack_state(stack, 'LOADING', host, port, owner, project) if transformFile is None: @@ -213,6 +221,16 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): + ''' + input: + stack -- stack to which tilespecs will be added + tilespecs -- list of tilespecs + sharedTransforms -- list of shared + referenced transforms to be ingested + poolsize -- degree of parallelism to use + subprocess_mode -- subprocess mode used when calling client side java + close_stack: mark render stack as COMPLETE after successful import + ''' set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial( import_tilespecs, stack, sharedTransforms=sharedTransforms, @@ -241,6 +259,7 @@ def local_to_world_array(stack, points, tileId, subprocess_mode=None, stack -- stack to which world coordinates are mapped points -- local points to map to world tileId -- tileId to which points correspond + subprocess_mode -- subprocess mode used when calling clientside java client outputs: list of points in world coordinates corresponding to local points ''' @@ -309,7 +328,9 @@ def importJsonClient(stack, tileFiles=None, transformFile=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - '''run ImportJsonClient.java''' + '''run ImportJsonClient.java + see render documentation (add link here) + ''' argvs = (make_stack_params(host, port, owner, project, stack) + (['--transformFile', transformFile] if transformFile else []) + (tileFiles if isinstance(tileFiles, list) @@ -333,7 +354,9 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - '''run TilePairClient.java''' + '''run TilePairClient.java + see render documentation (#add link here) + ''' if outjson is None: tempjson = tempfile.NamedTemporaryFile( suffix=".json", mode='r', delete=False) diff --git a/renderapi/stack.py b/renderapi/stack.py index 52f45efd..450256e6 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -12,6 +12,17 @@ class StackVersion: + '''StackVersion + cycleNumber -- cycleNumber, use as you wish to track versions + cycleStepNumber -- cycleStepNumber, use as you with to track versions + stackResolutionX -- stackResolutionX, resolution of scale = 1.0 in nm (float) + stackResolutionY -- stackResolutionY, resolution of scale = 1.0 in nm (float) + stackResolutionZ -- stackResolutionZ, resolution of scale = 1.0 in nm (float) + mipmapPathBuilder -- ? + materializedBoxRootPath -- ? + createTimeStamp -- time stamp of stack creation (default to now) + versionNotes -- string of notes about this stack (optional) + ''' def __init__(self, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, stackResolutionZ=None, @@ -30,6 +41,9 @@ def __init__(self, cycleNumber=None, cycleStepNumber=None, self.versionNotes = versionNotes def to_dict(self): + '''to_dict + turns this object into a json dictionary + ''' d = {} d.update(({'cycleNumber': self.cycleNumber} if self.cycleNumber is not None else {})) @@ -52,6 +66,9 @@ def to_dict(self): return d def from_dict(self, d): + '''from_dict + update the properties of this object with a json dictonary + ''' self.__dict__.update({k: v for k, v in d.items()}) @@ -59,6 +76,10 @@ def from_dict(self, d): def set_stack_metadata(stack, sv, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + ''' set_stack_metadata + --stack: stack to set the metadata for + --sv: StackVersion to set the metadata to + ''' request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) return post_json(session, request_url, sv.to_dict()) @@ -67,6 +88,13 @@ def set_stack_metadata(stack, sv, host=None, port=None, owner=None, @renderaccess def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + ''' get_stack_metadata + inputs: + --stack: render stack to get metadata from + outputs: + StackVersion object of the metadata of the stack + raises: RenderError + ''' request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) @@ -92,6 +120,13 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, OFFLINE: stack is not in use. READ_ONLY: stack cannot be changed. TODO there is a limited direction in which these stack changes can go + inputs: + --stack: name of render stack to input + --state: state to qset the stack + returns: + session.response object + raises: + RenderError if not successful ''' if state not in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY']: raise RenderError('state {} not in known states {}'.format( From d8e6794e59d5ab51941843389749c3747ad65d71 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 15 Apr 2017 15:17:48 -0700 Subject: [PATCH 347/766] stack docstrings --- renderapi/stack.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/renderapi/stack.py b/renderapi/stack.py index 450256e6..bac785d9 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -168,6 +168,12 @@ def make_stack_params(host, port, owner, project, stack): def delete_stack(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''deletes a stack from render + inputs: + --stack: render stack to delete + returns: + --r: response object of response from server + ''' request_url = format_preamble(host, port, owner, project, stack) r = session.delete(request_url) logger.debug(r.text) @@ -178,6 +184,13 @@ def delete_stack(stack, host=None, port=None, owner=None, def delete_section(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''removes a single z from a stack + inputs: + --stack: stack from which to remove + --z: z value to remove + returns: + response object from server + ''' request_url = '{}/z/{}'.format( format_preamble(host, port, owner, project, stack), z) r = session.delete(request_url) @@ -191,6 +204,18 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionZ=None, force_resolution=True, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''creates a new stack + inputs: + --stack: stack name to create + --cycleNumber: cycleNumber to use to track stages + --cycleStepNumber: cycleStepNumber to use to track stages + --stackResolutionX: resolution of x pixels at scale=1.0 + --stackResolutionY: resolution of y pixels at scale=1.0 + --stackResoluiontZ: resolution of z sections at scale=1.0 + --force_resolution: fill in resolution of 1.0 for missing resolutions (default True) + returns: + reponse object from server + ''' if force_resolution: stackResolutionX, stackResolutionY, stackResolutionZ = [ (1.0 if res is None else res) @@ -255,6 +280,12 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, def get_z_values_for_stack(stack, project=None, host=None, port=None, owner=None, session=requests.session(), render=None, **kwargs): + '''get a list of z values for which there are tiles in the stack + inputs: + --stack: stack to get z values for + returns: + list of z values + ''' request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" logger.debug(request_url) @@ -268,6 +299,13 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, def get_z_value_for_section(stack, sectionId, **kwargs): + '''get a list of z values for which there are tiles in the stack + inputs: + --stack: stack to get z values for + returns: + list of z values + ''' + logger.warning("Deprecated, use get_section_z_value instead") return get_section_z_value(stack, sectionId, **kwargs) @@ -286,6 +324,13 @@ def get_z_value_for_section(stack, sectionId, **kwargs): def get_bounds_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''get a bounds dictionary for a specific z + inputs: + --stack: stack to get bounds from + --z: z values (float) to get bounds from + return: + dictionary of bounds with keys minY,minY,maxX,maxY + ''' request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/bounds' % (z) @@ -301,6 +346,12 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, @renderaccess def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''get bounds of a whole stack + inputs: + --stack: stack to get bounds from + return: + dictionary of bounds with keys minY,minY,maxX,maxY,minZ,maxZ + ''' request_url = format_preamble( host, port, owner, project, stack) + '/bounds' r = session.get(request_url) @@ -316,6 +367,21 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, def get_stack_sectionData(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''returns information about the sectionIds of each slice in stack + inputs: + --stack: name of stack to get data about + returns: + list of dictionaries containing sectionData as below + [{ + "sectionId": "string", + "z": 0, + "tileCount": 0, + "minX": 0, + "maxX": 0, + "minY": 0, + "maxY": 0 + }] + ''' request_url = format_preamble( host, port, owner, project, stack) + '/sectionData' r = session.get(request_url) @@ -331,6 +397,15 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, def get_section_z_value(stack, sectionId, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''get the z value for a specific sectionId (string) + inputs: + --stack: name of stack to get info about + --sectionId: string of sectionId + outputs: + z value (float) that corresponds to that sectionId + raises: + RenderError + ''' request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % sectionId r = session.get(request_url) From 8a7eede9a6ddb9dcb69849932b3504f07a2749dc Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 15 Apr 2017 15:23:05 -0700 Subject: [PATCH 348/766] more docstrings --- renderapi/tilespec.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 22cb7435..845aa83d 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -12,6 +12,20 @@ class Layout: + '''Layout class to describe acquisition settings + inputs: + keyword arguments + --sectionId: unique string to describe sectionId this tile was taken from + --scopeId: string to track what microscope this came from + --cameraId: string to track what camera this was taken with + --imageRow: integer to track what row from a row,col layout this was taken + --imageCol: integer to track what column form a row,col layout this was taken + --stageX: X stage coordinates (float) for where this was taken + --stageY: Y stage coordinates (float) for where this taken from + --rotation: angle of camera when this was taken + --pixelsize: size of pixels in units of choice of camera at the magnification it was taken + --force_pixelsize: whether to default pixelsize to 0.1 (default True) + ''' def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, rotation=None, pixelsize=None, @@ -29,6 +43,11 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, self.pixelsize = pixelsize def to_dict(self): + '''return a dictionary representation of this object + no inputs + returns: + json dictionary of object + ''' d = {} d['sectionId'] = self.sectionId d['temca'] = self.scopeId @@ -43,6 +62,10 @@ def to_dict(self): return d def from_dict(self, d): + '''set this object equal to the fields found in dictionary + inputs: + --d:dictionary to use + ''' if d is not None: self.sectionId = d.get('sectionId') self.cameraId = d.get('camera') From 101eee19fb556470e79797fe46a8ea2200efb312 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 17 Apr 2017 09:55:14 -0700 Subject: [PATCH 349/766] chipping away at the documentation in tilespec --- renderapi/tilespec.py | 131 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 118 insertions(+), 13 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 845aa83d..ab08991f 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -64,23 +64,46 @@ def to_dict(self): def from_dict(self, d): '''set this object equal to the fields found in dictionary inputs: - --d:dictionary to use + --d:dictionary to use to update ''' if d is not None: - self.sectionId = d.get('sectionId') - self.cameraId = d.get('camera') - self.scopeId = d.get('temca') - self.imageRow = d.get('imageRow') - self.imageCol = d.get('imageCol') - self.stageX = d.get('stageX') - self.stageY = d.get('stageY') - self.rotation = d.get('rotation') - self.pixelsize = d.get('pixelsize') + self.sectionId = d.get('sectionId',self.sectionId) + self.cameraId = d.get('camera',self.cameraId) + self.scopeId = d.get('temca',self.scopeId) + self.imageRow = d.get('imageRow',self.imageRow) + self.imageCol = d.get('imageCol',self.imageCol) + self.stageX = d.get('stageX',self.stageX) + self.stageY = d.get('stageY',self.stageY) + self.rotation = d.get('rotation',self.rotation) + self.pixelsize = d.get('pixelsize',self.pixelsize) class TileSpec: + '''Fundamental class of render that store image tiles and their transformations + init: + Keyword arguments: + --tileId: unique string specifying a tile's identity + --z: z values this tile exists within (float) + --width: width in pixels of the raw tile + --height: height in pixels of the raw tile + --imageUrl: an image path that can be accessed by the render server, with an ImageJ.open command + or as an s3 url. files on disk should be specified with file: + --maskUrl: an image path that can be accessed by the render server which can be interpreted as an + alpha mask for the image (same as spec imageUrl) + --minint: pixel intensity value to display as black in a linear colormap (default 0) + --maxint: pixel intensity value to display as white in a linaer colormap (default 65535) + --layout: a Layout object for this tile + --tforms: a list of Transform objects (see AffineModel, TransformList, Polynomial2DTransform, Transform, ReferenceTransform) to apply to this tile + --inputfilters: a list of filters to apply to this tile (not yet implemented) + --scale3Url: url of a mipmap level 3 image of this tile (deprecated, see mipMapLevels, but will override) + --scale2Url: url of a mipmap level 2 image of this tile (deprecated, see mipMapLevels, but will override) + --scale1Url: url of a mipmap level 1 image of this tile (deprecated, see mipMapLevels, but will override) + --mipMapLevels: a list of MipMapLevel objects for this tile + --json: a json dictionary to initialize this object with (if not None overrides and ignores all keyword arguments) + + ''' def __init__(self, tileId=None, z=None, width=None, height=None, - imageUrl=None, frameId=None, maskUrl=None, + imageUrl=None, maskUrl=None, minint=0, maxint=65535, layout=None, tforms=[], inputfilters=[], scale3Url=None, scale2Url=None, scale1Url=None, json=None, mipMapLevels=[], **kwargs): @@ -95,7 +118,6 @@ def __init__(self, tileId=None, z=None, width=None, height=None, self.minint = minint self.maxint = maxint self.tforms = tforms - self.frameId = frameId self.inputfilters = inputfilters self.layout = Layout(**kwargs) if layout is None else layout @@ -127,6 +149,9 @@ def bbox(self): return box def to_dict(self): + '''method to produce a json tilespec for this tile + returns a json compatible dictionary + ''' thedict = {} thedict['tileId'] = self.tileId thedict['z'] = self.z @@ -134,7 +159,6 @@ def to_dict(self): thedict['height'] = self.height thedict['minIntensity'] = self.minint thedict['maxIntensity'] = self.maxint - thedict['frameId'] = self.frameId if self.layout is not None: thedict['layout'] = self.layout.to_dict() thedict['mipmapLevels'] = self.ip.to_ordered_dict() @@ -284,6 +308,21 @@ def __iter__(self): def get_tile_spec(stack, tile, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''renderapi call to get a specific tilespec by tileId + note that this will return a tilespec with resolved transform references + by accessing the render-parameters version of this tile + + args: + --stack: name of render stack to retrieve + --tile: tileId of tile to retrieve + keyword args: + --render: render connect object + (or host, port, owner, project) + --session: sessions object to connect with (default make a new one) + outputs: + A TileSpec object with dereferenced transforms + ''' + request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/render-parameters" % (tile) @@ -295,6 +334,34 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, logger.error(r.text) return TileSpec(json=tilespec_json['tileSpecs'][0]) +@renderaccess +def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + '''renderapi call to get a specific tilespec by tileId + note that this will return a tilespec without resolved transform references + + args: + --stack: name of render stack to retrieve + --tile: tileId of tile to retrieve + keyword args: + --render: render connect object + (or host, port, owner, project) + --session: sessions object to connect with (default make a new one) + outputs: + a TileSpec object with referenced transforms if present + ''' + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/render-parameters" % (tile) + r = session.get(request_url) + try: + tilespec_json = r.json() + except Exception as e: + logger.error(e) + logger.error(r.text) + return TileSpec(json=tilespec_json) @renderaccess def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, @@ -302,6 +369,25 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''renderapi call to get all tilespec that exist within a 2d bounding box + specified with min and max x,y values + note that this will return a tilespec with resolved transform references + + args: + --stack: name of render stack to retrieve + --z: z value of bounding box (float) + --xmin: minimum x value (float) + --ymin: minimum y value (float) + --xmax: maximum x value (float) + --ymax: maximum y value (float) + keyword args: + --scale: scale to use when retrieving render parameters (not important) + --render: render connect object + (or host, port, owner, project) + --session: sessions object to connect with (default make a new one) + outputs: + a list of TileSpec objects + ''' x = xmin y = ymin width = xmax - xmin @@ -317,6 +403,25 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, scale=1.0, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + '''renderapi call to get all tilespec that exist within a 2d bounding box + specified with min x,y values and width, height + note that this will return a tilespec with resolved transform references + + args: + --stack: name of render stack to retrieve + --z: z value of bounding box (float) + --xmin: minimum x value (float) + --ymin: minimum y value (float) + --xmax: maximum x value (float) + --ymax: maximum y value (float) + keyword args: + --scale: scale to use when retrieving render parameters (not important) + --render: render connect object + (or host, port, owner, project) + --session: sessions object to connect with (default make a new one) + outputs: + a list of TileSpec objects + ''' request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( From 1a228a20cba10ce4fd2800260a9e52a886d98a93 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 17 Apr 2017 10:07:05 -0700 Subject: [PATCH 350/766] removing from_dict that leaves old defaults --- renderapi/tilespec.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index ab08991f..71703372 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -67,15 +67,15 @@ def from_dict(self, d): --d:dictionary to use to update ''' if d is not None: - self.sectionId = d.get('sectionId',self.sectionId) - self.cameraId = d.get('camera',self.cameraId) - self.scopeId = d.get('temca',self.scopeId) - self.imageRow = d.get('imageRow',self.imageRow) - self.imageCol = d.get('imageCol',self.imageCol) - self.stageX = d.get('stageX',self.stageX) - self.stageY = d.get('stageY',self.stageY) - self.rotation = d.get('rotation',self.rotation) - self.pixelsize = d.get('pixelsize',self.pixelsize) + self.sectionId = d.get('sectionId') + self.cameraId = d.get('camera') + self.scopeId = d.get('temca') + self.imageRow = d.get('imageRow') + self.imageCol = d.get('imageCol') + self.stageX = d.get('stageX') + self.stageY = d.get('stageY') + self.rotation = d.get('rotation') + self.pixelsize = d.get('pixelsize') class TileSpec: From e76ec29f4c0023ab809fb35f152d6c3c6dae16f9 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 17 Apr 2017 18:21:15 -0700 Subject: [PATCH 351/766] added raw tilespec grab test --- integration_tests/test_stack_integrated.py | 7 ++++++- renderapi/tilespec.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 4aae2eb1..cce63d43 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -376,7 +376,12 @@ def test_get_tilespecs_from_z(render, teststack, tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] assert len(tiles) == len(tsz) - +def test_get_tilespec_raw( + render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + ts = renderapi.tilespec.get_tile_spec_raw(teststack, tilespecs[0].tileId) + assert ts.to_dict() == tilespecs[0].to_dict() + def test_get_tile_specs_from_minmax_box( render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 71703372..8710033e 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -354,7 +354,7 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, request_url = format_preamble( host, port, owner, project, stack) + \ - "/tile/%s/render-parameters" % (tile) + "/tile/%s/" % (tile) r = session.get(request_url) try: tilespec_json = r.json() From 167e3d914559ac931ef6c6cbe14be3eedc08b779 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 18 Apr 2017 07:52:39 -0700 Subject: [PATCH 352/766] fixed bug in test --- integration_tests/test_stack_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index cce63d43..1ea728da 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -379,9 +379,9 @@ def test_get_tilespecs_from_z(render, teststack, def test_get_tilespec_raw( render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms - ts = renderapi.tilespec.get_tile_spec_raw(teststack, tilespecs[0].tileId) + ts = renderapi.tilespec.get_tile_spec_raw(teststack, tilespecs[0].tileId, render=render) assert ts.to_dict() == tilespecs[0].to_dict() - + def test_get_tile_specs_from_minmax_box( render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms From 3d571e3db349b451fc6f50d0a9c0e961d2fdb820 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 19 Apr 2017 07:11:10 -0700 Subject: [PATCH 353/766] more documentation --- renderapi/stack.py | 174 ++++++++++++++++++++++++++++-------------- renderapi/tilespec.py | 15 +++- 2 files changed, 131 insertions(+), 58 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index bac785d9..91df56e0 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -13,8 +13,9 @@ class StackVersion: '''StackVersion - cycleNumber -- cycleNumber, use as you wish to track versions - cycleStepNumber -- cycleStepNumber, use as you with to track versions + keyword arguments: + cycleNumber -- cycleNumber, use as you wish to track versions (default None) + cycleStepNumber -- cycleStepNumber, use as you with to track versions (default None) stackResolutionX -- stackResolutionX, resolution of scale = 1.0 in nm (float) stackResolutionY -- stackResolutionY, resolution of scale = 1.0 in nm (float) stackResolutionZ -- stackResolutionZ, resolution of scale = 1.0 in nm (float) @@ -77,8 +78,12 @@ def set_stack_metadata(stack, sv, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): ''' set_stack_metadata - --stack: stack to set the metadata for - --sv: StackVersion to set the metadata to + inputs: + stack -- stack to set the metadata for + sv -- StackVersion to set the metadata to + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) ''' request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) @@ -90,7 +95,10 @@ def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): ''' get_stack_metadata inputs: - --stack: render stack to get metadata from + stack -- render stack to get metadata from + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) outputs: StackVersion object of the metadata of the stack raises: RenderError @@ -123,7 +131,10 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, inputs: --stack: name of render stack to input --state: state to qset the stack - returns: + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) + outputs: session.response object raises: RenderError if not successful @@ -145,7 +156,13 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, @renderaccess def likelyUniqueId(host=None, port=None, session=requests.session(), render=None, **kwargs): - '''return hex-code nearly-unique id from render server''' + '''return hex-code nearly-unique id from render server + keyword arguments: + render -- render connect object (or host, port) + session -- requests.session (default start a new one) + returns: + string representation of hex-code + ''' request_url = '{}/likelyUniqueId'.format(format_baseurl(host, port)) r = session.get(request_url, data=None, headers={"content-type": "text/plain"}) @@ -170,9 +187,12 @@ def delete_stack(stack, host=None, port=None, owner=None, render=None, **kwargs): '''deletes a stack from render inputs: - --stack: render stack to delete - returns: - --r: response object of response from server + stack -- render stack to delete + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) + outputs: + r -- response object of response from server ''' request_url = format_preamble(host, port, owner, project, stack) r = session.delete(request_url) @@ -186,10 +206,13 @@ def delete_section(stack, z, host=None, port=None, owner=None, render=None, **kwargs): '''removes a single z from a stack inputs: - --stack: stack from which to remove - --z: z value to remove - returns: - response object from server + stack -- stack from which to remove + z -- z value to remove + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) + outputs: + r -- response object from server ''' request_url = '{}/z/{}'.format( format_preamble(host, port, owner, project, stack), z) @@ -206,15 +229,20 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, session=requests.session(), render=None, **kwargs): '''creates a new stack inputs: - --stack: stack name to create - --cycleNumber: cycleNumber to use to track stages - --cycleStepNumber: cycleStepNumber to use to track stages - --stackResolutionX: resolution of x pixels at scale=1.0 - --stackResolutionY: resolution of y pixels at scale=1.0 - --stackResoluiontZ: resolution of z sections at scale=1.0 - --force_resolution: fill in resolution of 1.0 for missing resolutions (default True) + stack -- stack name to create + keyword arguments: + cycleNumber -- cycleNumber to use to track stages + cycleStepNumber -- cycleStepNumber to use to track stages + stackResolutionX -- resolution of x pixels at scale=1.0 + stackResolutionY -- resolution of y pixels at scale=1.0 + stackResoluiontZ -- resolution of z sections at scale=1.0 + force_resolution -- fill in resolution of 1.0 for missing resolutions (default True) + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) returns: - reponse object from server + r -- reponse object from server + raises: + RenderError ''' if force_resolution: stackResolutionX, stackResolutionY, stackResolutionZ = [ @@ -235,6 +263,8 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) + @renderaccess @@ -243,14 +273,19 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, owner=None, project=None, session=None, render=None, **kwargs): ''' input: - inputstack: string name of input stack to clone - outputstack: string name of destination stack. + inputstack -- string name of input stack to clone + outputstack -- string name of destination stack. if exists, must be LOADING - close_stack: boolean, whether to set stack to COMPLETE when finished - toProject: optional, string name of project - skipTransforms: optional, boolean whether to strip transformations + keyword arguments: + skipTransforms -- optional, boolean whether to strip transformations in new stack - zs: optional, list of selected z values to clone into stack + toProject -- optional, string name of project + zs -- optional, list of selected z values to clone into stack + close_stack -- boolean, whether to set stack to COMPLETE when finished + render -- render connect object (or host, port, owner, project) + session -- optional, requests.session (default start a new one) + outputs: + r -- reponse object from server ''' session = requests.session() if session is None else session sv = StackVersion(**kwargs) @@ -282,9 +317,14 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, render=None, **kwargs): '''get a list of z values for which there are tiles in the stack inputs: - --stack: stack to get z values for + stack -- stack to get z values for + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- optional, requests.session (default start a new one) returns: - list of z values + list of z values + raises: + RenderError ''' request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" @@ -299,11 +339,17 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, def get_z_value_for_section(stack, sectionId, **kwargs): - '''get a list of z values for which there are tiles in the stack + '''DEPRECATED (use get_section_z_value) instead get z values for a specific sectionId inputs: - --stack: stack to get z values for + stack -- render stack string to look within + sectionId -- string of sectionId to find z value + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- optional, requests.session (default start a new one) returns: - list of z values + list of z values + raises: + RenderErorr ''' logger.warning("Deprecated, use get_section_z_value instead") return get_section_z_value(stack, sectionId, **kwargs) @@ -326,10 +372,13 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, render=None, **kwargs): '''get a bounds dictionary for a specific z inputs: - --stack: stack to get bounds from - --z: z values (float) to get bounds from - return: - dictionary of bounds with keys minY,minY,maxX,maxY + stack -- stack to get bounds from + z -- z values (float) to get bounds from + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- optional, requests.session (default start a new one) + outputs: + dictionary of bounds with keys minY,minY,maxX,maxY ''' request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/bounds' % (z) @@ -348,9 +397,14 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''get bounds of a whole stack inputs: - --stack: stack to get bounds from - return: - dictionary of bounds with keys minY,minY,maxX,maxY,minZ,maxZ + stack -- stack to get bounds from + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- optional, requests.session (default start a new one) + outputs: + dictionary of bounds with keys minY,minY,maxX,maxY,minZ,maxZ + raises: + RenderError ''' request_url = format_preamble( host, port, owner, project, stack) + '/bounds' @@ -369,18 +423,23 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, render=None, **kwargs): '''returns information about the sectionIds of each slice in stack inputs: - --stack: name of stack to get data about + stack -- name of stack to get data about + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- optional, requests.session (default start a new one) returns: - list of dictionaries containing sectionData as below - [{ - "sectionId": "string", - "z": 0, - "tileCount": 0, - "minX": 0, - "maxX": 0, - "minY": 0, - "maxY": 0 - }] + list of dictionaries containing sectionData as below + [{ + "sectionId": "string", + "z": 0, + "tileCount": 0, + "minX": 0, + "maxX": 0, + "minY": 0, + "maxY": 0 + }] + raises: + RenderError ''' request_url = format_preamble( host, port, owner, project, stack) + '/sectionData' @@ -399,12 +458,15 @@ def get_section_z_value(stack, sectionId, host=None, port=None, render=None, **kwargs): '''get the z value for a specific sectionId (string) inputs: - --stack: name of stack to get info about - --sectionId: string of sectionId - outputs: - z value (float) that corresponds to that sectionId + stack -- render stack string to look within + sectionId -- string of sectionId to find z value + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- optional, requests.session (default start a new one) + returns: + list of z values raises: - RenderError + RenderError ''' request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % sectionId diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 8710033e..ec0e093e 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -445,7 +445,10 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, input: stack -- string render stack z -- render z - output: list of tilespec objects + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) + output: list of TileSpec objects from that stack at that z ''' request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) @@ -469,7 +472,15 @@ def get_tile_specs_from_stack(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''get flat list of tilespecs for stack using i for sl in l for i in sl''' + '''get flat list of tilespecs for stack using i for sl in l for i in sl + input: + stack -- string render stack + keyword arguments: + render -- render connect object (or host, port, owner, project) + session -- requests.session (default start a new one) + output: + List of TileSpec objects from the stack + ''' return [i for sl in [ get_tile_specs_from_z(stack, z, host=host, port=port, owner=owner, project=project, session=session) From d506d58519529c44b80f754cdc6acd1b8bc420f7 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 20 Apr 2017 10:54:02 -0700 Subject: [PATCH 354/766] made tilespec functions raise RenderError if tilespec doesn't exist or something else goes wrong --- renderapi/tilespec.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index ec0e093e..d4edb20e 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -3,6 +3,7 @@ from .utils import NullHandler from .stack import get_z_values_for_stack from .transform import TransformList +from .errors import RenderError from collections import OrderedDict import logging import requests @@ -332,6 +333,7 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) return TileSpec(json=tilespec_json['tileSpecs'][0]) @renderaccess @@ -361,6 +363,7 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) return TileSpec(json=tilespec_json) @renderaccess @@ -433,6 +436,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json['tileSpecs']] @@ -459,6 +463,7 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) if len(tilespecs_json) == 0: return None From 9d0e57a889e9d3914771ef41b37d7dd2a801a4bf Mon Sep 17 00:00:00 2001 From: RussTorres Date: Mon, 24 Apr 2017 10:42:09 -0700 Subject: [PATCH 355/766] PEP8 --- renderapi/client.py | 20 +++++++------ renderapi/coordinate.py | 9 +++--- renderapi/stack.py | 28 +++++++++++-------- renderapi/tilespec.py | 62 +++++++++++++++++++++++++---------------- 4 files changed, 72 insertions(+), 47 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 14f0903a..cb711aed 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -35,7 +35,8 @@ def import_single_json_file(stack, jsonfile, transformFile=None, calls client script to import given jsonfile: stack: stack to import into jsonfile: path to jsonfile to import - transformFile: path to a file that contains shared transform references if necessary + transformFile: path to a file that contains shared + transform references if necessary ''' if transformFile is None: transform_params = [] @@ -63,10 +64,11 @@ def import_jsonfiles_and_transforms_parallel_by_z( imports json files and transform files in parallel: stack: the stack to import within jsonfiles: "list of tilespec" jsons to import - transformfiles: "list of transform files" which matches in a 1-1 way with - jsonfiles, so referenced transforms are shared only within a single element - of these matched lists. Useful cases where there is as single z transforms shared - by all tiles within a single z, but not across z's + transformfiles: "list of transform files" which matches + in a 1-1 way with jsonfiles, so referenced transforms + are shared only within a single element of these matched lists. + Useful cases where there is as single z transforms shared + by all tiles within a single z, but not across z's poolsize: number of processes for multiprocessing pool close_stack: mark render stack as COMPLETE after successful import ''' @@ -96,7 +98,7 @@ def import_jsonfiles_parallel( close_stack: mark render stack as COMPLETE after successful import ''' set_stack_state(stack, 'LOADING', host, port, owner, project) - + partial_import = partial(import_single_json_file, stack, render=render, transformFile=transformFile, client_scripts=client_scripts, @@ -117,7 +119,8 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, ''' import jsons using client script serially jsonfiles: iterator of filenames to be uploaded - transformFile: path to a jsonfile that contains shared transform references (if necessary) + transformFile: path to a jsonfile that contains shared + transform references (if necessary) close_stack: mark render stack as COMPLETE after successful import ''' set_stack_state(stack, 'LOADING', host, port, owner, project) @@ -259,7 +262,8 @@ def local_to_world_array(stack, points, tileId, subprocess_mode=None, stack -- stack to which world coordinates are mapped points -- local points to map to world tileId -- tileId to which points correspond - subprocess_mode -- subprocess mode used when calling clientside java client + subprocess_mode -- subprocess mode used when calling + clientside java client outputs: list of points in world coordinates corresponding to local points ''' diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index ad8cba50..e0428d4f 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -216,8 +216,8 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, jsondata = package_point_match_data_into_json(dataarray, tileId, 'local') if doClientSide: json_answer = local_to_world_coordinates_clientside( - stack, [[lp] for lp in jsondata], z, host=host, port=port, owner=owner, - project=project, client_script=client_script, + stack, [[lp] for lp in jsondata], z, host=host, port=port, + owner=owner, project=project, client_script=client_script, number_of_threads=number_of_threads) else: json_answer = local_to_world_coordinates_batch( @@ -236,7 +236,6 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, logger.debug('jsondata:{}'.format(jsondata)) # d = json.loads(jsondata) json.dump(jsondata, fp) - fp.close() # fp.write(jsondata) # json.dump(jsondata,open(json_inpath,'w')) @@ -253,7 +252,9 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, client_script=client_script, memGB=memGB) # return the json results - return json.load(open(json_outpath, 'r')) + with open(json_outpath, 'r') as f: + j = json.load(f) + return j @renderaccess diff --git a/renderapi/stack.py b/renderapi/stack.py index 91df56e0..6cf2ef41 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -14,11 +14,16 @@ class StackVersion: '''StackVersion keyword arguments: - cycleNumber -- cycleNumber, use as you wish to track versions (default None) - cycleStepNumber -- cycleStepNumber, use as you with to track versions (default None) - stackResolutionX -- stackResolutionX, resolution of scale = 1.0 in nm (float) - stackResolutionY -- stackResolutionY, resolution of scale = 1.0 in nm (float) - stackResolutionZ -- stackResolutionZ, resolution of scale = 1.0 in nm (float) + cycleNumber -- cycleNumber, use as you wish to + track versions (default None) + cycleStepNumber -- cycleStepNumber, use as you with to + track versions (default None) + stackResolutionX -- stackResolutionX, + resolution of scale = 1.0 in nm (float) + stackResolutionY -- stackResolutionY, + resolution of scale = 1.0 in nm (float) + stackResolutionZ -- stackResolutionZ, + resolution of scale = 1.0 in nm (float) mipmapPathBuilder -- ? materializedBoxRootPath -- ? createTimeStamp -- time stamp of stack creation (default to now) @@ -236,7 +241,8 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionX -- resolution of x pixels at scale=1.0 stackResolutionY -- resolution of y pixels at scale=1.0 stackResoluiontZ -- resolution of z sections at scale=1.0 - force_resolution -- fill in resolution of 1.0 for missing resolutions (default True) + force_resolution -- fill in resolution of 1.0 for missing + resolutions (default True) render -- render connect object (or host, port, owner, project) session -- requests.session (default start a new one) returns: @@ -266,7 +272,6 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, raise RenderError(r.text) - @renderaccess def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, zs=None, close_stack=True, host=None, port=None, @@ -339,9 +344,10 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, def get_z_value_for_section(stack, sectionId, **kwargs): - '''DEPRECATED (use get_section_z_value) instead get z values for a specific sectionId + '''DEPRECATED (use get_section_z_value) instead get z + values for a specific sectionId inputs: - stack -- render stack string to look within + stack -- render stack string to look within sectionId -- string of sectionId to find z value keyword arguments: render -- render connect object (or host, port, owner, project) @@ -421,7 +427,7 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, def get_stack_sectionData(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''returns information about the sectionIds of each slice in stack + '''returns information about the sectionIds of each slice in stack inputs: stack -- name of stack to get data about keyword arguments: @@ -458,7 +464,7 @@ def get_section_z_value(stack, sectionId, host=None, port=None, render=None, **kwargs): '''get the z value for a specific sectionId (string) inputs: - stack -- render stack string to look within + stack -- render stack string to look within sectionId -- string of sectionId to find z value keyword arguments: render -- render connect object (or host, port, owner, project) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index d4edb20e..f8c0e88a 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -20,11 +20,13 @@ class Layout: --scopeId: string to track what microscope this came from --cameraId: string to track what camera this was taken with --imageRow: integer to track what row from a row,col layout this was taken - --imageCol: integer to track what column form a row,col layout this was taken + --imageCol: integer to track what column form a + row,col layout this was taken --stageX: X stage coordinates (float) for where this was taken --stageY: Y stage coordinates (float) for where this taken from - --rotation: angle of camera when this was taken - --pixelsize: size of pixels in units of choice of camera at the magnification it was taken + --rotation: angle of camera when this was taken + --pixelsize: size of pixels in units of choice of camera + at the magnification it was taken --force_pixelsize: whether to default pixelsize to 0.1 (default True) ''' def __init__(self, sectionId=None, scopeId=None, cameraId=None, @@ -87,21 +89,31 @@ class TileSpec: --z: z values this tile exists within (float) --width: width in pixels of the raw tile --height: height in pixels of the raw tile - --imageUrl: an image path that can be accessed by the render server, with an ImageJ.open command - or as an s3 url. files on disk should be specified with file: - --maskUrl: an image path that can be accessed by the render server which can be interpreted as an - alpha mask for the image (same as spec imageUrl) - --minint: pixel intensity value to display as black in a linear colormap (default 0) - --maxint: pixel intensity value to display as white in a linaer colormap (default 65535) + --imageUrl: an image path that can be accessed by the render server, + with an ImageJ.open command or as an s3 url. + Files on disk should be specified with file: + --maskUrl: an image path that can be accessed by the render server + which can be interpreted as an + alpha mask for the image (same as spec imageUrl) + --minint: pixel intensity value to display as black in a + linear colormap (default 0) + --maxint: pixel intensity value to display as white in a + linear colormap (default 65535) --layout: a Layout object for this tile - --tforms: a list of Transform objects (see AffineModel, TransformList, Polynomial2DTransform, Transform, ReferenceTransform) to apply to this tile - --inputfilters: a list of filters to apply to this tile (not yet implemented) - --scale3Url: url of a mipmap level 3 image of this tile (deprecated, see mipMapLevels, but will override) - --scale2Url: url of a mipmap level 2 image of this tile (deprecated, see mipMapLevels, but will override) - --scale1Url: url of a mipmap level 1 image of this tile (deprecated, see mipMapLevels, but will override) + --tforms: a list of Transform objects + (see AffineModel, TransformList, Polynomial2DTransform, + Transform, ReferenceTransform) to apply to this tile + --inputfilters: a list of filters to apply to + this tile (not yet implemented) + --scale3Url: url of a mipmap level 3 image of this tile + (deprecated, see mipMapLevels, but will override) + --scale2Url: url of a mipmap level 2 image of this tile + (deprecated, see mipMapLevels, but will override) + --scale1Url: url of a mipmap level 1 image of this tile + (deprecated, see mipMapLevels, but will override) --mipMapLevels: a list of MipMapLevel objects for this tile - --json: a json dictionary to initialize this object with (if not None overrides and ignores all keyword arguments) - + --json: a json dictionary to initialize this object with + (if not None overrides and ignores all keyword arguments) ''' def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, @@ -336,13 +348,14 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, raise RenderError(r.text) return TileSpec(json=tilespec_json['tileSpecs'][0]) + @renderaccess def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, - project=None, session=requests.session(), - render=None, **kwargs): + project=None, session=requests.session(), + render=None, **kwargs): '''renderapi call to get a specific tilespec by tileId note that this will return a tilespec without resolved transform references - + args: --stack: name of render stack to retrieve --tile: tileId of tile to retrieve @@ -366,6 +379,7 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, raise RenderError(r.text) return TileSpec(json=tilespec_json) + @renderaccess def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, scale=1.0, host=None, @@ -375,7 +389,7 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, '''renderapi call to get all tilespec that exist within a 2d bounding box specified with min and max x,y values note that this will return a tilespec with resolved transform references - + args: --stack: name of render stack to retrieve --z: z value of bounding box (float) @@ -390,7 +404,7 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, --session: sessions object to connect with (default make a new one) outputs: a list of TileSpec objects - ''' + ''' x = xmin y = ymin width = xmax - xmin @@ -409,7 +423,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, '''renderapi call to get all tilespec that exist within a 2d bounding box specified with min x,y values and width, height note that this will return a tilespec with resolved transform references - + args: --stack: name of render stack to retrieve --z: z value of bounding box (float) @@ -424,7 +438,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, --session: sessions object to connect with (default make a new one) outputs: a list of TileSpec objects - ''' + ''' request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( @@ -478,7 +492,7 @@ def get_tile_specs_from_stack(stack, host=None, port=None, session=requests.session(), render=None, **kwargs): '''get flat list of tilespecs for stack using i for sl in l for i in sl - input: + input: stack -- string render stack keyword arguments: render -- render connect object (or host, port, owner, project) From 996bb9ec78ea02c63ec803ee998ec125d6d8148b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 12 May 2017 15:40:48 -0700 Subject: [PATCH 356/766] added min and max intensity --- renderapi/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 14f0903a..0258b424 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -454,7 +454,8 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, @renderaccess -def renderSectionClient(stack, rootDirectory, zs, scale=None, format=None, +def renderSectionClient(stack, rootDirectory, zs, scale=None, + maxIntensity=None, minIntensity=None, format=None, doFilter=None, fillWithNoise=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, @@ -466,6 +467,8 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, format=None, ['--rootDirectory', rootDirectory] + get_param(scale, '--scale') + get_param(format, '--format') + get_param(doFilter, '--doFilter') + + get_param(minIntensity, '--minIntensity') + + get_param(maxIntensity, '--maxIntensity') + get_param(fillWithNoise, '--fillWithNoise') + zs) call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, From 2ac3632f5414306d484e62e0e7ed0cbc73976620 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 18 May 2017 15:21:42 -0700 Subject: [PATCH 357/766] transform: estimation for Rigid, Translation, and Similarity 2D with tests --- renderapi/transform.py | 195 +++++++++++++++++++++++++++++++---------- test/test_transform.py | 53 +++++++++++ 2 files changed, 202 insertions(+), 46 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index a40efd2b..f1f4aca1 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -80,9 +80,12 @@ def load_transform_json(d, default_type='leaf'): def load_leaf_json(d): handle_load_leaf = { - AffineModel.className: lambda x: AffineModel(json=d), + AffineModel.className: lambda x: AffineModel(json=x), Polynomial2DTransform.className: - lambda x: Polynomial2DTransform(json=d)} + lambda x: Polynomial2DTransform(json=x), + TranslationModel.className: lambda x: TranslationModel(json=x), + RigidModel.className: lambda x: RigidModel(json=x), + SimilarityModel.className: lambda x: SimilarityModel(json=x)} tform_type = d.get('type', 'leaf') if tform_type != 'leaf': @@ -242,13 +245,8 @@ def load_M(self): self.M[0, 2] = self.B0 self.M[1, 2] = self.B1 - def invert(self): - inv_M = np.linalg.inv(self.M) - Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], - inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) - return Ai - - def fit(self, A, B): + @staticmethod + def fit(A, B): if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): raise EstimationError( 'shape mismatch! A shape: {}, B shape {}'.format( @@ -267,10 +265,8 @@ def fit(self, A, B): (Tvec, residuals, rank, s) = np.linalg.lstsq(M, Y) return Tvec - def estimate(self, A, B): - Tvec = self.fit(A, B) - # t = numpy.array([Tvec[4,0],Tvec[5,0]]) - # R = numpy.array([[Tvec[0,0],Tvec[1,0]],[Tvec[2,0],Tvec[3,0]]]) + def estimate(self, A, B, return_params=True, **kwargs): + Tvec = self.fit(A, B, **kwargs) self.M00 = Tvec[0, 0] self.M10 = Tvec[2, 0] self.M01 = Tvec[1, 0] @@ -278,27 +274,8 @@ def estimate(self, A, B): self.B0 = Tvec[4, 0] self.B1 = Tvec[5, 0] self.load_M() - return self.M - - def convert_to_point_vector(self, points): - Np = points.shape[0] - onevec = np.ones((Np, 1), np.double) - - if points.shape[1] != 2: - raise ConversionError('Points must be of shape (:, 2) ' - '-- got {}'.format(points.shape)) - Nd = 2 - points = np.concatenate((points, onevec), axis=1) - return points, Nd - - def convert_points_vector_to_array(self, points, Nd): - points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T - return points - - def tform(self, points): - points, Nd = self.convert_to_point_vector(points) - pt = np.dot(self.M, points.T).T - return self.convert_points_vector_to_array(pt, Nd) + if return_params: + return self.M def concatenate(self, model): ''' @@ -324,6 +301,34 @@ def concatenate(self, model): newmodel = AffineModel(a00, a01, a10, a11, a02, a12) return newmodel + def invert(self): + inv_M = np.linalg.inv(self.M) + Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], + inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) + return Ai + + @staticmethod + def convert_to_point_vector(points): + Np = points.shape[0] + onevec = np.ones((Np, 1), np.double) + + if points.shape[1] != 2: + raise ConversionError('Points must be of shape (:, 2) ' + '-- got {}'.format(points.shape)) + Nd = 2 + points = np.concatenate((points, onevec), axis=1) + return points, Nd + + @staticmethod + def convert_points_vector_to_array(points, Nd): + points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T + return points + + def tform(self, points): + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(self.M, points.T).T + return self.convert_points_vector_to_array(pt, Nd) + def inverse_tform(self, points): points, Nd = self.convert_to_point_vector(points) pt = np.dot(np.linalg.inv(self.M), points.T).T @@ -356,6 +361,106 @@ def __str__(self): self.M[1, 1], self.M[0, 2], self.M[1, 2]) +class TranslationModel(AffineModel): + className = 'mpicbg.trakem2.transform.TranslationModel2D' + + def __init__(self, *args, **kwargs): + super(TranslationModel, self).__init__(*args, **kwargs) + # raise NotImplementedError( + # 'TranslationModel not implemented. please use Affine') + + def _process_dataString(self, dataString): + '''expected dataString is tx, ty''' + raise NotImplementedError + + @staticmethod + def fit(src, dst): + t = dst.mean(axis=0) - src.mean(axis=0) + T = np.eye(3) + T[:2, 2] = t + return T + + def estimate(self, src, dst, return_params=True): + self.M = self.fit(src, dst) + if return_params: + return self.M + + +class RigidModel(AffineModel): + className = 'mpicbg.trakem2.transform.RigidModel2D' + + def __init__(self, *args, **kwargs): + super(RigidModel, self).__init__(*args, **kwargs) + # raise NotImplementedError( + # 'RigidModel not implemented. please use Affine') + + def _process_dataString(self, dataString): + '''expected datastring is theta, tx, ty''' + raise NotImplementedError + + @staticmethod + def fit(src, dst, rigid=True): + ''' + Umeyama estimation of similarity transformation + ''' + # TODO shape assertion + num, dim = src.shape + src_cld = src - src.mean(axis=0) + dst_cld = dst - dst.mean(axis=0) + A = np.dot(dst_cld.T, src_cld) / num + d = np.ones((dim, ), dtype=np.double) + if np.linalg.det(A) < 0: + d[dim - 1] = -1 + T = np.eye(dim + 1, dtype=np.double) + + rank = np.linalg.matrix_rank(A) + if rank == 0: + raise EstimationError('zero rank matrix A unacceptable -- ' + 'likely poorly conditioned') + + U, S, V = svd(A) + + if rank == dim - 1: + if np.linalg.det(U) * np.linalg.det(V) > 0: + T[:dim, :dim] = np.dot(U, V) + else: + s = d[dim - 1] + d[dim - 1] = -1 + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V)) + d[dim - 1] = s + else: + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V.T)) + + fit_scale = (1.0 if rigid else + 1.0 / src_cld.var(axis=0).sum() * np.dot(S, d)) + + T[:dim, dim] = dst.mean(axis=0) - fit_scale * np.dot( + T[:dim, :dim], src.mean(axis=0).T) + T[:dim, :dim] *= fit_scale + return T + + def estimate(self, A, B, return_params=True, **kwargs): + self.M = self.fit(A, B, **kwargs) + if return_params: + return self.M + + +class SimilarityModel(RigidModel): + className = 'mpicbg.trakem2.transform.SimilarityModel2D' + + def __init__(self, *args, **kwargs): + super(SimilarityModel, self).__init__(*args, **kwargs) + # raise NotImplementedError( + # 'SimilarityModel not implemented. please use Affine') + + def _process_dataString(self, dataString): + '''expected datastring is s, theta, tx, ty''' + raise NotImplementedError + + def estimate(self, src, dst, **kwargs): + super(SimilarityModel, self).estimate(src, dst, rigid=False, **kwargs) + + class Polynomial2DTransform(Transform): ''' Polynomial2DTransform implemented as in skimage @@ -372,7 +477,6 @@ class Polynomial2DTransform(Transform): 2nd dataString = dataString representation of transform from mpicpg - 3rd identity = make this transform the identity @@ -387,7 +491,6 @@ class Polynomial2DTransform(Transform): TODO: fall back to Affine Model in special cases - robustness in estimation ''' className = 'mpicbg.trakem2.transform.PolynomialTransform2D' @@ -405,7 +508,7 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, elif params is not None: self.params = params elif src is not None and dst is not None: - self.params = self.estimate(src, dst, order, **kwargs) + self.estimate(src, dst, order, return_params=False, **kwargs) if not force_polynomial and self.is_affine: raise NotImplementedError('Falling back to Affine model is ' @@ -429,7 +532,6 @@ def dataString(self): @staticmethod def fit(src, dst, order=2): - '''This is unreliable -- add tests to ensure repeatability''' xs = src[:, 0] ys = src[:, 1] xd = dst[:, 0] @@ -458,19 +560,18 @@ def fit(src, dst, order=2): A[rows:, -1] = yd # right singular vector corresponding to smallest singular value - # TODO implement tests for this _, s, V = svd(A) Vsm = V[np.argmin(s), :] # never trust computers return (-Vsm[:-1] / Vsm[-1]).reshape((2, no_coeff // 2)) - # return (-V[-1, :-1] / V[-1, -1]).reshape((2, no_coeff // 2)) def estimate(self, src, dst, order=2, - convergence_test=None, max_tries=100, **kwargs): - def fitgood(src, dst, params): + test_coords=True, max_tries=100, return_params=True, + **kwargs): + def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): result = Polynomial2DTransform(params=params).tform(src) t = np.allclose( result, dst, - atol=1e-3, rtol=0) + atol=atol, rtol=rtol) return t estimated = False @@ -482,14 +583,16 @@ def fitgood(src, dst, params): except (LinAlgError, ValueError) as e: logger.debug('Encountered error {}'.format(e)) continue - estimated = fitgood(src, dst, params) + estimated = (fitgood(src, dst, params, **kwargs) if + test_coords else True) if tries == max_tries and not estimated: raise EstimationError('Could not fit Polynomial ' 'in {} attempts!'.format(tries)) logger.debug('fit parameters in {} attempts'.format(tries)) self.params = params - return self.params + if return_params: + return self.params @staticmethod def _dataStringfromParams(params=None): @@ -604,6 +707,6 @@ def flatten(l): if all([tform.className == AffineModel.className for tform in tforms]): am = AffineModel() - _ = am.estimate(A=src, B=dstpts) + am.estimate(A=src, B=dstpts, return_params=False) return am return Polynomial2DTransform(src=src, dst=dstpts, order=order) diff --git a/test/test_transform.py b/test/test_transform.py index 42c98c2b..99a0a288 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -301,3 +301,56 @@ def test_transformsum_identity_polynomial(): new_dstpts_comp = new_tform.tform(new_srcpts) new_dstpts = am2.concatenate(am1).tform(new_srcpts) assert np.allclose(new_dstpts_comp, new_dstpts) + + +def estimate_homography_transform( + do_scale=True, do_translate=True, do_rotate=True, transformclass=None): + scale = np.random.rand() + random_scale = (renderapi.transform.AffineModel( + M00=scale, M11=scale) + if do_scale else renderapi.transform.AffineModel()) + + random_translate = (renderapi.transform.AffineModel( + B0=np.random.rand(), B1=np.random.rand()) + if do_translate else renderapi.transform.AffineModel()) + + theta = np.random.rand() * 2 * np.pi + random_rotate = (renderapi.transform.AffineModel( + M00=np.cos(theta), M01=-np.sin(theta), + M10=np.sin(theta), M11=np.cos(theta)) + if do_rotate else renderapi.transform.AffineModel()) + + target_tform = random_translate.concatenate( + random_rotate.concatenate(random_scale)) + + src_pts = np.random.rand(50, 2) + dst_pts = target_tform.tform(src_pts) + tform = transformclass() + tform.estimate(src_pts, dst_pts, return_params=False) + + assert np.allclose(target_tform.M, tform.M) + if do_scale: + assert np.isclose(tform.scale[0], scale) + if do_translate: + assert np.allclose(tform.translation, random_translate.translation) + if do_rotate: + assert np.isclose(tform.rotation, random_rotate.rotation) + # currently forces as affines + am = renderapi.transform.AffineModel(json=tform.to_dict()) + assert am.to_dict() == tform.to_dict() + + +def test_estimate_similarity_transform(): + estimate_homography_transform( + transformclass=renderapi.transform.SimilarityModel) + + +def test_estimate_rigid_transform(): + estimate_homography_transform( + do_scale=False, transformclass=renderapi.transform.RigidModel) + + +def test_estimate_translation_transform(): + estimate_homography_transform( + do_scale=False, do_rotate=False, + transformclass=renderapi.transform.TranslationModel) From e64553658d85dbf0cc8bd529595f6e2cd2cc11bc Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 18 May 2017 15:26:17 -0700 Subject: [PATCH 358/766] client: add importTransformChanges, NotImplemented transformSection --- renderapi/client.py | 107 +++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index cb711aed..2eb53dce 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,6 +8,7 @@ import logging import subprocess import tempfile +from .errors import ClientScriptError from .utils import NullHandler, renderdump_temp from .render import RenderClient, renderaccess from .stack import set_stack_state, make_stack_params @@ -19,7 +20,12 @@ class WithPool(Pool): - '''pathos ProcessingPool with functioning __exit__ call''' + ''' + pathos ProcessingPool with functioning __exit__ call + usage: + with WithPool(*poolargs, **poolkwargs) as pool: + pool.map(*mapargs, **mapkwargs) + ''' def __init__(self, *args, **kwargs): super(WithPool, self).__init__(*args, **kwargs) @@ -77,7 +83,7 @@ def import_jsonfiles_and_transforms_parallel_by_z( client_scripts=client_scripts, host=host, port=port, owner=owner, project=project) with WithPool(poolsize) as pool: - rs = pool.map(partial_import, jsonfiles, transformfiles) + pool.map(partial_import, jsonfiles, transformfiles) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -362,11 +368,10 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, see render documentation (#add link here) ''' if outjson is None: - tempjson = tempfile.NamedTemporaryFile( - suffix=".json", mode='r', delete=False) - tempjson.close() + with tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) as f: + outjson = f.name delete_json = True - outjson = tempjson.name argvs = (make_stack_params(host, port, owner, project, stack) + get_param(baseowner, '--baseOwner') + @@ -400,31 +405,39 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, os.remove(outjson) return jsondata -# FIXME I do not know what input file this client takes -# @renderaccess -# def importTransformChangesClient(stack, targetStack, transformFile, -# targetOwner=None, targetProject=None, -# changeMode=None, subprocess_mode=None, -# host=None, port=None, owner=None, -# project=None, client_script=None, memGB=None, -# render=None, **kwargs): -# ''' -# run ImportTransformChangesClient.java -# ''' -# if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: -# raise ClientScriptError( -# 'changeMode {} is not valid!'.format(changeMode)) -# -# argvs = (make_stack_params(host, port, owner, project, stack) + -# ['--targetStack', targetStack] + -# ['--transformFile', transformFile] + -# get_param(targetOwner, '--targetOwner') + -# get_param(targetProject, '--targetProject') + -# get_param(changeMode, '--changeMode')) -# call_run_ws_client( -# 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, -# client_script=client_script, subprocess_mode=subprocess_mode, -# add_args=argvs) + +@renderaccess +def importTransformChangesClient(stack, targetStack, transformFile, + targetOwner=None, targetProject=None, + changeMode=None, close_stack=True, + subprocess_mode=None, + host=None, port=None, owner=None, + project=None, client_script=None, memGB=None, + render=None, **kwargs): + ''' + run ImportTransformChangesClient.java + transformFile: json file in format defined below + [{{"tileId": , + "transform": }}, + {{"tileId": ...}}, + ... + ] + ''' + if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: + raise ClientScriptError( + 'changeMode {} is not valid!'.format(changeMode)) + argvs = (make_stack_params(host, port, owner, project, stack) + + ['--targetStack', targetStack] + + ['--transformFile', transformFile] + + get_param(targetOwner, '--targetOwner') + + get_param(targetProject, '--targetProject') + + get_param(changeMode, '--changeMode')) + call_run_ws_client( + 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, + client_script=client_script, subprocess_mode=subprocess_mode, + add_args=argvs) + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) @renderaccess @@ -474,3 +487,35 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, format=None, call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) + + +@renderaccess +def transformSectionClient(stack, transformId, transformClass, transformData, + zValues, targetProject=None, targetStack=None, + replaceLast=None, subprocess_mode=None, + host=None, port=None, + owner=None, project=None, client_script=None, + memGB=None, render=None, **kwargs): + ''' + run TranformSectionClient.java + expects: + transformId -- string unique transform identifier + transformClass -- string representing mpicbg transform + transformData -- mpicbg datastring + zValues -- list of z values to apply tform + optional: + targetProject -- project to output the transformed sections + targetStack -- stack to ouput transformed sections + replaceLast -- bool whether to have transform replace + last in specList (default false) + ''' + raise NotImplementedError + argvs = (make_stack_params(host, port, owner, project, stack) + + (['--replaceLast'] if replaceLast else []) + + get_param(targetProject, '--targetProject') + + get_param(targetStack, '--targetStack') + + ['--transformId', transformId, '--transformClass', transformClass, + '--transformData', transformData] + zValues) + call_run_ws_client('org.janelia.render.client.TransformSectionClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs) From 2b7ed14b855683a526f04955fdf869b5106ecaef Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 18 May 2017 18:22:36 -0700 Subject: [PATCH 359/766] coordinate fixes, pointmatch Error handling, render connect tweaks --- renderapi/coordinate.py | 30 ++++++++++++++++++------------ renderapi/pointmatch.py | 11 +++++++++++ renderapi/render.py | 15 ++++++++++----- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index e0428d4f..a3037d96 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -5,12 +5,13 @@ from .render import format_preamble, renderaccess from .utils import NullHandler from .client import coordinateClient +from .errors import RenderError import requests import json import numpy as np import logging import tempfile - +import os logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -84,6 +85,7 @@ def local_to_world_coordinates_batch(stack, d, z, host=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) def package_point_match_data_into_json(dataarray, tileId, @@ -228,22 +230,21 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, def map_coordinates_clientside(stack, jsondata, z, host, port, owner, project, client_script, isLocalToWorld=False, + store_injson=False, store_outjson=False, number_of_threads=20, memGB='1G'): # write point match json to temp file on disk - json_infile, json_inpath = tempfile.mkstemp( - prefix='render_coordinates_in_', suffix='.json') - with open(json_inpath, 'w') as fp: + with tempfile.NamedTemporaryFile( + prefix='render_coordinates_in_', suffix='.json', + mode='w', delete=False) as f: logger.debug('jsondata:{}'.format(jsondata)) - # d = json.loads(jsondata) - json.dump(jsondata, fp) - # fp.write(jsondata) - - # json.dump(jsondata,open(json_inpath,'w')) + json_inpath = f.name + json.dump(jsondata, f) # get a temporary location for the output - json_outpath = tempfile.mktemp( - prefix='render_coordinates_out_', suffix='.json') - + with tempfile.NamedTemporaryFile( + prefix='render_coordinates_out_', suffix='.json', + delete=False) as f: + json_outpath = f.name # call the java client coordinateClient(stack, z, fromJson=json_inpath, toJson=json_outpath, localToWorld=isLocalToWorld, @@ -254,6 +255,11 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, # return the json results with open(json_outpath, 'r') as f: j = json.load(f) + if not store_injson: + os.remove(json_inpath) + if not store_outjson: + os.remove(json_outpath) + return j diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index f3add735..7ad7c727 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -24,6 +24,7 @@ def get_matchcollection_owners(host=None, port=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -37,6 +38,7 @@ def get_matchcollections(owner=None, host=None, port=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -51,6 +53,7 @@ def get_match_groupIds(matchCollection, owner=None, host=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -66,6 +69,7 @@ def get_matches_outside_group(matchCollection, groupId, owner=None, host=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -81,6 +85,7 @@ def get_matches_within_group(matchCollection, groupId, owner=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -97,6 +102,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -114,6 +120,7 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -129,6 +136,7 @@ def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -143,6 +151,7 @@ def get_match_groupIds_from_only(matchCollection, render=None, owner=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -157,6 +166,7 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -172,6 +182,7 @@ def get_matches_involving_tile(matchCollection, pGroupId, pTileId, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess diff --git a/renderapi/render.py b/renderapi/render.py index 774473fd..1e0fc5fb 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -3,8 +3,8 @@ import os from functools import wraps import requests -from .utils import defaultifNone, NullHandler, fitargspec, post_json, put_json -from .errors import ClientScriptError +from .utils import defaultifNone, NullHandler, fitargspec +from .errors import ClientScriptError, RenderError logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -103,7 +103,7 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, - force_http=True, validate_client=True, **kwargs): + force_http=True, validate_client=True, web_only=False, **kwargs): ''' helper function to connect to a render instance can default to using environment variables if not specified in call. @@ -129,6 +129,10 @@ def connect(host=None, port=None, owner=None, project=None, Can be set by environment variable RENDER_CLIENT_HEAP. force_http -- boolean determining whether to prepend 'http://' to render host + validate_client -- boolean whether to + validate existence of RenderClient run_ws_client.sh script + web_only -- boolean whether to check environment variables/prompt user + for client_scripts directory if not in arguments returns: RenderClient or Render object ''' @@ -172,8 +176,7 @@ def connect(host=None, port=None, owner=None, project=None, logger.critical('Render Owner must not be empty!') raise ValueError('Render Owner must not be empty!') - # TODO should client_scripts be required? - if client_scripts is None: + if client_scripts is None and not web_only: if 'RENDER_CLIENT_SCRIPTS' not in os.environ: client_scripts = str(raw_input( "Enter Render Client Scripts location: ")) @@ -262,6 +265,7 @@ def get_owners(host=None, port=None, session=requests.session(), except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess @@ -282,6 +286,7 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, except Exception as e: logger.error(e) logger.error(r.text) + raise RenderError(r.text) @renderaccess From ac0cc16559528065eababe8e808d186aa12892e6 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 18 May 2017 18:24:28 -0700 Subject: [PATCH 360/766] stack: add get_stack_tileIds --- renderapi/stack.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/renderapi/stack.py b/renderapi/stack.py index 6cf2ef41..0073a843 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -483,3 +483,18 @@ def get_section_z_value(stack, sectionId, host=None, port=None, logger.error(e) logger.error(r.text) raise RenderError(r.text) + + +@renderaccess +def get_stack_tileIds(stack, host=None, port=None, owner=None, project=None, + session=requests.session(), render=None, **kwargs): + '''get tileIds for a stack''' + request_url = '{}/tileIds'.format( + format_preamble(host, port, owner, project, stack)) + r = session.get(request_url) + try: + return r.json() + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError(r.text) From 4e6ef64e8d84f8f8a42fb3387f5baa074035c648 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 19 May 2017 02:59:59 -0700 Subject: [PATCH 361/766] transform: support Rigid, Similarity, Translation models from dataString --- renderapi/transform.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index f1f4aca1..07d08db4 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -370,8 +370,11 @@ def __init__(self, *args, **kwargs): # 'TranslationModel not implemented. please use Affine') def _process_dataString(self, dataString): - '''expected dataString is tx, ty''' - raise NotImplementedError + '''expected dataString is "tx ty"''' + tx, ty = map(float(dataString.split(' '))) + self.B0 = tx + self.B1 = ty + self.load_M() @staticmethod def fit(src, dst): @@ -395,8 +398,15 @@ def __init__(self, *args, **kwargs): # 'RigidModel not implemented. please use Affine') def _process_dataString(self, dataString): - '''expected datastring is theta, tx, ty''' - raise NotImplementedError + '''expected datastring is "theta tx ty"''' + theta, tx, ty = map(float(dataString.split(' '))) + self.M00 = np.cos(theta) + self.M01 = -np.sin(theta) + self.M10 = np.sin(theta) + self.M11 = np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() @staticmethod def fit(src, dst, rigid=True): @@ -454,8 +464,15 @@ def __init__(self, *args, **kwargs): # 'SimilarityModel not implemented. please use Affine') def _process_dataString(self, dataString): - '''expected datastring is s, theta, tx, ty''' - raise NotImplementedError + '''expected datastring is "s theta tx ty"''' + s, theta, tx, ty = map(float(dataString.split(' '))) + self.M00 = s * np.cos(theta) + self.M01 = -s * np.sin(theta) + self.M10 = s * np.sin(theta) + self.M11 = s * np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() def estimate(self, src, dst, **kwargs): super(SimilarityModel, self).estimate(src, dst, rigid=False, **kwargs) From 919add92b44a3c4c2fc0bbdb38ab14df741dfe3d Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 19 May 2017 03:32:24 -0700 Subject: [PATCH 362/766] client: add test for importTransformChangesClient, formatting --- integration_tests/test_client_integrated.py | 65 ++++++++++++++------- renderapi/client.py | 3 +- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index bf29d174..7c32fb2c 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -77,28 +77,30 @@ def test_import_jsonfiles_validate_client( renderapi.stack.delete_stack(stack, render=render) -def test_import_jsonfiles_parallel(render, - render_example_tilespec_and_transforms, - stack='test_import_jsonfiles_parallel', - poolsize = 5): +def test_import_jsonfiles_parallel( + render, render_example_tilespec_and_transforms, + stack='test_import_jsonfiles_parallel', poolsize=5): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files( render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles_parallel( - stack, tfiles, transformFile=transformFile, render=render, poolsize=poolsize) + stack, tfiles, transformFile=transformFile, + render=render, poolsize=poolsize) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) -def test_import_jsonfiles_parallel_multiple(render, - render_example_tilespec_and_transforms, - poolsize=5): - stacks = ['testmultiple1','testmultiple2','testmultiple3'] - mylist = range(10) - for stack in stacks: + +def test_import_jsonfiles_parallel_multiple( + render, render_example_tilespec_and_transforms, poolsize=5): + stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] + mylist = range(10) + for stack in stacks: with renderapi.client.WithPool(poolsize) as pool: - results=pool.map(lambda x:x**2,mylist) - test_import_jsonfiles_parallel(render,render_example_tilespec_and_transforms,stack,poolsize) + results = pool.map(lambda x: x**2, mylist) + test_import_jsonfiles_parallel( + render, render_example_tilespec_and_transforms, stack, poolsize) + def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, @@ -161,22 +163,45 @@ def test_renderSectionClient(render, teststack): pngfiles += [f for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) -# TODO importTransformChangesClient is not implemented + # def test_importTransformChangesClient(render, teststack, # render_example_tilespec_and_transforms): # (tilespecs, tforms) = render_example_tilespec_and_transforms # new_tforms = {} -# for tform in tforms: +# for tform in : # tformid = tform.transformId -# new_tforms[tformid] = AffineModel( +# new_tforms[tformid] = renderapi.transform.AffineModel( # B0=np.random.rand(), B1=np.random.rand()) -# transformFile = renderapi.utils.renderdump_temp(new_tforms.values()) -# +# transformFile = renderapi.utils.renderdump_temp(new_tforms) # renderapi.client.importTransformChangesClient(teststack, transformFile, # render=render) -# -# tilespecs = renderapi.tilespec.get_tile_specs_from_stack(stack) +# tilespecs = renderapi.tilespec.get_tile_specs_from_stack(teststack) # for ts in tilespecs: # for tf in ts.tforms: # if tf.transformId in new_tforms.keys(): # assert tf == new_tforms[tf.transformId] + +def test_importTransformChangesClient(render, teststack, + render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + deststack = 'test_stack_TCC' + + tform_to_append = renderapi.transform.AffineModel() + + TCCjson = renderapi.utils.renderdump_temp( + [{'tileId': tileId, 'transform': tform_to_append} + for tileId in renderapi.stack.get_stack_tileIds( + teststack, render=render)]) + renderapi.client.importTransformChangesClient( + teststack, deststack, TCCjson, changeMode='APPEND', render=render) + renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) + os.remove(TCCjson) + assert all([ts.tforms[-1].to_dict() == tform_to_append.to_dict() + for ts in renderapi.tilespec.get_tile_specs_from_stack( + deststack, render=render)]) + renderapi.stack.delete_stack(deststack, render=render) + + +def test_transformSectionClient(render, teststack, + render_example_tilespec_and_transforms): + raise NotImplementedError diff --git a/renderapi/client.py b/renderapi/client.py index b8232be8..f07e2a59 100644 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -471,7 +471,7 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, @renderaccess -def renderSectionClient(stack, rootDirectory, zs, scale=None, +def renderSectionClient(stack, rootDirectory, zs, scale=None, maxIntensity=None, minIntensity=None, format=None, doFilter=None, fillWithNoise=None, subprocess_mode=None, host=None, port=None, owner=None, @@ -512,7 +512,6 @@ def transformSectionClient(stack, transformId, transformClass, transformData, replaceLast -- bool whether to have transform replace last in specList (default false) ''' - raise NotImplementedError argvs = (make_stack_params(host, port, owner, project, stack) + (['--replaceLast'] if replaceLast else []) + get_param(targetProject, '--targetProject') + From a984b4af8e9a4a4e0702b90642ca16ffda3680fc Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 25 May 2017 13:27:36 -0700 Subject: [PATCH 363/766] stack: adding delete_tile by tileId --- renderapi/stack.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/renderapi/stack.py b/renderapi/stack.py index 0073a843..cc5be7a7 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -226,6 +226,25 @@ def delete_section(stack, z, host=None, port=None, owner=None, return r +@renderaccess +def delete_tile(stack, tileId, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + ''' + removes a tile from a stack + inputs: + stack -- stack from which to remove + tileId -- tile Id of tilespec to remove from stack + outputs: + r -- response from server + ''' + request_url = '{}/tile/{}'.format( + format_preamble(host, port, owner, project, stack), tileId) + r = session.delete(request_url) + logger.debug(r.text) + return r + + @renderaccess def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, From 574d9c37d2731bced53b601c792df82ec371d4d7 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 25 May 2017 13:30:18 -0700 Subject: [PATCH 364/766] tests: remove old transformchanges test --- integration_tests/test_client_integrated.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 7c32fb2c..e26dec5d 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -164,26 +164,7 @@ def test_renderSectionClient(render, teststack): assert len(pngfiles) == len(zvalues) -# def test_importTransformChangesClient(render, teststack, -# render_example_tilespec_and_transforms): -# (tilespecs, tforms) = render_example_tilespec_and_transforms -# new_tforms = {} -# for tform in : -# tformid = tform.transformId -# new_tforms[tformid] = renderapi.transform.AffineModel( -# B0=np.random.rand(), B1=np.random.rand()) -# transformFile = renderapi.utils.renderdump_temp(new_tforms) -# renderapi.client.importTransformChangesClient(teststack, transformFile, -# render=render) -# tilespecs = renderapi.tilespec.get_tile_specs_from_stack(teststack) -# for ts in tilespecs: -# for tf in ts.tforms: -# if tf.transformId in new_tforms.keys(): -# assert tf == new_tforms[tf.transformId] - -def test_importTransformChangesClient(render, teststack, - render_example_tilespec_and_transforms): - (tilespecs, tforms) = render_example_tilespec_and_transforms +def test_importTransformChangesClient(render, teststack): deststack = 'test_stack_TCC' tform_to_append = renderapi.transform.AffineModel() From 2296b5f052830d60f8f7bc29341bae3d237342d1 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 7 Jun 2017 11:43:25 -0700 Subject: [PATCH 365/766] transform: modify fit rather than estimate in SimilarityModel --- renderapi/transform.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 07d08db4..4ff886a0 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -4,12 +4,6 @@ Currently only implemented to facilitate Affine, Polynomial2D, and LensCorrection used in Khaled Khairy's EM aligner workflow -TODO: - interpolation functions - Affine as subset of Polynomial2D - approximation of other functions(TPS, meshtechniques) to Polynomial2D - ^ would this be better in Java using mpicbg implementation? - Allow reading datastring for Affine, Rigid, Translation into Affine ''' import json import logging @@ -409,7 +403,7 @@ def _process_dataString(self, dataString): self.load_M() @staticmethod - def fit(src, dst, rigid=True): + def fit(src, dst, rigid=True, **kwargs): ''' Umeyama estimation of similarity transformation ''' @@ -474,8 +468,12 @@ def _process_dataString(self, dataString): self.B1 = ty self.load_M() - def estimate(self, src, dst, **kwargs): - super(SimilarityModel, self).estimate(src, dst, rigid=False, **kwargs) + @staticmethod + def fit(src, dst, rigid=False, **kwargs): + return RigidModel.fit(src, dst, rigid=rigid) + + # def estimate(self, src, dst, **kwargs): + # super(SimilarityModel, self).estimate(src, dst, rigid=False, **kwargs) class Polynomial2DTransform(Transform): From 19564e09af47f67bea07f10a8aeeaed015b5c91e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 7 Jun 2017 12:15:43 -0700 Subject: [PATCH 366/766] image: getting images from ws raises rendererror on failure --- renderapi/image.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/renderapi/image.py b/renderapi/image.py index 1d41071e..e4381141 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -6,6 +6,7 @@ import numpy as np import logging from .render import format_preamble, renderaccess +from .errors import RenderError from .utils import NullHandler, jbool logger = logging.getLogger(__name__) @@ -69,6 +70,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, except Exception as e: logger.error(e) logger.error(r.text) + return RenderError(r.text) @renderaccess @@ -105,6 +107,7 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, scale=None, except Exception as e: logger.error(e) logger.error(r.text) + return RenderError(r.text) @renderaccess From fbe02df6f78a86ac2e0734741d7c1d49796d6c45 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 7 Jun 2017 12:16:55 -0700 Subject: [PATCH 367/766] utils: renderdum_temp now accepts args/kwargs --- renderapi/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index e860ed71..566b63fa 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -98,7 +98,7 @@ def renderdump(obj, *args, **kwargs): return json.dump(obj, *args, cls=cls_, **kwargs) -def renderdump_temp(obj): +def renderdump_temp(obj, *args, **kwargs): '''json.dump into a temporary file renderdump_temp(obj), obj will be dumped through renderdump into a temporary file @@ -106,7 +106,7 @@ def renderdump_temp(obj): with tempfile.NamedTemporaryFile( suffix=".json", mode='w', delete=False) as tf: tempfilename = tf.name - renderdump(obj, tf) + renderdump(obj, tf, *args, **kwargs) return tempfilename From 00b251ef2675c98938fd2828ae0029e5a8284c41 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Tue, 27 Jun 2017 08:56:23 -0700 Subject: [PATCH 368/766] fixed error handling in tilespec get_tilespec --- renderapi/tilespec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index d4edb20e..cdc33f0f 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -330,11 +330,12 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, r = session.get(request_url) try: tilespec_json = r.json() + return TileSpec(json=tilespec_json['tileSpecs'][0]) except Exception as e: logger.error(e) logger.error(r.text) raise RenderError(r.text) - return TileSpec(json=tilespec_json['tileSpecs'][0]) + @renderaccess def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, From d7ad711db925b2db5f7d7aac18aaab5ab4d8f9da Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 27 Jun 2017 11:38:38 -0700 Subject: [PATCH 369/766] trying to get develop merged --- renderapi/client.py | 1 + renderapi/image.py | 5 ++++- renderapi/tilespec.py | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) mode change 100644 => 100755 renderapi/client.py diff --git a/renderapi/client.py b/renderapi/client.py old mode 100644 new mode 100755 index 14f0903a..cdf0b83c --- a/renderapi/client.py +++ b/renderapi/client.py @@ -470,3 +470,4 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, format=None, call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) + \ No newline at end of file diff --git a/renderapi/image.py b/renderapi/image.py index 1d41071e..c1df307b 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -72,7 +72,8 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, @renderaccess -def get_tile_image_data(stack, tileId, normalizeForMatching=True, scale=None, +def get_tile_image_data(stack, tileId, normalizeForMatching=True, + removeAllOption=False, scale=None, filter=None, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): @@ -95,6 +96,8 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, scale=None, qparams['scale'] = scale if filter is not None: qparams['filter'] = jbool(filter) + if removeAllOption is not None: + qparams['removeAllOptions']=jbool(removeAllOption) logger.debug(request_url) r = session.get(request_url, params=qparams) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index d4edb20e..cfb07898 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -330,11 +330,13 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, r = session.get(request_url) try: tilespec_json = r.json() + return TileSpec(json=tilespec_json['tileSpecs'][0]) except Exception as e: logger.error(e) logger.error(r.text) raise RenderError(r.text) - return TileSpec(json=tilespec_json['tileSpecs'][0]) + return None + #return TileSpec(json=tilespec_json['tileSpecs'][0]) @renderaccess def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, From ef875e92ea27c409eab4369243cae5825ce3bd2a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 27 Jun 2017 11:39:51 -0700 Subject: [PATCH 370/766] adding ignore file changes --- .dockerignore | 1 + .gitignore | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6b8710a7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.git diff --git a/.gitignore b/.gitignore index 81b560a2..ec8f90cf 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ build/ *.egg-info *.egg dist/ -.DS_Store \ No newline at end of file +.DS_Store +.coverage From ae6c9548f5ee9cd89e11bc6c30ff4417edc97699 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 27 Jun 2017 11:40:12 -0700 Subject: [PATCH 371/766] ignore test reports --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ec8f90cf..f9180271 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build/ dist/ .DS_Store .coverage +test-reports/ \ No newline at end of file From 75a0f950716cd1d611fb3b0f4ae449cc25272373 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 27 Jun 2017 11:42:09 -0700 Subject: [PATCH 372/766] merging changes again --- renderapi/tilespec.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index d7ac7463..a8534128 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -335,12 +335,10 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, logger.error(e) logger.error(r.text) raise RenderError(r.text) -<<<<<<< HEAD + return None #return TileSpec(json=tilespec_json['tileSpecs'][0]) -======= - ->>>>>>> 00b251ef2675c98938fd2828ae0029e5a8284c41 + @renderaccess def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, From 3d979e2086b12d3b0c8b41ab4d6efcb6c8197bec Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 27 Jun 2017 15:11:09 -0700 Subject: [PATCH 373/766] removed odd HEAD --- renderapi/client.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 337a610b..c73a1669 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -490,8 +490,6 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) -<<<<<<< HEAD - @renderaccess def transformSectionClient(stack, transformId, transformClass, transformData, @@ -522,6 +520,3 @@ def transformSectionClient(stack, transformId, transformClass, transformData, call_run_ws_client('org.janelia.render.client.TransformSectionClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) -======= - ->>>>>>> develop From c016dee272bb72ec7e602d5939c835f07daef0b5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 27 Jun 2017 18:55:39 -0700 Subject: [PATCH 374/766] tests: sectiontransform test, test fixes --- integration_tests/test_client_integrated.py | 15 ++++++++++++++- renderapi/client.py | 2 +- renderapi/stack.py | 5 ++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index e26dec5d..e5261d67 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -185,4 +185,17 @@ def test_importTransformChangesClient(render, teststack): def test_transformSectionClient(render, teststack, render_example_tilespec_and_transforms): - raise NotImplementedError + deststack = 'test_stack_TSC' + transformId = 'TSC_testtransform' + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + tform = renderapi.transform.AffineModel() + + renderapi.client.transformSectionClient( + teststack, transformId, tform.className, + tform.dataString.replace(" ", ","), zvalues, targetStack=deststack, + render=render) + renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) + assert all([ts.tforms[-1].to_dict() == tform.to_dict() + for ts in renderapi.tilespec.get_tile_specs_from_stack( + deststack, render=render)]) + renderapi.stack.delete_stack(deststack, render=render) diff --git a/renderapi/client.py b/renderapi/client.py index c73a1669..abe6d2f0 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -503,7 +503,7 @@ def transformSectionClient(stack, transformId, transformClass, transformData, expects: transformId -- string unique transform identifier transformClass -- string representing mpicbg transform - transformData -- mpicbg datastring + transformData -- mpicbg datastring delimited by "," instead of " " zValues -- list of z values to apply tform optional: targetProject -- project to output the transformed sections diff --git a/renderapi/stack.py b/renderapi/stack.py index cc5be7a7..7d88ded6 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -6,6 +6,7 @@ from .utils import jbool, NullHandler, post_json, put_json from .render import (format_baseurl, format_preamble, renderaccess) +import json logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -512,7 +513,9 @@ def get_stack_tileIds(stack, host=None, port=None, owner=None, project=None, format_preamble(host, port, owner, project, stack)) r = session.get(request_url) try: - return r.json() + # FIXME render bug return non-json formatted answer + # return r.json() + return json.loads(r.text.replace("'", '"')) except Exception as e: logger.error(e) logger.error(r.text) From 4aebb37b595c52f25712dbacb9654182182f3521 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 27 Jun 2017 19:13:10 -0700 Subject: [PATCH 375/766] PEP8 --- renderapi/client.py | 1 + renderapi/image.py | 2 +- renderapi/tilespec.py | 4 ---- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index abe6d2f0..7f2459ed 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -491,6 +491,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) + @renderaccess def transformSectionClient(stack, transformId, transformClass, transformData, zValues, targetProject=None, targetStack=None, diff --git a/renderapi/image.py b/renderapi/image.py index 7789be2f..366a42c1 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -99,7 +99,7 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, if filter is not None: qparams['filter'] = jbool(filter) if removeAllOption is not None: - qparams['removeAllOptions']=jbool(removeAllOption) + qparams['removeAllOptions'] = jbool(removeAllOption) logger.debug(request_url) r = session.get(request_url, params=qparams) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index e021dbd7..f878335c 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -348,10 +348,6 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, logger.error(r.text) raise RenderError(r.text) - return None - #return TileSpec(json=tilespec_json['tileSpecs'][0]) - - @renderaccess def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, From cef91480032b78a38733e16949e7a1fc6ab49202 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 28 Jun 2017 12:25:34 -0700 Subject: [PATCH 376/766] fixed bug in removeAllOption --- renderapi/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/image.py b/renderapi/image.py index 366a42c1..f63b0189 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -99,7 +99,7 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, if filter is not None: qparams['filter'] = jbool(filter) if removeAllOption is not None: - qparams['removeAllOptions'] = jbool(removeAllOption) + qparams['removeAllOption'] = jbool(removeAllOption) logger.debug(request_url) r = session.get(request_url, params=qparams) From 620fe881281a8df6a4574364f4c22cb1844a6b38 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 29 Jun 2017 08:50:58 -0700 Subject: [PATCH 377/766] added method to get sectionId from a z value plus test --- integration_tests/test_stack_integrated.py | 5 +++++ renderapi/stack.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 1ea728da..2c9c03c8 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -414,3 +414,8 @@ def test_get_tile_specs_from_stack(render, teststack, (tilespecs, tforms) = render_example_tilespec_and_transforms ts = renderapi.tilespec.get_tile_specs_from_stack(teststack, render=render) assert len(ts) == len(tilespecs) + +def test_get_sectionId_for_z(render, teststack, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + sectionId = render.run(renderapi.stack.get_sectionId_for_z, teststack, tilespecs[0].z) + assert (sectionId == tilespecs[0].z) diff --git a/renderapi/stack.py b/renderapi/stack.py index 7d88ded6..fae7360b 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -443,6 +443,26 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, raise RenderError(r.text) +@renderaccess +def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + '''returns the sectionId associated with a particular z value + inputs: + stack -- name of the stack to get zvalues about + z -- z values to look for + render -- render connect object (or host, port, owner, project) + session -- options, requests.session + returns: + z values that have that has sectionId + ''' + sectionData=get_stack_sectionData(stack,host,port,owner,project,session) + try: + return next(sd['sectionId'] for sd in sectionData if sd['z']==z) + except: + raise RenderError('Could not find z value %f in stack %s'%(z,stack)) + + @renderaccess def get_stack_sectionData(stack, host=None, port=None, owner=None, project=None, session=requests.session(), From cf00e795cdad4a5e5823c79ed474356965bbc42a Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 29 Jun 2017 09:09:58 -0700 Subject: [PATCH 378/766] fixed bug in test --- integration_tests/test_stack_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 2c9c03c8..c22c210b 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -418,4 +418,4 @@ def test_get_tile_specs_from_stack(render, teststack, def test_get_sectionId_for_z(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms sectionId = render.run(renderapi.stack.get_sectionId_for_z, teststack, tilespecs[0].z) - assert (sectionId == tilespecs[0].z) + assert (sectionId == tilespecs[0].layout.sectionId) From 7416d2f323b9b32631a9f7a7e6ec2e62c28e6abf Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 29 Jun 2017 09:37:34 -0700 Subject: [PATCH 379/766] PEP8 and gitignore changes --- .gitignore | 6 +++--- renderapi/stack.py | 17 ++++++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index f9180271..497eb9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ -*.pyc +**/*.pyc *source_me.sh build/ .eggs* *.egg-info *.egg dist/ -.DS_Store +**/.DS_Store .coverage -test-reports/ \ No newline at end of file +test-reports/ diff --git a/renderapi/stack.py b/renderapi/stack.py index fae7360b..c02bdcd2 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -385,7 +385,8 @@ def get_z_value_for_section(stack, sectionId, **kwargs): # @renderaccess # def put_resolved_tilespecs(stack, json_dict, host=None, port=None, # owner=None, project=None, -# session=requests.session(), render=None, **kwargs): +# session=requests.session(), +# render=None, **kwargs): # request_url = format_preamble( # host, port, owner, project, stack) + "/resolvedTiles" # r = post_json(session, request_url, json_dict) @@ -456,13 +457,15 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, returns: z values that have that has sectionId ''' - sectionData=get_stack_sectionData(stack,host,port,owner,project,session) + sectionData = get_stack_sectionData( + stack, host, port, owner, project, session) try: - return next(sd['sectionId'] for sd in sectionData if sd['z']==z) - except: - raise RenderError('Could not find z value %f in stack %s'%(z,stack)) - - + return next(sd['sectionId'] for sd in sectionData if sd['z'] == z) + except Exception as e: + logger.error(e) + raise RenderError('Could not find z value %f in stack %s' % (z, stack)) + + @renderaccess def get_stack_sectionData(stack, host=None, port=None, owner=None, project=None, session=requests.session(), From d71f45a82e6b1118f9639c5fd1d0d091eb492b46 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Thu, 29 Jun 2017 20:59:37 -0700 Subject: [PATCH 380/766] changed version and added mergecollection to pointmatch --- renderapi/pointmatch.py | 37 ++++++++++++++++++++++++++++++------- setup.py | 2 +- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 7ad7c727..9938544e 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -57,12 +57,14 @@ def get_match_groupIds(matchCollection, owner=None, host=None, @renderaccess -def get_matches_outside_group(matchCollection, groupId, owner=None, host=None, +def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( owner, matchCollection, groupId) + request_url=add_merge_collections(request_url,mergeCollections) + r = session.get(request_url) try: return r.json() @@ -73,12 +75,14 @@ def get_matches_outside_group(matchCollection, groupId, owner=None, host=None, @renderaccess -def get_matches_within_group(matchCollection, groupId, owner=None, +def get_matches_within_group(matchCollection, groupId,mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( owner, matchCollection, groupId) + request_url=add_merge_collections(request_url,mergeCollections) + r = session.get(request_url) try: return r.json() @@ -89,13 +93,15 @@ def get_matches_within_group(matchCollection, groupId, owner=None, @renderaccess -def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, +def get_matches_from_group_to_group(matchCollection, pgroup, qgroup,mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( owner, matchCollection, pgroup, qgroup) + request_url=add_merge_collections(request_url,mergeCollections) + r = session.get(request_url) try: return r.json() @@ -104,16 +110,23 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, logger.error(r.text) raise RenderError(r.text) +def add_merge_collections(request_url,mergeCollections): + if mergeCollections is not None: + if type(mergeCollections) is list: + request_url+="?"+"&".join(mergeCollections) + return request_url @renderaccess def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, - qgroup, qid, render=None, owner=None, + qgroup, qid, mergeCollections=None,render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ ("/owner/%s/matchCollection/%s/group/%s/id/%s/" "matchesWith/%s/id/%s" % ( owner, matchCollection, pgroup, pid, qgroup, qid)) + request_url=add_merge_collections(request_url,mergeCollections) + r = session.get(request_url) try: return r.json() @@ -124,12 +137,15 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, @renderaccess -def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, +def get_matches_with_group(matchCollection, pgroup,mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( owner, matchCollection, pgroup) + request_url=add_merge_collections(request_url,mergeCollections) + + r = session.get(request_url) try: return r.json() @@ -140,11 +156,13 @@ def get_matches_with_group(matchCollection, pgroup, render=None, owner=None, @renderaccess -def get_match_groupIds_from_only(matchCollection, render=None, owner=None, +def get_match_groupIds_from_only(matchCollection, mergeCollections=None,render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) + request_url=add_merge_collections(request_url,mergeCollections) + r = session.get(request_url) try: return r.json() @@ -155,11 +173,13 @@ def get_match_groupIds_from_only(matchCollection, render=None, owner=None, @renderaccess -def get_match_groupIds_to_only(matchCollection, render=None, owner=None, +def get_match_groupIds_to_only(matchCollection, mergeCollections=None,render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) + request_url=add_merge_collections(request_url,mergeCollections) + r = session.get(request_url) try: return r.json() @@ -171,11 +191,14 @@ def get_match_groupIds_to_only(matchCollection, render=None, owner=None, @renderaccess def get_matches_involving_tile(matchCollection, pGroupId, pTileId, + mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/id/{}/".format( owner, matchCollection, pGroupId, pTileId) + request_url=add_merge_collections(request_url,mergeCollections) + r = session.get(request_url) try: return r.json() diff --git a/setup.py b/setup.py index af99635d..9cdc089a 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.0', + version='1.1', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From 119fb4043712be05b04187b99f1cde303acbceca Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 30 Jun 2017 07:14:12 -0700 Subject: [PATCH 381/766] fixed bug in mergecollections --- renderapi/pointmatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 9938544e..726a51a3 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -113,7 +113,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup,mergeCollect def add_merge_collections(request_url,mergeCollections): if mergeCollections is not None: if type(mergeCollections) is list: - request_url+="?"+"&".join(mergeCollections) + request_url+="?"+"&".join(['mergeCollection=%s'%mc for mc in mergeCollections]) return request_url @renderaccess From 856f464a11caf45cb47696ee75a47f0ee5bf66a1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 3 Jul 2017 10:36:37 -0700 Subject: [PATCH 382/766] added a render-parameters call for individual tiles --- renderapi/tilespec.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index f878335c..a88a6352 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -318,13 +318,11 @@ def __iter__(self): @renderaccess -def get_tile_spec(stack, tile, host=None, port=None, owner=None, +def get_tile_spec_renderparameters(stack, tile, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''renderapi call to get a specific tilespec by tileId - note that this will return a tilespec with resolved transform references - by accessing the render-parameters version of this tile - + '''renderapi call to get the render parameters of a specific tileId + args: --stack: name of render stack to retrieve --tile: tileId of tile to retrieve @@ -333,7 +331,7 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, (or host, port, owner, project) --session: sessions object to connect with (default make a new one) outputs: - A TileSpec object with dereferenced transforms + A render-parameters json with the tilespec for that tile and dereferenced transforms ''' request_url = format_preamble( @@ -342,12 +340,38 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, r = session.get(request_url) try: tilespec_json = r.json() - return TileSpec(json=tilespec_json['tileSpecs'][0]) + return tilespec_json except Exception as e: logger.error(e) logger.error(r.text) raise RenderError(r.text) +@renderaccess +def get_tile_spec(stack, tile, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + '''renderapi call to get a specific tilespec by tileId + note that this will return a tilespec with resolved transform references + by accessing the render-parameters version of this tile + + args: + --stack: name of render stack to retrieve + --tile: tileId of tile to retrieve + keyword args: + --render: render connect object + (or host, port, owner, project) + --session: sessions object to connect with (default make a new one) + outputs: + A TileSpec object with dereferenced transforms + ''' + + try: + tilespec_json = get_tile_spec_renderparameters(stack,tile,host,port,owner,project,session) + return TileSpec(json=tilespec_json['tileSpecs'][0]) + except Exception as e: + logger.error(e) + + @renderaccess def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, From 98d783e295adbd5e7a080e5863060a119bd3624b Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 5 Jul 2017 10:49:48 -0700 Subject: [PATCH 383/766] PEP8 --- renderapi/pointmatch.py | 47 ++++++++++++++++++++++++----------------- renderapi/tilespec.py | 19 ++++++++++------- renderapi/utils.py | 7 ++++++ setup.py | 2 +- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 726a51a3..9be4ef9f 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -57,13 +57,14 @@ def get_match_groupIds(matchCollection, owner=None, host=None, @renderaccess -def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, +def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, + owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( owner, matchCollection, groupId) - request_url=add_merge_collections(request_url,mergeCollections) + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: @@ -75,13 +76,14 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, o @renderaccess -def get_matches_within_group(matchCollection, groupId,mergeCollections=None, owner=None, - host=None, port=None, session=requests.session(), +def get_matches_within_group(matchCollection, groupId, mergeCollections=None, + owner=None, host=None, port=None, + session=requests.session(), render=None, **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( owner, matchCollection, groupId) - request_url=add_merge_collections(request_url,mergeCollections) + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: @@ -93,14 +95,15 @@ def get_matches_within_group(matchCollection, groupId,mergeCollections=None, own @renderaccess -def get_matches_from_group_to_group(matchCollection, pgroup, qgroup,mergeCollections=None, +def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, + mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( owner, matchCollection, pgroup, qgroup) - request_url=add_merge_collections(request_url,mergeCollections) + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: @@ -110,22 +113,26 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup,mergeCollect logger.error(r.text) raise RenderError(r.text) -def add_merge_collections(request_url,mergeCollections): + +def add_merge_collections(request_url, mergeCollections): if mergeCollections is not None: if type(mergeCollections) is list: - request_url+="?"+"&".join(['mergeCollection=%s'%mc for mc in mergeCollections]) + request_url += "?"+"&".join( + ['mergeCollection=%s' % mc for mc in mergeCollections]) return request_url + @renderaccess def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, - qgroup, qid, mergeCollections=None,render=None, owner=None, + qgroup, qid, mergeCollections=None, + render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ ("/owner/%s/matchCollection/%s/group/%s/id/%s/" "matchesWith/%s/id/%s" % ( owner, matchCollection, pgroup, pid, qgroup, qid)) - request_url=add_merge_collections(request_url,mergeCollections) + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: @@ -137,14 +144,14 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, @renderaccess -def get_matches_with_group(matchCollection, pgroup,mergeCollections=None, render=None, owner=None, +def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, + render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( owner, matchCollection, pgroup) - request_url=add_merge_collections(request_url,mergeCollections) - + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: @@ -156,12 +163,13 @@ def get_matches_with_group(matchCollection, pgroup,mergeCollections=None, render @renderaccess -def get_match_groupIds_from_only(matchCollection, mergeCollections=None,render=None, owner=None, +def get_match_groupIds_from_only(matchCollection, mergeCollections=None, + render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) - request_url=add_merge_collections(request_url,mergeCollections) + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: @@ -173,12 +181,13 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None,render=N @renderaccess -def get_match_groupIds_to_only(matchCollection, mergeCollections=None,render=None, owner=None, +def get_match_groupIds_to_only(matchCollection, mergeCollections=None, + render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) - request_url=add_merge_collections(request_url,mergeCollections) + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: @@ -197,7 +206,7 @@ def get_matches_involving_tile(matchCollection, pGroupId, pTileId, request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/id/{}/".format( owner, matchCollection, pGroupId, pTileId) - request_url=add_merge_collections(request_url,mergeCollections) + request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) try: diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index a88a6352..88d1153b 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -318,11 +318,12 @@ def __iter__(self): @renderaccess -def get_tile_spec_renderparameters(stack, tile, host=None, port=None, owner=None, - project=None, session=requests.session(), - render=None, **kwargs): +def get_tile_spec_renderparameters(stack, tile, host=None, port=None, + owner=None, project=None, + session=requests.session(), + render=None, **kwargs): '''renderapi call to get the render parameters of a specific tileId - + args: --stack: name of render stack to retrieve --tile: tileId of tile to retrieve @@ -331,7 +332,8 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, owner=None (or host, port, owner, project) --session: sessions object to connect with (default make a new one) outputs: - A render-parameters json with the tilespec for that tile and dereferenced transforms + A render-parameters json with the tilespec for that tile and + dereferenced transforms ''' request_url = format_preamble( @@ -346,6 +348,7 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, owner=None logger.error(r.text) raise RenderError(r.text) + @renderaccess def get_tile_spec(stack, tile, host=None, port=None, owner=None, project=None, session=requests.session(), @@ -364,13 +367,13 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, outputs: A TileSpec object with dereferenced transforms ''' - + try: - tilespec_json = get_tile_spec_renderparameters(stack,tile,host,port,owner,project,session) + tilespec_json = get_tile_spec_renderparameters( + stack, tile, host, port, owner, project, session) return TileSpec(json=tilespec_json['tileSpecs'][0]) except Exception as e: logger.error(e) - @renderaccess diff --git a/renderapi/utils.py b/renderapi/utils.py index 566b63fa..181c585d 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -49,6 +49,13 @@ def default(self, obj): def post_json(session, request_url, d, params=None): + ''' + POST requests with RenderError handling + session: requests session + request_url: url + d: data payload (will be json dumpsed) + params: requests parameters + ''' headers = {"content-type": "application/json"} if d is not None: payload = json.dumps(d) diff --git a/setup.py b/setup.py index 9cdc089a..af99635d 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.1', + version='1.0', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From 6157e8eff8e27a9e746d0e36c13e0f2e6a3c47e7 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:38:35 -0700 Subject: [PATCH 384/766] added documentation --- .gitignore | 1 + docs/Makefile | 20 +++++ docs/api/modules.rst | 7 ++ docs/api/renderapi.rst | 94 ++++++++++++++++++++++ docs/conf.py | 176 +++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 28 +++++++ 6 files changed, 326 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/api/modules.rst create mode 100644 docs/api/renderapi.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst diff --git a/.gitignore b/.gitignore index 497eb9ae..9ff0bb95 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ dist/ **/.DS_Store .coverage test-reports/ +docs/_build/ \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..7b7dda3a --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = render-python +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/api/modules.rst b/docs/api/modules.rst new file mode 100644 index 00000000..0695f8b4 --- /dev/null +++ b/docs/api/modules.rst @@ -0,0 +1,7 @@ +renderapi +========= + +.. toctree:: + :maxdepth: 4 + + renderapi diff --git a/docs/api/renderapi.rst b/docs/api/renderapi.rst new file mode 100644 index 00000000..15e1cd8f --- /dev/null +++ b/docs/api/renderapi.rst @@ -0,0 +1,94 @@ +renderapi package +================= + +Submodules +---------- + +renderapi\.client module +------------------------ + +.. automodule:: renderapi.client + :members: + :undoc-members: + :show-inheritance: + +renderapi\.coordinate module +---------------------------- + +.. automodule:: renderapi.coordinate + :members: + :undoc-members: + :show-inheritance: + +renderapi\.errors module +------------------------ + +.. automodule:: renderapi.errors + :members: + :undoc-members: + :show-inheritance: + +renderapi\.image module +----------------------- + +.. automodule:: renderapi.image + :members: + :undoc-members: + :show-inheritance: + +renderapi\.pointmatch module +---------------------------- + +.. automodule:: renderapi.pointmatch + :members: + :undoc-members: + :show-inheritance: + +renderapi\.render module +------------------------ + +.. automodule:: renderapi.render + :members: + :undoc-members: + :show-inheritance: + +renderapi\.stack module +----------------------- + +.. automodule:: renderapi.stack + :members: + :undoc-members: + :show-inheritance: + +renderapi\.tilespec module +-------------------------- + +.. automodule:: renderapi.tilespec + :members: + :undoc-members: + :show-inheritance: + +renderapi\.transform module +--------------------------- + +.. automodule:: renderapi.transform + :members: + :undoc-members: + :show-inheritance: + +renderapi\.utils module +----------------------- + +.. automodule:: renderapi.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: renderapi + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..acf0b2d1 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# +# render-python documentation build configuration file, created by +# sphinx-quickstart on Sat Jul 22 11:57:57 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'render-python' +copyright = u'2017, Forrest Collman, Russel Torres, Eric Perlman' +author = u'Forrest Collman, Russel Torres, Eric Perlman' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'1.0' +# The full version, including alpha/beta/rc tags. +release = u'1.0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'about.html', + 'navigation.html', + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + 'donate.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'render-pythondoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'render-python.tex', u'render-python Documentation', + u'Forrest Collman, Russel Torres, Eric Perlman', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'render-python', u'render-python Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'render-python', u'render-python Documentation', + author, 'render-python', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..6f49525e --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,28 @@ +.. render-python documentation master file, created by + sphinx-quickstart on Sat Jul 22 11:57:57 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to render-python's documentation! +========================================= + + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +api +--- +.. toctree:: + :maxdepth: 2 + + api/renderapi + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From 7f8c1789b80192d9c9085b36aee290b9a34374ad Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:38:47 -0700 Subject: [PATCH 385/766] added documentation requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 83589c97..6f82661f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ numpy pillow dill>=0.2.6 pathos +sphinxcontrib-napoleon \ No newline at end of file From a1ce66b1a8adcee05631c4c7ed0f676c181f41a6 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:39:06 -0700 Subject: [PATCH 386/766] altered docstrings of render and tilespec --- renderapi/render.py | 159 ++++++++++++++++++---- renderapi/tilespec.py | 298 +++++++++++++++++++++++------------------- 2 files changed, 298 insertions(+), 159 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 1e0fc5fb..b7f0bfc7 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -5,14 +5,16 @@ import requests from .utils import defaultifNone, NullHandler, fitargspec from .errors import ClientScriptError, RenderError - +from decorator import decorator logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) class Render(object): ''' - Render object to store connection settings for render server + Render object to store connection settings for render server. + Baseclass that doesn't require client_scripts definition for client side java processing. + See :func:`connect` for parameter definitions. ''' def __init__(self, host=None, port=None, owner=None, project=None, client_scripts=None): @@ -35,6 +37,9 @@ def DEFAULT_KWARGS(self): kwargs to which the render object falls back. Depends on: self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS + + Returns: + dict: default keyword arguments ''' return self.make_kwargs() @@ -43,6 +48,16 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, ''' make kwargs using this render object's defaults and any designated kwargs passed in + Args: + host (str or None): + port (int or None): + owner (str or None): + project (str or None): + client_scripts (str or None): + **kwargs: all other keyword arguments passed through + Returns: + dict: keyword arguments with missing host,port,owner,project,client_scripts + filled in with defaults ''' processed_kwargs = { 'host': self.DEFAULT_HOST if host is None else host, @@ -58,6 +73,18 @@ def run(self, f, *args, **kwargs): ''' run function from object technically shorter than adding render=Render to kwargs + + allows this syntax for running renderapi + + render = Render('server',8080) + metadata = render.run(renderapi.render.get_stack_metadata_by_owner,'myowner') + + Args: + f (func): renderapi function you want to call + *args: args passed to that function + **kwargs: kwargs passed to that function + Returns: + func: function with this :class:`Render` instance in keyword arguments as render= ''' # FIXME WARNING I think renderaccess can default to # another render if defined in args (test/squash) @@ -68,7 +95,8 @@ def run(self, f, *args, **kwargs): class RenderClient(Render): ''' Render object to run java client commands via a wrapped client script. - This is a work in progress. + This is a work in progress. + Should use :func:`connect` to create and for documentation of parameters. ''' def __init__(self, client_script=None, memGB=None, validate_client=True, *args, **kwargs): @@ -91,6 +119,14 @@ def __init__(self, client_script=None, memGB=None, validate_client=True, self.memGB = memGB def make_kwargs(self, *args, **kwargs): + '''method to fill in default properties of RenderClient object + + Args: + *args: args used to initialize RenderClient + **kwargs: kwargs used to initialize RenderClient + Returns: + Render: Render instance with default values filled in + ''' # hack to get dictionary defaults to work client_script = defaultifNone( kwargs.pop('client_script', None), self.client_script) @@ -105,36 +141,38 @@ def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, force_http=True, validate_client=True, web_only=False, **kwargs): ''' - helper function to connect to a render instance - can default to using environment variables if not specified in call. - input: - host -- string hostname for target render server -- will prepend + helper function to create a :class:`RenderClient` instance. + Will default to using environment variables if not specified in call, + and prompt user for any parameters that are not given. + + Args: + host (str): hostname for target render server -- will prepend "http://" if not found and force_http keyword evaluates True. Can be set by environment variable RENDER_HOST. - port -- string, int, or None port for target render server. + port (str, int, or None): port for target render server. Optional as in 'http://hostname[:port]'. Can be set by environment variable RENDER_PORT. - owner -- string owner for render-ws. + owner (str): owner for render-ws. Can be set by environment variable RENDER_OWNER. - project -- string project for render webservice. + project (str): project for render webservice. Can be set by environment variable RENDER_PROJECT. - client_scripts -- string specifying directory path + client_scripts (str): directory path for render-ws-java-client scripts. Can be set by environment variable RENDER_CLIENT_SCRIPTS. - client_script -- string path to a wrapper for java client classes. + client_script (str): path to a wrapper for java client classes. Used only in RenderClient. Can be set by environment variable RENDER_CLIENT_SCRIPT. - memGB -- string specifying heap size in GB for java client scripts, + memGB (str):heap size in GB for java client scripts, example for 1 GB: '1G'. Used only in RenderClient. Can be set by environment variable RENDER_CLIENT_HEAP. - force_http -- boolean determining whether to prepend + force_http (bool): whether to prepend 'http://' to render host - validate_client -- boolean whether to + validate_client (bool): whether to validate existence of RenderClient run_ws_client.sh script - web_only -- boolean whether to check environment variables/prompt user + web_only (bool): whether to check environment variables/prompt user for client_scripts directory if not in arguments - returns: - RenderClient or Render object + Returns: + RenderClient: a connect object to simplify specifying what render server to connect to ''' if host is None: if 'RENDER_HOST' not in os.environ: @@ -213,11 +251,27 @@ def connect(host=None, port=None, owner=None, project=None, return Render(host=host, port=port, owner=owner, project=project, client_scripts=client_scripts) - +@decorator def renderaccess(f): ''' decorator allowing functions asking for host, port, owner, project - to default to a connection defined by a render object kwarg + to default to a connection defined by a :class:`Render` object + using its :func:`RenderClient.make_kwargs` method. + + Example: + render = renderapi.render.connect('server',80,'me','my_project', + client_scripts='/usr/local/render/render-ws/src/scripts') + owners = renderapi.render.get_stacks_by_owner_project(render=render) + + this works because renderapi.render.get_stacks_by_owner_project has the + renderaccess decorator to fill in the default values for + host, owner, port and project. + + you can if you wish specify any of the arguments, in which case they will not + be filled in by the default values, but you don't have to. + + As such, the documentation omits describing the parameters which are natural + to expect will be filled in by the renderaccess decorator. ''' @wraps(f) def wrapper(*args, **kwargs): @@ -236,7 +290,14 @@ def wrapper(*args, **kwargs): def format_baseurl(host, port): - '''format host and port to a standard template render-ws url''' + '''format host and port to a standard template render-ws url + + Args: + host (str):host of render server + port (int or None): port of render server + Returns + str: a url to the render endpoint at that host/port combination (append render-ws/v1) + ''' # return 'http://%s:%d/render-ws/v1' % (host, port) server = '{}{}'.format(host, ('' if port is None else ':{}'.format(port))) return '{}/render-ws/v1'.format(server) @@ -246,6 +307,15 @@ def format_preamble(host, port, owner, project, stack): ''' format host, port, owner, project, and stack parameters to the access point to stack-based apis + + Args: + host (str): render host + port (int): render host port + owner (str): render owner + project (str): render project + stack (str): render stack + Returns: + str: a url to the endpoint for that host, port, owner, project, stack combination ''' preamble = "%s/owner/%s/project/%s/stack/%s" % ( format_baseurl(host, port), owner, project, stack) @@ -257,6 +327,17 @@ def get_owners(host=None, port=None, session=requests.session(), render=None, **kwargs): ''' return list of owners across all Projects and Stacks for a render server + + :func:`renderaccess` decorated function + + Args: + host (str): render host (defaults to host from render) + port (int): render port (default to port from render) + session (requests.Session): requests session + render (RenderClient): RenderClient connection object + Returns: + list: list of strings containing all render owners + ''' request_url = "%s/owners/" % format_baseurl(host, port) r = session.get(request_url) @@ -275,7 +356,17 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, ''' return metadata for all stacks belonging to particular owner on render server - TODO example + + :func:`renderaccess` decorated function + + Args: + owner (str): render owner + render (RenderClient): render connect object + session (requests.sessions.Session): http session to use + Returns: + dict: stackInfo metadata, TODO example + + ''' request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) @@ -292,7 +383,18 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, @renderaccess def get_projects_by_owner(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''return list of projects belonging to a single owner for render stack''' + '''return list of projects belonging to a single owner for render stack + + :func:`renderaccess` decorated function + + Args: + owner (str): render owner + render (RenderClient): render connect object + session (requests.sessions.Session): http session to use + Returns: + list[str]: render projects by this owner + + ''' metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) projects = list(set([m['stackId']['project'] for m in metadata])) @@ -305,6 +407,17 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, render=None, **kwargs): ''' return list of stacks belonging to an owner's project on render server + + :func:`renderaccess` decorated function + + Args: + owner (str): render owner + project (str): render project + render (RenderClient): render connect object + session (requests.sessions.Session): http session to use + Returns: + list[str]: render stacks by this owner in this project + ''' metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 88d1153b..c89f17f9 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -15,19 +15,19 @@ class Layout: '''Layout class to describe acquisition settings inputs: - keyword arguments - --sectionId: unique string to describe sectionId this tile was taken from - --scopeId: string to track what microscope this came from - --cameraId: string to track what camera this was taken with - --imageRow: integer to track what row from a row,col layout this was taken - --imageCol: integer to track what column form a - row,col layout this was taken - --stageX: X stage coordinates (float) for where this was taken - --stageY: Y stage coordinates (float) for where this taken from - --rotation: angle of camera when this was taken - --pixelsize: size of pixels in units of choice of camera - at the magnification it was taken - --force_pixelsize: whether to default pixelsize to 0.1 (default True) + + Args: + sectionId (str): sectionId this tile was taken from + scopeId (str): what microscope this came from + cameraId (str): camera this was taken with + imageRow (int): what row from a row,col layout this was taken + imageCol (int): column from a row,col layout this was taken + stageX (float): X stage coordinates for where this was taken + stageY (float): Y stage coordinates for where this taken + rotation (float): angle of camera when this was taken + pixelsize (float): effective size of pixels (in units of choice) + force_pixelsize (bool): whether to default pixelsize to 0.1 + ''' def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, @@ -47,9 +47,8 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, def to_dict(self): '''return a dictionary representation of this object - no inputs - returns: - json dictionary of object + Returns: + dict: json compatible dictionary of this object ''' d = {} d['sectionId'] = self.sectionId @@ -66,8 +65,9 @@ def to_dict(self): def from_dict(self, d): '''set this object equal to the fields found in dictionary - inputs: - --d:dictionary to use to update + + Args: + d (dict): dictionary to use to update ''' if d is not None: self.sectionId = d.get('sectionId') @@ -84,36 +84,38 @@ def from_dict(self, d): class TileSpec: '''Fundamental class of render that store image tiles and their transformations init: - Keyword arguments: - --tileId: unique string specifying a tile's identity - --z: z values this tile exists within (float) - --width: width in pixels of the raw tile - --height: height in pixels of the raw tile - --imageUrl: an image path that can be accessed by the render server, - with an ImageJ.open command or as an s3 url. - Files on disk should be specified with file: - --maskUrl: an image path that can be accessed by the render server - which can be interpreted as an - alpha mask for the image (same as spec imageUrl) - --minint: pixel intensity value to display as black in a - linear colormap (default 0) - --maxint: pixel intensity value to display as white in a - linear colormap (default 65535) - --layout: a Layout object for this tile - --tforms: a list of Transform objects - (see AffineModel, TransformList, Polynomial2DTransform, - Transform, ReferenceTransform) to apply to this tile - --inputfilters: a list of filters to apply to - this tile (not yet implemented) - --scale3Url: url of a mipmap level 3 image of this tile - (deprecated, see mipMapLevels, but will override) - --scale2Url: url of a mipmap level 2 image of this tile - (deprecated, see mipMapLevels, but will override) - --scale1Url: url of a mipmap level 1 image of this tile - (deprecated, see mipMapLevels, but will override) - --mipMapLevels: a list of MipMapLevel objects for this tile - --json: a json dictionary to initialize this object with - (if not None overrides and ignores all keyword arguments) + + Args: + tileId (str): unique string specifying a tile's identity + z (float): z values this tile exists within + width (int): width in pixels of the raw tile + height (int): height in pixels of the raw tile + imageUrl (str): an image path that can be accessed by the render server, + with an ImageJ.open command or as an s3 url. + Files on disk should be specified with file: + maskUrl (str): an image path that can be accessed by the render server + which can be interpreted as an + alpha mask for the image (same as spec imageUrl) + minint (int): pixel intensity value to display as black in a + linear colormap (default 0) + maxint (int): pixel intensity value to display as white in a + linear colormap (default 65535) + layout (Layout): a Layout object for this tile + tforms (list[Transform]): Transform objects + (see :class:`AffineModel`, :class:`TransformList`, :class:`Polynomial2DTransform`, + :class:`Transform`, :class:`ReferenceTransform`) to apply to this tile + inputfilters (list): a list of filters to apply to + this tile (not yet implemented) + mipMapLevels (list[MipMapLevel]): :class:`MipMapLevel` objects for this tile + json (dict or None): dictionary to initialize this object with + (if not None overrides and ignores all keyword arguments) + scale3Url (str): url of a mipmap level 3 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + scale2Url (str): url of a mipmap level 2 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + scale1Url (str): url of a mipmap level 1 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + ''' def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, @@ -164,6 +166,9 @@ def bbox(self): def to_dict(self): '''method to produce a json tilespec for this tile returns a json compatible dictionary + + Returns: + dict: json compatible dictionary representation of this object ''' thedict = {} thedict['tileId'] = self.tileId @@ -203,7 +208,11 @@ def to_dict(self): return thedict def from_dict(self, d): - '''Method to load tilespec from json dictionary''' + '''Method to load tilespec from json dictionary + + Args: + d (dict): dictionary to use to set properties of this object + ''' self.tileId = d['tileId'] self.z = d['z'] self.width = d['width'] @@ -241,10 +250,10 @@ class MipMapLevel: MipMapLevel class to represent a level of an image pyramid. Can be put in dictionary formatting using dict(mML) - init: - level -- integer level of 2x downsampling represented by mipmaplevel - imageUrl (optional) -- url corresponding to image - maskUrl (optional) -- url corresponding to mask + Args: + level (int): level of 2x downsampling represented by mipmaplevel + imageUrl (str or None): url corresponding to image + maskUrl (str or None): url corresponding to mask ''' def __init__(self, level, imageUrl=None, maskUrl=None): self.level = level @@ -273,43 +282,54 @@ class ImagePyramid: of an image at level 0 Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) - init: - mipMapLevels -- list of MipMapLevel objects - append: - adds MipmapLevel without checking if it exists - input: MipMapLevel object - update: - adds MipMapLevel object replacing a corresponding level if it exists - input: MipMapLevel object - to_ordered_dict: - input: key(optional) -- key to sort ordered dictionary - default sort by level via lambda x: x[0] + Args: + mipMapLevels (list[MipMapLevel]): list of :class:`MipMapLevel` objects + ''' def __init__(self, mipMapLevels=[]): self.mipMapLevels = mipMapLevels def to_dict(self): + '''return dictionary representation of this object''' return dict(self.__iter__()) def to_ordered_dict(self, key=None): - '''defaults to order by mipmapLevel''' + '''returns :class:`OrderedDict` represention of this object, ordered by mipmapLevel''' return OrderedDict(sorted( self.__iter__(), key=((lambda x: x[0]) if key is None else key))) def append(self, mmL): + '''appends a MipMapLevel to this ImagePyramid + + Args: + mml (MipMapLevel): MipMapLevel to append + ''' self.mipMapLevels.append(mmL) def update(self, mmL): + '''updates the ImagePyramid with this MipMapLevel + + Args: + mml (MipMapLevel): mipmap level to update in pyramid + ''' self.mipMapLevels = [ l for l in self.mipMapLevels if l.level != mmL.level] self.append(mmL) def get(self, to_get): + '''gets a specific mipmap level in dictionary form + + Args: + to_get (int): mipmaplevel to get + Returns: + dict: dictionary representation of requested MipMapLevel + ''' return self.to_dict()[to_get] # TODO should this default @property def levels(self): + '''list of MipMapLevels in this ImagePyramid''' return [int(i.level) for i in self.mipMapLevels] def __iter__(self): @@ -324,16 +344,16 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, render=None, **kwargs): '''renderapi call to get the render parameters of a specific tileId - args: - --stack: name of render stack to retrieve - --tile: tileId of tile to retrieve - keyword args: - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - A render-parameters json with the tilespec for that tile and - dereferenced transforms + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + tile (str): tileId of tile to retrieve + render (renderapi.render.RenderClient): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + dict: render-parameters json with the tilespec for that tile and dereferenced transforms + TODO provide example ''' request_url = format_preamble( @@ -357,15 +377,15 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, note that this will return a tilespec with resolved transform references by accessing the render-parameters version of this tile - args: - --stack: name of render stack to retrieve - --tile: tileId of tile to retrieve - keyword args: - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - A TileSpec object with dereferenced transforms + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + tile (str): tileId of tile to retrieve + render (renderapi.render.RenderClient): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + TileSepc: TileSpec with dereferenced transforms ''' try: @@ -383,15 +403,15 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, '''renderapi call to get a specific tilespec by tileId note that this will return a tilespec without resolved transform references - args: - --stack: name of render stack to retrieve - --tile: tileId of tile to retrieve - keyword args: - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - a TileSpec object with referenced transforms if present + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + tile (str): tileId of tile to retrieve + render (renderapi.render.RenderClient): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + TileSepc: TileSpec with referenced transforms intact ''' request_url = format_preamble( @@ -417,20 +437,20 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, specified with min and max x,y values note that this will return a tilespec with resolved transform references - args: - --stack: name of render stack to retrieve - --z: z value of bounding box (float) - --xmin: minimum x value (float) - --ymin: minimum y value (float) - --xmax: maximum x value (float) - --ymax: maximum y value (float) - keyword args: - --scale: scale to use when retrieving render parameters (not important) - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - a list of TileSpec objects + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + z (float): z value of bounding box + xmin (float): minimum x value + ymin (float): minimum y value + xmax (float): maximum x value + ymax (float): maximum y value + scale (float): scale to use when retrieving render parameters (not important) + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]:TileSpec objects with dereferenced tansforms ''' x = xmin y = ymin @@ -448,23 +468,23 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, project=None, session=requests.session(), render=None, **kwargs): '''renderapi call to get all tilespec that exist within a 2d bounding box - specified with min x,y values and width, height + specified with min x,y values and width, height note that this will return a tilespec with resolved transform references - args: - --stack: name of render stack to retrieve - --z: z value of bounding box (float) - --xmin: minimum x value (float) - --ymin: minimum y value (float) - --xmax: maximum x value (float) - --ymax: maximum y value (float) - keyword args: - --scale: scale to use when retrieving render parameters (not important) - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - a list of TileSpec objects + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + z (float): z value of bounding box + x (float): minimum x value + y (float): minimum y value + width (float): width of box (in scale=1.0 units) + height (float): height of box (in scale=1.0 units) + scale (float): scale to use when retrieving render parameters (not important) + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]:TileSpec objects with dereferenced tansforms ''' request_url = format_preamble( host, port, owner, project, stack) + \ @@ -487,13 +507,17 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): ''' - input: - stack -- string render stack - z -- render z - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - output: list of TileSpec objects from that stack at that z + Get all TileSpecs in a specific z values. Returns referenced transforms. + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): render stack + z (float): render z + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]: list of TileSpec objects from that stack at that z ''' request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) @@ -519,13 +543,15 @@ def get_tile_specs_from_stack(stack, host=None, port=None, session=requests.session(), render=None, **kwargs): '''get flat list of tilespecs for stack using i for sl in l for i in sl - input: - stack -- string render stack - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - output: - List of TileSpec objects from the stack + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): render stack + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]: list of TileSpec objects from that stack ''' return [i for sl in [ get_tile_specs_from_z(stack, z, host=host, port=port, From 7b1a4537dd530fd79da76680df85abdc30f93b3f Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:38:35 -0700 Subject: [PATCH 387/766] added documentation --- .gitignore | 1 + docs/Makefile | 20 +++++ docs/api/modules.rst | 7 ++ docs/api/renderapi.rst | 94 ++++++++++++++++++++++ docs/conf.py | 176 +++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 28 +++++++ 6 files changed, 326 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/api/modules.rst create mode 100644 docs/api/renderapi.rst create mode 100644 docs/conf.py create mode 100644 docs/index.rst diff --git a/.gitignore b/.gitignore index 497eb9ae..9ff0bb95 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ dist/ **/.DS_Store .coverage test-reports/ +docs/_build/ \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..7b7dda3a --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = render-python +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/api/modules.rst b/docs/api/modules.rst new file mode 100644 index 00000000..0695f8b4 --- /dev/null +++ b/docs/api/modules.rst @@ -0,0 +1,7 @@ +renderapi +========= + +.. toctree:: + :maxdepth: 4 + + renderapi diff --git a/docs/api/renderapi.rst b/docs/api/renderapi.rst new file mode 100644 index 00000000..15e1cd8f --- /dev/null +++ b/docs/api/renderapi.rst @@ -0,0 +1,94 @@ +renderapi package +================= + +Submodules +---------- + +renderapi\.client module +------------------------ + +.. automodule:: renderapi.client + :members: + :undoc-members: + :show-inheritance: + +renderapi\.coordinate module +---------------------------- + +.. automodule:: renderapi.coordinate + :members: + :undoc-members: + :show-inheritance: + +renderapi\.errors module +------------------------ + +.. automodule:: renderapi.errors + :members: + :undoc-members: + :show-inheritance: + +renderapi\.image module +----------------------- + +.. automodule:: renderapi.image + :members: + :undoc-members: + :show-inheritance: + +renderapi\.pointmatch module +---------------------------- + +.. automodule:: renderapi.pointmatch + :members: + :undoc-members: + :show-inheritance: + +renderapi\.render module +------------------------ + +.. automodule:: renderapi.render + :members: + :undoc-members: + :show-inheritance: + +renderapi\.stack module +----------------------- + +.. automodule:: renderapi.stack + :members: + :undoc-members: + :show-inheritance: + +renderapi\.tilespec module +-------------------------- + +.. automodule:: renderapi.tilespec + :members: + :undoc-members: + :show-inheritance: + +renderapi\.transform module +--------------------------- + +.. automodule:: renderapi.transform + :members: + :undoc-members: + :show-inheritance: + +renderapi\.utils module +----------------------- + +.. automodule:: renderapi.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: renderapi + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..acf0b2d1 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# +# render-python documentation build configuration file, created by +# sphinx-quickstart on Sat Jul 22 11:57:57 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'render-python' +copyright = u'2017, Forrest Collman, Russel Torres, Eric Perlman' +author = u'Forrest Collman, Russel Torres, Eric Perlman' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'1.0' +# The full version, including alpha/beta/rc tags. +release = u'1.0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'about.html', + 'navigation.html', + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + 'donate.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'render-pythondoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'render-python.tex', u'render-python Documentation', + u'Forrest Collman, Russel Torres, Eric Perlman', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'render-python', u'render-python Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'render-python', u'render-python Documentation', + author, 'render-python', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..6f49525e --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,28 @@ +.. render-python documentation master file, created by + sphinx-quickstart on Sat Jul 22 11:57:57 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to render-python's documentation! +========================================= + + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +api +--- +.. toctree:: + :maxdepth: 2 + + api/renderapi + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From e3b9ca28f9cb266203ba4503cfa1f98b9ab97db9 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:38:47 -0700 Subject: [PATCH 388/766] added documentation requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 83589c97..6f82661f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ numpy pillow dill>=0.2.6 pathos +sphinxcontrib-napoleon \ No newline at end of file From be1986098f0223e673a169396c65ac72fc29478a Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:39:06 -0700 Subject: [PATCH 389/766] altered docstrings of render and tilespec --- renderapi/render.py | 159 ++++++++++++++++++---- renderapi/tilespec.py | 298 +++++++++++++++++++++++------------------- 2 files changed, 298 insertions(+), 159 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 1e0fc5fb..b7f0bfc7 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -5,14 +5,16 @@ import requests from .utils import defaultifNone, NullHandler, fitargspec from .errors import ClientScriptError, RenderError - +from decorator import decorator logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) class Render(object): ''' - Render object to store connection settings for render server + Render object to store connection settings for render server. + Baseclass that doesn't require client_scripts definition for client side java processing. + See :func:`connect` for parameter definitions. ''' def __init__(self, host=None, port=None, owner=None, project=None, client_scripts=None): @@ -35,6 +37,9 @@ def DEFAULT_KWARGS(self): kwargs to which the render object falls back. Depends on: self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS + + Returns: + dict: default keyword arguments ''' return self.make_kwargs() @@ -43,6 +48,16 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, ''' make kwargs using this render object's defaults and any designated kwargs passed in + Args: + host (str or None): + port (int or None): + owner (str or None): + project (str or None): + client_scripts (str or None): + **kwargs: all other keyword arguments passed through + Returns: + dict: keyword arguments with missing host,port,owner,project,client_scripts + filled in with defaults ''' processed_kwargs = { 'host': self.DEFAULT_HOST if host is None else host, @@ -58,6 +73,18 @@ def run(self, f, *args, **kwargs): ''' run function from object technically shorter than adding render=Render to kwargs + + allows this syntax for running renderapi + + render = Render('server',8080) + metadata = render.run(renderapi.render.get_stack_metadata_by_owner,'myowner') + + Args: + f (func): renderapi function you want to call + *args: args passed to that function + **kwargs: kwargs passed to that function + Returns: + func: function with this :class:`Render` instance in keyword arguments as render= ''' # FIXME WARNING I think renderaccess can default to # another render if defined in args (test/squash) @@ -68,7 +95,8 @@ def run(self, f, *args, **kwargs): class RenderClient(Render): ''' Render object to run java client commands via a wrapped client script. - This is a work in progress. + This is a work in progress. + Should use :func:`connect` to create and for documentation of parameters. ''' def __init__(self, client_script=None, memGB=None, validate_client=True, *args, **kwargs): @@ -91,6 +119,14 @@ def __init__(self, client_script=None, memGB=None, validate_client=True, self.memGB = memGB def make_kwargs(self, *args, **kwargs): + '''method to fill in default properties of RenderClient object + + Args: + *args: args used to initialize RenderClient + **kwargs: kwargs used to initialize RenderClient + Returns: + Render: Render instance with default values filled in + ''' # hack to get dictionary defaults to work client_script = defaultifNone( kwargs.pop('client_script', None), self.client_script) @@ -105,36 +141,38 @@ def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, force_http=True, validate_client=True, web_only=False, **kwargs): ''' - helper function to connect to a render instance - can default to using environment variables if not specified in call. - input: - host -- string hostname for target render server -- will prepend + helper function to create a :class:`RenderClient` instance. + Will default to using environment variables if not specified in call, + and prompt user for any parameters that are not given. + + Args: + host (str): hostname for target render server -- will prepend "http://" if not found and force_http keyword evaluates True. Can be set by environment variable RENDER_HOST. - port -- string, int, or None port for target render server. + port (str, int, or None): port for target render server. Optional as in 'http://hostname[:port]'. Can be set by environment variable RENDER_PORT. - owner -- string owner for render-ws. + owner (str): owner for render-ws. Can be set by environment variable RENDER_OWNER. - project -- string project for render webservice. + project (str): project for render webservice. Can be set by environment variable RENDER_PROJECT. - client_scripts -- string specifying directory path + client_scripts (str): directory path for render-ws-java-client scripts. Can be set by environment variable RENDER_CLIENT_SCRIPTS. - client_script -- string path to a wrapper for java client classes. + client_script (str): path to a wrapper for java client classes. Used only in RenderClient. Can be set by environment variable RENDER_CLIENT_SCRIPT. - memGB -- string specifying heap size in GB for java client scripts, + memGB (str):heap size in GB for java client scripts, example for 1 GB: '1G'. Used only in RenderClient. Can be set by environment variable RENDER_CLIENT_HEAP. - force_http -- boolean determining whether to prepend + force_http (bool): whether to prepend 'http://' to render host - validate_client -- boolean whether to + validate_client (bool): whether to validate existence of RenderClient run_ws_client.sh script - web_only -- boolean whether to check environment variables/prompt user + web_only (bool): whether to check environment variables/prompt user for client_scripts directory if not in arguments - returns: - RenderClient or Render object + Returns: + RenderClient: a connect object to simplify specifying what render server to connect to ''' if host is None: if 'RENDER_HOST' not in os.environ: @@ -213,11 +251,27 @@ def connect(host=None, port=None, owner=None, project=None, return Render(host=host, port=port, owner=owner, project=project, client_scripts=client_scripts) - +@decorator def renderaccess(f): ''' decorator allowing functions asking for host, port, owner, project - to default to a connection defined by a render object kwarg + to default to a connection defined by a :class:`Render` object + using its :func:`RenderClient.make_kwargs` method. + + Example: + render = renderapi.render.connect('server',80,'me','my_project', + client_scripts='/usr/local/render/render-ws/src/scripts') + owners = renderapi.render.get_stacks_by_owner_project(render=render) + + this works because renderapi.render.get_stacks_by_owner_project has the + renderaccess decorator to fill in the default values for + host, owner, port and project. + + you can if you wish specify any of the arguments, in which case they will not + be filled in by the default values, but you don't have to. + + As such, the documentation omits describing the parameters which are natural + to expect will be filled in by the renderaccess decorator. ''' @wraps(f) def wrapper(*args, **kwargs): @@ -236,7 +290,14 @@ def wrapper(*args, **kwargs): def format_baseurl(host, port): - '''format host and port to a standard template render-ws url''' + '''format host and port to a standard template render-ws url + + Args: + host (str):host of render server + port (int or None): port of render server + Returns + str: a url to the render endpoint at that host/port combination (append render-ws/v1) + ''' # return 'http://%s:%d/render-ws/v1' % (host, port) server = '{}{}'.format(host, ('' if port is None else ':{}'.format(port))) return '{}/render-ws/v1'.format(server) @@ -246,6 +307,15 @@ def format_preamble(host, port, owner, project, stack): ''' format host, port, owner, project, and stack parameters to the access point to stack-based apis + + Args: + host (str): render host + port (int): render host port + owner (str): render owner + project (str): render project + stack (str): render stack + Returns: + str: a url to the endpoint for that host, port, owner, project, stack combination ''' preamble = "%s/owner/%s/project/%s/stack/%s" % ( format_baseurl(host, port), owner, project, stack) @@ -257,6 +327,17 @@ def get_owners(host=None, port=None, session=requests.session(), render=None, **kwargs): ''' return list of owners across all Projects and Stacks for a render server + + :func:`renderaccess` decorated function + + Args: + host (str): render host (defaults to host from render) + port (int): render port (default to port from render) + session (requests.Session): requests session + render (RenderClient): RenderClient connection object + Returns: + list: list of strings containing all render owners + ''' request_url = "%s/owners/" % format_baseurl(host, port) r = session.get(request_url) @@ -275,7 +356,17 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, ''' return metadata for all stacks belonging to particular owner on render server - TODO example + + :func:`renderaccess` decorated function + + Args: + owner (str): render owner + render (RenderClient): render connect object + session (requests.sessions.Session): http session to use + Returns: + dict: stackInfo metadata, TODO example + + ''' request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) @@ -292,7 +383,18 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, @renderaccess def get_projects_by_owner(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''return list of projects belonging to a single owner for render stack''' + '''return list of projects belonging to a single owner for render stack + + :func:`renderaccess` decorated function + + Args: + owner (str): render owner + render (RenderClient): render connect object + session (requests.sessions.Session): http session to use + Returns: + list[str]: render projects by this owner + + ''' metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) projects = list(set([m['stackId']['project'] for m in metadata])) @@ -305,6 +407,17 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, render=None, **kwargs): ''' return list of stacks belonging to an owner's project on render server + + :func:`renderaccess` decorated function + + Args: + owner (str): render owner + project (str): render project + render (RenderClient): render connect object + session (requests.sessions.Session): http session to use + Returns: + list[str]: render stacks by this owner in this project + ''' metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 88d1153b..c89f17f9 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -15,19 +15,19 @@ class Layout: '''Layout class to describe acquisition settings inputs: - keyword arguments - --sectionId: unique string to describe sectionId this tile was taken from - --scopeId: string to track what microscope this came from - --cameraId: string to track what camera this was taken with - --imageRow: integer to track what row from a row,col layout this was taken - --imageCol: integer to track what column form a - row,col layout this was taken - --stageX: X stage coordinates (float) for where this was taken - --stageY: Y stage coordinates (float) for where this taken from - --rotation: angle of camera when this was taken - --pixelsize: size of pixels in units of choice of camera - at the magnification it was taken - --force_pixelsize: whether to default pixelsize to 0.1 (default True) + + Args: + sectionId (str): sectionId this tile was taken from + scopeId (str): what microscope this came from + cameraId (str): camera this was taken with + imageRow (int): what row from a row,col layout this was taken + imageCol (int): column from a row,col layout this was taken + stageX (float): X stage coordinates for where this was taken + stageY (float): Y stage coordinates for where this taken + rotation (float): angle of camera when this was taken + pixelsize (float): effective size of pixels (in units of choice) + force_pixelsize (bool): whether to default pixelsize to 0.1 + ''' def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, @@ -47,9 +47,8 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, def to_dict(self): '''return a dictionary representation of this object - no inputs - returns: - json dictionary of object + Returns: + dict: json compatible dictionary of this object ''' d = {} d['sectionId'] = self.sectionId @@ -66,8 +65,9 @@ def to_dict(self): def from_dict(self, d): '''set this object equal to the fields found in dictionary - inputs: - --d:dictionary to use to update + + Args: + d (dict): dictionary to use to update ''' if d is not None: self.sectionId = d.get('sectionId') @@ -84,36 +84,38 @@ def from_dict(self, d): class TileSpec: '''Fundamental class of render that store image tiles and their transformations init: - Keyword arguments: - --tileId: unique string specifying a tile's identity - --z: z values this tile exists within (float) - --width: width in pixels of the raw tile - --height: height in pixels of the raw tile - --imageUrl: an image path that can be accessed by the render server, - with an ImageJ.open command or as an s3 url. - Files on disk should be specified with file: - --maskUrl: an image path that can be accessed by the render server - which can be interpreted as an - alpha mask for the image (same as spec imageUrl) - --minint: pixel intensity value to display as black in a - linear colormap (default 0) - --maxint: pixel intensity value to display as white in a - linear colormap (default 65535) - --layout: a Layout object for this tile - --tforms: a list of Transform objects - (see AffineModel, TransformList, Polynomial2DTransform, - Transform, ReferenceTransform) to apply to this tile - --inputfilters: a list of filters to apply to - this tile (not yet implemented) - --scale3Url: url of a mipmap level 3 image of this tile - (deprecated, see mipMapLevels, but will override) - --scale2Url: url of a mipmap level 2 image of this tile - (deprecated, see mipMapLevels, but will override) - --scale1Url: url of a mipmap level 1 image of this tile - (deprecated, see mipMapLevels, but will override) - --mipMapLevels: a list of MipMapLevel objects for this tile - --json: a json dictionary to initialize this object with - (if not None overrides and ignores all keyword arguments) + + Args: + tileId (str): unique string specifying a tile's identity + z (float): z values this tile exists within + width (int): width in pixels of the raw tile + height (int): height in pixels of the raw tile + imageUrl (str): an image path that can be accessed by the render server, + with an ImageJ.open command or as an s3 url. + Files on disk should be specified with file: + maskUrl (str): an image path that can be accessed by the render server + which can be interpreted as an + alpha mask for the image (same as spec imageUrl) + minint (int): pixel intensity value to display as black in a + linear colormap (default 0) + maxint (int): pixel intensity value to display as white in a + linear colormap (default 65535) + layout (Layout): a Layout object for this tile + tforms (list[Transform]): Transform objects + (see :class:`AffineModel`, :class:`TransformList`, :class:`Polynomial2DTransform`, + :class:`Transform`, :class:`ReferenceTransform`) to apply to this tile + inputfilters (list): a list of filters to apply to + this tile (not yet implemented) + mipMapLevels (list[MipMapLevel]): :class:`MipMapLevel` objects for this tile + json (dict or None): dictionary to initialize this object with + (if not None overrides and ignores all keyword arguments) + scale3Url (str): url of a mipmap level 3 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + scale2Url (str): url of a mipmap level 2 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + scale1Url (str): url of a mipmap level 1 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + ''' def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, @@ -164,6 +166,9 @@ def bbox(self): def to_dict(self): '''method to produce a json tilespec for this tile returns a json compatible dictionary + + Returns: + dict: json compatible dictionary representation of this object ''' thedict = {} thedict['tileId'] = self.tileId @@ -203,7 +208,11 @@ def to_dict(self): return thedict def from_dict(self, d): - '''Method to load tilespec from json dictionary''' + '''Method to load tilespec from json dictionary + + Args: + d (dict): dictionary to use to set properties of this object + ''' self.tileId = d['tileId'] self.z = d['z'] self.width = d['width'] @@ -241,10 +250,10 @@ class MipMapLevel: MipMapLevel class to represent a level of an image pyramid. Can be put in dictionary formatting using dict(mML) - init: - level -- integer level of 2x downsampling represented by mipmaplevel - imageUrl (optional) -- url corresponding to image - maskUrl (optional) -- url corresponding to mask + Args: + level (int): level of 2x downsampling represented by mipmaplevel + imageUrl (str or None): url corresponding to image + maskUrl (str or None): url corresponding to mask ''' def __init__(self, level, imageUrl=None, maskUrl=None): self.level = level @@ -273,43 +282,54 @@ class ImagePyramid: of an image at level 0 Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) - init: - mipMapLevels -- list of MipMapLevel objects - append: - adds MipmapLevel without checking if it exists - input: MipMapLevel object - update: - adds MipMapLevel object replacing a corresponding level if it exists - input: MipMapLevel object - to_ordered_dict: - input: key(optional) -- key to sort ordered dictionary - default sort by level via lambda x: x[0] + Args: + mipMapLevels (list[MipMapLevel]): list of :class:`MipMapLevel` objects + ''' def __init__(self, mipMapLevels=[]): self.mipMapLevels = mipMapLevels def to_dict(self): + '''return dictionary representation of this object''' return dict(self.__iter__()) def to_ordered_dict(self, key=None): - '''defaults to order by mipmapLevel''' + '''returns :class:`OrderedDict` represention of this object, ordered by mipmapLevel''' return OrderedDict(sorted( self.__iter__(), key=((lambda x: x[0]) if key is None else key))) def append(self, mmL): + '''appends a MipMapLevel to this ImagePyramid + + Args: + mml (MipMapLevel): MipMapLevel to append + ''' self.mipMapLevels.append(mmL) def update(self, mmL): + '''updates the ImagePyramid with this MipMapLevel + + Args: + mml (MipMapLevel): mipmap level to update in pyramid + ''' self.mipMapLevels = [ l for l in self.mipMapLevels if l.level != mmL.level] self.append(mmL) def get(self, to_get): + '''gets a specific mipmap level in dictionary form + + Args: + to_get (int): mipmaplevel to get + Returns: + dict: dictionary representation of requested MipMapLevel + ''' return self.to_dict()[to_get] # TODO should this default @property def levels(self): + '''list of MipMapLevels in this ImagePyramid''' return [int(i.level) for i in self.mipMapLevels] def __iter__(self): @@ -324,16 +344,16 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, render=None, **kwargs): '''renderapi call to get the render parameters of a specific tileId - args: - --stack: name of render stack to retrieve - --tile: tileId of tile to retrieve - keyword args: - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - A render-parameters json with the tilespec for that tile and - dereferenced transforms + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + tile (str): tileId of tile to retrieve + render (renderapi.render.RenderClient): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + dict: render-parameters json with the tilespec for that tile and dereferenced transforms + TODO provide example ''' request_url = format_preamble( @@ -357,15 +377,15 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, note that this will return a tilespec with resolved transform references by accessing the render-parameters version of this tile - args: - --stack: name of render stack to retrieve - --tile: tileId of tile to retrieve - keyword args: - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - A TileSpec object with dereferenced transforms + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + tile (str): tileId of tile to retrieve + render (renderapi.render.RenderClient): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + TileSepc: TileSpec with dereferenced transforms ''' try: @@ -383,15 +403,15 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, '''renderapi call to get a specific tilespec by tileId note that this will return a tilespec without resolved transform references - args: - --stack: name of render stack to retrieve - --tile: tileId of tile to retrieve - keyword args: - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - a TileSpec object with referenced transforms if present + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + tile (str): tileId of tile to retrieve + render (renderapi.render.RenderClient): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + TileSepc: TileSpec with referenced transforms intact ''' request_url = format_preamble( @@ -417,20 +437,20 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, specified with min and max x,y values note that this will return a tilespec with resolved transform references - args: - --stack: name of render stack to retrieve - --z: z value of bounding box (float) - --xmin: minimum x value (float) - --ymin: minimum y value (float) - --xmax: maximum x value (float) - --ymax: maximum y value (float) - keyword args: - --scale: scale to use when retrieving render parameters (not important) - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - a list of TileSpec objects + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + z (float): z value of bounding box + xmin (float): minimum x value + ymin (float): minimum y value + xmax (float): maximum x value + ymax (float): maximum y value + scale (float): scale to use when retrieving render parameters (not important) + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]:TileSpec objects with dereferenced tansforms ''' x = xmin y = ymin @@ -448,23 +468,23 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, project=None, session=requests.session(), render=None, **kwargs): '''renderapi call to get all tilespec that exist within a 2d bounding box - specified with min x,y values and width, height + specified with min x,y values and width, height note that this will return a tilespec with resolved transform references - args: - --stack: name of render stack to retrieve - --z: z value of bounding box (float) - --xmin: minimum x value (float) - --ymin: minimum y value (float) - --xmax: maximum x value (float) - --ymax: maximum y value (float) - keyword args: - --scale: scale to use when retrieving render parameters (not important) - --render: render connect object - (or host, port, owner, project) - --session: sessions object to connect with (default make a new one) - outputs: - a list of TileSpec objects + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to retrieve + z (float): z value of bounding box + x (float): minimum x value + y (float): minimum y value + width (float): width of box (in scale=1.0 units) + height (float): height of box (in scale=1.0 units) + scale (float): scale to use when retrieving render parameters (not important) + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]:TileSpec objects with dereferenced tansforms ''' request_url = format_preamble( host, port, owner, project, stack) + \ @@ -487,13 +507,17 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): ''' - input: - stack -- string render stack - z -- render z - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - output: list of TileSpec objects from that stack at that z + Get all TileSpecs in a specific z values. Returns referenced transforms. + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): render stack + z (float): render z + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]: list of TileSpec objects from that stack at that z ''' request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) @@ -519,13 +543,15 @@ def get_tile_specs_from_stack(stack, host=None, port=None, session=requests.session(), render=None, **kwargs): '''get flat list of tilespecs for stack using i for sl in l for i in sl - input: - stack -- string render stack - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - output: - List of TileSpec objects from the stack + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): render stack + render: render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + list[TileSpec]: list of TileSpec objects from that stack ''' return [i for sl in [ get_tile_specs_from_z(stack, z, host=host, port=port, From 041bf20f83aa2863827c1b683cd51c9388639619 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:50:46 -0700 Subject: [PATCH 390/766] minor modifications to docstring --- renderapi/tilespec.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index c89f17f9..9a246971 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -47,6 +47,7 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, def to_dict(self): '''return a dictionary representation of this object + Returns: dict: json compatible dictionary of this object ''' @@ -102,8 +103,11 @@ class TileSpec: linear colormap (default 65535) layout (Layout): a Layout object for this tile tforms (list[Transform]): Transform objects - (see :class:`AffineModel`, :class:`TransformList`, :class:`Polynomial2DTransform`, - :class:`Transform`, :class:`ReferenceTransform`) to apply to this tile + (see :class:`.transform.AffineModel`, + :class:`.transform.TransformList`, + :class:`.transform.Polynomial2DTransform`, + :class:`.transform.Transform`, + :class:`.transform.ReferenceTransform`) to apply to this tile inputfilters (list): a list of filters to apply to this tile (not yet implemented) mipMapLevels (list[MipMapLevel]): :class:`MipMapLevel` objects for this tile @@ -261,6 +265,10 @@ def __init__(self, level, imageUrl=None, maskUrl=None): self.maskUrl = maskUrl def to_dict(self): + ''' + Returns: + dict: json compatible dictionary representaton + ''' return dict(self.__iter__()) def _formatUrls(self): @@ -322,6 +330,7 @@ def get(self, to_get): Args: to_get (int): mipmaplevel to get + Returns: dict: dictionary representation of requested MipMapLevel ''' From 142a86b951bc6d498766a6025a8078dae979f273 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:51:04 -0700 Subject: [PATCH 391/766] added renderapi link to index.rst --- docs/index.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 6f49525e..fc3c8971 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,12 +12,14 @@ Welcome to render-python's documentation! :maxdepth: 2 :caption: Contents: -api +API --- + .. toctree:: - :maxdepth: 2 + :maxdepth: 3 + + api/renderapi - api/renderapi Indices and tables From e4f4bf9388994da480015ce05515bb4425092165 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:55:41 -0700 Subject: [PATCH 392/766] added more to gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9ff0bb95..1d304750 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ dist/ **/.DS_Store .coverage test-reports/ -docs/_build/ \ No newline at end of file +docs/_build/ +htmlcov/ +.cache \ No newline at end of file From aab29b43e9a9550eedfb9176d73774378b2bc634 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 13:57:10 -0700 Subject: [PATCH 393/766] gave parameters to renderaccess --- renderapi/render.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/renderapi/render.py b/renderapi/render.py index b7f0bfc7..a1734d81 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -272,6 +272,11 @@ def renderaccess(f): As such, the documentation omits describing the parameters which are natural to expect will be filled in by the renderaccess decorator. + + Args: + f (func): function to decorate + Returns: + func: decorated function ''' @wraps(f) def wrapper(*args, **kwargs): From 734a46c2e490988197f311035777ccc2026e77c4 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 14:47:58 -0700 Subject: [PATCH 394/766] added decorator to requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f82661f..f56d3209 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ numpy pillow dill>=0.2.6 pathos -sphinxcontrib-napoleon \ No newline at end of file +sphinxcontrib-napoleon +decorator From eb73adad0b0adf49b551605bf46b4d1d45bedf06 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 15:34:44 -0700 Subject: [PATCH 395/766] added stack documentation --- renderapi/stack.py | 423 +++++++++++++++++++++++++-------------------- 1 file changed, 240 insertions(+), 183 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index c02bdcd2..11cb4867 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -13,22 +13,23 @@ class StackVersion: - '''StackVersion - keyword arguments: - cycleNumber -- cycleNumber, use as you wish to - track versions (default None) - cycleStepNumber -- cycleStepNumber, use as you with to - track versions (default None) - stackResolutionX -- stackResolutionX, - resolution of scale = 1.0 in nm (float) - stackResolutionY -- stackResolutionY, - resolution of scale = 1.0 in nm (float) - stackResolutionZ -- stackResolutionZ, - resolution of scale = 1.0 in nm (float) - mipmapPathBuilder -- ? - materializedBoxRootPath -- ? - createTimeStamp -- time stamp of stack creation (default to now) - versionNotes -- string of notes about this stack (optional) + '''StackVersion, metadata about a stack + + Args: + cycleNumber (int):cycleNumber, use as you wish to + track versions (default None) + cycleStepNumber (int): cycleStepNumber, use as you with to + track versions (default None) + stackResolutionX (float): stackResolutionX, + resolution of scale = 1.0 in nm + stackResolutionY (float): stackResolutionY, + resolution of scale = 1.0 in nm + stackResolutionZ (float): stackResolutionZ, + resolution of scale = 1.0 in nm + mipmapPathBuilder (str): path to mipmap builder (?) + materializedBoxRootPath (str): path to materializer (?) + createTimeStamp (str): time stamp of stack creation (default to now) + versionNotes (str): notes about this stack (optional) ''' def __init__(self, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, @@ -48,8 +49,10 @@ def __init__(self, cycleNumber=None, cycleStepNumber=None, self.versionNotes = versionNotes def to_dict(self): - '''to_dict - turns this object into a json dictionary + '''serialization function + + Returns: + dict: json compatible verson of this object ''' d = {} d.update(({'cycleNumber': self.cycleNumber} @@ -73,8 +76,10 @@ def to_dict(self): return d def from_dict(self, d): - '''from_dict - update the properties of this object with a json dictonary + '''deserialization function + + Args: + d (dict):dictionary to update the properties of this object ''' self.__dict__.update({k: v for k, v in d.items()}) @@ -83,13 +88,19 @@ def from_dict(self, d): def set_stack_metadata(stack, sv, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - ''' set_stack_metadata - inputs: - stack -- stack to set the metadata for - sv -- StackVersion to set the metadata to - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) + '''sets the stack metadata for a stack + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to set the metadata for + sv (StackVersion): metadata for the stack + render (renderapi.render.RenderClient): render connect object + session (requests.sessions.Session): session object (default start a new one) + + Returns: + requests.response: response from server + ''' request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) @@ -99,15 +110,18 @@ def set_stack_metadata(stack, sv, host=None, port=None, owner=None, @renderaccess def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - ''' get_stack_metadata - inputs: - stack -- render stack to get metadata from - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - outputs: - StackVersion object of the metadata of the stack - raises: RenderError + ''' get the stack metadata for a stack + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to get the metadata for + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + StackVersion: metadata of the stack + Raises: + .errors.RenderError ''' request_url = format_preamble(host, port, owner, project, stack) @@ -128,22 +142,21 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): ''' - set state of selected stack. Acceptable states are listed below: - LOADING: stack is accepting additional information. - COMPLETE: stack is finished loading. - OFFLINE: stack is not in use. - READ_ONLY: stack cannot be changed. + set state of selected stack. + TODO there is a limited direction in which these stack changes can go - inputs: - --stack: name of render stack to input - --state: state to qset the stack - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - outputs: - session.response object - raises: - RenderError if not successful + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to set state for + state (str): state of stack, one of ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'] + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + requests.session.response: server response + Raises: + .errors.RenderError ''' if state not in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY']: raise RenderError('state {} not in known states {}'.format( @@ -163,11 +176,14 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, def likelyUniqueId(host=None, port=None, session=requests.session(), render=None, **kwargs): '''return hex-code nearly-unique id from render server - keyword arguments: - render -- render connect object (or host, port) - session -- requests.session (default start a new one) - returns: - string representation of hex-code + + :func:`renderapi.render.renderaccess` decorated function + + Args: + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + str: string representation of hex-code ''' request_url = '{}/likelyUniqueId'.format(format_baseurl(host, port)) r = session.get(request_url, data=None, @@ -178,7 +194,15 @@ def likelyUniqueId(host=None, port=None, def make_stack_params(host, port, owner, project, stack): '''utility function to turn host,port,owner,project,stack combinations to java CLI based argument list for subprocess calling - returns [--baseDataUrl,self.format_baseurl(host,port),--owner + + Args: + host (str): render server + port (int): render server port + owner (str): render owner + project (str): render project + stack (str): render stack + Returns: + list[str]: java CLI list of arguments for subprocess calling ''' baseurl = format_baseurl(host, port) project_params = ['--baseDataUrl', baseurl, @@ -191,14 +215,17 @@ def make_stack_params(host, port, owner, project, stack): def delete_stack(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''deletes a stack from render - inputs: - stack -- render stack to delete - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - outputs: - r -- response object of response from server + '''deletes a stack from render server + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to delete + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + requests.session.response: server response + ''' request_url = format_preamble(host, port, owner, project, stack) r = session.delete(request_url) @@ -211,14 +238,16 @@ def delete_section(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''removes a single z from a stack - inputs: - stack -- stack from which to remove - z -- z value to remove - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - outputs: - r -- response object from server + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to delete section from + z (float): z value to delete + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + requests.session.response: server response ''' request_url = '{}/z/{}'.format( format_preamble(host, port, owner, project, stack), z) @@ -233,11 +262,16 @@ def delete_tile(stack, tileId, host=None, port=None, owner=None, render=None, **kwargs): ''' removes a tile from a stack - inputs: - stack -- stack from which to remove - tileId -- tile Id of tilespec to remove from stack - outputs: - r -- response from server + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to delete tile from + tileId (str): tileId of tilespec to remove from stack + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + requests.session.response: server response ''' request_url = '{}/tile/{}'.format( format_preamble(host, port, owner, project, stack), tileId) @@ -253,22 +287,24 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''creates a new stack - inputs: - stack -- stack name to create - keyword arguments: - cycleNumber -- cycleNumber to use to track stages - cycleStepNumber -- cycleStepNumber to use to track stages - stackResolutionX -- resolution of x pixels at scale=1.0 - stackResolutionY -- resolution of y pixels at scale=1.0 - stackResoluiontZ -- resolution of z sections at scale=1.0 - force_resolution -- fill in resolution of 1.0 for missing - resolutions (default True) - render -- render connect object (or host, port, owner, project) - session -- requests.session (default start a new one) - returns: - r -- reponse object from server - raises: - RenderError + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): render stack name to create + cycleNumber (int): cycleNumber to use to track stages + cycleStepNumber (int):cycleStepNumber to use to track stages + stackResolutionX (float): resolution of x pixels at scale=1.0 + stackResolutionY (float): resolution of y pixels at scale=1.0 + stackResoluiontZ (float): resolution of z sections at scale=1.0 + force_resolution (bool): fill in resolution of 1.0 for missing + resolutions + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + requests.session.response: server response + Raises: + .errors.RenderError ''' if force_resolution: stackResolutionX, stackResolutionY, stackResolutionZ = [ @@ -296,21 +332,23 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, zs=None, close_stack=True, host=None, port=None, owner=None, project=None, session=None, render=None, **kwargs): - ''' - input: - inputstack -- string name of input stack to clone - outputstack -- string name of destination stack. + '''clone a stack + + :func:`renderapi.render.renderaccess` decorated function + + Args: + inputstack (str): name of input stack to clone + outputstack (str): name of destination stack. if exists, must be LOADING - keyword arguments: - skipTransforms -- optional, boolean whether to strip transformations - in new stack - toProject -- optional, string name of project - zs -- optional, list of selected z values to clone into stack - close_stack -- boolean, whether to set stack to COMPLETE when finished - render -- render connect object (or host, port, owner, project) - session -- optional, requests.session (default start a new one) - outputs: - r -- reponse object from server + skipTransforms (bool): boolean whether to strip transformations + in new stack (default=False) + toProject (str): string name of destination project (default same as inputstack) + zs (list[float] or None): list of selected z values to clone into stack (optional) + close_stack (boolean): whether to set stack to COMPLETE when finished + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + requests.session.response: server response ''' session = requests.session() if session is None else session sv = StackVersion(**kwargs) @@ -341,15 +379,17 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, owner=None, session=requests.session(), render=None, **kwargs): '''get a list of z values for which there are tiles in the stack - inputs: - stack -- stack to get z values for - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- optional, requests.session (default start a new one) - returns: - list of z values - raises: - RenderError + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to get z values for + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + list[float]: z values in stack + Raises: + .errors.RenderError ''' request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" @@ -364,20 +404,9 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, def get_z_value_for_section(stack, sectionId, **kwargs): - '''DEPRECATED (use get_section_z_value) instead get z - values for a specific sectionId - inputs: - stack -- render stack string to look within - sectionId -- string of sectionId to find z value - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- optional, requests.session (default start a new one) - returns: - list of z values - raises: - RenderErorr + '''DEPRECATED (use :func:`get_section_z_value`) instead ''' - logger.warning("Deprecated, use get_section_z_value instead") + logger.warning("DEPRECATED, use renderapi.stack.get_section_z_value instead") return get_section_z_value(stack, sectionId, **kwargs) @@ -398,14 +427,18 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''get a bounds dictionary for a specific z - inputs: - stack -- stack to get bounds from - z -- z values (float) to get bounds from - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- optional, requests.session (default start a new one) - outputs: - dictionary of bounds with keys minY,minY,maxX,maxY + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to get bounds from + z (float): z value to get bounds for + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + dict: bounds with keys minY,minY,maxX,maxY + Raises: + .errorr.RenderError ''' request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/bounds' % (z) @@ -423,15 +456,17 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''get bounds of a whole stack - inputs: - stack -- stack to get bounds from - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- optional, requests.session (default start a new one) - outputs: - dictionary of bounds with keys minY,minY,maxX,maxY,minZ,maxZ - raises: - RenderError + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to get bounds from + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + dict: bounds with keys minY,minY,maxX,maxY,minZ,maxZ + Raises: + .errors.RenderError ''' request_url = format_preamble( host, port, owner, project, stack) + '/bounds' @@ -449,13 +484,18 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''returns the sectionId associated with a particular z value - inputs: - stack -- name of the stack to get zvalues about - z -- z values to look for - render -- render connect object (or host, port, owner, project) - session -- options, requests.session - returns: - z values that have that has sectionId + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): stack to look within + sectionId (str): sectionId to find z value + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + float: z value of sectionId + Raises: + .errors.RenderError ''' sectionData = get_stack_sectionData( stack, host, port, owner, project, session) @@ -471,24 +511,27 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''returns information about the sectionIds of each slice in stack - inputs: - stack -- name of stack to get data about - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- optional, requests.session (default start a new one) - returns: - list of dictionaries containing sectionData as below - [{ - "sectionId": "string", - "z": 0, - "tileCount": 0, - "minX": 0, - "maxX": 0, - "minY": 0, - "maxY": 0 - }] - raises: - RenderError + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of stack to get data about + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + dict:sectionData as below + :: + [{ + "sectionId": "string", + "z": 0, + "tileCount": 0, + "minX": 0, + "maxX": 0, + "minY": 0, + "maxY": 0 + }] + Raises: + .errors.RenderError: ''' request_url = format_preamble( host, port, owner, project, stack) + '/sectionData' @@ -506,16 +549,18 @@ def get_section_z_value(stack, sectionId, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): '''get the z value for a specific sectionId (string) - inputs: - stack -- render stack string to look within - sectionId -- string of sectionId to find z value - keyword arguments: - render -- render connect object (or host, port, owner, project) - session -- optional, requests.session (default start a new one) - returns: - list of z values - raises: - RenderError + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str):render stack string to look within + sectionId (str): of sectionId to find z value + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + float: z value of section + Raises: + .errors.RenderError: ''' request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % sectionId @@ -531,7 +576,19 @@ def get_section_z_value(stack, sectionId, host=None, port=None, @renderaccess def get_stack_tileIds(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''get tileIds for a stack''' + '''get tileIds for a stack + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str):stack to get tileIds + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): session object (default start a new one) + Returns: + list[str]: list of tileIds in stack + Raises: + .errors.RenderError: + ''' request_url = '{}/tileIds'.format( format_preamble(host, port, owner, project, stack)) r = session.get(request_url) From cfb537af5e71bcef659f0658de5bfa0a6936ff10 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 15:36:57 -0700 Subject: [PATCH 396/766] added example link to renderaccess --- renderapi/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index a1734d81..abb063dd 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -263,7 +263,7 @@ def renderaccess(f): client_scripts='/usr/local/render/render-ws/src/scripts') owners = renderapi.render.get_stacks_by_owner_project(render=render) - this works because renderapi.render.get_stacks_by_owner_project has the + this works because :func:`renderapi.render.get_stacks_by_owner_project` has the renderaccess decorator to fill in the default values for host, owner, port and project. From e8e1bc99e6fff0f09c0fa09c9fc93c4a4e35697d Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 16:12:33 -0700 Subject: [PATCH 397/766] added image documentation --- renderapi/image.py | 75 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index f63b0189..c8ffcf6f 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -33,14 +33,26 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, render=None, **kwargs): ''' render image from a bounding box defined in xy and return numpy array: - z: layer - x: leftmost point of bounding rectangle - y: topmost pont of bounding rectangle - width: extent to right in x - height: extent down in y - binaryMask: optional, boolean whether to treat maskimage as binary - maxTileSpecsToRender: optional, int number of tilespecs to render - filter: optional, boolean whether to use Khaled's preferred filter + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to get image from + z (float): z value to render + x (int): leftmost point of bounding rectangle + y (int): topmost pont of bounding rectangle + width (int): number of units @scale=1.0 to right (+x() of bounding box to render + height (int): number of units @scale=1.0 down (+y) of bounding box to render + scale (float): scale to render image at (default 1.0) + binaryMask (boolean): whether to treat maskimage as binary + maxTileSpecsToRender (int): max number of tilespecs to render + filter (boolean): whether to use server side filtering + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + numpy.array: [N,M,3] array of image data from render + Raises: + .errors.RenderError ''' try: image_ext = IMAGE_FORMATS[img_format] @@ -50,7 +62,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%f/%s" % ( - z, x, y, width, height, scale, image_ext) + z, x, y, width, height, scale, image_ext) qparams = {} if minIntensity is not None: qparams['minIntensity'] = minIntensity @@ -81,6 +93,25 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, session=requests.session(), render=None, **kwargs): ''' render image from a tile with all transforms and return numpy array + + :func:`renderapi.render.renderaccess` decorated function + + Args: + stack (str): name of render stack to get tile from + tileId (str): tileId of tile to render + normalizeForMatching (bool): whether to render the tile with transformations removed ('local' coordinates) + removeAllOption (bool): whether to remove all transforms from image when doing normalizeForMatching + some versions of render only remove the last transform from list. + (or remove till there are max 3 transforms) + scale (float): force scale of image + filter (bool): whether to apply server side filtering to image + img_format (str): image format: one of IMAGE_FORMATS = 'png','.png','jpg','jpeg','.jpg','tif','.tif','tiff' + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + numpy.array: [N,M,3] array of image data from render + Raises: + .errors.RenderError ''' try: image_ext = IMAGE_FORMATS[img_format] @@ -120,11 +151,27 @@ def get_section_image(stack, z, scale=1.0, filter=False, session=requests.session(), render=None, **kwargs): ''' - z: layer Z - scale: float -- linear scale at which to render image (e.g. 0.5) - filter: boolean -- whether or not to apply Khaled's preferred filter - maxTileSpecsToRender: int -- maximum number of tile specs in rendering - img_format: string -- format defined by IMAGE_FORMATS + render an section of image + + :func:`renderapi.render.renderaccess` decorated function + + Example: + :: + + import renderapi + render = renderapi.render.connect('server',8080,'me','myproject') + img = render.run(renderapi.stack.get_section_image,'mystack',3.0) + Args: + stack (str): name of render stack to render image from + z (float): layer Z + scale (float): linear scale at which to render image (e.g. 0.5) + filter (boolean): whether or not to apply server side filtering + maxTileSpecsToRender (int): maximum number of tile specs in rendering + img_format (str): one of IMAGE_FORMATS 'png','.png','jpg','jpeg','.jpg','tif','.tif','tiff' + render (renderapi.render.Render): render connect object + session (requests.sessions.Session): sessions object to connect with + Returns: + numpy.array: [N,M,3] array of image data of section from render ''' try: image_ext = IMAGE_FORMATS[img_format] From 89f12e89f3d5a835e515bb58af606f7440dba6be Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 16:12:48 -0700 Subject: [PATCH 398/766] slight tweaks to render and tilespec documentation --- renderapi/render.py | 44 +++++++++++++++++++++---------------------- renderapi/tilespec.py | 14 +++++++------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index abb063dd..2c9443ee 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -16,6 +16,7 @@ class Render(object): Baseclass that doesn't require client_scripts definition for client side java processing. See :func:`connect` for parameter definitions. ''' + def __init__(self, host=None, port=None, owner=None, project=None, client_scripts=None): self.DEFAULT_HOST = host @@ -27,9 +28,9 @@ def __init__(self, host=None, port=None, owner=None, project=None, logger.debug('Render object created with ' 'host={h}, port={p}, project={pr}, ' 'owner={o}, scripts={s}'.format( - h=self.DEFAULT_HOST, p=self.DEFAULT_PORT, - pr=self.DEFAULT_PROJECT, o=self.DEFAULT_OWNER, - s=self.DEFAULT_CLIENT_SCRIPTS)) + h=self.DEFAULT_HOST, p=self.DEFAULT_PORT, + pr=self.DEFAULT_PROJECT, o=self.DEFAULT_OWNER, + s=self.DEFAULT_CLIENT_SCRIPTS)) @property def DEFAULT_KWARGS(self): @@ -37,7 +38,7 @@ def DEFAULT_KWARGS(self): kwargs to which the render object falls back. Depends on: self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS - + Returns: dict: default keyword arguments ''' @@ -73,7 +74,7 @@ def run(self, f, *args, **kwargs): ''' run function from object technically shorter than adding render=Render to kwargs - + allows this syntax for running renderapi render = Render('server',8080) @@ -98,6 +99,7 @@ class RenderClient(Render): This is a work in progress. Should use :func:`connect` to create and for documentation of parameters. ''' + def __init__(self, client_script=None, memGB=None, validate_client=True, *args, **kwargs): super(RenderClient, self).__init__(**kwargs) @@ -120,7 +122,7 @@ def __init__(self, client_script=None, memGB=None, validate_client=True, def make_kwargs(self, *args, **kwargs): '''method to fill in default properties of RenderClient object - + Args: *args: args used to initialize RenderClient **kwargs: kwargs used to initialize RenderClient @@ -251,28 +253,28 @@ def connect(host=None, port=None, owner=None, project=None, return Render(host=host, port=port, owner=owner, project=project, client_scripts=client_scripts) + @decorator def renderaccess(f): ''' - decorator allowing functions asking for host, port, owner, project + Decorator allowing functions asking for host, port, owner, project to default to a connection defined by a :class:`Render` object using its :func:`RenderClient.make_kwargs` method. Example: - render = renderapi.render.connect('server',80,'me','my_project', - client_scripts='/usr/local/render/render-ws/src/scripts') - owners = renderapi.render.get_stacks_by_owner_project(render=render) - - this works because :func:`renderapi.render.get_stacks_by_owner_project` has the - renderaccess decorator to fill in the default values for - host, owner, port and project. - - you can if you wish specify any of the arguments, in which case they will not + :: + + render = renderapi.render.connect('server',8080,'me','my_project') + stacks = renderapi.render.get_stacks_by_owner_project(render=render) + This works because :func:`renderapi.render.get_stacks_by_owner_project` has the + renderaccess decorator to fill in the default values. + + You can if you wish specify any of the arguments, in which case they will not be filled in by the default values, but you don't have to. As such, the documentation omits describing the parameters which are natural to expect will be filled in by the renderaccess decorator. - + Args: f (func): function to decorate Returns: @@ -296,7 +298,7 @@ def wrapper(*args, **kwargs): def format_baseurl(host, port): '''format host and port to a standard template render-ws url - + Args: host (str):host of render server port (int or None): port of render server @@ -370,8 +372,6 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, session (requests.sessions.Session): http session to use Returns: dict: stackInfo metadata, TODO example - - ''' request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) @@ -389,7 +389,7 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, def get_projects_by_owner(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): '''return list of projects belonging to a single owner for render stack - + :func:`renderaccess` decorated function Args: @@ -412,7 +412,7 @@ def get_stacks_by_owner_project(owner=None, project=None, host=None, render=None, **kwargs): ''' return list of stacks belonging to an owner's project on render server - + :func:`renderaccess` decorated function Args: diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 9a246971..04bcd3b6 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -358,7 +358,7 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, Args: stack (str): name of render stack to retrieve tile (str): tileId of tile to retrieve - render (renderapi.render.RenderClient): render connect object + render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: dict: render-parameters json with the tilespec for that tile and dereferenced transforms @@ -391,7 +391,7 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, Args: stack (str): name of render stack to retrieve tile (str): tileId of tile to retrieve - render (renderapi.render.RenderClient): render connect object + render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: TileSepc: TileSpec with dereferenced transforms @@ -417,7 +417,7 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, Args: stack (str): name of render stack to retrieve tile (str): tileId of tile to retrieve - render (renderapi.render.RenderClient): render connect object + render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: TileSepc: TileSpec with referenced transforms intact @@ -456,7 +456,7 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, xmax (float): maximum x value ymax (float): maximum y value scale (float): scale to use when retrieving render parameters (not important) - render: render connect object + render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: list[TileSpec]:TileSpec objects with dereferenced tansforms @@ -490,7 +490,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, width (float): width of box (in scale=1.0 units) height (float): height of box (in scale=1.0 units) scale (float): scale to use when retrieving render parameters (not important) - render: render connect object + render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: list[TileSpec]:TileSpec objects with dereferenced tansforms @@ -523,7 +523,7 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, Args: stack (str): render stack z (float): render z - render: render connect object + render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: list[TileSpec]: list of TileSpec objects from that stack at that z @@ -557,7 +557,7 @@ def get_tile_specs_from_stack(stack, host=None, port=None, Args: stack (str): render stack - render: render connect object + render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: list[TileSpec]: list of TileSpec objects from that stack From 2251f6b353a4bfe19f65a009bb2ad887aebc7e27 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 16:15:43 -0700 Subject: [PATCH 399/766] put example in a code-block --- renderapi/render.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 2c9443ee..cdabc0e8 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -75,11 +75,11 @@ def run(self, f, *args, **kwargs): run function from object technically shorter than adding render=Render to kwargs - allows this syntax for running renderapi - - render = Render('server',8080) - metadata = render.run(renderapi.render.get_stack_metadata_by_owner,'myowner') - + Example: + allows this syntax for running renderapi:: + + render = Render('server',8080) + metadata = render.run(renderapi.render.get_stack_metadata_by_owner,'myowner') Args: f (func): renderapi function you want to call *args: args passed to that function From 27102dd657e5878312197505221f9b38342c0ac2 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 22 Jul 2017 18:03:55 -0700 Subject: [PATCH 400/766] adding more transform documentation --- renderapi/transform.py | 272 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 243 insertions(+), 29 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 4ff886a0..418ec401 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -26,6 +26,13 @@ class TransformList: + ''' A list of Transforms + + Args: + tforms (Transform): transforms to apply + transformId (str): uniqueId for this TransformList + json (dict): json compatible dictionary to create TransformList via :method:`from_dict` + ''' def __init__(self, tforms=None, transformId=None, json=None): if json is not None: self.from_dict(json) @@ -41,6 +48,10 @@ def __init__(self, tforms=None, transformId=None, json=None): self.transformId = transformId def to_dict(self): + ''' serialization function + Returns: + dict: json & render compatible representation of this TransformList + ''' d = {} d['type'] = 'list' d['specList'] = [tform.to_dict() for tform in self.tforms] @@ -49,9 +60,17 @@ def to_dict(self): return d def to_json(self): + ''' serialization function + Returns: + str: string representation of the json & render representation of this TransformList + ''' return json.dumps(self.to_dict()) def from_dict(self, d): + ''' deserialization function + Args: + d (dict): json compatible dictionary representation of this TransformList + ''' self.tforms = [] if d is not None: self.transformId = d.get('id') @@ -61,6 +80,17 @@ def from_dict(self, d): def load_transform_json(d, default_type='leaf'): + '''function to get the proper deserialization function + + Args: + d (dict): json compatible representation of Transform + default_type (str): what kind of transform should we assume this + if it is not specified in 'type' ('leaf','list','ref','interpolated') + Returns: + func: proper function to deserialize this transformation + Raises: + KeyError: if d['type'] isn't one of ('leaf','list','ref','interpolated') + ''' handle_load_tform = {'leaf': load_leaf_json, 'list': lambda x: TransformList(json=x), 'ref': lambda x: ReferenceTransform(json=x), @@ -73,6 +103,17 @@ def load_transform_json(d, default_type='leaf'): def load_leaf_json(d): + '''function to get the proper deserialization function for leaf transforms + + Args: + d (dict): json compatible representation of leaf transform to deserialize + Returns: + func: proper function to deserialize this transformation + Raises: + RenderError: if d['type']!= leaf or is omitted + KeyError: if d['type'] does not indicate one of AffineModel, Polynomial2DTransform + TranslationModel, RigidModel, or SimilarityModel + ''' handle_load_leaf = { AffineModel.className: lambda x: AffineModel(json=x), Polynomial2DTransform.className: @@ -97,11 +138,12 @@ def load_leaf_json(d): class InterpolatedTransform: ''' Transform spec defined by linear interpolation of two other transform specs - inputs: - a -- transform spec at minimum weight - b -- transform spec at maximum weight - lambda_ -- float value (0.-1.) which defines evaluation of the + Args: + a (Transform):transform at minimum weight + b (Transform):transform at maximum weight + lambda_ (float): value (0.-1.) which defines evaluation of the linear interpolation between a (at 0) and b (at 1) + json (dict): json compatible representation of this transform to initialize via :method:`self.from_dict` ''' def __init__(self, a=None, b=None, lambda_=None, json=None): if json is not None: @@ -112,9 +154,17 @@ def __init__(self, a=None, b=None, lambda_=None, json=None): self.lambda_ = lambda_ def to_dict(self): + '''serialization routine + Returns: + dict: json compatible representation + ''' return dict(self) def from_dict(self, d): + ''' deserialization routine + Args: + d (dict): json compatible representation + ''' self.a = load_transform_json(d['a']) self.b = load_transform_json(d['b']) self.lambda_ = d['lambda'] @@ -127,6 +177,12 @@ def __iter__(self): class ReferenceTransform: + '''Transform which is simply a reference to a transform stored elsewhere + + Args: + refId (str): transformId of the referenced transform + json (dict): json compatible representation of this transform + ''' def __init__(self, refId=None, json=None): if json is not None: self.from_dict(json) @@ -134,12 +190,20 @@ def __init__(self, refId=None, json=None): self.refId = refId def to_dict(self): + '''serialization routine + Returns: + dict: json compatible representation of this transform + ''' d = {} d['type'] = 'ref' d['refId'] = self.refId return d def from_dict(self, d): + ''' deserialization routine + Args: + d (dict): json compatible representation of this transform + ''' self.refId = d['refId'] def __str__(self): @@ -153,6 +217,14 @@ def __iter__(self): class Transform(object): + '''Base transformation class + + Args: + className (str): mpicbg java classname of this transform + dataString (str): string reprsentation of this transform as speced by mpicbg java class library + transformId (str or None): unique Id for this transform (optional) + json (dict): json compatible representation of this transform + ''' def __init__(self, className=None, dataString=None, transformId=None, json=None): if json is not None: @@ -163,6 +235,10 @@ def __init__(self, className=None, dataString=None, self.transformId = transformId def to_dict(self): + '''serialization routine + Returns: + dict: json compatible representation of this transform + ''' d = {} d['type'] = 'leaf' d['className'] = self.className @@ -172,11 +248,18 @@ def to_dict(self): return d def from_dict(self, d): + ''' deserialization routine + Args: + d (dict): json compatible representation of this transform + ''' self.className = d['className'] self.transformId = d.get('transformId', None) self._process_dataString(d['dataString']) def _process_dataString(self, datastring): + ''' method meant to set state of transform from datastring + generic implementation only saves datastring at self.dataString. + should rewrite for all transform classes that want to implement tform,fit,etc''' self.dataString = datastring def __str__(self): @@ -194,6 +277,22 @@ def __hash__(self): class AffineModel(Transform): + '''Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Args: + M00 (float): x'+=M00*x + M01 (float): x'+=M01*y + M10 (float): y'+=M10*x + M11 (float): y'+=M11*y + B0 (float): x'+=B0 + B1 (float): y'+=B1 + transformId (str): unique transformId for this transform (optional) + json (dict): json compatible representation of this transform + ''' className = 'mpicbg.trakem2.transform.AffineModel2D' def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, @@ -213,6 +312,7 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, @property def dataString(self): + '''dataString string for this transform''' return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( self.M[0, 0], self.M[1, 0], self.M[0, 1], self.M[1, 1], self.M[0, 2], self.M[1, 2]) @@ -231,6 +331,7 @@ def _process_dataString(self, datastring): self.load_M() def load_M(self): + '''method to take the attribute of self and fill in self.M''' self.M = np.identity(3, np.double) self.M[0, 0] = self.M00 self.M[0, 1] = self.M01 @@ -241,6 +342,14 @@ def load_M(self): @staticmethod def fit(A, B): + '''function to fit this transform given the corresponding sets of points A & B + + Args: + A (numpy.array): a Nx2 matrix of source points + B (numpy.array): a Nx2 matrix of destination points + Returns: + numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 + ''' if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): raise EstimationError( 'shape mismatch! A shape: {}, B shape {}'.format( @@ -260,6 +369,17 @@ def fit(A, B): return Tvec def estimate(self, A, B, return_params=True, **kwargs): + '''method for setting this transformation with the best fit given the corresponding points A,B + + Args: + A (numpy.array): a Nx2 matrix of source points + B (numpy.array): a Nx2 matrix of destination points + return_params (boolean): whether to return the parameter matrix + **kwargs (dict): keyword arguments to pass to self.fit + Returns: + numpy.array: a 2x3 matrix of parameters for this matrix, laid out (x,y) x (x,y,offset) + (or None if return_params=False) + ''' Tvec = self.fit(A, B, **kwargs) self.M00 = Tvec[0, 0] self.M10 = Tvec[2, 0] @@ -273,7 +393,8 @@ def estimate(self, A, B, return_params=True, **kwargs): def concatenate(self, model): ''' - concatenate a model to this model -- ported from trakEM2 below: + concatenate a model to this model -- ported from trakEM2 below:: + final double a00 = m00 * model.m00 + m01 * model.m10; final double a01 = m00 * model.m01 + m01 * model.m11; final double a02 = m00 * model.m02 + m01 * model.m12 + m02; @@ -281,6 +402,11 @@ def concatenate(self, model): final double a10 = m10 * model.m00 + m11 * model.m10; final double a11 = m10 * model.m01 + m11 * model.m11; final double a12 = m10 * model.m02 + m11 * model.m12 + m12; + + Args: + model (AffineModel): model to concatenate to this one + Returns: + AffineModel: model after concatenating model with this model ''' a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] @@ -296,6 +422,11 @@ def concatenate(self, model): return newmodel def invert(self): + '''return an inverted version of this transformation + + Returns: + AffineModel: an inverted version of this transformation + ''' inv_M = np.linalg.inv(self.M) Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) @@ -303,6 +434,13 @@ def invert(self): @staticmethod def convert_to_point_vector(points): + '''method to help reshape x,y points to x,y,1 vectors + + Args: + points (numpy.array): a Nx2 array of x,y points + Returns: + numpy.arary: a Nx3 array of x,y,1 points used for transformations + ''' Np = points.shape[0] onevec = np.ones((Np, 1), np.double) @@ -314,16 +452,38 @@ def convert_to_point_vector(points): return points, Nd @staticmethod - def convert_points_vector_to_array(points, Nd): + def convert_points_vector_to_array(points, Nd=2): + '''method for convertion x,y,K points to x,y vectors + + Args: + points (numpy.array): a Nx3 vector of points after transformation + Nd (int): the number of dimensions to cutoff (should be 2) + Returns: + numpy.array: a Nx2 array of x,y points + ''' points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T return points def tform(self, points): + '''transform a set of points through this transformation + + Args: + points (numpy.array): a Nx2 array of x,y points + Returns: + numpy.array: a Nx2 array of x,y points after transformation + ''' points, Nd = self.convert_to_point_vector(points) pt = np.dot(self.M, points.T).T return self.convert_points_vector_to_array(pt, Nd) def inverse_tform(self, points): + '''transform a set of points through the inverse of this transformation + + Args: + points (numpy.array): a Nx2 array of x,y points + Returns: + numpy.array: a Nx2 array of x,y points after inverse transformation + ''' points, Nd = self.convert_to_point_vector(points) pt = np.dot(np.linalg.inv(self.M), points.T).T return self.convert_points_vector_to_array(pt, Nd) @@ -372,20 +532,48 @@ def _process_dataString(self, dataString): @staticmethod def fit(src, dst): + '''function to fit this transform given the corresponding sets of points src & dst + + Args: + src (numpy.array): a Nx2 matrix of source points + dst (numpy.array): a Nx2 matrix of destination points + Returns: + numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 + ''' t = dst.mean(axis=0) - src.mean(axis=0) T = np.eye(3) T[:2, 2] = t return T def estimate(self, src, dst, return_params=True): + '''method for setting this transformation with the best fit given the corresponding points src,dst + + Args: + src (numpy.array): a Nx2 matrix of source points + dst (numpy.array): a Nx2 matrix of destination points + return_params (boolean): whether to return the parameter matrix + Returns: + numpy.array: a 2x3 matrix of parameters for this matrix, laid out (x,y) x (x,y,offset) + (or None if return_params=False) + ''' self.M = self.fit(src, dst) if return_params: return self.M class RigidModel(AffineModel): + '''model for storing Rigid only transformations + (rotation+translation) + or + (determinate=1, orthonormal eigenvectors) + + For now only default (identity) or json based initialization is supported + + Args: + json (dict): json compatible representation of this transform + ''' className = 'mpicbg.trakem2.transform.RigidModel2D' - + def __init__(self, *args, **kwargs): super(RigidModel, self).__init__(*args, **kwargs) # raise NotImplementedError( @@ -404,8 +592,15 @@ def _process_dataString(self, dataString): @staticmethod def fit(src, dst, rigid=True, **kwargs): - ''' + '''function to fit this transform given the corresponding sets of points src & dst Umeyama estimation of similarity transformation + + Args: + src (numpy.array): a Nx2 matrix of source points + dst (numpy.array): a Nx2 matrix of destination points + rigid (boolean): whether to constrain this transform to be rigid + Returns: + numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 ''' # TODO shape assertion num, dim = src.shape @@ -444,12 +639,29 @@ def fit(src, dst, rigid=True, **kwargs): return T def estimate(self, A, B, return_params=True, **kwargs): + '''method for setting this transformation with the best fit given the corresponding points src,dst + + Args: + A (numpy.array): a Nx2 matrix of source points + B (numpy.array): a Nx2 matrix of destination points + return_params (boolean): whether to return the parameter matrix + Returns: + numpy.array: a 2x3 matrix of parameters for this matrix, laid out (x,y) x (x,y,offset) + (or None if return_params=False) + ''' self.M = self.fit(A, B, **kwargs) if return_params: return self.M class SimilarityModel(RigidModel): + '''class for reprsenting Similarity transformations + (translation+rotation+scaling) + or + (orthogonal eigen vectors with equal eigenvalues) + + + ''' className = 'mpicbg.trakem2.transform.SimilarityModel2D' def __init__(self, *args, **kwargs): @@ -470,6 +682,16 @@ def _process_dataString(self, dataString): @staticmethod def fit(src, dst, rigid=False, **kwargs): + '''function to fit this transform given the corresponding sets of points src & dst + Umeyama estimation of similarity transformation + + Args: + src (numpy.array): a Nx2 matrix of source points + dst (numpy.array): a Nx2 matrix of destination points + rigid (boolean): whether to constrain this transform to be rigid + Returns: + numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 + ''' return RigidModel.fit(src, dst, rigid=rigid) # def estimate(self, src, dst, **kwargs): @@ -483,27 +705,19 @@ class Polynomial2DTransform(Transform): force_polynomial=True, params=None, identity=False, json=None) This provides 5 different ways to initialize the transform which are - mutually exclusive and applied in the following order. - - 1st - json = a json dictonary representation of the Polynomial2DTransform - generally used by TransformList - - 2nd - dataString = dataString representation of transform from mpicpg - - 3rd - identity = make this transform the identity - - 4th - params = 2xK np.array of polynomial coefficents up to order K - - 5th - src,dst = Nx2 np.array of source and dst points to use to estimate - transformation - order = integer degree of polynomial to fit when using src,dst - - + mutually exclusive and applied in the order specified here. + + 1)json2)dataString,3)identity,4)params,5)(src,dst) + + Args: + json (dict): dictonary representation of the Polynomial2DTransform generally used by TransformList + dataString (dict): dataString representation of transform from mpicpg + identity (boolean): whether to make this transform the identity + params (numpy.array): 2xK matrix of polynomial coefficents up to order K + src (numpy.array): Nx2 array of source points to use for fitting (used with dst) + dst (numpy.array): Nx2 array of destination points to use for fitting (used with src) + order (int): degree of polynomial to store + force_polynomial (boolean): whether to force this representation to be stored with degree order TODO: fall back to Affine Model in special cases ''' From 4b3160c42118d0199f30b357864a740e9f9260bd Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 23 Jul 2017 09:18:51 -0700 Subject: [PATCH 401/766] more transform documentation --- renderapi/transform.py | 104 ++++++++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 18 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 418ec401..4f398d83 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -746,21 +746,33 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, @property def is_affine(self): - '''TODO allow default to Affine''' + '''(boolean) TODO allow default to Affine''' return False # return self.order @property def order(self): + '''(int) order of polynomial''' no_coeffs = len(self.params.ravel()) return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) @property def dataString(self): + '''dataString of polynomial''' return Polynomial2DTransform._dataStringfromParams(self.params) @staticmethod def fit(src, dst, order=2): + '''function to fit this transform given the corresponding sets of points src & dst + polynomial fit + + Args: + src (numpy.array): a Nx2 matrix of source points + dst (numpy.array): a Nx2 matrix of destination points + order (boolean): order of polynomial to fit + Returns: + numpy.array: a (2,(order+1)*(order+2)/2) matrix with the best fit parameters + ''' xs = src[:, 0] ys = src[:, 1] xd = dst[:, 0] @@ -796,7 +808,34 @@ def fit(src, dst, order=2): def estimate(self, src, dst, order=2, test_coords=True, max_tries=100, return_params=True, **kwargs): + '''method for setting this transformation with the best fit given the corresponding points src,dst + + Args: + src (numpy.array): a Nx2 matrix of source points + dst (numpy.array): a Nx2 matrix of destination points + order (int): order of polynomial to fit + test_coords (bool): whether to test model after fitting to make sure it is good (see fitgood) + max_tries (int): how many times to attempt to fit the model (see fitgood) + return_params (bool): whether to return the parameter matrix + Returns: + numpy.array: a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix + (or None if return_params=False) + ''' def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): + '''check if model produces a 'good' result + True if all but rtol src points + are transformed to within atol of dst points + by model described by params, otherwise False + + Args: + src (numpy.array): a Nx2 matrix of source points + dst (numpy.array): a Nx2 matrix of destination points + params (numpy.array): a Kx2 matrix of parameters + atol (float): tolerance of how close is close enough + rtol (int): tolerance of how many outliers is acceptable + Result: + bool: whether the goodness condition is met + ''' result = Polynomial2DTransform(params=params).tform(src) t = np.allclose( result, dst, @@ -825,6 +864,7 @@ def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): @staticmethod def _dataStringfromParams(params=None): + '''method for producing a dataString from the parameters''' return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') for i in params.flatten()]).replace('e', 'E') @@ -837,11 +877,26 @@ def _process_dataString(self, datastring): @staticmethod def _format_raveled_params(raveled_params): + '''method to reshape linear parameters into parameter matrix + + Args: + ravled_params (numpy.array): an K long vector of parameters + Returns: + numpy.array: a (2,K/2) matrix of parameters, with first row for x and 2nd row for y + ''' + return np.array( [[float(d) for d in raveled_params[:len(raveled_params)/2]], [float(d) for d in raveled_params[len(raveled_params)/2:]]]) def tform(self, points): + '''transform a set of points through this transformation + + Args: + points (numpy.array): a Nx2 array of x,y points + Returns: + numpy.array: a Nx2 array of x,y points after transformation + ''' dst = np.zeros(points.shape) x = points[:, 0] y = points[:, 1] @@ -858,18 +913,23 @@ def tform(self, points): def coefficients(self, order=None): ''' determine number of coefficient terms in transform for a given order - input: order of polynomial -- defaults to self.order - output: integer number of coefficient terms expected in transform + + Args: + order (int or None): order of polynomial, defaults to self.order + Returns: + int: number of coefficient terms expected in transform ''' if order is None: order = self.order return (order + 1) * (order + 2) def asorder(self, order): - ''' - input: order > current order - output: new Transform object of selected order with coefficients - from self + '''return polynomial transform appoximation of this transformation with a lower order + + Args: + order (int): desired order (must have order> current order) + Returns: + Transform: transform of lower order ''' if self.order > order: raise ConversionError( @@ -882,9 +942,12 @@ def asorder(self, order): @staticmethod def fromAffine(aff): - ''' - input: AffineModel - output: Polynomial2DTransform defined by Affine model + ''' return a polynomial transformation equavalent to a given Affine + + Args: + aff (AffineModel): transform to become equivalent to + Returns: + Polynomial2DTransform: transform equal in effect to aff ''' if not isinstance(aff, AffineModel): raise ConversionError('attempting to convert a nonaffine model!') @@ -896,10 +959,12 @@ def fromAffine(aff): def estimate_dstpts(transformlist, src=None): ''' estimate destination points for list of transforms - input: - transformlist -- list of transform classes with tform method - src -- Nx2 numpy array of source points - output: Nx2 numpt array of destination points + + Args: + transformlist (list[Transform]):transforms that have a tform method implemented + src (numpy.array): a Nx2 array of source points + Returns: + numpy.array:Nx2 array of destination points ''' dstpts = src for tform in transformlist: @@ -916,11 +981,14 @@ def estimate_transformsum(transformlist, src=None, order=2): transformation and a single estimation. Will produce an Affine Model if all input transforms are Affine, otherwise will produce a Polynomial of specified order - inputs: - transformlist: recursive list of transform objects - src: Nx2 numpy array of source points for estimation - order: optional, order of Polynomial output if transformlist + + Args: + transformlist (list[Transform]): list of transform objects that implement tform + src (numpy.array): Nx2 array of source points for estimation + order (int): order of Polynomial output if transformlist inputs are non-Affine + Returns: + Polynomial2DTransform: best estimate of transformlist in a single transform of this order ''' def flatten(l): for i in l: From e90ed79b0c5b8a346354cfb705c468270aa2d0b5 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 23 Jul 2017 11:48:59 -0700 Subject: [PATCH 402/766] utils documentation --- renderapi/utils.py | 97 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 181c585d..0523953a 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -29,6 +29,15 @@ class RenderEncoder(json.JSONEncoder): obj.__dict__ ''' def default(self, obj): + '''default encoder for that handles Render objects + + Args: + obj (obj): any object that implements to_dict, dict(obj), + JsonEncoder.default(obj), or __dict__ (in order) + Returns: + (dict): json encodable dictionary + + ''' to_dict = getattr(obj, "to_dict", None) if callable(to_dict): return obj.to_dict() @@ -51,10 +60,16 @@ def default(self, obj): def post_json(session, request_url, d, params=None): ''' POST requests with RenderError handling - session: requests session - request_url: url - d: data payload (will be json dumpsed) - params: requests parameters + + Args: + session (requests.session.Session): requests session + request_url (str): url + d (dict): data payload (will be json dumps-ed) + params (dict): requests parameters + Returns: + requests.response: server response + Raises: + RenderError: if cannot post ''' headers = {"content-type": "application/json"} if d is not None: @@ -75,6 +90,20 @@ def post_json(session, request_url, d, params=None): def put_json(session, request_url, d, params=None): + ''' + PUT requests with RenderError handling + + Args: + session (requests.session.Session): requests session + request_url (str): url + d (dict): data payload (will be json dumps-ed) + params (dict): requests parameters + Returns: + requests.response: server response + Raises: + RenderError: if cannot post + ''' + headers = {"content-type": "application/json"} if d is not None: payload = json.dumps(d) @@ -94,13 +123,27 @@ def put_json(session, request_url, d, params=None): def renderdumps(obj, *args, **kwargs): - '''json.dumps using the RenderEncoder''' + '''json.dumps using the RenderEncode + + Args: + obj (obj): object to dumps + *args: json.dumps args + **kwargs: json.dumps kwargs + Returns: + str: serialized object + ''' cls_ = kwargs.pop('cls', RenderEncoder) return json.dumps(obj, *args, cls=cls_, **kwargs) def renderdump(obj, *args, **kwargs): - '''json.dump using the RenderEncoder''' + '''json.dump using the RenderEncoder + + Args: + obj (obj): object to dumps + *args: json.dump args + **kwargs: json.dump kwargs + ''' cls_ = kwargs.pop('cls', RenderEncoder) return json.dump(obj, *args, cls=cls_, **kwargs) @@ -109,7 +152,15 @@ def renderdump_temp(obj, *args, **kwargs): '''json.dump into a temporary file renderdump_temp(obj), obj will be dumped through renderdump into a temporary file - returns tempfilename, path to file it was dumped into''' + + Args: + obj (obj): object to dump + *args: json.dump args + **kwargs: json.dump kwargs + Returns: + str: path to location where temporary file was dumped + ''' + with tempfile.NamedTemporaryFile( suffix=".json", mode='w', delete=False) as tf: tempfilename = tf.name @@ -118,7 +169,14 @@ def renderdump_temp(obj, *args, **kwargs): def jbool(val): - '''return string representing java string values of py booleans''' + '''return string representing java string values of py booleans + + Args: + val (bool): boolean to encode + Returns: + str: 'true' or 'false' + + ''' if not isinstance(val, bool): logger.warning('Evaluating javastring of non-boolean {} {}'.format( type(val), val)) @@ -128,8 +186,8 @@ def jbool(val): def stripLogger(logger_tostrip): # pragma: no cover ''' remove all handlers from a logger -- useful for redefining - input: - logger_tostrip: logging logger as from logging.getLogger + Args: + logger_tostrip (logging.Logger): logging logger to strip ''' if logger_tostrip.handlers: for handler in logger_tostrip.handlers: @@ -137,11 +195,28 @@ def stripLogger(logger_tostrip): # pragma: no cover def defaultifNone(val, default=None): + '''simple default handler + + Args: + val (obj): value to fill in default + default (obj): default value + Returns: + obj: val if val is not None, else default + ''' return val if val is not None else default def fitargspec(f, oldargs, oldkwargs): - ''' fit function argspec given input args tuple and kwargs dict''' + ''' fit function argspec given input args tuple and kwargs dict + + Args: + f (func): function to inspect + oldargs (tuple): arguments passed to func + oldkwards (dict): keyword args passed to func + Returns + new_args: args with values filled in according to f spec + new_kwargs: kwargs with values filled in according to f spec + ''' try: args, varargs, keywords, defaults = inspect.getargspec(f) num_expected_args = len(args) - len(defaults) From ad92a4146de1ad8a08b3e505da00d3c4a52c0f3a Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 23 Jul 2017 12:16:36 -0700 Subject: [PATCH 403/766] pointmatch documentation v1 --- renderapi/pointmatch.py | 236 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 234 insertions(+), 2 deletions(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 9be4ef9f..e56fd56d 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -16,6 +16,19 @@ def get_matchcollection_owners(host=None, port=None, session=requests.session(), render=None, **kwargs): + + '''get all the matchCollection owners + + :func:`renderapi.render.renderaccess` decorated function + + Args: + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[str]: matchCollection owners + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/matchCollectionOwners" r = session.get(request_url) @@ -30,6 +43,20 @@ def get_matchcollection_owners(host=None, port=None, @renderaccess def get_matchcollections(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + '''get all the matchCollections owned by owner + + :func:`renderapi.render.renderaccess` decorated function + + Args: + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[str]: matchcollections owned by owner + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollections" % owner r = session.get(request_url) @@ -45,6 +72,21 @@ def get_matchcollections(owner=None, host=None, port=None, def get_match_groupIds(matchCollection, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + '''get all the groupIds in a matchCollection + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[str]: groupIds in matchCollection + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) r = session.get(request_url) @@ -61,6 +103,24 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + '''get all the matches outside a groupId in a matchCollection + returns all matches where pGroupId == groupId and qGroupId != groupId + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + groupId (str): groupId to query + mergeCollections (list[str]): other matchCollections to aggregate into answer + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[dict]: list of matches (see matches definition) + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( owner, matchCollection, groupId) @@ -80,6 +140,25 @@ def get_matches_within_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + '''get all the matches within a groupId in a matchCollection + returns all matches where pGroupId == groupId and qGroupId == groupId + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + groupId (str): groupId to query + mergeCollections (list[str] or None): other matchCollections + to aggregate into answer + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[dict]: list of matches (see matches definition) + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( owner, matchCollection, groupId) @@ -100,6 +179,28 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): + '''get all the matches between two specific groups + returns all matches where pgroup == pGroupId and qgroup == qGroupId + + OR pgroup == qGroupId and qgroup == pGroupId + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + pgroup (str): first group + qgroup (str): second group + mergeCollections (list[str] or None): other matchCollections + to aggregate into answer + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[dict]: list of matches (see matches definition) + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( owner, matchCollection, pgroup, qgroup) @@ -128,6 +229,31 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): + '''get all the matches between two specific tiles + returns all matches where + pgroup == pGroupId and pid=pId and qgroup == qGroupId and qid == qId + OR + qgroup == pGroupId and Qid=pId and Pgroup == qGroupId and pid == qId + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + pgroup (str): first group + pid (str): first id + qgroup (str): second group + qid (str): second id + mergeCollections (list[str] or None): other matchCollections + to aggregate into answer + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[dict]: list of matches (see matches definition) + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ ("/owner/%s/matchCollection/%s/group/%s/id/%s/" "matchesWith/%s/id/%s" % ( @@ -148,6 +274,25 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): + '''get all the matches from a specific groups + returns all matches where pgroup == pGroupId + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + pgroup (str): source group to query + mergeCollections (list[str] or None): other matchCollections + to aggregate into answer + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[dict]: list of matches (see matches definition) + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( owner, matchCollection, pgroup) @@ -167,6 +312,21 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): + '''get all the source pGroupIds in a matchCollection + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[str]: pGroupIds in matchCollection + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) @@ -184,7 +344,22 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, def get_match_groupIds_to_only(matchCollection, mergeCollections=None, render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): + session=requests.session(), **kwargs): + '''get all the destination qGroupIds in a matchCollection + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[str]: qGroupIds in matchCollection + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) @@ -199,10 +374,32 @@ def get_match_groupIds_to_only(matchCollection, mergeCollections=None, @renderaccess -def get_matches_involving_tile(matchCollection, pGroupId, pTileId, +def get_matches_involving_tile(matchCollection, groupId, id, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): + '''get all the matches involving a specific tile + returns all matches where groupId == pGroupId and id == pId + OR + groupId == qGroupId and id == qId + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + groupId (str): groupId to query + id (str): id to query + mergeCollections (list[str] or None): other matchCollections + to aggregate into answer + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[dict]: list of matches (see matches definition) + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/id/{}/".format( owner, matchCollection, pGroupId, pTileId) @@ -222,6 +419,27 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): + '''delete all the matches between two specific groups + deletes all matches where (pgroup == pGroupId and qgroup == qGroupId) + OR (pgroup == qGroupId and qgroup == pGroupId() + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + pgroup (str): first group + qgroup (str): second group + mergeCollections (list[str] or None): other matchCollections + to aggregate into answer + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[dict]: list of matches (see matches definition) + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/matchesWith/{}".format( owner, matchCollection, pGroupId, qGroupId) @@ -238,6 +456,20 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, @renderaccess def import_matches(matchCollection, data, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): + '''import matches into render database + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + data (list[dict]): list of matches to import (see matches definition) + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + requests.response.Reponse: server response + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/matches" % (owner, matchCollection) logger.debug(request_url) From fc980a1fa59155b5feaf39ca18cff5da6c3e79a5 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 23 Jul 2017 12:22:51 -0700 Subject: [PATCH 404/766] pointmatch doc refinement --- renderapi/pointmatch.py | 70 +++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index e56fd56d..f427c84f 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -72,21 +72,21 @@ def get_matchcollections(owner=None, host=None, port=None, def get_match_groupIds(matchCollection, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the groupIds in a matchCollection - - :func:`renderapi.render.renderaccess` decorated function - - Args: - matchCollection (str): matchCollection name - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[str]: groupIds in matchCollection - Raises: - RenderError: if cannot get a reponse from server - ''' + '''get all the groupIds in a matchCollection + + :func:`renderapi.render.renderaccess` decorated function + + Args: + matchCollection (str): matchCollection name + owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render (RenderClient): RenderClient connection object + session (requests.session.Session): requests session + Returns: + list[str]: groupIds in matchCollection + Raises: + RenderError: if cannot get a reponse from server + ''' request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) r = session.get(request_url) @@ -103,8 +103,8 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the matches outside a groupId in a matchCollection - returns all matches where pGroupId == groupId and qGroupId != groupId + '''get all the matches outside a groupId in a matchCollection + returns all matches where pGroupId == groupId and qGroupId != groupId :func:`renderapi.render.renderaccess` decorated function @@ -140,8 +140,8 @@ def get_matches_within_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the matches within a groupId in a matchCollection - returns all matches where pGroupId == groupId and qGroupId == groupId + '''get all the matches within a groupId in a matchCollection + returns all matches where pGroupId == groupId and qGroupId == groupId :func:`renderapi.render.renderaccess` decorated function @@ -215,11 +215,19 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, raise RenderError(r.text) -def add_merge_collections(request_url, mergeCollections): - if mergeCollections is not None: - if type(mergeCollections) is list: +def add_merge_collections(request_url, mcs): + '''utility function to add mergeCollections to request_url + + Args: + request_url (str): request url + mcs (list[str]): list of mergeCollections to add + Returns: + str: request_url with ?mergeCollection=mc[0]&mergeCollection=mc[1]... appended + ''' + if mcs is not None: + if type(mcs) is list: request_url += "?"+"&".join( - ['mergeCollection=%s' % mc for mc in mergeCollections]) + ['mergeCollection=%s' % mc for mc in mcs]) return request_url @@ -229,13 +237,13 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''get all the matches between two specific tiles - returns all matches where - pgroup == pGroupId and pid=pId and qgroup == qGroupId and qid == qId - OR - qgroup == pGroupId and Qid=pId and Pgroup == qGroupId and pid == qId + '''get all the matches between two specific tiles + returns all matches where + pgroup == pGroupId and pid=pId and qgroup == qGroupId and qid == qId + OR + qgroup == pGroupId and Qid=pId and Pgroup == qGroupId and pid == qId - :func:`renderapi.render.renderaccess` decorated function + :func:`renderapi.render.renderaccess` decorated function Args: matchCollection (str): matchCollection name @@ -274,8 +282,8 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''get all the matches from a specific groups - returns all matches where pgroup == pGroupId + '''get all the matches from a specific groups + returns all matches where pgroup == pGroupId :func:`renderapi.render.renderaccess` decorated function From 0eb278f08f53a9c9e0942e18c12cd9d85dbfb020 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 23 Jul 2017 12:23:31 -0700 Subject: [PATCH 405/766] pep8 --- renderapi/transform.py | 49 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 4f398d83..110694ea 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -33,6 +33,7 @@ class TransformList: transformId (str): uniqueId for this TransformList json (dict): json compatible dictionary to create TransformList via :method:`from_dict` ''' + def __init__(self, tforms=None, transformId=None, json=None): if json is not None: self.from_dict(json) @@ -111,8 +112,8 @@ def load_leaf_json(d): func: proper function to deserialize this transformation Raises: RenderError: if d['type']!= leaf or is omitted - KeyError: if d['type'] does not indicate one of AffineModel, Polynomial2DTransform - TranslationModel, RigidModel, or SimilarityModel + KeyError: if d['type'] does not indicate one of AffineModel, + Polynomial2DTransform, TranslationModel, RigidModel, or SimilarityModel ''' handle_load_leaf = { AffineModel.className: lambda x: AffineModel(json=x), @@ -145,6 +146,7 @@ class InterpolatedTransform: linear interpolation between a (at 0) and b (at 1) json (dict): json compatible representation of this transform to initialize via :method:`self.from_dict` ''' + def __init__(self, a=None, b=None, lambda_=None, json=None): if json is not None: self.from_dict(json) @@ -183,6 +185,7 @@ class ReferenceTransform: refId (str): transformId of the referenced transform json (dict): json compatible representation of this transform ''' + def __init__(self, refId=None, json=None): if json is not None: self.from_dict(json) @@ -225,6 +228,7 @@ class Transform(object): transformId (str or None): unique Id for this transform (optional) json (dict): json compatible representation of this transform ''' + def __init__(self, className=None, dataString=None, transformId=None, json=None): if json is not None: @@ -343,7 +347,7 @@ def load_M(self): @staticmethod def fit(A, B): '''function to fit this transform given the corresponding sets of points A & B - + Args: A (numpy.array): a Nx2 matrix of source points B (numpy.array): a Nx2 matrix of destination points @@ -533,7 +537,7 @@ def _process_dataString(self, dataString): @staticmethod def fit(src, dst): '''function to fit this transform given the corresponding sets of points src & dst - + Args: src (numpy.array): a Nx2 matrix of source points dst (numpy.array): a Nx2 matrix of destination points @@ -566,14 +570,14 @@ class RigidModel(AffineModel): (rotation+translation) or (determinate=1, orthonormal eigenvectors) - + For now only default (identity) or json based initialization is supported Args: json (dict): json compatible representation of this transform ''' className = 'mpicbg.trakem2.transform.RigidModel2D' - + def __init__(self, *args, **kwargs): super(RigidModel, self).__init__(*args, **kwargs) # raise NotImplementedError( @@ -594,7 +598,7 @@ def _process_dataString(self, dataString): def fit(src, dst, rigid=True, **kwargs): '''function to fit this transform given the corresponding sets of points src & dst Umeyama estimation of similarity transformation - + Args: src (numpy.array): a Nx2 matrix of source points dst (numpy.array): a Nx2 matrix of destination points @@ -684,7 +688,7 @@ def _process_dataString(self, dataString): def fit(src, dst, rigid=False, **kwargs): '''function to fit this transform given the corresponding sets of points src & dst Umeyama estimation of similarity transformation - + Args: src (numpy.array): a Nx2 matrix of source points dst (numpy.array): a Nx2 matrix of destination points @@ -706,7 +710,7 @@ class Polynomial2DTransform(Transform): json=None) This provides 5 different ways to initialize the transform which are mutually exclusive and applied in the order specified here. - + 1)json2)dataString,3)identity,4)params,5)(src,dst) Args: @@ -765,7 +769,7 @@ def dataString(self): def fit(src, dst, order=2): '''function to fit this transform given the corresponding sets of points src & dst polynomial fit - + Args: src (numpy.array): a Nx2 matrix of source points dst (numpy.array): a Nx2 matrix of destination points @@ -820,7 +824,7 @@ def estimate(self, src, dst, order=2, Returns: numpy.array: a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix (or None if return_params=False) - ''' + ''' def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): '''check if model produces a 'good' result True if all but rtol src points @@ -878,7 +882,7 @@ def _process_dataString(self, datastring): @staticmethod def _format_raveled_params(raveled_params): '''method to reshape linear parameters into parameter matrix - + Args: ravled_params (numpy.array): an K long vector of parameters Returns: @@ -886,8 +890,8 @@ def _format_raveled_params(raveled_params): ''' return np.array( - [[float(d) for d in raveled_params[:len(raveled_params)/2]], - [float(d) for d in raveled_params[len(raveled_params)/2:]]]) + [[float(d) for d in raveled_params[:len(raveled_params) / 2]], + [float(d) for d in raveled_params[len(raveled_params) / 2:]]]) def tform(self, points): '''transform a set of points through this transformation @@ -913,7 +917,7 @@ def tform(self, points): def coefficients(self, order=None): ''' determine number of coefficient terms in transform for a given order - + Args: order (int or None): order of polynomial, defaults to self.order Returns: @@ -925,7 +929,7 @@ def coefficients(self, order=None): def asorder(self, order): '''return polynomial transform appoximation of this transformation with a lower order - + Args: order (int): desired order (must have order> current order) Returns: @@ -978,10 +982,9 @@ def estimate_dstpts(transformlist, src=None): def estimate_transformsum(transformlist, src=None, order=2): ''' pseudo-composition of transforms in list of transforms using source point - transformation and a single estimation. - Will produce an Affine Model if all input transforms are Affine, - otherwise will produce a Polynomial of specified order - + transformation and a single estimation. Will produce an Affine Model if all + input transforms are Affine, otherwise will produce a Polynomial of specified order + Args: transformlist (list[Transform]): list of transform objects that implement tform src (numpy.array): Nx2 array of source points for estimation @@ -1003,7 +1006,7 @@ def flatten(l): tforms = flatten(transformlist) if all([tform.className == AffineModel.className for tform in tforms]): - am = AffineModel() - am.estimate(A=src, B=dstpts, return_params=False) - return am + am = AffineModel() + am.estimate(A=src, B=dstpts, return_params=False) + return am return Polynomial2DTransform(src=src, dst=dstpts, order=order) From 275356289ded488020d8a9ac9ea0b694ba5b3d70 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 08:59:55 -0700 Subject: [PATCH 406/766] documentation updates --- docs/guide/index.rst | 316 +++++++++++++++++++++++++++ docs/guide/pointmatchassumptions.rst | 70 ++++++ docs/index.rst | 5 +- 3 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 docs/guide/index.rst create mode 100644 docs/guide/pointmatchassumptions.rst diff --git a/docs/guide/index.rst b/docs/guide/index.rst new file mode 100644 index 00000000..1a9e3a08 --- /dev/null +++ b/docs/guide/index.rst @@ -0,0 +1,316 @@ +User Guide +========== + +Getting Started +--------------- + +First you must have a render server running and accessible to you. +For instructions on getting a render server up and running +http://www.github.com/saalfeldlab/render + +For the purposes of this tutorial i'm going to assume you have +a render-server installed and up and running on localhost and +your render dashboard is viewable at +http://localhost:8080/render-ws/view/index.html + +You also should have available on your local system a path to the +render-ws-java-client scripts with a compiled jar file. +One way to get these things, with render-python already installed +is to use get the render-python Docker image (fcollman\render-python), +where it will be installed at \usr\local\render\render-ws-java-client\src\scripts + +.. code-block:: bash + + $ docker pull fcollman\render-python + $ docker run -t fcollman\render-python \ + -v .:\scripts \ + python \scripts\my_script.py + +Making a new stack +------------------ + +First we have to setup our default connection properties for our render server +:: + + import renderapi + + #create a renderapi.connect.Render object + render_connect_params ={ + 'host':'localhost', + 'port':8080, + 'owner':'myowner', + 'project':'myproject', + 'client_scripts':'\usr\local\render\render-ws-java-client\src\scripts' + 'memGB':'2G' + } + render = renderapi.connect(**render_connect_params) + + +You can simplify your call to :func:`renderapi.connect` if you wish by setting up some or all of +the following environment variables +DEFAULT_RENDER_HOST, DEFAULT_RENDER_PORT, DEFAULT_RENDER_CLIENT_SCRIPTS, +DEFAULT_RENDER_PROJECT, DEFAULT_RENDER_OWNER, RENDER_CLIENT_HEAP. + +Now we can create a new stack on the render server + +:: + + #make a new stack + stack = 'mystack' + renderapi.stack.create_stack(stack,render=render) + +now you should be able to see your stack at http://localhost:8080/render-ws/view/stacks.html. +It is presently in the LOADING state as it is awaiting tiles to be loaded. +In order to give it some tiles you must fill out some metadata information about the tile. +Many fields are technically optional, but it's best to fill them out if you can. +In this example we will do it manually, of course most users will likely write code to +create this metadata from existing metadata. + +:: + + #define a tile layout + layout = Layout(sectionId='1', + scopeId='myscope', + cameraId='mycamera', + imageRow=0, + imageCol=0, + stageX=100.0, + stageY=300.0, + rotation=0.0, + pixelsize=3.0) + +Next you have to define the set of transformations that should be applied to the raw image. +In this example we will use :class:`renderapi.transform.AffineModel`. +However, you can use any of the transforms defined in :class:`renderapi.transform` +or any transformation in the mpicpg library (https://github.com/axtimwalde/mpicbg) +installed with your render server via the :class:`renderapi.transform.Transform` class, +provided you know its classname and dataString. + +:: + + #define a simple transformation, here a translation based upon layout + at = AffineTransform(B0=layout.stageX/layout.pixelsize, + B1=layout.stageY/layout.pixelsize) + +Now we can define the actual tile + +:: + + tilespec = TileSpec(tileId='000000000000', + z=0.0, + width=2048, + height=2048, + imageUrl='/data/images/0_0_0.tif', + maskUrl=None, + layout=layout, + tforms=[at]) + +Note that the path to the imageUrl needs to be a path that is readable by the render server +if you want server side rendering of images to function correctly. If you only want to use +render to store metadata information then this isn't strictly necessary, but other web-services +such as https://github.com/neurodata/ndviz will require this. + + +Importing tilespecs +------------------- +Now we can import the tile to our stack using :class:`renderapi.client` which uses the +render-ws-java-client scripts to perform client side validation and bounding box estimation +of your tiles before uploading them to the server. + +:: + + #use the simple non-parallelized upload option + renderapi.client.import_tilespecs(stack, + [tilespec], + render = render) + + #now close the stack + renderapi.stack.set_stack_state(stack, 'COMPLETE', render = render) + +If you have many tilespecs to import, it often makes sense to parallelize the client side +validation and bounding box estimation. So lets simulate the importing of many tiles + +:: + + rows = 10 + cols = 20 + sections = 500 + overlap = .8 #20% overlap + pix = 3.0 #nm + img_width = 2048 #pixels + img_height = img_width + + tilespecs = [] + for section in range(sections): + for r in range(rows): + for c in range(cols): + layout = Layout(sectionId='%05d'%section, + scopeId='myscope', + cameraId='mycamera', + imageRow=0, + imageCol=0, + stageX=c*img_width*overlap*pix, + stageY=r*img_height*overlap*pix, + rotation=0.0, + pixelsize=pix) + + + + #define a simple transformation, here a translation based upon layout + at = AffineTransform(B0=c*img_width*overlap, + B1=r*img_height*overlap) + + tileId = '%d_%d_%d'%(section,r,c) + + ts = TileSpec(tileId=, + z=section, + width=img_width, + height=img_height, + imageUrl='/data/images/%s.tif'%tileId, + maskUrl=None, + layout=layout, + tforms=[at]) + tilespecs.append(ts) + +This would of course would need to be adapted to suit the needs of your +specific situation, but assuming you have a large number of tilespecs, +they can be imported more efficently using +:func:`renderapi.client.import_tilespecs_parallel` +which will also close the stack for you if you'd like. +:: + + renderapi.client.import_tilespecs_parallel(stack, + tilespecs, + close_stack=True, + render = render) + + +When you are done, you should be able to see your stack on the render dashboard. + +Transformations +--------------- + +The idea behind render is that it serves as a central place to store the metadata data +about image tiles and how they should be tranformed. Central to that concept is what +transformations it supports. Render is written in java and uses the mpicbg library +(https://github.com/axtimwalde/mpicbg), the same library that backs TrakEM2, to +perform all server side image transformation. + +Render-python is client side library and can assist you in managing and setting +those tranformations, and performing some calculations using the +:class:`renderapi.transorm` module. We have focused our initial +efforts at supporting the most commonly used types of transformations. + +Some transformation types presently support `tform` and 'inverse_tform` methods +for calculating where numpy array sets of points map to and from these tranformations. +Some presently support `estimate` methods which given a set of source and destination +points, allow the estimation of a best fit transformation. + +:func:`renderapi.transform.estimate_dstpts` simplifies mapping points through an +ordered list of transformations. :func:`renderapi.transform.estimate_transformsum` +provides a general way to produce a single :class:`renderapi.transform.Polynomial2DTransform` +that approximates a list of tranforms which have implemented `.tform` methods. + +One aspect to keep in mind about render is that it supports +:class:`renderapi.transform.ReferenceTransform` which allows many tiles to share a common +transformation without having to store its parameters directly. This is a conveient way to +save on database storage and ensure many tiles are identically manipulated. +However, depending on what you want to do, it can make things slightly more complicated. + +For example, presently, the java client side scripts used to calculate bounding boxes and +validate transformations need to have access to the referenced transformations in order to +do their work, even if they already exist in the database. This is what the `sharedTransforms` +argument in import methods within :class:`renderapi.client` is for. + +The :func:`renderapi.transform.ReferenceTransform.tform` method does not exist because +the :class:`renderapi.transform.ReferenceTransform` transform doesn't have any data about what +kind the transform parameters are. Most render-python calls default to returning dereferenced +transforms, which avoids this issue. However, this will break the efficency gains if you simply +upload those dereferenced transforms. This is why :func:`renderapi.tilespec.get_tile_spec_raw` +exists, in order to give you referenced transforms if you so wish. Also, you can use the +:class:`renderapi.client.importTransformChangesClient` to accelerate and simplify many transform +modification tasks, as it won't do any client side validation or bounding box calculations. + + +Pointmatch database +------------------- + +Pointmatchs are locations between two images that correspond. Render has two groups +of web services that are both available on the same web interface, the 'tile' +based services and the 'pointmatch' services. + +They are loosely coupled in the sense that they are stored in distinct databases, +and make no assumptions about how the other is structured. This allows them to be +deployed and maintained seperately, but can make things confusing if you assume +that one knows more about what the other is doing than it does. To make things +less confusing there is a set of reccomendations that you can read at :doc:`pointmatchassumptions`. + +Pointmatches are stored by collection, group's and id's (see :ref:`reccomendation `) +and have a source 'p' and a destination 'q', thus each set of matches in a collection is specified by a +pGroupId, pId, qGroupId, qId combination. Functions in the :class:`renderapi.pointmatch` module +allow you to make queries on these point matches in various ways, and upload new matches. +You might place some point matches between tile 0_0_0 on section 0, and tile 1_0_0 on section 1, +using :func:`renderapi.pointmatch.import_matches` +and then retrieve them using +:func:`renderapi.pointmatch.get_matches_from_tile_to_tile`. + +:: + + matches_in={ + 'pGroupId':'0', + 'qGroupId':'1', + 'pId':'0_0_0', + 'qId':'1_0_0', + 'matches':{ + 'p':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'q':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'w':[1,1,1,1] + } + } + renderapi.pointmatch.import_matches('mycollection', + [matches_in], + render=render) + matches_out=renderapi.pointmatch.get_matches_from_tile_to_tile('mycollection' + '0', + '0_0_0, + '1', + '1_0_0', + render = render) + + print(matches_out) + >> [{ + 'pGroupId':'0', + 'qGroupId':'1', + 'pId':'0_0_0', + 'qId':'1_0_0', + 'matches':{ + 'p':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'q':[[0,0],[1000,1000],[1000,0],[0,1000]], + 'w':[1,1,1,1] + } + }] + + + + + + + + + + + + + + + + +Installation +------------ +from source + +.. code-block:: bash + + $ git clone https://www.github.com/fcollman/render-python + $ cd render-python + $ python setup.py install \ No newline at end of file diff --git a/docs/guide/pointmatchassumptions.rst b/docs/guide/pointmatchassumptions.rst new file mode 100644 index 00000000..9b14b8a7 --- /dev/null +++ b/docs/guide/pointmatchassumptions.rst @@ -0,0 +1,70 @@ +Pointmatch Assumptions +====================== +Generally, I would reccomend adopting a set of conventions that constrain the +relationship between data in the 'tile' and 'pointmatch' databases. These conventions are informed +in large part based upon how the EMAligner https://github.com/khaledkhairy/EM_aligner +assumes they are related, when using them to solve alignment problems. + + +tile_owners = pointmatch_owners +------------------------------- +Particularly on deployments which have only a single render service, this will save you +from having to redefine the default owner in the :class:`renderapi.render.Render` or override +it at each step. Make all stacks related to a dataset owned by a single owner, and make all +pointmatch collections owned by that same owner. + +.. _group-section-explanation: + +groupId=sectionId and id=tileId +---------------------------- +Or more verbosely, groupId/qGroupId/pGroupId = sectionId and pId/qId/id = tileId. +Groups thus correspond to what section the point match is from and the id's correspond to what tile +they are from. +Technically, there is no need to make this association, and none of the render web services strictly require it. +However, if you want to use the pointmatch results in combination with the tile services, +it will be far easier if there is a strict mapping between these two databases. +In addition, tools like EMAligner and ndviz are presently written in a way that assumes this mapping is held, +so you need to make the same assumption if you want to use those services. + +Know your 'local' +------------------------------------------------------------------- +Write all pointmatches between tiles in a consistent 'local' coordinate system +and make that local coordinate system the raw image space given by rendering the tile +using :func:`renderapi.image.get_tile_image_data` with normalizeForMatching=True and scale=1.0. + +This would be conceptually simple, if the 'local' meant the same as +:class:`renderapi.coordinate` module defines local to mean, +namely the raw image space, with the upper left hand pixel at 0,0 and positive x to the right +and positive y down. + +However, in some deployments of render this is not the case, and you might find that +rendering a tile using normalizeForMatching=True does not produce a raw image tile. +In fact it might render blank data in some circumstances if you have more than 1 transformation. + +This is because, at Janelia the EMAligner was developed on TEM images that need a lens correction +transformation, and the pointmatches are defined on the 'local' coordinate system after lens correction. +This simplifies solving for the non-lens component of the transformation, as the EMAligner only +needs to specify the single transformation that brings 'local' pointmatches into 'global' alignment +and can safely disregard the non-linear effects of the lens correction. + +However, it produces the confusing result that mapping the 'local' point matches coordinates +through :func:`renderapi.coordinate.local_to_world_coordinates` does not give the correct result, +and you have to have a second stack with lens correction transformations removed in order to map +point match coordinates from local to world coordinates accurately using the coordinate mapping service. + +More discussion on this at https://github.com/saalfeldlab/render/issues/13, +https://github.com/saalfeldlab/render/issues/31. + +I implemented an alternative strategy at +https://github.com/saalfeldlab/render/pull/29 +which adds a removeAllOption=True to many render calls +which simply removes all transformations from the tilespec before returning or rendering. +In applications where non-linear lens corrections are minimal, this simplifies things. + +However for TEM or other applications with stereotyped non-linear transformations, +it will make using the EMAligner to solve alignment problems more difficult, +as the EMAligner doesn't know that it should map the pointmatches into the post non-linear correction +space before attemptign to solve and isn't presently written to do this. + + + diff --git a/docs/index.rst b/docs/index.rst index fc3c8971..656d6784 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,9 +18,10 @@ API .. toctree:: :maxdepth: 3 + guide/index + guide/pointmatchassumptions api/renderapi - - + Indices and tables ================== From 1188ad71718255a2ec47bec497d271e7957195ce Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 09:00:09 -0700 Subject: [PATCH 407/766] editing client docstrings --- renderapi/client.py | 104 +++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 39 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 7f2459ed..8590b519 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -38,11 +38,14 @@ def import_single_json_file(stack, jsonfile, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, render=None, **kwargs): ''' - calls client script to import given jsonfile: - stack: stack to import into - jsonfile: path to jsonfile to import - transformFile: path to a file that contains shared + calls client script to import given jsonfile + + Args: + stack (str): stack to import into + jsonfile (str): path to jsonfile to import + transformFile (str): path to a file that contains shared transform references if necessary + render (renderapi.render.RenderClient): render connect object ''' if transformFile is None: transform_params = [] @@ -67,16 +70,20 @@ def import_jsonfiles_and_transforms_parallel_by_z( client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): ''' - imports json files and transform files in parallel: - stack: the stack to import within - jsonfiles: "list of tilespec" jsons to import - transformfiles: "list of transform files" which matches + imports json files and transform files in parallel + + Args: + stack (str): the stack to import within + jsonfiles (list[str]): "list of tilespec" json paths to import + transformfiles (list[str]): "list of transform files" paths which matches in a 1-1 way with jsonfiles, so referenced transforms are shared only within a single element of these matched lists. Useful cases where there is as single z transforms shared by all tiles within a single z, but not across z's - poolsize: number of processes for multiprocessing pool - close_stack: mark render stack as COMPLETE after successful import + poolsize (int): number of processes for multiprocessing pool + close_stack (bool): whether to mark render stack as COMPLETE after successful import + render (renderapi.render.RenderClient): render connect object + ''' set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial(import_single_json_file, stack, render=render, @@ -96,12 +103,15 @@ def import_jsonfiles_parallel( project=None, close_stack=True, render=None, **kwargs): ''' import jsons using client script in parallel - stack: the stack to upload into - jsonfiles: list of jsonfiles to upload - poolsize: number of upload processes spawned by multiprocessing pool - transformFile: a single json file containing transforms referenced + + Args: + stack (str): the stack to upload into + jsonfiles (list[str]): list of jsonfile paths to upload + poolsize (int): number of upload processes spawned by multiprocessing pool + transformFile (str): a single json file path containing transforms referenced in the jsonfiles - close_stack: mark render stack as COMPLETE after successful import + close_stack (boolean): whether to mark render stack as COMPLETE after successful import + render (renderapi.render.RenderClient): render connect object ''' set_stack_state(stack, 'LOADING', host, port, owner, project) @@ -124,11 +134,15 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, render=None, **kwargs): ''' import jsons using client script serially - jsonfiles: iterator of filenames to be uploaded - transformFile: path to a jsonfile that contains shared + + Args: + jsonfiles (list): iterator of filenames to be uploaded + transformFile (str): path to a jsonfile that contains shared transform references (if necessary) - close_stack: mark render stack as COMPLETE after successful import + close_stack (boolean): mark render stack as COMPLETE after successful import + render (renderapi.render.RenderClient): render connect object ''' + set_stack_state(stack, 'LOADING', host, port, owner, project) if transformFile is None: transform_params = [] @@ -158,6 +172,8 @@ def import_jsonfiles_validate_client(stack, jsonfiles, render=None, **kwargs): ''' Uses java client for parallelization and validation + + ''' transform_params = (['--transformFile', transformFile] if transformFile is not None else []) @@ -201,11 +217,14 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): ''' - input: - stack -- stack to which tilespecs will be added - tilespecs -- list of tilespecs - sharedTransforms -- list of shared + method to import tilesepcs directly from :class:`renderapi.tilespec.TileSpec` objects + + Args: + stack (str): stack to which tilespecs will be added + tilespecs (list[renderapi.tilespec.TileSpec]): list of tilespecs to import + sharedTransforms (list[renderapi.transform.Transform]): list of shared referenced transforms to be ingested + render (renderapi.render.RenderClient): render connect object ''' tsjson = renderdump_temp(tilespecs) @@ -231,14 +250,18 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, client_script=None, memGB=None, render=None, **kwargs): ''' - input: - stack -- stack to which tilespecs will be added - tilespecs -- list of tilespecs - sharedTransforms -- list of shared + method to import tilesepcs directly from :class:`renderapi.tilespec.TileSpec` objects + using pathos.multiprocessing to parallelize + + Args: + stack (str): stack to which tilespecs will be added + tilespecs (list[renderapi.tilespec.TileSpec]): list of tilespecs to import + sharedTransforms (list[renderapi.transform.Transform]): list of shared referenced transforms to be ingested - poolsize -- degree of parallelism to use - subprocess_mode -- subprocess mode used when calling client side java - close_stack: mark render stack as COMPLETE after successful import + poolsize (int): degree of parallelism to use + subprocess_mode (str): subprocess mode used when calling client side java + close_stack (boolean): mark render stack as COMPLETE after successful import + render (renderapi.render.RenderClient): render connect object ''' set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial( @@ -264,11 +287,11 @@ def local_to_world_array(stack, points, tileId, subprocess_mode=None, ''' placeholder function for coordinateClient localtoworld - inputs: - stack -- stack to which world coordinates are mapped - points -- local points to map to world - tileId -- tileId to which points correspond - subprocess_mode -- subprocess mode used when calling + Args: + stack (str): stack to which world coordinates are mapped + points (dict): local points to map to world + tileId (str): tileId to which points correspond + subprocess_mode (str): subprocess mode used when calling clientside java client outputs: list of points in world coordinates corresponding to local points @@ -284,11 +307,14 @@ def world_to_local_array(stack, points, subprocess_mode=None, ''' placeholder function for coordinateClient worldtolocal - inputs: - stack -- stack to which world coordinates are mapped - points -- world points in stack to map to local - outputs: - list of list of dictionaries defining local coordinates + Args: + stack (str): stack to which world coordinates are mapped + points (dict): local points to map to world + subprocess_mode (str): subprocess mode used when calling client side java + render (renderapi.render.RenderClient): render connect object + + Returns: + list[list]: dictionaries defining local coordinates and tileIds corresponding to world point ''' raise NotImplementedError('Whoops.') From 0495bc1e9956a8f6fdfb7858d27ba5332da49467 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 20:41:06 -0700 Subject: [PATCH 408/766] made changes to specify that 3rd dimension of numpy array is variabe in docs --- renderapi/image.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index c8ffcf6f..4babaa35 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -50,7 +50,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: - numpy.array: [N,M,3] array of image data from render + numpy.array: [N,M,:] array of image data from render Raises: .errors.RenderError ''' @@ -109,7 +109,7 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: - numpy.array: [N,M,3] array of image data from render + numpy.array: [N,M,:] array of image data from render Raises: .errors.RenderError ''' @@ -171,7 +171,7 @@ def get_section_image(stack, z, scale=1.0, filter=False, render (renderapi.render.Render): render connect object session (requests.sessions.Session): sessions object to connect with Returns: - numpy.array: [N,M,3] array of image data of section from render + numpy.array: [N,M,:] array of image data of section from render ''' try: image_ext = IMAGE_FORMATS[img_format] From 5663f391c4dce92f8025aa47faa0980373f511a1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 20:43:34 -0700 Subject: [PATCH 409/766] changing owners to unicode in documentation --- renderapi/pointmatch.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index f427c84f..370ba8d2 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -25,7 +25,7 @@ def get_matchcollection_owners(host=None, port=None, render (RenderClient): RenderClient connection object session (requests.session.Session): requests session Returns: - list[str]: matchCollection owners + list[unicode]: matchCollection owners Raises: RenderError: if cannot get a reponse from server ''' @@ -48,12 +48,12 @@ def get_matchcollections(owner=None, host=None, port=None, :func:`renderapi.render.renderaccess` decorated function Args: - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session Returns: - list[str]: matchcollections owned by owner + list[unicode]: matchcollections owned by owner Raises: RenderError: if cannot get a reponse from server ''' @@ -112,7 +112,7 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, matchCollection (str): matchCollection name groupId (str): groupId to query mergeCollections (list[str]): other matchCollections to aggregate into answer - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -150,7 +150,7 @@ def get_matches_within_group(matchCollection, groupId, mergeCollections=None, groupId (str): groupId to query mergeCollections (list[str] or None): other matchCollections to aggregate into answer - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -192,7 +192,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, qgroup (str): second group mergeCollections (list[str] or None): other matchCollections to aggregate into answer - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -253,7 +253,7 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, qid (str): second id mergeCollections (list[str] or None): other matchCollections to aggregate into answer - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -292,7 +292,7 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, pgroup (str): source group to query mergeCollections (list[str] or None): other matchCollections to aggregate into answer - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -326,7 +326,7 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, Args: matchCollection (str): matchCollection name - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -359,7 +359,7 @@ def get_match_groupIds_to_only(matchCollection, mergeCollections=None, Args: matchCollection (str): matchCollection name - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -399,7 +399,7 @@ def get_matches_involving_tile(matchCollection, groupId, id, id (str): id to query mergeCollections (list[str] or None): other matchCollections to aggregate into answer - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -439,7 +439,7 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, qgroup (str): second group mergeCollections (list[str] or None): other matchCollections to aggregate into answer - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session @@ -471,7 +471,7 @@ def import_matches(matchCollection, data, owner=None, host=None, port=None, Args: matchCollection (str): matchCollection name data (list[dict]): list of matches to import (see matches definition) - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) + owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) render (RenderClient): RenderClient connection object session (requests.session.Session): requests session From 27e15c6077c8659c30b5f208cabe804ea7fdf7f3 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 20:45:35 -0700 Subject: [PATCH 410/766] added clarification to connect --- renderapi/render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index cdabc0e8..b8bc9ade 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -174,7 +174,8 @@ def connect(host=None, port=None, owner=None, project=None, web_only (bool): whether to check environment variables/prompt user for client_scripts directory if not in arguments Returns: - RenderClient: a connect object to simplify specifying what render server to connect to + Render: a connect object to simplify specifying what render server to connect to + (returns :class:`RenderClient` if sufficent parameters are passed) ''' if host is None: if 'RENDER_HOST' not in os.environ: From fe6a5cdcee0ca03ac3004ed5a2225a628e8bfa0a Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 20:46:33 -0700 Subject: [PATCH 411/766] added minZ,maxZ --- renderapi/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 11cb4867..d81a782b 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -436,7 +436,7 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, render (renderapi.render.Render): render connect object session (requests.sessions.Session): session object (default start a new one) Returns: - dict: bounds with keys minY,minY,maxX,maxY + dict: bounds with keys minY,minY,maxX,maxY,minZ,maxZ Raises: .errorr.RenderError ''' From d979ad35c988c28a40aa09ec2b1dcac362d8cd91 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 20:47:39 -0700 Subject: [PATCH 412/766] added interval clarification --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 110694ea..10d00b2d 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -142,7 +142,7 @@ class InterpolatedTransform: Args: a (Transform):transform at minimum weight b (Transform):transform at maximum weight - lambda_ (float): value (0.-1.) which defines evaluation of the + lambda_ (float): value in interval [0.,1.] which defines evaluation of the linear interpolation between a (at 0) and b (at 1) json (dict): json compatible representation of this transform to initialize via :method:`self.from_dict` ''' From 79e8ab8637307c87b75105aeaeda62527efc465f Mon Sep 17 00:00:00 2001 From: forrest collman Date: Mon, 24 Jul 2017 20:49:14 -0700 Subject: [PATCH 413/766] added clarification to connect docstring --- renderapi/render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/render.py b/renderapi/render.py index b8bc9ade..25413768 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -143,7 +143,8 @@ def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, force_http=True, validate_client=True, web_only=False, **kwargs): ''' - helper function to create a :class:`RenderClient` instance. + helper function to create a :class:`Render` instance, or :class:`RenderClient` + if sufficent parameters are provided. Will default to using environment variables if not specified in call, and prompt user for any parameters that are not given. From 18116f6ed72dd05559eda317546a77249e2fa9a3 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 26 Jul 2017 16:19:40 -0700 Subject: [PATCH 414/766] fixed pointmatch typo --- renderapi/pointmatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 370ba8d2..bd0c8c1c 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -410,7 +410,7 @@ def get_matches_involving_tile(matchCollection, groupId, id, ''' request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/id/{}/".format( - owner, matchCollection, pGroupId, pTileId) + owner, matchCollection, groupId, id) request_url = add_merge_collections(request_url, mergeCollections) r = session.get(request_url) From 79c80c63e3845b721cb4689ff088d59303330fce Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 26 Jul 2017 16:31:22 -0700 Subject: [PATCH 415/766] added decorator test --- test/test_client.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/test/test_client.py b/test/test_client.py index 94646281..b4acad94 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -2,15 +2,14 @@ import renderapi import rendersettings - +args = { + 'host': 'renderhost', + 'port': 8080, + 'owner': 'renderowner', + 'project': 'renderproject', + 'client_scripts': '/path/to/client_scripts' + } def test_render_client(): - args = { - 'host': 'renderhost', - 'port': 8080, - 'owner': 'renderowner', - 'project': 'renderproject', - 'client_scripts': '/path/to/client_scripts' - } r = renderapi.render.connect(**args) @@ -51,3 +50,15 @@ def test_environment_variables_client(): rkwargs=rendersettings.DEFAULT_RENDER_CLIENT, renvkwargs=rendersettings.DEFAULT_RENDER_CLIENT_ENVIRONMENT_VARIABLES, validate_client=False) + +@renderapi.render.renderaccess +def my_decorated(myparameter, owner=None, host=None, port=None, + project=None,client_scripts=None, **kwargs): + return (owner,host,port,project,client_scripts) + +def test_decorator(): + r = renderapi.render.connect(**args) + (owner,host,port,project,client_scripts)=my_decorated(5,render=r) + assert(owner == args['owner']) + (owner,host,port,project,client_scripts)=my_decorated(5,owner='newowner',render=r) + assert(owner == 'newowner') \ No newline at end of file From 5252fbc21e1813968058543e21d4cf6c019a1381 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 26 Jul 2017 16:32:10 -0700 Subject: [PATCH 416/766] fixed broken wraps and decorator decorator --- renderapi/render.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 25413768..e6fa9d96 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -257,7 +257,7 @@ def connect(host=None, port=None, owner=None, project=None, @decorator -def renderaccess(f): +def renderaccess(f,*args,**kwargs): ''' Decorator allowing functions asking for host, port, owner, project to default to a connection defined by a :class:`Render` object @@ -282,20 +282,18 @@ def renderaccess(f): Returns: func: decorated function ''' - @wraps(f) - def wrapper(*args, **kwargs): - args, kwargs = fitargspec(f, args, kwargs) - render = kwargs.get('render') - if render is not None: - if isinstance(render, Render): - return f(*args, **render.make_kwargs(**kwargs)) - else: - raise ValueError( - 'invalid Render object type {} specified!'.format( - type(render))) + args, kwargs = fitargspec(f, args, kwargs) + render = kwargs.get('render') + if render is not None: + if isinstance(render, Render): + return f(*args, **render.make_kwargs(**kwargs)) else: - return f(*args, **kwargs) - return wrapper + raise ValueError( + 'invalid Render object type {} specified!'.format( + type(render))) + else: + return f(*args, **kwargs) + def format_baseurl(host, port): From d9bf09eb4b8f82bf4b638ba81d61b1aca095eadc Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 27 Jul 2017 13:14:32 -0700 Subject: [PATCH 417/766] documentation: switch to numpy-style doctrings and add documentation --- renderapi/client.py | 540 +++++++++++++------ renderapi/image.py | 165 +++--- renderapi/pointmatch.py | 547 +++++++++++++------- renderapi/render.py | 476 +++++++++++------ renderapi/stack.py | 596 +++++++++++++-------- renderapi/tilespec.py | 518 ++++++++++++------- renderapi/transform.py | 1091 ++++++++++++++++++++++++++------------- renderapi/utils.py | 248 +++++---- 8 files changed, 2770 insertions(+), 1411 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 8590b519..7c0332c8 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -20,12 +20,21 @@ class WithPool(Pool): - ''' - pathos ProcessingPool with functioning __exit__ call - usage: - with WithPool(*poolargs, **poolkwargs) as pool: - pool.map(*mapargs, **mapkwargs) - ''' + """pathos ProcessingPool with functioning __exit__ call + + Parameters + ---------- + *args + variable length argument list matching input + to pathos.multiprocessing.Pool + **kwargs + keyword argument input matching pathos.multiprocessing.Pool + + Examples + -------- + >>> with WithPool(number_processes) as pool: + >>> pool.map(myfunc, myInput) + """ def __init__(self, *args, **kwargs): super(WithPool, self).__init__(*args, **kwargs) @@ -37,16 +46,20 @@ def __exit__(self, *args, **kwargs): def import_single_json_file(stack, jsonfile, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, render=None, **kwargs): - ''' - calls client script to import given jsonfile - - Args: - stack (str): stack to import into - jsonfile (str): path to jsonfile to import - transformFile (str): path to a file that contains shared - transform references if necessary - render (renderapi.render.RenderClient): render connect object - ''' + """calls client script to import given jsonfile + + Parameters + ---------- + stack : str + stack to import into + jsonfile : str + path to jsonfile to import + transformFile : str + path to a file that contains shared + transform references if necessary + render : renderapi.render.RenderClient + render connect object + """ if transformFile is None: transform_params = [] else: @@ -69,22 +82,30 @@ def import_jsonfiles_and_transforms_parallel_by_z( stack, jsonfiles, transformfiles, poolsize=20, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): - ''' - imports json files and transform files in parallel - - Args: - stack (str): the stack to import within - jsonfiles (list[str]): "list of tilespec" json paths to import - transformfiles (list[str]): "list of transform files" paths which matches - in a 1-1 way with jsonfiles, so referenced transforms - are shared only within a single element of these matched lists. - Useful cases where there is as single z transforms shared - by all tiles within a single z, but not across z's - poolsize (int): number of processes for multiprocessing pool - close_stack (bool): whether to mark render stack as COMPLETE after successful import - render (renderapi.render.RenderClient): render connect object - - ''' + """imports json files and transform files in parallel + + Parameters + ---------- + stack : str + the stack to import within + jsonfiles : :obj:`list` of :obj:`str` + "list of tilespec" json paths to import + transformfiles : :obj:`list` of :obj:`str` + "list of transform files" paths which matches + in a 1-1 way with jsonfiles, so referenced transforms + are shared only within a single element of these matched lists. + Useful cases where there is as single z transforms shared + by all tiles within a single z, but not across z's + poolsize : int, optional + number of processes for multiprocessing pool + close_stack : bool, optional + whether to mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + **kwargs + arbitrary keyword arguments + + """ set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial(import_single_json_file, stack, render=render, client_scripts=client_scripts, host=host, @@ -101,18 +122,27 @@ def import_jsonfiles_parallel( stack, jsonfiles, poolsize=20, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): - ''' - import jsons using client script in parallel - - Args: - stack (str): the stack to upload into - jsonfiles (list[str]): list of jsonfile paths to upload - poolsize (int): number of upload processes spawned by multiprocessing pool - transformFile (str): a single json file path containing transforms referenced - in the jsonfiles - close_stack (boolean): whether to mark render stack as COMPLETE after successful import - render (renderapi.render.RenderClient): render connect object - ''' + """import jsons using client script in parallel + + Parameters + ---------- + stack : str + the stack to upload into + jsonfiles : :obj:`list` of :obj:`str` + list of jsonfile paths to upload + poolsize : int + number of upload processes spawned by multiprocessing pool + transformFile : str + a single json file path containing transforms referenced + in the jsonfiles + close_stack : bool + whether to mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + **kwargs + arbitrary keyword arguments + + """ set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial(import_single_json_file, stack, render=render, @@ -132,16 +162,21 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): - ''' - import jsons using client script serially - - Args: - jsonfiles (list): iterator of filenames to be uploaded - transformFile (str): path to a jsonfile that contains shared - transform references (if necessary) - close_stack (boolean): mark render stack as COMPLETE after successful import - render (renderapi.render.RenderClient): render connect object - ''' + """import jsons using client script serially + + Parameters + ---------- + jsonfiles : :obj:`list` of :obj:`str` + iterator of filenames to be uploaded + transformFile : str + path to a jsonfile that contains shared + transform references (if necessary) + close_stack : bool + mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + + """ set_stack_state(stack, 'LOADING', host, port, owner, project) if transformFile is None: @@ -170,11 +205,19 @@ def import_jsonfiles_validate_client(stack, jsonfiles, project=None, close_stack=True, mem=6, validator=None, render=None, **kwargs): - ''' - Uses java client for parallelization and validation - - - ''' + """Uses java client for parallelization and validation + + Parameters + ---------- + stack: str + stack to which jsonfiles should be uploaded + jsonfiles: :obj:`list` of :obj:`str` + tilespecs in json files + transformFile: str, optional + json file listing transformspecs with ids which are referenced + in tilespecs contained in jsonfiles + + """ transform_params = (['--transformFile', transformFile] if transformFile is not None else []) if validator is None: @@ -202,11 +245,6 @@ def import_jsonfiles_validate_client(stack, jsonfiles, subprocess.call(cmd, env=my_env) - ''' - proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - proc.wait() - logger.debug(proc.stdout.read()) - ''' if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -216,16 +254,21 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - method to import tilesepcs directly from :class:`renderapi.tilespec.TileSpec` objects - - Args: - stack (str): stack to which tilespecs will be added - tilespecs (list[renderapi.tilespec.TileSpec]): list of tilespecs to import - sharedTransforms (list[renderapi.transform.Transform]): list of shared - referenced transforms to be ingested - render (renderapi.render.RenderClient): render connect object - ''' + """method to import tilesepcs directly from + :class:`renderapi.tilespec.TileSpec` objects + + Parameters + ---------- + stack : str + stack to which tilespecs will be added + tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` + list of tilespecs to import + sharedTransforms : :obj:`list` of :class:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional + list of shared referenced transforms to be ingested + render : renderapi.render.Render + render connect object + + """ tsjson = renderdump_temp(tilespecs) if sharedTransforms is not None: @@ -249,20 +292,27 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - method to import tilesepcs directly from :class:`renderapi.tilespec.TileSpec` objects - using pathos.multiprocessing to parallelize - - Args: - stack (str): stack to which tilespecs will be added - tilespecs (list[renderapi.tilespec.TileSpec]): list of tilespecs to import - sharedTransforms (list[renderapi.transform.Transform]): list of shared - referenced transforms to be ingested - poolsize (int): degree of parallelism to use - subprocess_mode (str): subprocess mode used when calling client side java - close_stack (boolean): mark render stack as COMPLETE after successful import - render (renderapi.render.RenderClient): render connect object - ''' + """method to import tilesepcs directly from + :class:`renderapi.tilespec.TileSpec` objects using + pathos.multiprocessing to parallelize + + Parameters + ---------- + stack : str + stack to which tilespecs will be added + tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` + list of tilespecs to import + sharedTransforms :obj:`list` of :obj:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional + list of shared referenced transforms to be ingested + poolsize : int + degree of parallelism to use + subprocess_mode : str + subprocess mode used when calling client side java + close_stack : bool + mark render stack as COMPLETE after successful import + render : :class:renderapi.render.Render + render connect object + """ set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial( import_tilespecs, stack, sharedTransforms=sharedTransforms, @@ -284,18 +334,24 @@ def local_to_world_array(stack, points, tileId, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - placeholder function for coordinateClient localtoworld - - Args: - stack (str): stack to which world coordinates are mapped - points (dict): local points to map to world - tileId (str): tileId to which points correspond - subprocess_mode (str): subprocess mode used when calling - clientside java client - outputs: - list of points in world coordinates corresponding to local points - ''' + """placeholder function for coordinateClient localtoworld + + Parameters + ---------- + stack : str + stack to which world coordinates are mapped + points : dict + local points to map to world + tileId : str + tileId to which points correspond + subprocess_mode : str + subprocess mode used when calling + clientside java client + Returns + ------- + list + points in world coordinates corresponding to local points + """ raise NotImplementedError('Whoops') @@ -304,28 +360,57 @@ def world_to_local_array(stack, points, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - placeholder function for coordinateClient worldtolocal - - Args: - stack (str): stack to which world coordinates are mapped - points (dict): local points to map to world - subprocess_mode (str): subprocess mode used when calling client side java - render (renderapi.render.RenderClient): render connect object - - Returns: - list[list]: dictionaries defining local coordinates - and tileIds corresponding to world point - ''' + """placeholder function for coordinateClient worldtolocal + + Parameters + ---------- + stack : str + stack to which world coordinates are mapped + points : dict + local points to map to world + subprocess_mode : str + subprocess mode used when calling client side java + render : :class:`renderapi.render.Render` + render connect object + + Returns + ------- + :obj:`list` of :obj:`list` + dictionaries defining local coordinates + and tileIds corresponding to world point + """ raise NotImplementedError('Whoops.') def call_run_ws_client(className, add_args=[], renderclient=None, memGB=None, client_script=None, subprocess_mode=None, **kwargs): - ''' - simple call for run_ws_client.sh -- all arguments set in add_args - ''' + """simple call for run_ws_client.sh -- all arguments set in add_args + + Parameters + ---------- + className : str + Render java client classname to call as first argv for + Render's call_run_ws_client.sh wrapper script + add_args : :obj:`list` of :obj:`str`, optional + command line arguments + renderclient : :class:`renderapi.render.RenderClient`, optional + render client connection object + memGB : str, optional + GB memory for this java process + (defaults to '1G' or value defined in renderclient) + client_script : str, optional + client script to be used as the Render library's call_run_ws_client.sh + wrapper script (this option overrides value in renderclient) + subprocess_mode: str, optional + subprocess mode 'call', 'check_call', 'check_output' (default 'call') + + + Returns + ------- + obj + result of subprocess_mode call + """ logger.debug('call_run_ws_client -- classname:{} add_args:{} ' 'client_script:{} memGB:{}'.format( className, add_args, client_script, memGB)) @@ -364,9 +449,21 @@ def importJsonClient(stack, tileFiles=None, transformFile=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - '''run ImportJsonClient.java - see render documentation (add link here) - ''' + """run ImportJsonClient.java + see render documentation (add link here) + + Parameters + ---------- + stack : str + stack to which tilespecs in tileFiles will be imported + tileFiles : :obj:`list` of :obj:`str` + json files containing tilespecs to import + transformFile : str, optional + json file containing transform specs which are + referenced by tilespecs in tileFiles + render : :class:`renderapi.render.Render` + render connection object + """ argvs = (make_stack_params(host, port, owner, project, stack) + (['--transformFile', transformFile] if transformFile else []) + (tileFiles if isinstance(tileFiles, list) @@ -390,9 +487,64 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - '''run TilePairClient.java - see render documentation (#add link here) - ''' + """run TilePairClient.java + see render documentation (#add link here) + + This client selects a set of tiles 'p' based on its position in + a stack and then searches for nearby 'q' tiles using geometric parameters + + Parameters + ---------- + stack : str + stack from which tilepairs should be considered + minz : str + minimum z bound from which tile 'p' is selected + maxz : str + maximum z bound from which tile 'p' is selected + outjson : str or None + json to which tile pair file should be written + (defaults to using temporary file and deleting after completion) + delete_json : bool + whether to delete outjson on function exit (True if outjson is None) + baseowner : str + owner of stack from which stack was derived + baseproject : str + project of stack from which stack was derived + basestack : str + stack from which stack was derived + xyNeighborFactor : float + factor to multiply by max(width, height) of tile 'p' in order + to generate search radius in z (0.9 if None) + zNeighborDistance : int + number of z sections defining the half-height of search cylinder + for tile 'p' (2 if None) + excludeCornerNeighbors : bool + whether to exclude potential 'q' tiles based on center points + falling outside search (True if None) + excludeCompletelyObscuredTiles : bool + whether to exclude potential 'q' tiles that are obscured by other tiles + based on Render's sorting (True if None) + excludeSameLayerNeighbors : bool + whether to exclude potential 'q' tiles in the same z layer as 'p' + excludeSameSectionNeighbors : bool + whether to exclude potential 'q' tiles with the same sectionId as 'p' + excludePairsInMatchCollection : str + a matchCollection whose 'p' and 'q' pairs will be ignored + if generated using this client + minx : float + minimum x bound from which tile 'p' is selected + maxx : float + maximum x bound from wich tile 'p' is selected + miny : float + minimum y bound from which tile 'p' is selected + maxy : float + maximum y bound from wich tile 'p' is selected + + Returns + ------- + :obj:`list` of :obj:`dict` + list of tilepairs + """ if outjson is None: with tempfile.NamedTemporaryFile( suffix=".json", mode='r', delete=False) as f: @@ -440,15 +592,39 @@ def importTransformChangesClient(stack, targetStack, transformFile, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - run ImportTransformChangesClient.java - transformFile: json file in format defined below - [{{"tileId": , - "transform": }}, - {{"tileId": ...}}, - ... - ] - ''' + """run ImportTransformChangesClient.java + + Parameters + ---------- + stack : str + stack from which tiles will be transformed + targetStack : str + stack that will hold results of transforms + transformFile : str + locaiton of json file in format defined below + :: + [{{"tileId": , + "transform": }}, + {{"tileId": ...}}, + ... + ] + targetOwner : str + owner of target stack + targetProject : str + project of target stack + changeMode : str + method to apply transform to tiles. Options are: + 'APPEND' -- add transform to tilespec's list of transforms + 'REPLACE_LAST' -- change last transform in tilespec's + list of transforms to match transform + 'REPLACE_ALL' -- overwrite tilespec's transforms field to match + transform + + Raises + ------ + ClientScriptError + if changeMode is not valid + """ if changeMode not in ['APPEND', 'REPLACE_LAST', 'REPLACE_ALL']: raise ClientScriptError( 'changeMode {} is not valid!'.format(changeMode)) @@ -472,16 +648,32 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - run CoordinateClient.java - expects: - fromJson -- json in format defined by list of - coordinate dictionaries (world) or - list of list of coordinate dictionaries (local) - toJson -- json to save results of mapping - localToWorld -- flag defaults to accepting world coordinates - numberOfThreads -- java-based threads for client script - ''' + """run CoordinateClient.java + + map coordinates between local and world systems + + Parameters + ---------- + stack : str + stack representing the world coordinates + z : str + z value of the section containing the tiles to map + fromJson : str + input json file in format defined by + list of coordinate dictionaries (for world to local) + or list of list of coordinate dictionaries (local to world) + toJson : str + json to save results of mapping coordinates + localToWorld : bool + whether to transform form local to world coordinates (False if None) + numberOfThreads : int + number of threads for java process (1 if None) + + Returns + ------- + :obj:`list` of :obj:`dict` for local to world or :obj:`list` of :obj:`list` of :obj:`dict` for world to local + list representing mapped coordinates + """ argvs = (make_stack_params(host, port, owner, project, stack) + ['--z', z, '--fromJson', fromJson, '--toJson', toJson] + (['--localToWorld'] if localToWorld else []) + @@ -503,9 +695,33 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - run RenderSectionClient.java - ''' + """run RenderSectionClient.java + + Parameters + ---------- + stack : str + stack to which zs to render belong + rootDirectory : str + directory to which rendered sections should be generated + zs : :obj:`list` of :obj:`str` + z indices of sections to render + scale : float + factor by which section image should be scaled + (this materialization is 32-bit limited) + maxIntensity : int + value todisplay as white on a linear colormap + minIntensity : int + value to display as black on a linear colormap + format : str + output image format in 'PNG', 'TIFF', 'JPEG' + doFilter : str + string representing java boolean for whether to render image + with default filter (varies with render version) + fillWithNoise : str + string representing java boolean for whether to replace saturated + image values with uniform noise + + """ argvs = (make_stack_params(host, port, owner, project, stack) + ['--rootDirectory', rootDirectory] + get_param(scale, '--scale') + get_param(format, '--format') + @@ -525,19 +741,29 @@ def transformSectionClient(stack, transformId, transformClass, transformData, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): - ''' - run TranformSectionClient.java - expects: - transformId -- string unique transform identifier - transformClass -- string representing mpicbg transform - transformData -- mpicbg datastring delimited by "," instead of " " - zValues -- list of z values to apply tform - optional: - targetProject -- project to output the transformed sections - targetStack -- stack to ouput transformed sections - replaceLast -- bool whether to have transform replace - last in specList (default false) - ''' + """run TranformSectionClient.java + + Parameters + ---------- + stack : str + stack containing section to transform + transformId : str + unique transform identifier + transformClass : str + transform className defined by the java mpicbg library + transformData : str + mpicbg datastring delimited by "," instead of " " + zValues : list + z values to which transform should be applied + targetProject : str, optional + project to which transformed sections should be added + targetStack : str, optional + stack to which transformed sections should be added + replaceLast : bool, optional + whether to replace the last transform in the section + with this transform + + """ argvs = (make_stack_params(host, port, owner, project, stack) + (['--replaceLast'] if replaceLast else []) + get_param(targetProject, '--targetProject') + diff --git a/renderapi/image.py b/renderapi/image.py index 4babaa35..5a967ef2 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -31,29 +31,46 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): - ''' - render image from a bounding box defined in xy and return numpy array: + """render image from a bounding box defined in xy and return numpy array: :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): name of render stack to get image from - z (float): z value to render - x (int): leftmost point of bounding rectangle - y (int): topmost pont of bounding rectangle - width (int): number of units @scale=1.0 to right (+x() of bounding box to render - height (int): number of units @scale=1.0 down (+y) of bounding box to render - scale (float): scale to render image at (default 1.0) - binaryMask (boolean): whether to treat maskimage as binary - maxTileSpecsToRender (int): max number of tilespecs to render - filter (boolean): whether to use server side filtering - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - numpy.array: [N,M,:] array of image data from render - Raises: - .errors.RenderError - ''' + Parameters + ---------- + stack : str + name of render stack to get image from + z : float + z value to render + x : int + leftmost point of bounding rectangle + y : int + topmost pont of bounding rectangle + width : int + number of units @scale=1.0 to right (+x() of bounding box to render + height : int + number of units @scale=1.0 down (+y) of bounding box to render + scale : float + scale to render image at (default 1.0) + binaryMask : bool + whether to treat maskimage as binary + maxTileSpecsToRender : int + max number of tilespecs to render + filter : bool + whether to use server side filtering + render : :class:`renderapi.render.Render` + render connect object + session : :class:`requests.sessions.Session` + sessions object to connect with + + Returns + ------- + numpy.array + [N,M,:] array of image data from render + + Raises + ------ + RenderError + """ try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: # pragma: no cover @@ -91,28 +108,46 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, filter=None, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): - ''' - render image from a tile with all transforms and return numpy array + """render image from a tile with all transforms and return numpy array :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): name of render stack to get tile from - tileId (str): tileId of tile to render - normalizeForMatching (bool): whether to render the tile with transformations removed ('local' coordinates) - removeAllOption (bool): whether to remove all transforms from image when doing normalizeForMatching - some versions of render only remove the last transform from list. + Parameters + ---------- + stack : str + name of render stack to get tile from + tileId : str + tileId of tile to render + normalizeForMatching : bool + whether to render the tile with transformations + removed ('local' coordinates) + removeAllOption : bool + whether to remove all transforms from image when + doing normalizeForMatching some versions of render + only remove the last transform from list. (or remove till there are max 3 transforms) - scale (float): force scale of image - filter (bool): whether to apply server side filtering to image - img_format (str): image format: one of IMAGE_FORMATS = 'png','.png','jpg','jpeg','.jpg','tif','.tif','tiff' - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - numpy.array: [N,M,:] array of image data from render - Raises: - .errors.RenderError - ''' + scale : float + force scale of image + filter : bool + whether to apply server side filtering to image + img_format : str + image format: one of IMAGE_FORMATS = 'png','.png','jpg', + 'jpeg','.jpg','tif','.tif','tiff' + render : :obj:`renderapi.render.Render` + render connect object + session : :obj:`requests.sessions.Session` + sessions object to connect with + + Returns + ------- + numpy.array + [N,M,:] array of image data from render + + Raises + ------ + RenderError + + """ try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: # pragma: no cover @@ -150,29 +185,43 @@ def get_section_image(stack, z, scale=1.0, filter=False, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - ''' - render an section of image + """render an section of image :func:`renderapi.render.renderaccess` decorated function - Example: - :: - - import renderapi - render = renderapi.render.connect('server',8080,'me','myproject') - img = render.run(renderapi.stack.get_section_image,'mystack',3.0) - Args: - stack (str): name of render stack to render image from - z (float): layer Z - scale (float): linear scale at which to render image (e.g. 0.5) - filter (boolean): whether or not to apply server side filtering - maxTileSpecsToRender (int): maximum number of tile specs in rendering - img_format (str): one of IMAGE_FORMATS 'png','.png','jpg','jpeg','.jpg','tif','.tif','tiff' - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - numpy.array: [N,M,:] array of image data of section from render - ''' + + Parameters + ---------- + stack : str + name of render stack to render image from + z : float + layer Z + scale : float + linear scale at which to render image (e.g. 0.5) + filter : bool + whether or not to apply server side filtering + maxTileSpecsToRender : int + maximum number of tile specs in rendering + img_format : str + one of IMAGE_FORMATS 'png','.png','jpg','jpeg', + '.jpg','tif','.tif','tiff' + render : :obj:`renderapi.render.Render` + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + numpy.array + [N,M,:] array of image data of section from render + + Examples + -------- + >>>import renderapi + >>>render = renderapi.render.connect('server',8080,'me','myproject') + >>>img = render.run(renderapi.stack.get_section_image,'mystack',3.0) + + """ try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: # pragma: no cover diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index bd0c8c1c..0f7cfd13 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -8,6 +8,7 @@ from .errors import RenderError from .utils import NullHandler import json + logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -17,18 +18,27 @@ def get_matchcollection_owners(host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the matchCollection owners + """get all the matchCollection owners :func:`renderapi.render.renderaccess` decorated function - Args: - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[unicode]: matchCollection owners - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + render : renderapi.render.Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`unicode` + matchCollection owners + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/matchCollectionOwners" r = session.get(request_url) @@ -43,20 +53,30 @@ def get_matchcollection_owners(host=None, port=None, @renderaccess def get_matchcollections(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the matchCollections owned by owner + """get all the matchCollections owned by owner :func:`renderapi.render.renderaccess` decorated function - Args: - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[unicode]: matchcollections owned by owner - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`unicode` + matchcollections owned by owner + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollections" % owner r = session.get(request_url) @@ -72,21 +92,32 @@ def get_matchcollections(owner=None, host=None, port=None, def get_match_groupIds(matchCollection, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the groupIds in a matchCollection + """get all the groupIds in a matchCollection :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - owner (str): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[str]: groupIds in matchCollection - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + owner : str + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`str` + groupIds in matchCollection + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) r = session.get(request_url) @@ -103,24 +134,37 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the matches outside a groupId in a matchCollection + """get all the matches outside a groupId in a matchCollection returns all matches where pGroupId == groupId and qGroupId != groupId :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - groupId (str): groupId to query - mergeCollections (list[str]): other matchCollections to aggregate into answer - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[dict]: list of matches (see matches definition) - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + groupId : str + groupId to query + mergeCollections : :obj:`list` of :obj:`str` + other matchCollections to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesOutsideGroup" % ( owner, matchCollection, groupId) @@ -140,25 +184,38 @@ def get_matches_within_group(matchCollection, groupId, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''get all the matches within a groupId in a matchCollection + """get all the matches within a groupId in a matchCollection returns all matches where pGroupId == groupId and qGroupId == groupId :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - groupId (str): groupId to query - mergeCollections (list[str] or None): other matchCollections - to aggregate into answer - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[dict]: list of matches (see matches definition) - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + groupId : str + groupId to query + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections + to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWithinGroup" % ( owner, matchCollection, groupId) @@ -179,28 +236,42 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''get all the matches between two specific groups + """get all the matches between two specific groups returns all matches where pgroup == pGroupId and qgroup == qGroupId - OR pgroup == qGroupId and qgroup == pGroupId :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - pgroup (str): first group - qgroup (str): second group - mergeCollections (list[str] or None): other matchCollections - to aggregate into answer - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[dict]: list of matches (see matches definition) - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + first group + qgroup : str + second group + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections + to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/group/%s/matchesWith/%s" % ( owner, matchCollection, pgroup, qgroup) @@ -216,14 +287,20 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, def add_merge_collections(request_url, mcs): - '''utility function to add mergeCollections to request_url - - Args: - request_url (str): request url - mcs (list[str]): list of mergeCollections to add - Returns: - str: request_url with ?mergeCollection=mc[0]&mergeCollection=mc[1]... appended - ''' + """utility function to add mergeCollections to request_url + + Parameters + ---------- + request_url : str + request url + mcs : :obj:`list` of :obj:`str` + list of mergeCollections to add + Returns + ------- + str + request_url with ?mergeCollection=mc[0]&mergeCollection=mc[1]... + appended + """ if mcs is not None: if type(mcs) is list: request_url += "?"+"&".join( @@ -237,31 +314,46 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''get all the matches between two specific tiles - returns all matches where + """get all the matches between two specific tiles + returns all matches where pgroup == pGroupId and pid=pId and qgroup == qGroupId and qid == qId OR qgroup == pGroupId and Qid=pId and Pgroup == qGroupId and pid == qId - + :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - pgroup (str): first group - pid (str): first id - qgroup (str): second group - qid (str): second id - mergeCollections (list[str] or None): other matchCollections - to aggregate into answer - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[dict]: list of matches (see matches definition) - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + first group + pid : str + first id + qgroup : str + second group + qid : str + second id + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ ("/owner/%s/matchCollection/%s/group/%s/id/%s/" "matchesWith/%s/id/%s" % ( @@ -282,25 +374,37 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''get all the matches from a specific groups - returns all matches where pgroup == pGroupId + """get all the matches from a specific groups + returns all matches where pgroup == pGroupId :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - pgroup (str): source group to query - mergeCollections (list[str] or None): other matchCollections - to aggregate into answer - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[dict]: list of matches (see matches definition) - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + source group to query + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroup/%s/matches/" % ( owner, matchCollection, pgroup) @@ -320,21 +424,32 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''get all the source pGroupIds in a matchCollection + """get all the source pGroupIds in a matchCollection :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[str]: pGroupIds in matchCollection - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : RenderClient + RenderClient connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`str` + pGroupIds in matchCollection + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) @@ -352,22 +467,34 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, def get_match_groupIds_to_only(matchCollection, mergeCollections=None, render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): - '''get all the destination qGroupIds in a matchCollection + session=requests.session(), **kwargs): + """get all the destination qGroupIds in a matchCollection :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[str]: qGroupIds in matchCollection - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`str` + qGroupIds in matchCollection + + Raises + ------ + RenderError + if cannot get a reponse from server + + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) @@ -386,28 +513,40 @@ def get_matches_involving_tile(matchCollection, groupId, id, mergeCollections=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''get all the matches involving a specific tile + """get all the matches involving a specific tile returns all matches where groupId == pGroupId and id == pId - OR - groupId == qGroupId and id == qId + OR groupId == qGroupId and id == qId :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - groupId (str): groupId to query - id (str): id to query - mergeCollections (list[str] or None): other matchCollections - to aggregate into answer - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[dict]: list of matches (see matches definition) - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + groupId : str + groupId to query + id : str + id to query + mergeCollections : :obj:`list` of :obj:`str`, optional + other matchCollections to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + """ request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/id/{}/".format( owner, matchCollection, groupId, id) @@ -427,27 +566,41 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): - '''delete all the matches between two specific groups + """delete all the matches between two specific groups deletes all matches where (pgroup == pGroupId and qgroup == qGroupId) OR (pgroup == qGroupId and qgroup == pGroupId() :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - pgroup (str): first group - qgroup (str): second group - mergeCollections (list[str] or None): other matchCollections - to aggregate into answer - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - list[dict]: list of matches (see matches definition) - Raises: - RenderError: if cannot get a reponse from server - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + pgroup : str + first group + qgroup : str + second group + mergeCollections : :obj:`list` of :obj:`str` or None + other matchCollections to aggregate into answer + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + :obj:`list` of :obj:`dict` + list of matches (see matches definition) + + Raises + ------ + RenderError + if cannot get a reponse from server + + """ request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/matchesWith/{}".format( owner, matchCollection, pGroupId, qGroupId) @@ -464,20 +617,30 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, @renderaccess def import_matches(matchCollection, data, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''import matches into render database + """import matches into render database :func:`renderapi.render.renderaccess` decorated function - Args: - matchCollection (str): matchCollection name - data (list[dict]): list of matches to import (see matches definition) - owner (unicode): matchCollection owner (fallback to render.DEFAULT_OWNER) - (note match owner != stack owner always) - render (RenderClient): RenderClient connection object - session (requests.session.Session): requests session - Returns: - requests.response.Reponse: server response - ''' + Parameters + ---------- + matchCollection : str + matchCollection name + data : :obj:`list` of :obj:`dict` + list of matches to import (see matches definition) + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + requests.response.Reponse + server response + + """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/matches" % (owner, matchCollection) logger.debug(request_url) diff --git a/renderapi/render.py b/renderapi/render.py index e6fa9d96..02451849 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -6,16 +6,32 @@ from .utils import defaultifNone, NullHandler, fitargspec from .errors import ClientScriptError, RenderError from decorator import decorator + logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) class Render(object): - ''' - Render object to store connection settings for render server. - Baseclass that doesn't require client_scripts definition for client side java processing. + """Render object to store connection settings for render server. + Baseclass that doesn't require client_scripts definition + for client side java processing. + See :func:`connect` for parameter definitions. - ''' + + Attributes + ---------- + DEFAULT_HOST : str + render host to which make_kwargs will default + DEFAULT_PORT : int + render port to which make_kwargs will default + DEFAULT_OWNER : str + render owner to which make_kwargs will default + DEFAULT_PROJECT : str + render project to which make_kwargs will default + DEFAULT_CLIENT_SCRIPTS : str + render client scripts path to which make_kwargs will default + + """ def __init__(self, host=None, port=None, owner=None, project=None, client_scripts=None): @@ -34,32 +50,43 @@ def __init__(self, host=None, port=None, owner=None, project=None, @property def DEFAULT_KWARGS(self): - ''' - kwargs to which the render object falls back. Depends on: - self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, - self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS - - Returns: - dict: default keyword arguments - ''' + """"kwargs to which the render object falls back. Depends on: + self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, + self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS + + Returns + ------- + dict + default keyword arguments + """ return self.make_kwargs() def make_kwargs(self, host=None, port=None, owner=None, project=None, client_scripts=None, **kwargs): - ''' - make kwargs using this render object's defaults and any - designated kwargs passed in - Args: - host (str or None): - port (int or None): - owner (str or None): - project (str or None): - client_scripts (str or None): - **kwargs: all other keyword arguments passed through - Returns: - dict: keyword arguments with missing host,port,owner,project,client_scripts - filled in with defaults - ''' + """make kwargs using this render object's defaults and any + designated kwargs passed in + + Parameters + ---------- + host : str or None + render webservice host + port : int or None + render webservice port + owner : str or None + render webservice owner + project : str or None + render webservice project + client_scripts : str or None + render java client script location + **kwargs + all other keyword arguments passed through + + Returns + ------- + dict + keyword arguments with missing + host,port,owner,project,client_scripts filled in with defaults + """ processed_kwargs = { 'host': self.DEFAULT_HOST if host is None else host, 'port': self.DEFAULT_PORT if port is None else port, @@ -71,22 +98,30 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, return processed_kwargs def run(self, f, *args, **kwargs): - ''' - run function from object + """run function from object technically shorter than adding render=Render to kwargs - Example: - allows this syntax for running renderapi:: - - render = Render('server',8080) - metadata = render.run(renderapi.render.get_stack_metadata_by_owner,'myowner') - Args: - f (func): renderapi function you want to call - *args: args passed to that function - **kwargs: kwargs passed to that function - Returns: - func: function with this :class:`Render` instance in keyword arguments as render= - ''' + Parameters + ---------- + f : func + renderapi function you want to call + *args + args passed to that function + **kwargs + kwargs passed to that function + + Returns + ------- + func + function with this :class:`Render` instance in + keyword arguments as render= + + Examples + -------- + >>> render = Render('server',8080) + >>> metadata = render.run(renderapi.render.get_stack_metadata_by_owner, 'myowner') + + """ # FIXME WARNING I think renderaccess can default to # another render if defined in args (test/squash) kwargs['render'] = self @@ -94,14 +129,50 @@ def run(self, f, *args, **kwargs): class RenderClient(Render): - ''' - Render object to run java client commands via a wrapped client script. - This is a work in progress. - Should use :func:`connect` to create and for documentation of parameters. - ''' + """Render object to run java client commands via a wrapped client script. + Should use :func:`connect` to create and for documentation of parameters. + + Attributes + ---------- + DEFAULT_HOST : str + render host to which make_kwargs will default + DEFAULT_PORT : int + render port to which make_kwargs will default + DEFAULT_OWNER : str + render owner to which make_kwargs will default + DEFAULT_PROJECT : str + render project to which make_kwargs will default + DEFAULT_CLIENT_SCRIPTS : str + render client scripts path to which make_kwargs will default + client_script : str + location of wrapper script for java client with input same as Render + java client's run_ws_client.sh + memGB : str + string defining heap in GB to be utilized by + java clients (default '1G' for 1 GB) + """ def __init__(self, client_script=None, memGB=None, validate_client=True, *args, **kwargs): + """Initialize RenderClient object extending Render to + running java client scripts + + Parameters + ---------- + client_script : str + path to script with same inputs as + Render Java Client 'run_ws_client.sh' + memGB : str + string defining heap to be utilized by java clients in GB + (defaults to '1G' for 1GB) + validate_client : bool + whether to validate that client script is a file + + Raises + ------ + ClientScriptError + if render client script cannot be found + """ super(RenderClient, self).__init__(**kwargs) if validate_client: if client_script is None: @@ -121,14 +192,22 @@ def __init__(self, client_script=None, memGB=None, validate_client=True, self.memGB = memGB def make_kwargs(self, *args, **kwargs): - '''method to fill in default properties of RenderClient object - - Args: - *args: args used to initialize RenderClient - **kwargs: kwargs used to initialize RenderClient - Returns: - Render: Render instance with default values filled in - ''' + """method to fill in default properties of RenderClient object + + Parameters + ---------- + *args + args used to initialize RenderClient + **kwargs + kwargs used to initialize RenderClient + + Returns + ------- + dict + keyword arguments with missing + host,port,owner,project,client_scripts,client_script,memGB + filled in with defaults + """ # hack to get dictionary defaults to work client_script = defaultifNone( kwargs.pop('client_script', None), self.client_script) @@ -142,42 +221,61 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, force_http=True, validate_client=True, web_only=False, **kwargs): - ''' - helper function to create a :class:`Render` instance, or :class:`RenderClient` - if sufficent parameters are provided. + """helper function to create a :class:`Render` instance, or + :class:`RenderClient` if sufficent parameters are provided. Will default to using environment variables if not specified in call, and prompt user for any parameters that are not given. - Args: - host (str): hostname for target render server -- will prepend - "http://" if not found and force_http keyword evaluates True. - Can be set by environment variable RENDER_HOST. - port (str, int, or None): port for target render server. - Optional as in 'http://hostname[:port]'. - Can be set by environment variable RENDER_PORT. - owner (str): owner for render-ws. - Can be set by environment variable RENDER_OWNER. - project (str): project for render webservice. - Can be set by environment variable RENDER_PROJECT. - client_scripts (str): directory path - for render-ws-java-client scripts. - Can be set by environment variable RENDER_CLIENT_SCRIPTS. - client_script (str): path to a wrapper for java client classes. - Used only in RenderClient. - Can be set by environment variable RENDER_CLIENT_SCRIPT. - memGB (str):heap size in GB for java client scripts, - example for 1 GB: '1G'. Used only in RenderClient. - Can be set by environment variable RENDER_CLIENT_HEAP. - force_http (bool): whether to prepend - 'http://' to render host - validate_client (bool): whether to - validate existence of RenderClient run_ws_client.sh script - web_only (bool): whether to check environment variables/prompt user - for client_scripts directory if not in arguments - Returns: - Render: a connect object to simplify specifying what render server to connect to - (returns :class:`RenderClient` if sufficent parameters are passed) - ''' + Parameters + ---------- + host : str + hostname for target render server -- will prepend + "http://" if host does not begin with 'http' and + force_http keyword evaluates True. + Can be set by environment variable RENDER_HOST. + port : str, int, or None + port for target render server. + Optional as in 'http://hostname[:port]'. + Can be set by environment variable RENDER_PORT. + owner : str + owner for render-ws. + Can be set by environment variable RENDER_OWNER. + project : str + project for render webservice. + Can be set by environment variable RENDER_PROJECT. + client_scripts : str + directory path for render-ws-java-client scripts. + Can be set by environment variable RENDER_CLIENT_SCRIPTS. + client_script : str, optional + path to a wrapper for java client classes. + Used only in RenderClient. + Can be set by environment variable RENDER_CLIENT_SCRIPT. + memGB : str + heap size in GB for java client scripts, + example for 1 GB: '1G'. Used only in RenderClient. + Can be set by environment variable RENDER_CLIENT_HEAP. + force_http : bool + whether to prepend + 'http://' to render host if it does not begin with 'http' + validate_client : bool + whether to validate existence of RenderClient run_ws_client.sh script + web_only : bool + whether to check environment variables/prompt user + for client_scripts directory if not in arguments + + Returns + ------- + Render + a connect object to simplify specifying what render + server to connect to + (returns :class:`RenderClient` if sufficent parameters are passed) + + Raises + ------ + ValueError + if empty user input is given for required field + + """ if host is None: if 'RENDER_HOST' not in os.environ: host = str(raw_input("Enter Render Host: ")) @@ -257,31 +355,31 @@ def connect(host=None, port=None, owner=None, project=None, @decorator -def renderaccess(f,*args,**kwargs): - ''' - Decorator allowing functions asking for host, port, owner, project - to default to a connection defined by a :class:`Render` object - using its :func:`RenderClient.make_kwargs` method. - - Example: - :: - - render = renderapi.render.connect('server',8080,'me','my_project') - stacks = renderapi.render.get_stacks_by_owner_project(render=render) - This works because :func:`renderapi.render.get_stacks_by_owner_project` has the - renderaccess decorator to fill in the default values. - - You can if you wish specify any of the arguments, in which case they will not - be filled in by the default values, but you don't have to. - - As such, the documentation omits describing the parameters which are natural - to expect will be filled in by the renderaccess decorator. - - Args: - f (func): function to decorate - Returns: - func: decorated function - ''' +def renderaccess(f, *args, **kwargs): + """Decorator allowing functions asking for host, port, owner, project + to default to a connection defined by a :class:`Render` object + using its :func:`RenderClient.make_kwargs` method. + + You can if you wish specify any of the arguments, in which case they + will not be filled in by the default values, but you don't have to. + + As such, the documentation omits describing the parameters which are + natural to expect will be filled in by the renderaccess decorator. + + Parameters + ---------- + f : func + function to decorate + Returns + ------- + func + decorated function + + Examples + -------- + >>> render = renderapi.render.connect('server',8080,'me','my_project') + >>> stacks = renderapi.render.get_stacks_by_owner_project(render=render) + """ args, kwargs = fitargspec(f, args, kwargs) render = kwargs.get('render') if render is not None: @@ -293,37 +391,52 @@ def renderaccess(f,*args,**kwargs): type(render))) else: return f(*args, **kwargs) - def format_baseurl(host, port): - '''format host and port to a standard template render-ws url + """format host and port to a standard template render-ws url + + Parameters + ---------- + host : str + host of render server + port : int or None + port of render server - Args: - host (str):host of render server - port (int or None): port of render server Returns - str: a url to the render endpoint at that host/port combination (append render-ws/v1) - ''' + ------- + str + a url to the render endpoint at that + host/port combination (append render-ws/v1) + """ # return 'http://%s:%d/render-ws/v1' % (host, port) server = '{}{}'.format(host, ('' if port is None else ':{}'.format(port))) return '{}/render-ws/v1'.format(server) def format_preamble(host, port, owner, project, stack): - ''' - format host, port, owner, project, and stack parameters - to the access point to stack-based apis - - Args: - host (str): render host - port (int): render host port - owner (str): render owner - project (str): render project - stack (str): render stack - Returns: - str: a url to the endpoint for that host, port, owner, project, stack combination - ''' + """format host, port, owner, project, and stack parameters + to the access point to stack-based apis + + Parameters + ---------- + host : str + render host + port : int + render host port + owner : str + render owner + project : str + render project + stack : str + render stack + + Returns + ------- + str + a url to the endpoint for that host, port, + owner, project, stack combination + """ preamble = "%s/owner/%s/project/%s/stack/%s" % ( format_baseurl(host, port), owner, project, stack) return preamble @@ -332,20 +445,27 @@ def format_preamble(host, port, owner, project, stack): @renderaccess def get_owners(host=None, port=None, session=requests.session(), render=None, **kwargs): - ''' - return list of owners across all Projects and Stacks for a render server + """return list of owners across all Projects and Stacks for a render server :func:`renderaccess` decorated function - Args: - host (str): render host (defaults to host from render) - port (int): render port (default to port from render) - session (requests.Session): requests session - render (RenderClient): RenderClient connection object - Returns: - list: list of strings containing all render owners + Parameters + ---------- + host : str + render host (defaults to host from render) + port : int + render port (default to port from render) + session : requests.Session + requests session + render : RenderClient + RenderClient connection object - ''' + Returns + ------- + list + list of strings containing all render owners + + """ request_url = "%s/owners/" % format_baseurl(host, port) r = session.get(request_url) try: @@ -360,19 +480,25 @@ def get_owners(host=None, port=None, session=requests.session(), def get_stack_metadata_by_owner(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - ''' - return metadata for all stacks belonging to particular + """return metadata for all stacks belonging to particular owner on render server :func:`renderaccess` decorated function - Args: - owner (str): render owner - render (RenderClient): render connect object - session (requests.sessions.Session): http session to use - Returns: - dict: stackInfo metadata, TODO example - ''' + Parameters + ---------- + owner : str + render owner + render : RenderClient + render connect object + session : requests.sessions.Session + http session to use + + Returns + ------- + dict + stackInfo metadata, TODO example + """ request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) logger.debug(request_url) @@ -388,18 +514,25 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, @renderaccess def get_projects_by_owner(owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - '''return list of projects belonging to a single owner for render stack + """return list of projects belonging to a single owner for render stack :func:`renderaccess` decorated function - Args: - owner (str): render owner - render (RenderClient): render connect object - session (requests.sessions.Session): http session to use - Returns: - list[str]: render projects by this owner + Parameters + ---------- + owner : str + render owner + render : RenderClient + render connect object + session : requests.sessions.Session + http session to use + + Returns + ------- + :obj:`list` of :obj:`unicode` + render projects by this owner - ''' + """ metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) projects = list(set([m['stackId']['project'] for m in metadata])) @@ -410,20 +543,27 @@ def get_projects_by_owner(owner=None, host=None, port=None, def get_stacks_by_owner_project(owner=None, project=None, host=None, port=None, session=requests.session(), render=None, **kwargs): - ''' - return list of stacks belonging to an owner's project on render server + """return list of stacks belonging to an owner's project on render server :func:`renderaccess` decorated function - Args: - owner (str): render owner - project (str): render project - render (RenderClient): render connect object - session (requests.sessions.Session): http session to use - Returns: - list[str]: render stacks by this owner in this project + Parameters + ---------- + owner : str + render owner + project : str + render project + render : RenderClient + render connect object + session : requests.sessions.Session + http session to use + + Returns + ------- + :obj:`list` of :obj:`str` + render stacks by this owner in this project - ''' + """ metadata = get_stack_metadata_by_owner(owner=owner, host=host, port=port, session=session) stacks = ([m['stackId']['stack'] for m in metadata diff --git a/renderapi/stack.py b/renderapi/stack.py index d81a782b..17afc51e 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -13,24 +13,29 @@ class StackVersion: - '''StackVersion, metadata about a stack - - Args: - cycleNumber (int):cycleNumber, use as you wish to - track versions (default None) - cycleStepNumber (int): cycleStepNumber, use as you with to - track versions (default None) - stackResolutionX (float): stackResolutionX, - resolution of scale = 1.0 in nm - stackResolutionY (float): stackResolutionY, - resolution of scale = 1.0 in nm - stackResolutionZ (float): stackResolutionZ, - resolution of scale = 1.0 in nm - mipmapPathBuilder (str): path to mipmap builder (?) - materializedBoxRootPath (str): path to materializer (?) - createTimeStamp (str): time stamp of stack creation (default to now) - versionNotes (str): notes about this stack (optional) - ''' + """StackVersion, metadata about a stack + + Attributes + ---------- + cycleNumber : int + cycleNumber, use as you wish to track versions (default None) + cycleStepNumber : int + cycleStepNumber, use as you with to track versions (default None) + stackResolutionX : float + stackResolutionX, resolution of scale = 1.0 in nm + stackResolutionY : float + stackResolutionY, resolution of scale = 1.0 in nm + stackResolutionZ : float + stackResolutionZ, resolution of scale = 1.0 in nm + mipmapPathBuilder : str + path to mipmap builder (?) + materializedBoxRootPath : str + path to materializer (?) + createTimeStamp : str + time stamp of stack creation (default to now) + versionNotes : str + notes about this stack (optional) + """ def __init__(self, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, stackResolutionZ=None, @@ -49,11 +54,13 @@ def __init__(self, cycleNumber=None, cycleStepNumber=None, self.versionNotes = versionNotes def to_dict(self): - '''serialization function + """serialization function - Returns: - dict: json compatible verson of this object - ''' + Returns + ------- + dict + json compatible verson of this object + """ d = {} d.update(({'cycleNumber': self.cycleNumber} if self.cycleNumber is not None else {})) @@ -76,11 +83,13 @@ def to_dict(self): return d def from_dict(self, d): - '''deserialization function + """deserialization function - Args: - d (dict):dictionary to update the properties of this object - ''' + Parameters + ---------- + d : dict + dictionary to update the properties of this object + """ self.__dict__.update({k: v for k, v in d.items()}) @@ -88,20 +97,27 @@ def from_dict(self, d): def set_stack_metadata(stack, sv, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''sets the stack metadata for a stack + """sets the stack metadata for a stack :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to set the metadata for - sv (StackVersion): metadata for the stack - render (renderapi.render.RenderClient): render connect object - session (requests.sessions.Session): session object (default start a new one) - - Returns: - requests.response: response from server - - ''' + Parameters + ---------- + stack : str + stack to set the metadata for + sv : StackVersion + metadata for the stack + render : renderapi.render.RenderClient + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.response + response from server + + """ request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) return post_json(session, request_url, sv.to_dict()) @@ -110,19 +126,29 @@ def set_stack_metadata(stack, sv, host=None, port=None, owner=None, @renderaccess def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - ''' get the stack metadata for a stack + """get the stack metadata for a stack :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to get the metadata for - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - StackVersion: metadata of the stack - Raises: - .errors.RenderError - ''' + Parameters + ---------- + stack : str + stack to get the metadata for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + StackVersion + metadata of the stack + + Raises + ------ + RenderError + + """ request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) @@ -141,23 +167,33 @@ def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - ''' + """ set state of selected stack. TODO there is a limited direction in which these stack changes can go :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to set state for - state (str): state of stack, one of ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'] - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - requests.session.response: server response - Raises: - .errors.RenderError - ''' + Parameters + ---------- + stack : str + stack to set state for + state : str + state of stack, one of ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'] + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + Raises + ------ + RenderError + """ if state not in ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY']: raise RenderError('state {} not in known states {}'.format( state, ['LOADING', 'COMPLETE', 'OFFLINE', 'READ_ONLY'])) @@ -175,16 +211,22 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, @renderaccess def likelyUniqueId(host=None, port=None, session=requests.session(), render=None, **kwargs): - '''return hex-code nearly-unique id from render server + """return hex-code nearly-unique id from render server :func:`renderapi.render.renderaccess` decorated function - Args: - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - str: string representation of hex-code - ''' + Parameters + ---------- + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + str + string representation of hex-code + """ request_url = '{}/likelyUniqueId'.format(format_baseurl(host, port)) r = session.get(request_url, data=None, headers={"content-type": "text/plain"}) @@ -192,18 +234,28 @@ def likelyUniqueId(host=None, port=None, def make_stack_params(host, port, owner, project, stack): - '''utility function to turn host,port,owner,project,stack combinations + """utility function to turn host,port,owner,project,stack combinations to java CLI based argument list for subprocess calling - - Args: - host (str): render server - port (int): render server port - owner (str): render owner - project (str): render project - stack (str): render stack - Returns: - list[str]: java CLI list of arguments for subprocess calling - ''' + + Parameters + ---------- + host : str + render server + port : int + render server port + owner : str + render owner + project : str + render project + stack : str + render stack + + Returns + ------- + :obj:`list` of :obj:`str` + java CLI list of arguments for subprocess calling + + """ baseurl = format_baseurl(host, port) project_params = ['--baseDataUrl', baseurl, '--owner', owner, '--project', project] @@ -215,18 +267,25 @@ def make_stack_params(host, port, owner, project, stack): def delete_stack(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''deletes a stack from render server + """deletes a stack from render server :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to delete - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - requests.session.response: server response - - ''' + Parameters + ---------- + stack : str + stack to delete + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + """ request_url = format_preamble(host, port, owner, project, stack) r = session.delete(request_url) logger.debug(r.text) @@ -237,18 +296,26 @@ def delete_stack(stack, host=None, port=None, owner=None, def delete_section(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''removes a single z from a stack + """removes a single z from a stack :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to delete section from - z (float): z value to delete - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - requests.session.response: server response - ''' + Parameters + ---------- + stack : str + stack to delete section from + z : float + z value to delete + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + """ request_url = '{}/z/{}'.format( format_preamble(host, port, owner, project, stack), z) r = session.delete(request_url) @@ -260,19 +327,28 @@ def delete_section(stack, z, host=None, port=None, owner=None, def delete_tile(stack, tileId, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - ''' + """ removes a tile from a stack :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to delete tile from - tileId (str): tileId of tilespec to remove from stack - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - requests.session.response: server response - ''' + Parameters + ---------- + stack : str + stack to delete tile from + tileId : str + tileId of tilespec to remove from stack + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + """ request_url = '{}/tile/{}'.format( format_preamble(host, port, owner, project, stack), tileId) r = session.delete(request_url) @@ -286,26 +362,40 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionZ=None, force_resolution=True, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''creates a new stack + """creates a new stack :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): render stack name to create - cycleNumber (int): cycleNumber to use to track stages - cycleStepNumber (int):cycleStepNumber to use to track stages - stackResolutionX (float): resolution of x pixels at scale=1.0 - stackResolutionY (float): resolution of y pixels at scale=1.0 - stackResoluiontZ (float): resolution of z sections at scale=1.0 - force_resolution (bool): fill in resolution of 1.0 for missing - resolutions - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - requests.session.response: server response - Raises: - .errors.RenderError - ''' + Parameters + ---------- + stack : str + render stack name to create + cycleNumber : int + cycleNumber to use to track stages + cycleStepNumber : int + cycleStepNumber to use to track stages + stackResolutionX : float + resolution of x pixels at scale=1.0 + stackResolutionY : float + resolution of y pixels at scale=1.0 + stackResoluiontZ : float + resolution of z sections at scale=1.0 + force_resolution : bool + fill in resolution of 1.0 for missing resolutions + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + + Raises + ------ + RenderError + """ if force_resolution: stackResolutionX, stackResolutionY, stackResolutionZ = [ (1.0 if res is None else res) @@ -332,24 +422,34 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, zs=None, close_stack=True, host=None, port=None, owner=None, project=None, session=None, render=None, **kwargs): - '''clone a stack + """clone a stack :func:`renderapi.render.renderaccess` decorated function - Args: - inputstack (str): name of input stack to clone - outputstack (str): name of destination stack. - if exists, must be LOADING - skipTransforms (bool): boolean whether to strip transformations - in new stack (default=False) - toProject (str): string name of destination project (default same as inputstack) - zs (list[float] or None): list of selected z values to clone into stack (optional) - close_stack (boolean): whether to set stack to COMPLETE when finished - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - requests.session.response: server response - ''' + Parameters + ---------- + inputstack : str + name of input stack to clone + outputstack : str + name of destination stack. if exists, must be LOADING + skipTransforms : bool + boolean whether to strip transformations in new stack (default=False) + toProject : str + string name of destination project (default same as inputstack) + zs : :obj:`list` of :obj:`float` or None + list of selected z values to clone into stack (optional) + close_stack : bool + whether to set stack to COMPLETE when finished + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + """ session = requests.session() if session is None else session sv = StackVersion(**kwargs) newstack_project = project @@ -378,19 +478,28 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, def get_z_values_for_stack(stack, project=None, host=None, port=None, owner=None, session=requests.session(), render=None, **kwargs): - '''get a list of z values for which there are tiles in the stack - + """get a list of z values for which there are tiles in the stack + :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to get z values for - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - list[float]: z values in stack - Raises: - .errors.RenderError - ''' + Parameters + ---------- + stack : str + stack to get z values for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + :obj:`list` of :obj:`float` + z values in stack + + Raises + ------ + RenderError + """ request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" logger.debug(request_url) @@ -404,9 +513,9 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, def get_z_value_for_section(stack, sectionId, **kwargs): - '''DEPRECATED (use :func:`get_section_z_value`) instead - ''' - logger.warning("DEPRECATED, use renderapi.stack.get_section_z_value instead") + """DEPRECATED (use :func:`get_section_z_value`) instead""" + logger.warning( + "DEPRECATED, use renderapi.stack.get_section_z_value instead") return get_section_z_value(stack, sectionId, **kwargs) @@ -426,20 +535,31 @@ def get_z_value_for_section(stack, sectionId, **kwargs): def get_bounds_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''get a bounds dictionary for a specific z + """get a bounds dictionary for a specific z :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to get bounds from - z (float): z value to get bounds for - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - dict: bounds with keys minY,minY,maxX,maxY,minZ,maxZ - Raises: - .errorr.RenderError - ''' + Parameters + ---------- + stack : str + stack to get bounds from + z : float + z value to get bounds for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + dict + bounds with keys minY,minY,maxX,maxY,minZ,maxZ + + Raises + ------ + RenderError + + """ request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/bounds' % (z) @@ -455,19 +575,29 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, @renderaccess def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''get bounds of a whole stack + """get bounds of a whole stack :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to get bounds from - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - dict: bounds with keys minY,minY,maxX,maxY,minZ,maxZ - Raises: - .errors.RenderError - ''' + Parameters + ---------- + stack : str + stack to get bounds from + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + dict + bounds with keys minY,minY,maxX,maxY,minZ,maxZ + + Raises + ------ + RenderError + + """ request_url = format_preamble( host, port, owner, project, stack) + '/bounds' r = session.get(request_url) @@ -483,20 +613,31 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''returns the sectionId associated with a particular z value + """returns the sectionId associated with a particular z value :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): stack to look within - sectionId (str): sectionId to find z value - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - float: z value of sectionId - Raises: - .errors.RenderError - ''' + Parameters + ---------- + stack : str + stack to look within + sectionId : str + sectionId to find z value + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + float + z value of sectionId + + Raises + ------ + RenderError + + """ sectionData = get_stack_sectionData( stack, host, port, owner, project, session) try: @@ -510,16 +651,23 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, def get_stack_sectionData(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''returns information about the sectionIds of each slice in stack - + """returns information about the sectionIds of each slice in stack + :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): name of stack to get data about - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - dict:sectionData as below + Parameters + ---------- + stack : str + name of stack to get data about + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + dict + sectionData as below :: [{ "sectionId": "string", @@ -530,9 +678,10 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, "minY": 0, "maxY": 0 }] - Raises: - .errors.RenderError: - ''' + Raises + ------ + RenderError + """ request_url = format_preamble( host, port, owner, project, stack) + '/sectionData' r = session.get(request_url) @@ -548,20 +697,30 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, def get_section_z_value(stack, sectionId, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''get the z value for a specific sectionId (string) + """get the z value for a specific sectionId (string) :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str):render stack string to look within - sectionId (str): of sectionId to find z value - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - float: z value of section - Raises: - .errors.RenderError: - ''' + Parameters + ---------- + stack : str + render stack string to look within + sectionId : str + sectionId to find z value + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + float + z value of section + + Raises + ------ + RenderError + """ request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % sectionId r = session.get(request_url) @@ -576,19 +735,28 @@ def get_section_z_value(stack, sectionId, host=None, port=None, @renderaccess def get_stack_tileIds(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''get tileIds for a stack - + """get tileIds for a stack + :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str):stack to get tileIds - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): session object (default start a new one) - Returns: - list[str]: list of tileIds in stack - Raises: - .errors.RenderError: - ''' + Parameters + ---------- + stack : str + stack to get tileIds + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + :obj:`list` of :obj:`str` + list of tileIds in stack + + Raises + ------ + RenderError + """ request_url = '{}/tileIds'.format( format_preamble(host, port, owner, project, stack)) r = session.get(request_url) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 04bcd3b6..004a7ab4 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -13,26 +13,60 @@ class Layout: - '''Layout class to describe acquisition settings - inputs: - - Args: - sectionId (str): sectionId this tile was taken from - scopeId (str): what microscope this came from - cameraId (str): camera this was taken with - imageRow (int): what row from a row,col layout this was taken - imageCol (int): column from a row,col layout this was taken - stageX (float): X stage coordinates for where this was taken - stageY (float): Y stage coordinates for where this taken - rotation (float): angle of camera when this was taken - pixelsize (float): effective size of pixels (in units of choice) - force_pixelsize (bool): whether to default pixelsize to 0.1 - - ''' + """Layout class to describe acquisition settings + + Attributes + ---------- + sectionId : str + sectionId this tile was taken from + scopeId : str + what microscope this came from + cameraId : str + camera this was taken with + imageRow : int + what row from a row,col layout this was taken + imageCol : int + column from a row,col layout this was taken + stageX : float + X stage coordinates for where this was taken + stageY : float + Y stage coordinates for where this taken + rotation : float + angle of camera when this was taken + pixelsize : float + effective size of pixels (in units of choice) + + """ def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, rotation=None, pixelsize=None, force_pixelsize=True, **kwargs): + """Initialize Layout + + Parameters + ---------- + sectionId : str + sectionId this tile was taken from + scopeId : str + what microscope this came from + cameraId : str + camera this was taken with + imageRow : int + what row from a row,col layout this was taken + imageCol : int + column from a row,col layout this was taken + stageX : float + X stage coordinates for where this was taken + stageY : float + Y stage coordinates for where this taken + rotation : float + angle of camera when this was taken + pixelsize : float + effective size of pixels (in units of choice) + force_pixelsize : bool + whether to default pixelsize to 0.1 + + """ self.sectionId = sectionId self.scopeId = scopeId self.cameraId = cameraId @@ -46,11 +80,13 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, self.pixelsize = pixelsize def to_dict(self): - '''return a dictionary representation of this object - - Returns: - dict: json compatible dictionary of this object - ''' + """return a dictionary representation of this object + + Returns + ------- + dict + json compatible dictionary of this object + """ d = {} d['sectionId'] = self.sectionId d['temca'] = self.scopeId @@ -65,11 +101,13 @@ def to_dict(self): return d def from_dict(self, d): - '''set this object equal to the fields found in dictionary + """set this object equal to the fields found in dictionary - Args: - d (dict): dictionary to use to update - ''' + Parameters + ---------- + d : dict + dictionary to use to update + """ if d is not None: self.sectionId = d.get('sectionId') self.cameraId = d.get('camera') @@ -83,42 +121,60 @@ def from_dict(self, d): class TileSpec: - '''Fundamental class of render that store image tiles and their transformations - init: - - Args: - tileId (str): unique string specifying a tile's identity - z (float): z values this tile exists within - width (int): width in pixels of the raw tile - height (int): height in pixels of the raw tile - imageUrl (str): an image path that can be accessed by the render server, - with an ImageJ.open command or as an s3 url. - Files on disk should be specified with file: - maskUrl (str): an image path that can be accessed by the render server - which can be interpreted as an - alpha mask for the image (same as spec imageUrl) - minint (int): pixel intensity value to display as black in a - linear colormap (default 0) - maxint (int): pixel intensity value to display as white in a - linear colormap (default 65535) - layout (Layout): a Layout object for this tile - tforms (list[Transform]): Transform objects - (see :class:`.transform.AffineModel`, - :class:`.transform.TransformList`, - :class:`.transform.Polynomial2DTransform`, - :class:`.transform.Transform`, - :class:`.transform.ReferenceTransform`) to apply to this tile - inputfilters (list): a list of filters to apply to - this tile (not yet implemented) - mipMapLevels (list[MipMapLevel]): :class:`MipMapLevel` objects for this tile - json (dict or None): dictionary to initialize this object with - (if not None overrides and ignores all keyword arguments) - scale3Url (str): url of a mipmap level 3 image of this tile - (DEPRECATED, use mipMapLevels, but will override) - scale2Url (str): url of a mipmap level 2 image of this tile - (DEPRECATED, use mipMapLevels, but will override) - scale1Url (str): url of a mipmap level 1 image of this tile - (DEPRECATED, use mipMapLevels, but will override) + '''Fundamental class of render that store image tiles + and their transformations + + Attributes + ---------- + tileId : str + unique string specifying a tile's identity + z : float + z values this tile exists within + width : int + width in pixels of the raw tile + height : int + height in pixels of the raw tile + imageUrl : str + an image path URI that can be accessed by the render server, + with an ImageJ.open command. Preceded by , e.g. 'file://' for files + or s3:// for s3 + maskUrl : str + an image path that can be accessed by the render server + which can be interpreted as an + alpha mask for the image (same as spec imageUrl) + minint : int + pixel intensity value to display as black in a + linear colormap (default 0) + maxint : int + pixel intensity value to display as white in a + linear colormap (default 65535) + layout : :class:`Layout` + a :class:`Layout` object for this tile + tforms : :obj:`list` of :obj:`Transform` + or :obj:`list` of :obj:`TransformList` + or :obj:`list` of :obj:`InterpolatedTransform` + Transform objects + (see :class:`.transform.AffineModel`, + :class:`.transform.TransformList`, + :class:`.transform.Polynomial2DTransform`, + :class:`.transform.Transform`, + :class:`.transform.ReferenceTransform`) to apply to this tile + inputfilters : list + a list of filters to apply to this tile (not yet implemented) + mipMapLevels :obj:`list` of :obj:`MipMapLevel` + :class:`MipMapLevel` objects for this tile + json : dict or None + dictionary to initialize this object with + (if not None overrides and ignores all keyword arguments) + scale3Url : str + uri of a mipmap level 3 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + scale2Url : str + uri of a mipmap level 2 image of this tile + (DEPRECATED, use mipMapLevels, but will override) + scale1Url : str + uri of a mipmap level 1 image of this tile + (DEPRECATED, use mipMapLevels, but will override) ''' def __init__(self, tileId=None, z=None, width=None, height=None, @@ -160,7 +216,7 @@ def __init__(self, tileId=None, z=None, width=None, height=None, @property def bbox(self): - '''bbox defined to fit shapely call''' + """bbox defined to fit shapely call""" box = (self.minX, self.minY, self.maxX, self.maxY) if any([v is None for v in box]): logger.error( @@ -168,12 +224,14 @@ def bbox(self): return box def to_dict(self): - '''method to produce a json tilespec for this tile + """method to produce a json tilespec for this tile returns a json compatible dictionary - Returns: - dict: json compatible dictionary representation of this object - ''' + Returns + ------- + dict + json compatible dictionary representation of this object + """ thedict = {} thedict['tileId'] = self.tileId thedict['z'] = self.z @@ -212,11 +270,13 @@ def to_dict(self): return thedict def from_dict(self, d): - '''Method to load tilespec from json dictionary - - Args: - d (dict): dictionary to use to set properties of this object - ''' + """Method to load tilespec from json dictionary + + Paramters + --------- + d : dict + dictionary to use to set properties of this object + """ self.tileId = d['tileId'] self.z = d['z'] self.width = d['width'] @@ -250,25 +310,31 @@ def from_dict(self, d): class MipMapLevel: - ''' - MipMapLevel class to represent a level of an image pyramid. + """MipMapLevel class to represent a level of an image pyramid. Can be put in dictionary formatting using dict(mML) - Args: - level (int): level of 2x downsampling represented by mipmaplevel - imageUrl (str or None): url corresponding to image - maskUrl (str or None): url corresponding to mask - ''' + Attributes + ---------- + level : int + level of 2x downsampling represented by mipmaplevel + imageUrl : str or None + uri corresponding to image + maskUrl : str or None + uri corresponding to mask + + """ def __init__(self, level, imageUrl=None, maskUrl=None): self.level = level self.imageUrl = imageUrl self.maskUrl = maskUrl def to_dict(self): - ''' - Returns: - dict: json compatible dictionary representaton - ''' + """ + Returns + ------- + dict + json compatible dictionary representaton + """ return dict(self.__iter__()) def _formatUrls(self): @@ -284,61 +350,83 @@ def __iter__(self): class ImagePyramid: - ''' - Image Pyramid class representing a set of MipMapLevels which correspond - to mipmapped (continuously downsmapled by 2x) representations - of an image at level 0 + '''Image Pyramid class representing a set of MipMapLevels which correspond + to mipmapped (continuously downsmapled by 2x) representations + of an image at level 0 Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) - Args: - mipMapLevels (list[MipMapLevel]): list of :class:`MipMapLevel` objects - + Attributes + ---------- + mipMapLevels : :obj:`list` of :class:`MipMapLevel` + list of :class:`MipMapLevel` objects defining image pyramid + ''' def __init__(self, mipMapLevels=[]): self.mipMapLevels = mipMapLevels def to_dict(self): - '''return dictionary representation of this object''' + """return dictionary representation of this object""" return dict(self.__iter__()) def to_ordered_dict(self, key=None): - '''returns :class:`OrderedDict` represention of this object, ordered by mipmapLevel''' + """returns :class:`OrderedDict` represention of this object, + ordered by mipmapLevel + + Parameters + ---------- + key : func + function to sort ordered dict of + :class:`mipMapLevel` dicts (default is by level) + + Returns + ------- + OrderedDict + sorted dictionary of :class:`mipMapLevels` in ImagePyramid + + """ return OrderedDict(sorted( self.__iter__(), key=((lambda x: x[0]) if key is None else key))) def append(self, mmL): - '''appends a MipMapLevel to this ImagePyramid + """appends a MipMapLevel to this ImagePyramid - Args: - mml (MipMapLevel): MipMapLevel to append - ''' + Parameters + ---------- + mml : :class:`MipMapLevel` + :class:`MipMapLevel` to append + """ self.mipMapLevels.append(mmL) def update(self, mmL): - '''updates the ImagePyramid with this MipMapLevel + """updates the ImagePyramid with this MipMapLevel. + will overwrite existing mipMapLevels with same level Args: mml (MipMapLevel): mipmap level to update in pyramid - ''' + """ self.mipMapLevels = [ l for l in self.mipMapLevels if l.level != mmL.level] self.append(mmL) def get(self, to_get): - '''gets a specific mipmap level in dictionary form - - Args: - to_get (int): mipmaplevel to get - - Returns: - dict: dictionary representation of requested MipMapLevel - ''' + """gets a specific mipmap level in dictionary form + + Parameters + ---------- + to_get : int + level to get + + Returns + ------- + dict + representation of requested MipMapLevel + """ return self.to_dict()[to_get] # TODO should this default @property def levels(self): - '''list of MipMapLevels in this ImagePyramid''' + """list of MipMapLevels in this ImagePyramid""" return [int(i.level) for i in self.mipMapLevels] def __iter__(self): @@ -351,19 +439,29 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''renderapi call to get the render parameters of a specific tileId + """renderapi call to get the render parameters of a specific tileId :func:`renderapi.render.renderaccess` decorated function - - Args: - stack (str): name of render stack to retrieve - tile (str): tileId of tile to retrieve - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - dict: render-parameters json with the tilespec for that tile and dereferenced transforms - TODO provide example - ''' + TODO provide example of return + + Parameters + ---------- + stack : str + name of render stack to retrieve + tile : str + tileId of tile to retrieve + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + dict + render-parameters json with the tilespec for that + tile and dereferenced transforms + + """ request_url = format_preamble( host, port, owner, project, stack) + \ @@ -382,20 +480,28 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, def get_tile_spec(stack, tile, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''renderapi call to get a specific tilespec by tileId + """renderapi call to get a specific tilespec by tileId note that this will return a tilespec with resolved transform references by accessing the render-parameters version of this tile :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): name of render stack to retrieve - tile (str): tileId of tile to retrieve - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - TileSepc: TileSpec with dereferenced transforms - ''' + Parameters + ---------- + stack : str + name of render stack to retrieve + tile : str + tileId of tile to retrieve + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + TileSpec + TileSpec with dereferenced transforms + """ try: tilespec_json = get_tile_spec_renderparameters( @@ -409,19 +515,27 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''renderapi call to get a specific tilespec by tileId + """renderapi call to get a specific tilespec by tileId note that this will return a tilespec without resolved transform references :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): name of render stack to retrieve - tile (str): tileId of tile to retrieve - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - TileSepc: TileSpec with referenced transforms intact - ''' + Parameters + ---------- + stack : str + name of render stack to retrieve + tile : str + tileId of tile to retrieve + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + TileSpec + TileSpec with referenced transforms intact + """ request_url = format_preamble( host, port, owner, project, stack) + \ @@ -442,25 +556,38 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''renderapi call to get all tilespec that exist within a 2d bounding box + """renderapi call to get all tilespec that exist within a 2d bounding box specified with min and max x,y values note that this will return a tilespec with resolved transform references :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): name of render stack to retrieve - z (float): z value of bounding box - xmin (float): minimum x value - ymin (float): minimum y value - xmax (float): maximum x value - ymax (float): maximum y value - scale (float): scale to use when retrieving render parameters (not important) - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - list[TileSpec]:TileSpec objects with dereferenced tansforms - ''' + Parameters + ---------- + stack : str + name of render stack to retrieve + z : float + z value of bounding box + xmin : float + minimum x value + ymin : float + minimum y value + xmax : float + maximum x value + ymax : float + maximum y value + scale : float + scale to use when retrieving render parameters (not important) + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + :class:`TileSpec` objects with dereferenced tansforms + """ x = xmin y = ymin width = xmax - xmin @@ -476,25 +603,38 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, scale=1.0, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''renderapi call to get all tilespec that exist within a 2d bounding box + """renderapi call to get all tilespec that exist within a 2d bounding box specified with min x,y values and width, height note that this will return a tilespec with resolved transform references :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): name of render stack to retrieve - z (float): z value of bounding box - x (float): minimum x value - y (float): minimum y value - width (float): width of box (in scale=1.0 units) - height (float): height of box (in scale=1.0 units) - scale (float): scale to use when retrieving render parameters (not important) - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - list[TileSpec]:TileSpec objects with dereferenced tansforms - ''' + Parameters + ---------- + stack : str + name of render stack to retrieve + z : float + z value of bounding box + x : float + minimum x value + y : float + minimum y value + width : float + width of box (in scale=1.0 units) + height : float + height of box (in scale=1.0 units) + scale : float + scale to use when retrieving render parameters (not important) + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + TileSpec objects with dereferenced tansforms + """ request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( @@ -515,19 +655,26 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, def get_tile_specs_from_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - ''' - Get all TileSpecs in a specific z values. Returns referenced transforms. + """Get all TileSpecs in a specific z values. Returns referenced transforms. :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): render stack - z (float): render z - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - list[TileSpec]: list of TileSpec objects from that stack at that z - ''' + Parameters + ---------- + stack : str + render stack + z : float + render z + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + list of TileSpec objects from that stack at that z + """ request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) logger.debug(request_url) @@ -551,17 +698,24 @@ def get_tile_specs_from_stack(stack, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''get flat list of tilespecs for stack using i for sl in l for i in sl - + """get flat list of tilespecs for stack using i for sl in l for i in sl + :func:`renderapi.render.renderaccess` decorated function - Args: - stack (str): render stack - render (renderapi.render.Render): render connect object - session (requests.sessions.Session): sessions object to connect with - Returns: - list[TileSpec]: list of TileSpec objects from that stack - ''' + Parameters + ---------- + stack : str + render stack + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`list` of :class:`TileSpec` + list of TileSpec objects from that stack + """ return [i for sl in [ get_tile_specs_from_z(stack, z, host=host, port=port, owner=owner, project=project, session=session) diff --git a/renderapi/transform.py b/renderapi/transform.py index 10d00b2d..2e493965 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -''' -handling mpicbg transforms in python +"""handling mpicbg transforms in python -Currently only implemented to facilitate Affine, Polynomial2D, - and LensCorrection used in Khaled Khairy's EM aligner workflow -''' +Currently only implemented to facilitate Affine and Polynomial2D + used in Khaled Khairy's EM aligner workflow +""" import json import logging from collections import Iterable @@ -26,15 +25,30 @@ class TransformList: - ''' A list of Transforms + """A list of Transforms - Args: - tforms (Transform): transforms to apply - transformId (str): uniqueId for this TransformList - json (dict): json compatible dictionary to create TransformList via :method:`from_dict` - ''' + Attributes + ---------- + tforms : :obj:`list` of :class:`Transform` + transforms to apply + transformId : str, optional + uniqueId for this TransformList + """ def __init__(self, tforms=None, transformId=None, json=None): + """Initialize TransformList + + Parameters + ---------- + tforms : :obj:`list` of :class:`Transform` + transforms to apply + transformId : str, optional + uniqueId for this TransformList + json : dict, optional + json compatible dictionary to create + :class:`TransformList` via :method:`from_dict` + (will supersede tforms and transformId if not None) + """ if json is not None: self.from_dict(json) else: @@ -49,10 +63,13 @@ def __init__(self, tforms=None, transformId=None, json=None): self.transformId = transformId def to_dict(self): - ''' serialization function - Returns: - dict: json & render compatible representation of this TransformList - ''' + """serialization function + + Returns + ------- + dict + json & render compatible representation of this TransformList + """ d = {} d['type'] = 'list' d['specList'] = [tform.to_dict() for tform in self.tforms] @@ -61,17 +78,24 @@ def to_dict(self): return d def to_json(self): - ''' serialization function - Returns: - str: string representation of the json & render representation of this TransformList - ''' + """serialization function + + Returns + ------- + str + string representation of the json & render + representation of this TransformList + """ return json.dumps(self.to_dict()) def from_dict(self, d): - ''' deserialization function - Args: - d (dict): json compatible dictionary representation of this TransformList - ''' + """deserialization function + + Parameters + ---------- + d : dict + json compatible dictionary representation of this TransformList + """ self.tforms = [] if d is not None: self.transformId = d.get('id') @@ -81,17 +105,26 @@ def from_dict(self, d): def load_transform_json(d, default_type='leaf'): - '''function to get the proper deserialization function - - Args: - d (dict): json compatible representation of Transform - default_type (str): what kind of transform should we assume this + """function to get the proper deserialization function + + Parameters + ---------- + d : dict + json compatible representation of Transform + default_type : str + what kind of transform should we assume this if it is not specified in 'type' ('leaf','list','ref','interpolated') - Returns: - func: proper function to deserialize this transformation - Raises: - KeyError: if d['type'] isn't one of ('leaf','list','ref','interpolated') - ''' + + Returns + ------- + func + proper function to deserialize this transformation + + Raises + ------ + RenderError + if d['type'] isn't one of ('leaf','list','ref','interpolated') + """ handle_load_tform = {'leaf': load_leaf_json, 'list': lambda x: TransformList(json=x), 'ref': lambda x: ReferenceTransform(json=x), @@ -104,17 +137,24 @@ def load_transform_json(d, default_type='leaf'): def load_leaf_json(d): - '''function to get the proper deserialization function for leaf transforms - - Args: - d (dict): json compatible representation of leaf transform to deserialize - Returns: - func: proper function to deserialize this transformation - Raises: - RenderError: if d['type']!= leaf or is omitted - KeyError: if d['type'] does not indicate one of AffineModel, - Polynomial2DTransform, TranslationModel, RigidModel, or SimilarityModel - ''' + """function to get the proper deserialization function for leaf transforms + + Parameters + ---------- + d : dict + json compatible representation of leaf transform to deserialize + + Returns + ------- + func + proper function to deserialize this transformation + + Raises + ------ + RenderError + if d['type'] != leaf or is omitted + + """ handle_load_leaf = { AffineModel.className: lambda x: AffineModel(json=x), Polynomial2DTransform.className: @@ -137,17 +177,40 @@ def load_leaf_json(d): class InterpolatedTransform: - ''' - Transform spec defined by linear interpolation of two other transform specs - Args: - a (Transform):transform at minimum weight - b (Transform):transform at maximum weight - lambda_ (float): value in interval [0.,1.] which defines evaluation of the - linear interpolation between a (at 0) and b (at 1) - json (dict): json compatible representation of this transform to initialize via :method:`self.from_dict` - ''' + """Transform spec defined by linear interpolation of + two other transform specs + + Attributes + ---------- + a : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + """ def __init__(self, a=None, b=None, lambda_=None, json=None): + """Initialize InterpolatedTransform + + Parameters + ---------- + a : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + json : dict + json compatible representation of this transform to + initialize via :method:`self.from_dict` + (will supersede a, b, and lambda_ if not None) + + """ if json is not None: self.from_dict(json) else: @@ -156,17 +219,23 @@ def __init__(self, a=None, b=None, lambda_=None, json=None): self.lambda_ = lambda_ def to_dict(self): - '''serialization routine - Returns: - dict: json compatible representation - ''' + """serialization routine + + Returns + ------- + dict + json compatible representation + """ return dict(self) def from_dict(self, d): - ''' deserialization routine - Args: - d (dict): json compatible representation - ''' + """deserialization routine + + Parameters + ---------- + d : dict + json compatible representation + """ self.a = load_transform_json(d['a']) self.b = load_transform_json(d['b']) self.lambda_ = d['lambda'] @@ -179,34 +248,53 @@ def __iter__(self): class ReferenceTransform: - '''Transform which is simply a reference to a transform stored elsewhere + """Transform which is simply a reference to a transform stored elsewhere - Args: - refId (str): transformId of the referenced transform - json (dict): json compatible representation of this transform - ''' + Attributes + ---------- + refId : str + transformId of the referenced transform + + """ def __init__(self, refId=None, json=None): + """Initialize ReferenceTransform + + Parameters + ---------- + refId : str + transformId of the referenced transform + json : dict + json compatible representation of this transform + (will supersede refId if not None) + + """ if json is not None: self.from_dict(json) else: self.refId = refId def to_dict(self): - '''serialization routine - Returns: - dict: json compatible representation of this transform - ''' + """serialization routine + + Returns + ------- + dict + json compatible representation of this transform + """ d = {} d['type'] = 'ref' d['refId'] = self.refId return d def from_dict(self, d): - ''' deserialization routine - Args: - d (dict): json compatible representation of this transform - ''' + """deserialization routine + + Parameters + ---------- + d : dict + json compatible representation of this transform + """ self.refId = d['refId'] def __str__(self): @@ -220,17 +308,36 @@ def __iter__(self): class Transform(object): - '''Base transformation class - - Args: - className (str): mpicbg java classname of this transform - dataString (str): string reprsentation of this transform as speced by mpicbg java class library - transformId (str or None): unique Id for this transform (optional) - json (dict): json compatible representation of this transform - ''' + """Base transformation class + + Attributes + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced by + mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + """ def __init__(self, className=None, dataString=None, transformId=None, json=None): + """Initialize Transform + + Parameters + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced + by mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + json : dict + json compatible representation of this transform + (supersedes className, dataString, and transformId if not None) + """ if json is not None: self.from_dict(json) else: @@ -239,10 +346,13 @@ def __init__(self, className=None, dataString=None, self.transformId = transformId def to_dict(self): - '''serialization routine - Returns: - dict: json compatible representation of this transform - ''' + """serialization routine + + Returns + ------- + dict + json compatible representation of this transform + """ d = {} d['type'] = 'leaf' d['className'] = self.className @@ -252,18 +362,28 @@ def to_dict(self): return d def from_dict(self, d): - ''' deserialization routine - Args: - d (dict): json compatible representation of this transform - ''' + """deserialization routine + + Parameters + ---------- + d : dict + json compatible representation of this transform + """ self.className = d['className'] self.transformId = d.get('transformId', None) self._process_dataString(d['dataString']) def _process_dataString(self, datastring): - ''' method meant to set state of transform from datastring + """method meant to set state of transform from datastring generic implementation only saves datastring at self.dataString. - should rewrite for all transform classes that want to implement tform,fit,etc''' + should rewrite for all transform classes that want to + implement tform,fit,etc + + Parameters + ---------- + dataString : str + string which can be used to initialize mpicbg transforms in java + """ self.dataString = datastring def __str__(self): @@ -281,26 +401,61 @@ def __hash__(self): class AffineModel(Transform): - '''Linear 2d Transformation + """Linear 2d Transformation mpicbg classname: mpicbg.trakem2.transform.AffineModel2D implements this simple math x'=M00*x + M01*x + B0 y'=M10*x + M11*y + B1 - Args: - M00 (float): x'+=M00*x - M01 (float): x'+=M01*y - M10 (float): y'+=M10*x - M11 (float): y'+=M11*y - B0 (float): x'+=B0 - B1 (float): y'+=B1 - transformId (str): unique transformId for this transform (optional) - json (dict): json compatible representation of this transform - ''' + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + className = 'mpicbg.trakem2.transform.AffineModel2D' def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, transformId=None, json=None): + """Initialize AffineModel, defaulting to identity + + Parameters + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str + unique transformId for this transform (optional) + json : dict + json compatible representation of this transform + (will supersede all other parameters if not None) + + """ if json is not None: self.from_dict(json) else: @@ -316,15 +471,13 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, @property def dataString(self): - '''dataString string for this transform''' + """dataString string for this transform""" return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( self.M[0, 0], self.M[1, 0], self.M[0, 1], self.M[1, 1], self.M[0, 2], self.M[1, 2]) def _process_dataString(self, datastring): - ''' - generate datastring and param attributes from datastring - ''' + """generate datastring and param attributes from datastring""" dsList = datastring.split() self.M00 = float(dsList[0]) self.M10 = float(dsList[1]) @@ -335,7 +488,7 @@ def _process_dataString(self, datastring): self.load_M() def load_M(self): - '''method to take the attribute of self and fill in self.M''' + """method to take the attribute of self and fill in self.M""" self.M = np.identity(3, np.double) self.M[0, 0] = self.M00 self.M[0, 1] = self.M01 @@ -346,14 +499,21 @@ def load_M(self): @staticmethod def fit(A, B): - '''function to fit this transform given the corresponding sets of points A & B - - Args: - A (numpy.array): a Nx2 matrix of source points - B (numpy.array): a Nx2 matrix of destination points - Returns: - numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 - ''' + """function to fit this transform given the corresponding sets of points A & B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): raise EstimationError( 'shape mismatch! A shape: {}, B shape {}'.format( @@ -373,17 +533,27 @@ def fit(A, B): return Tvec def estimate(self, A, B, return_params=True, **kwargs): - '''method for setting this transformation with the best fit given the corresponding points A,B - - Args: - A (numpy.array): a Nx2 matrix of source points - B (numpy.array): a Nx2 matrix of destination points - return_params (boolean): whether to return the parameter matrix - **kwargs (dict): keyword arguments to pass to self.fit - Returns: - numpy.array: a 2x3 matrix of parameters for this matrix, laid out (x,y) x (x,y,offset) + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the parameter matrix + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) (or None if return_params=False) - ''' + """ Tvec = self.fit(A, B, **kwargs) self.M00 = Tvec[0, 0] self.M10 = Tvec[2, 0] @@ -396,8 +566,7 @@ def estimate(self, A, B, return_params=True, **kwargs): return self.M def concatenate(self, model): - ''' - concatenate a model to this model -- ported from trakEM2 below:: + """concatenate a model to this model -- ported from trakEM2 below: final double a00 = m00 * model.m00 + m01 * model.m10; final double a01 = m00 * model.m01 + m01 * model.m11; @@ -407,11 +576,16 @@ def concatenate(self, model): final double a11 = m10 * model.m01 + m11 * model.m11; final double a12 = m10 * model.m02 + m11 * model.m12 + m12; - Args: - model (AffineModel): model to concatenate to this one - Returns: - AffineModel: model after concatenating model with this model - ''' + Parameters + ---------- + model : AffineModel + model to concatenate to this one + + Returns + ------- + AffineModel + model after concatenating model with this model + """ a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + @@ -426,11 +600,13 @@ def concatenate(self, model): return newmodel def invert(self): - '''return an inverted version of this transformation + """return an inverted version of this transformation - Returns: - AffineModel: an inverted version of this transformation - ''' + Returns + ------- + AffineModel + an inverted version of this transformation + """ inv_M = np.linalg.inv(self.M) Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) @@ -438,13 +614,18 @@ def invert(self): @staticmethod def convert_to_point_vector(points): - '''method to help reshape x,y points to x,y,1 vectors - - Args: - points (numpy.array): a Nx2 array of x,y points - Returns: - numpy.arary: a Nx3 array of x,y,1 points used for transformations - ''' + """method to help reshape x,y points to x,y,1 vectors + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx3 array of x,y,1 points used for transformations + """ Np = points.shape[0] onevec = np.ones((Np, 1), np.double) @@ -457,60 +638,75 @@ def convert_to_point_vector(points): @staticmethod def convert_points_vector_to_array(points, Nd=2): - '''method for convertion x,y,K points to x,y vectors - - Args: - points (numpy.array): a Nx3 vector of points after transformation - Nd (int): the number of dimensions to cutoff (should be 2) - Returns: - numpy.array: a Nx2 array of x,y points - ''' + """method for convertion x,y,K points to x,y vectors + + Parameters + ---------- + points : numpy.array + a Nx3 vector of points after transformation + Nd : int + the number of dimensions to cutoff (should be 2) + + Returns + ------- + numpy.array: a Nx2 array of x,y points + """ points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T return points def tform(self, points): - '''transform a set of points through this transformation - - Args: - points (numpy.array): a Nx2 array of x,y points - Returns: - numpy.array: a Nx2 array of x,y points after transformation - ''' + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ points, Nd = self.convert_to_point_vector(points) pt = np.dot(self.M, points.T).T return self.convert_points_vector_to_array(pt, Nd) def inverse_tform(self, points): - '''transform a set of points through the inverse of this transformation - - Args: - points (numpy.array): a Nx2 array of x,y points - Returns: - numpy.array: a Nx2 array of x,y points after inverse transformation - ''' + """transform a set of points through the inverse of this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after inverse transformation + """ points, Nd = self.convert_to_point_vector(points) pt = np.dot(np.linalg.inv(self.M), points.T).T return self.convert_points_vector_to_array(pt, Nd) @property def scale(self): - '''tuple of scale for x, y''' + """tuple of scale for x, y""" return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) for j in range(self.M.shape[1])])[:2] @property def shear(self): - '''counter-clockwise shear angle''' + """counter-clockwise shear angle""" return np.arctan2(-self.M[0, 1], self.M[1, 1]) - self.rotation @property def translation(self): - '''tuple of translation in x, y''' + """tuple of translation in x, y""" return tuple(self.M[:2, 2]) @property def rotation(self): - '''counter-clockwise rotation''' + """counter-clockwise rotation""" return np.arctan2(self.M[1, 0], self.M[0, 0]) def __str__(self): @@ -520,6 +716,34 @@ def __str__(self): class TranslationModel(AffineModel): + """Translation fitting and estimation as an :class:`AffineModel` + Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + """ + className = 'mpicbg.trakem2.transform.TranslationModel2D' def __init__(self, *args, **kwargs): @@ -528,7 +752,7 @@ def __init__(self, *args, **kwargs): # 'TranslationModel not implemented. please use Affine') def _process_dataString(self, dataString): - '''expected dataString is "tx ty"''' + """expected dataString is 'tx ty'""" tx, ty = map(float(dataString.split(' '))) self.B0 = tx self.B1 = ty @@ -536,46 +760,81 @@ def _process_dataString(self, dataString): @staticmethod def fit(src, dst): - '''function to fit this transform given the corresponding sets of points src & dst - - Args: - src (numpy.array): a Nx2 matrix of source points - dst (numpy.array): a Nx2 matrix of destination points - Returns: - numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 - ''' + """function to fit Translation transform given + the corresponding sets of points src & dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ t = dst.mean(axis=0) - src.mean(axis=0) T = np.eye(3) T[:2, 2] = t return T def estimate(self, src, dst, return_params=True): - '''method for setting this transformation with the best fit given the corresponding points src,dst - - Args: - src (numpy.array): a Nx2 matrix of source points - dst (numpy.array): a Nx2 matrix of destination points - return_params (boolean): whether to return the parameter matrix - Returns: - numpy.array: a 2x3 matrix of parameters for this matrix, laid out (x,y) x (x,y,offset) + """method for setting this transformation with the best fit + given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) (or None if return_params=False) - ''' + """ self.M = self.fit(src, dst) if return_params: return self.M class RigidModel(AffineModel): - '''model for storing Rigid only transformations + """model for fitting Rigid only transformations (rotation+translation) or (determinate=1, orthonormal eigenvectors) - - For now only default (identity) or json based initialization is supported - - Args: - json (dict): json compatible representation of this transform - ''' + implemented as an :class:`AffineModel` + + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ className = 'mpicbg.trakem2.transform.RigidModel2D' def __init__(self, *args, **kwargs): @@ -584,7 +843,7 @@ def __init__(self, *args, **kwargs): # 'RigidModel not implemented. please use Affine') def _process_dataString(self, dataString): - '''expected datastring is "theta tx ty"''' + """expected datastring is 'theta tx ty'""" theta, tx, ty = map(float(dataString.split(' '))) self.M00 = np.cos(theta) self.M01 = -np.sin(theta) @@ -596,16 +855,25 @@ def _process_dataString(self, dataString): @staticmethod def fit(src, dst, rigid=True, **kwargs): - '''function to fit this transform given the corresponding sets of points src & dst + """function to fit this transform given the corresponding + sets of points src & dst Umeyama estimation of similarity transformation - Args: - src (numpy.array): a Nx2 matrix of source points - dst (numpy.array): a Nx2 matrix of destination points - rigid (boolean): whether to constrain this transform to be rigid - Returns: - numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 - ''' + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ # TODO shape assertion num, dim = src.shape src_cld = src - src.mean(axis=0) @@ -643,29 +911,59 @@ def fit(src, dst, rigid=True, **kwargs): return T def estimate(self, A, B, return_params=True, **kwargs): - '''method for setting this transformation with the best fit given the corresponding points src,dst - - Args: - A (numpy.array): a Nx2 matrix of source points - B (numpy.array): a Nx2 matrix of destination points - return_params (boolean): whether to return the parameter matrix - Returns: - numpy.array: a 2x3 matrix of parameters for this matrix, laid out (x,y) x (x,y,offset) + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) (or None if return_params=False) - ''' + """ self.M = self.fit(A, B, **kwargs) if return_params: return self.M class SimilarityModel(RigidModel): - '''class for reprsenting Similarity transformations + """class for fitting Similarity transformations (translation+rotation+scaling) - or + or (orthogonal eigen vectors with equal eigenvalues) - - ''' + implemented as an :class:`AffineModel` + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ className = 'mpicbg.trakem2.transform.SimilarityModel2D' def __init__(self, *args, **kwargs): @@ -674,7 +972,7 @@ def __init__(self, *args, **kwargs): # 'SimilarityModel not implemented. please use Affine') def _process_dataString(self, dataString): - '''expected datastring is "s theta tx ty"''' + """expected datastring is 's theta tx ty'""" s, theta, tx, ty = map(float(dataString.split(' '))) self.M00 = s * np.cos(theta) self.M01 = -s * np.sin(theta) @@ -686,50 +984,70 @@ def _process_dataString(self, dataString): @staticmethod def fit(src, dst, rigid=False, **kwargs): - '''function to fit this transform given the corresponding sets of points src & dst + """function to fit this transform given the corresponding + sets of points src & dst Umeyama estimation of similarity transformation - Args: - src (numpy.array): a Nx2 matrix of source points - dst (numpy.array): a Nx2 matrix of destination points - rigid (boolean): whether to constrain this transform to be rigid - Returns: - numpy.array: a 6x1 matrix with the best fit parameters ordered M00,M01,M10,M11,B0,B1 - ''' + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ return RigidModel.fit(src, dst, rigid=rigid) - # def estimate(self, src, dst, **kwargs): - # super(SimilarityModel, self).estimate(src, dst, rigid=False, **kwargs) - class Polynomial2DTransform(Transform): - ''' - Polynomial2DTransform implemented as in skimage - Polynomial2DTransform(dataString=None, src=None, dst=None, order=2, - force_polynomial=True, params=None, identity=False, - json=None) - This provides 5 different ways to initialize the transform which are - mutually exclusive and applied in the order specified here. - - 1)json2)dataString,3)identity,4)params,5)(src,dst) - - Args: - json (dict): dictonary representation of the Polynomial2DTransform generally used by TransformList - dataString (dict): dataString representation of transform from mpicpg - identity (boolean): whether to make this transform the identity - params (numpy.array): 2xK matrix of polynomial coefficents up to order K - src (numpy.array): Nx2 array of source points to use for fitting (used with dst) - dst (numpy.array): Nx2 array of destination points to use for fitting (used with src) - order (int): degree of polynomial to store - force_polynomial (boolean): whether to force this representation to be stored with degree order - TODO: - fall back to Affine Model in special cases - ''' + """Polynomial2DTransform implemented as in skimage + + Attributes + ---------- + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + + """ className = 'mpicbg.trakem2.transform.PolynomialTransform2D' def __init__(self, dataString=None, src=None, dst=None, order=2, force_polynomial=True, params=None, identity=False, json=None, **kwargs): + """Initialize Polynomial2DTransform + This provides 5 different ways to initialize the transform which are + mutually exclusive and applied in the order specified here. + 1)json2)dataString,3)identity,4)params,5)(src,dst) + + Parameters + ---------- + json : dict + dictionary representation of the Polynomial2DTransform + generally used by TransformList + dataString : str + dataString representation of transform from mpicpg + identity : bool + whether to make this transform the identity + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + src : numpy.array + Nx2 array of source points to use for fitting (used with dst) + dst : numpy.array + Nx2 array of destination points to use for fitting (used with src) + order : int + degree of polynomial to store + force_polynomial : bool + whether to force this representation to return a Polynomial + regardless of degree (not implemented) + + + """ if json is not None: self.from_dict(json) else: @@ -750,33 +1068,41 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, @property def is_affine(self): - '''(boolean) TODO allow default to Affine''' + """(boolean) TODO allow default to Affine""" return False # return self.order @property def order(self): - '''(int) order of polynomial''' + """(int) order of polynomial""" no_coeffs = len(self.params.ravel()) return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) @property def dataString(self): - '''dataString of polynomial''' + """dataString of polynomial""" return Polynomial2DTransform._dataStringfromParams(self.params) @staticmethod def fit(src, dst, order=2): - '''function to fit this transform given the corresponding sets of points src & dst + """function to fit this transform given the corresponding sets + of points src & dst polynomial fit - Args: - src (numpy.array): a Nx2 matrix of source points - dst (numpy.array): a Nx2 matrix of destination points - order (boolean): order of polynomial to fit - Returns: - numpy.array: a (2,(order+1)*(order+2)/2) matrix with the best fit parameters - ''' + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : bool + order of polynomial to fit + + Returns + ------- + numpy.array + a [2,(order+1)*(order+2)/2] array with the best fit parameters + """ xs = src[:, 0] ys = src[:, 1] xd = dst[:, 0] @@ -812,34 +1138,57 @@ def fit(src, dst, order=2): def estimate(self, src, dst, order=2, test_coords=True, max_tries=100, return_params=True, **kwargs): - '''method for setting this transformation with the best fit given the corresponding points src,dst - - Args: - src (numpy.array): a Nx2 matrix of source points - dst (numpy.array): a Nx2 matrix of destination points - order (int): order of polynomial to fit - test_coords (bool): whether to test model after fitting to make sure it is good (see fitgood) - max_tries (int): how many times to attempt to fit the model (see fitgood) - return_params (bool): whether to return the parameter matrix - Returns: - numpy.array: a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : int + order of polynomial to fit + test_coords : bool + whether to test model after fitting to + make sure it is good (see fitgood) + max_tries : int + how many times to attempt to fit the model (see fitgood) + return_params : bool + whether to return the parameter matrix + **kwargs + dictionary of keyword arguments including those + that can be passed to fitgood + + Returns + ------- + numpy.array + a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix (or None if return_params=False) - ''' + """ def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): - '''check if model produces a 'good' result - True if all but rtol src points - are transformed to within atol of dst points - by model described by params, otherwise False - - Args: - src (numpy.array): a Nx2 matrix of source points - dst (numpy.array): a Nx2 matrix of destination points - params (numpy.array): a Kx2 matrix of parameters - atol (float): tolerance of how close is close enough - rtol (int): tolerance of how many outliers is acceptable - Result: - bool: whether the goodness condition is met - ''' + """check if model produces a 'good' result + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + params : numpy.array + a Kx2 matrix of parameters + atol : float + absolute tolerance as in numpy.allclose for + transformed sample points + rtol : float + relative tolerance as in numpy.allclose for + transformed sample points + + Returns + ------- + bool + whether the goodness condition is met + """ result = Polynomial2DTransform(params=params).tform(src) t = np.allclose( result, dst, @@ -868,39 +1217,48 @@ def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): @staticmethod def _dataStringfromParams(params=None): - '''method for producing a dataString from the parameters''' + """method for producing a dataString from the parameters""" return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') for i in params.flatten()]).replace('e', 'E') def _process_dataString(self, datastring): - ''' - generate datastring and param attributes from datastring - ''' + """generate datastring and param attributes from datastring""" dsList = datastring.split(' ') self.params = Polynomial2DTransform._format_raveled_params(dsList) @staticmethod def _format_raveled_params(raveled_params): - '''method to reshape linear parameters into parameter matrix + """method to reshape linear parameters into parameter matrix - Args: - ravled_params (numpy.array): an K long vector of parameters - Returns: - numpy.array: a (2,K/2) matrix of parameters, with first row for x and 2nd row for y - ''' + Parameters + ---------- + raveled_params : numpy.array + an K long vector of parameters + + Returns + ------- + numpy.array + a (2,K/2) matrix of parameters, with + first row for x and 2nd row for y + """ return np.array( [[float(d) for d in raveled_params[:len(raveled_params) / 2]], [float(d) for d in raveled_params[len(raveled_params) / 2:]]]) def tform(self, points): - '''transform a set of points through this transformation - - Args: - points (numpy.array): a Nx2 array of x,y points - Returns: - numpy.array: a Nx2 array of x,y points after transformation - ''' + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ dst = np.zeros(points.shape) x = points[:, 0] y = points[:, 1] @@ -915,25 +1273,41 @@ def tform(self, points): return dst def coefficients(self, order=None): - ''' - determine number of coefficient terms in transform for a given order + """determine number of coefficient terms in transform for a given order - Args: - order (int or None): order of polynomial, defaults to self.order - Returns: - int: number of coefficient terms expected in transform - ''' + Parameters + ---------- + order : int, optional + order of polynomial, defaults to self.order + + Returns + ------- + int + number of coefficient terms expected in transform + + """ if order is None: order = self.order return (order + 1) * (order + 2) def asorder(self, order): - '''return polynomial transform appoximation of this transformation with a lower order - - Args: - order (int): desired order (must have order> current order) - Returns: - Transform: transform of lower order + '''return polynomial transform appoximation of this + transformation with a lower order + + Parameters + ---------- + order :int + desired order (must have order> current order) + + Returns + ------- + :class:`Polynomial2DTransform` + transform of lower order + + Raises + ------ + ConversionError + if target order < input order ''' if self.order > order: raise ConversionError( @@ -946,13 +1320,23 @@ def asorder(self, order): @staticmethod def fromAffine(aff): - ''' return a polynomial transformation equavalent to a given Affine - - Args: - aff (AffineModel): transform to become equivalent to - Returns: - Polynomial2DTransform: transform equal in effect to aff - ''' + """return a polynomial transformation equavalent to a given Affine + + Parameters + ---------- + aff : AffineModel + transform to become equivalent to + + Returns + ------- + Polynomial2DTransform + Order 1 transform equal in effect to aff + + Raises + ------ + ConversionError + if input model is not AffineModel + """ if not isinstance(aff, AffineModel): raise ConversionError('attempting to convert a nonaffine model!') return Polynomial2DTransform(order=1, params=np.array([ @@ -961,15 +1345,21 @@ def fromAffine(aff): def estimate_dstpts(transformlist, src=None): - ''' - estimate destination points for list of transforms - - Args: - transformlist (list[Transform]):transforms that have a tform method implemented - src (numpy.array): a Nx2 array of source points - Returns: - numpy.array:Nx2 array of destination points - ''' + """estimate destination points for list of transforms. Recurses + through lists. + + Parameters + ---------- + transformlist : :obj:list of :obj:Transform + transforms that have a tform method implemented + src : numpy.array + a Nx2 array of source points + + Returns + ------- + numpy.array + Nx2 array of destination points + """ dstpts = src for tform in transformlist: if isinstance(tform, list): @@ -980,20 +1370,27 @@ def estimate_dstpts(transformlist, src=None): def estimate_transformsum(transformlist, src=None, order=2): - ''' - pseudo-composition of transforms in list of transforms using source point - transformation and a single estimation. Will produce an Affine Model if all - input transforms are Affine, otherwise will produce a Polynomial of specified order - - Args: - transformlist (list[Transform]): list of transform objects that implement tform - src (numpy.array): Nx2 array of source points for estimation - order (int): order of Polynomial output if transformlist - inputs are non-Affine - Returns: - Polynomial2DTransform: best estimate of transformlist in a single transform of this order - ''' + """pseudo-composition of transforms in list of transforms + using source point transformation and a single estimation. + Will produce an Affine Model if all input transforms are Affine, + otherwise will produce a Polynomial of specified order + + Parameters + ---------- + transformlist : :obj:`list` of :obj:`Transform` + list of transform objects that implement tform + src : numpy.array + Nx2 array of source points for estimation + order : int + order of Polynomial output if transformlist + inputs are non-Affine + Returns + ------- + :class:`AffineModel` or :class:`Polynomial2DTransform` + best estimate of transformlist in a single transform of this order + """ def flatten(l): + """generator-iterator to flatten deep lists of lists""" for i in l: if (isinstance(i, Iterable) and not isinstance(i, basestring)): diff --git a/renderapi/utils.py b/renderapi/utils.py index 0523953a..afa2a795 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -11,7 +11,7 @@ class NullHandler(logging.Handler): - '''handler to avoid logging errors for, e.g., missing logger setup''' + """handler to avoid logging errors for, e.g., missing logger setup""" def emit(self, record): pass @@ -21,23 +21,26 @@ def emit(self, record): class RenderEncoder(json.JSONEncoder): - ''' - json Encoder in the following hierarchy for serialization: + """json Encoder in the following hierarchy for serialization: obj.to_dict() dict(obj) JsonEncoder.default(obj) obj.__dict__ - ''' + """ def default(self, obj): - '''default encoder for that handles Render objects - - Args: - obj (obj): any object that implements to_dict, dict(obj), - JsonEncoder.default(obj), or __dict__ (in order) - Returns: - (dict): json encodable dictionary - - ''' + """default encoder for that handles Render objects + + Parameters + ---------- + obj : obj + any object that implements to_dict, dict(obj), + JsonEncoder.default(obj), or __dict__ (in order) + Returns + ------- + dict or list + json encodable datatype + + """ to_dict = getattr(obj, "to_dict", None) if callable(to_dict): return obj.to_dict() @@ -58,19 +61,29 @@ def default(self, obj): def post_json(session, request_url, d, params=None): - ''' - POST requests with RenderError handling - - Args: - session (requests.session.Session): requests session - request_url (str): url - d (dict): data payload (will be json dumps-ed) - params (dict): requests parameters - Returns: - requests.response: server response - Raises: - RenderError: if cannot post - ''' + """POST requests with RenderError handling + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + d : dict + data payload (will be json dumps-ed) + params : dict + requests parameters + + Returns + ------- + requests.response: server response + + Raises + ------ + RenderError + if cannot post + """ + headers = {"content-type": "application/json"} if d is not None: payload = json.dumps(d) @@ -90,19 +103,29 @@ def post_json(session, request_url, d, params=None): def put_json(session, request_url, d, params=None): - ''' - PUT requests with RenderError handling - - Args: - session (requests.session.Session): requests session - request_url (str): url - d (dict): data payload (will be json dumps-ed) - params (dict): requests parameters - Returns: - requests.response: server response - Raises: - RenderError: if cannot post - ''' + """PUT requests with RenderError handling + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + d : dict + data payload (will be json dumps-ed) + params : dict + requests parameters + + Returns + ------- + requests.response + server response + + Raises + ------ + RenderError + if cannot post + """ headers = {"content-type": "application/json"} if d is not None: @@ -123,43 +146,61 @@ def put_json(session, request_url, d, params=None): def renderdumps(obj, *args, **kwargs): - '''json.dumps using the RenderEncode - - Args: - obj (obj): object to dumps - *args: json.dumps args - **kwargs: json.dumps kwargs - Returns: - str: serialized object - ''' + """json.dumps using the RenderEncode + + Parameters + ---------- + obj : obj + object to dumps + *args + json.dumps args + **kwargs + json.dumps kwargs + + Returns + ------- + str + serialized object + """ cls_ = kwargs.pop('cls', RenderEncoder) return json.dumps(obj, *args, cls=cls_, **kwargs) def renderdump(obj, *args, **kwargs): - '''json.dump using the RenderEncoder - - Args: - obj (obj): object to dumps - *args: json.dump args - **kwargs: json.dump kwargs - ''' + """json.dump using the RenderEncoder + + Parameters + ---------- + obj : obj + object to dumps + *args + json.dump args + **kwargs + json.dump kwargs + """ cls_ = kwargs.pop('cls', RenderEncoder) return json.dump(obj, *args, cls=cls_, **kwargs) def renderdump_temp(obj, *args, **kwargs): - '''json.dump into a temporary file + """json.dump into a temporary file renderdump_temp(obj), obj will be dumped through renderdump into a temporary file - Args: - obj (obj): object to dump - *args: json.dump args - **kwargs: json.dump kwargs - Returns: - str: path to location where temporary file was dumped - ''' + Parameters + ---------- + obj : obj + object to dump + *args + json.dump args + **kwargs + json.dump kwargs + + Returns + ------- + str + path to location where temporary file was dumped + """ with tempfile.NamedTemporaryFile( suffix=".json", mode='w', delete=False) as tf: @@ -169,14 +210,19 @@ def renderdump_temp(obj, *args, **kwargs): def jbool(val): - '''return string representing java string values of py booleans - - Args: - val (bool): boolean to encode - Returns: - str: 'true' or 'false' - - ''' + """return string representing java string values of py booleans + + Parameters + ---------- + val : bool + boolean to encode + + Returns + ------- + str + 'true' or 'false' + + """ if not isinstance(val, bool): logger.warning('Evaluating javastring of non-boolean {} {}'.format( type(val), val)) @@ -184,39 +230,55 @@ def jbool(val): def stripLogger(logger_tostrip): # pragma: no cover - ''' - remove all handlers from a logger -- useful for redefining - Args: - logger_tostrip (logging.Logger): logging logger to strip - ''' + """remove all handlers from a logger -- useful for redefining + + Parameters + ---------- + logger_tostrip : :class:`logging.Logger` + logging logger to strip + """ if logger_tostrip.handlers: for handler in logger_tostrip.handlers: logger_tostrip.removeHandler(handler) def defaultifNone(val, default=None): - '''simple default handler - - Args: - val (obj): value to fill in default - default (obj): default value - Returns: - obj: val if val is not None, else default - ''' + """simple default handler + + Parameters + ---------- + val : obj + value to fill in default + default : obj + default value + + Returns + ------- + obj + val if val is not None, else default + """ return val if val is not None else default def fitargspec(f, oldargs, oldkwargs): - ''' fit function argspec given input args tuple and kwargs dict - - Args: - f (func): function to inspect - oldargs (tuple): arguments passed to func - oldkwards (dict): keyword args passed to func + """fit function argspec given input args tuple and kwargs dict + + Parameters + ---------- + f : func + function to inspect + oldargs : tuple + arguments passed to func + oldkwards : dict + keyword args passed to func + Returns - new_args: args with values filled in according to f spec - new_kwargs: kwargs with values filled in according to f spec - ''' + ------- + new_args + args with values filled in according to f spec + new_kwargs + kwargs with values filled in according to f spec + """ try: args, varargs, keywords, defaults = inspect.getargspec(f) num_expected_args = len(args) - len(defaults) From 5c75dfcb391df22b706388542f08bc8c985657ef Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 28 Jul 2017 11:01:06 -0700 Subject: [PATCH 418/766] added coordinate documentation --- renderapi/coordinate.py | 342 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 337 insertions(+), 5 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index a3037d96..079bc24e 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -22,7 +22,40 @@ def world_to_local_coordinates(stack, z, x, y, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''''' + """ + maps an world x,y,z coordinate in stack to a local coordinate + Parameters + ---------- + stack : str + render stack to map coordinates through + z : float + z coordinate to map + x : float + x coordinate to map + y : float + y coordinate to map + session : + (Default value = requests.session() + render : renderapi.render.Render + render connect object + Returns + ------- + json + list of dictionaries of local coordinates following this pattern + :: + + [ + { + "tileId": "string", + "visible": false, + "local": [ + [0,0], + [1,0]... + ], + "error": "string" + } + ] + """ request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) @@ -39,7 +72,39 @@ def local_to_world_coordinates(stack, tileId, x, y, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - '''''' + """ + + Parameters + ---------- + stack : str + render stack to map coordinates through + z : float + z coordinate to map + x : float + x coordinate to map + y : float + y coordinate to map + session : + (Default value = requests.session() + render : renderapi.render.Render + render connect object + Returns + ------- + dict + dictionary of world coordinates following this pattern + :: + + { + "tileId": "string", + "visible": false, + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + } + + """ request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) @@ -58,7 +123,52 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, session=requests.session(), render=None, **kwargs): - '''''' + """ + + Parameters + ---------- + stack : str + stack to map coordinates + d : list[dict] + list of dictionary of world coordinates to map following this schema + :: + + [ { + "tileId": "string", + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + z : float + z coordinate to map + execute_local : boolean + (Default value = False) + session : requests.session.Session + (Default value = requests.session() + render : renderapi.render.Render + render connect object + Returns + ------- + list[list[dict]] + list of lists of dictionaries containing local positions + that overlap with this point, following.. + :: + + + [[ { + "tileId": "string", + "visible": True,False, + "local": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + ] + + """ if (execute_local is True): raise NotImplementedError("local execution not yet implemented") @@ -75,6 +185,47 @@ def local_to_world_coordinates_batch(stack, d, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): + """ + + Parameters + ---------- + stack : str + + d : list[dict] + list of dictionary of local coordinates to map + :: + [ { + "tileId": "string", + "local": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + + z : float + z coordinate to map from + session : + (Default value = requests.session() + render : renderapi.render.Render + render connect object + + Returns + ------- + list[dict] + list of dictionaries containing world coordinates + + :: + + [ { + "tileId": "string", + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + }] + """ request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%s/local-to-world-coordinates" % (str(z)) @@ -90,6 +241,37 @@ def local_to_world_coordinates_batch(stack, d, z, host=None, def package_point_match_data_into_json(dataarray, tileId, local_or_world='local'): + """ + Convert a set of points defined by a numpy array and a tileId to a json + for use in the renderapi + + Parameters + ---------- + dataarray : numpy.array + a Nx2 array of points + + tileId : str + a tileId to package them into + local_or_world : + whether this should be represented as a local or world coordinate + (Default value = 'local') + + Returns + ------- + dict + dictionary representation of those points and tileId + following + + :: + { + "tileId": "string", + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + } + """ dlist = [] for i in range(dataarray.shape[0]): d = {} @@ -100,6 +282,23 @@ def package_point_match_data_into_json(dataarray, tileId, def unpackage_world_to_local_point_match_from_json(json_answer, tileId): + """ + Converts a dictionary answer from a world>local coordinates call from a dictionary + to numpy array format + + Parameters + ---------- + json_answer : list[dict] + json reponse from a world>local call (N long) + + tileId : str + tileId to extract, usually the world tileId passed in + + Returns + ------- + numpy.array + Nx2 array of local points + """ answer = np.zeros((len(json_answer), 2)) for i, local_answer in enumerate(json_answer): coord = next(ans for ans in local_answer if ans['tileId'] == tileId) @@ -143,6 +342,19 @@ def unpackage_world_to_local_point_match_from_json(json_answer, tileId): def unpackage_local_to_world_point_match_from_json(json_answer): + """ + converts a local>world call json response into a numpy array + + Parameters + ---------- + json_answer : list[dict] + response from a local>world call (N long) + + Returns + ------- + numpy.array + Nx2 numpy array of coordinates + """ logger.debug("json_answer_length %d" % len(json_answer)) answer = np.zeros((len(json_answer), 2)) for i, coord in enumerate(json_answer): @@ -159,7 +371,33 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, client_script=None, doClientSide=False, number_of_threads=20, session=requests.session(), **kwargs): - '''''' + """ + map world to local coordinates using numpy array + + Parameters + ---------- + stack : str + render stack to map + dataarray : numpy.array + Nx2 numpy array of points to world points to map + tileId : str + tileId to map from and to + z : float + z coordinate to map + render : renderapi.render.RenderClient + render connect object + doClientSide : boolean + (Default value = False) + number_of_threads : int + (Default value = 20) + session : + (Default value = requests.session() + + Returns + ------- + numpy.array: + Nx2 numpy array of points in local coordinates + """ jsondata = package_point_match_data_into_json(dataarray, tileId, 'world') if doClientSide: json_answer = world_to_local_coordinates_clientside( @@ -214,7 +452,34 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, client_script=None, doClientSide=False, number_of_threads=20, session=requests.session(), **kwargs): - '''''' + """ + + Parameters + ---------- + stack : str + render stack to map + dataarray : numpy.array + Nx2 array of points in local coordinates + tileId : str + tile to map points from + z : float + z position to map + render : renderapi.render.Render + render connect object + doClientSide : boolean + (Default value = False) + number_of_threads : int + (Default value = 20) + session : requests.session.Session + (Default value = requests.session() + render : renderapi.render.RenderClient + + Returns + ------- + numpy.array + Nx2 numpy array in world coordinates + + """ jsondata = package_point_match_data_into_json(dataarray, tileId, 'local') if doClientSide: json_answer = local_to_world_coordinates_clientside( @@ -232,6 +497,31 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, project, client_script, isLocalToWorld=False, store_injson=False, store_outjson=False, number_of_threads=20, memGB='1G'): + """ + + Parameters + ---------- + stack : str + stack to map + jsondata : dict + json dictionary to map following the pattern of local>world or world>local + z : float + z position to map + isLocalToWorld : boolean + (Default value = False) + store_injson : boolean + (Default value = False) + store_outjson : boolean + (Default value = False) + number_of_threads : int + (Default value = 20) + render : renderapi.render.RenderClient + render connect object + Returns + ------- + json + json data as would be returned by client calls of local>world or world>local + """ # write point match json to temp file on disk with tempfile.NamedTemporaryFile( prefix='render_coordinates_in_', suffix='.json', @@ -270,6 +560,28 @@ def world_to_local_coordinates_clientside(stack, jsondata, z, number_of_threads=20, session=requests.session(), render=None, **kwargs): + """ + + Parameters + ---------- + stack : str + render stack to map + jsondata : dict + world coordinates in dictionary format + z : float + z coordinate to map + number_of_threads : int + number of threads to use when doing parallelization + (Default value = 20) + session : requests.session.Session + (Default value = requests.session() + render : renderapi.render.RenderClient + render connect object + Returns + ------- + json + local coordinates in dictionary format + """ return map_coordinates_clientside(stack, jsondata, z, host=host, port=port, owner=owner, @@ -286,6 +598,26 @@ def local_to_world_coordinates_clientside(stack, jsondata, z, number_of_threads=20, session=requests.session(), render=None, **kwargs): + """ + + Parameters + ---------- + stack : str + render stack to map + jsondata : list[dict] + local coordinates in dictionary format + z : float + z position to map + number_of_threads : int + (Default value = 20) + session : request.session.Session + (Default value = requests.session() + + Returns + ------- + dict + world coordinates in dictionary format + """ return map_coordinates_clientside(stack, jsondata, z, host=host, port=port, owner=owner, project=project, From 86fbc281f1b6c1e17fe1d7913518c1c46b04a740 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 28 Jul 2017 14:26:39 -0700 Subject: [PATCH 419/766] trying to upgrade setuptools in docker --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index b42d493d..74e03ec5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,7 @@ RUN pip install multiprocess==0.70.5 RUN pip install pathos==0.2.0 RUN pip install pillow RUN conda install scipy +RUN pip install setuptools --upgrade #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python From ba9a5910be6d414fe3493c1539061b0181ca3ff0 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 28 Jul 2017 14:26:55 -0700 Subject: [PATCH 420/766] relaxing testing requirements --- test/requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/requirements.txt b/test/requirements.txt index 8a5a5ccc..18650732 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,10 +1,10 @@ -coverage==4.1 -mock==2.0.0 -pep8==1.7.0 -pytest==3.0.5 -pytest-cov==2.2.1 -pytest-pep8==1.0.6 -pytest-xdist==1.14 +coverage>=4.1 +mock>=2.0.0 +pep8>=1.7.0 +pytest>=3.0.5 +pytest-cov>=2.2.1 +pytest-pep8>=1.0.6 +pytest-xdist>=1.14 flake8>=3.0.4 pylint>=1.5.4 scipy \ No newline at end of file From b78085e619cec619f79c01c74d1c023106500832 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 28 Jul 2017 14:29:05 -0700 Subject: [PATCH 421/766] addressing setuptools upgrade test --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 74e03ec5..05eacf88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,7 +69,7 @@ RUN pip install multiprocess==0.70.5 RUN pip install pathos==0.2.0 RUN pip install pillow RUN conda install scipy -RUN pip install setuptools --upgrade +RUN pip install setuptools --upgrade --disable-pip-version-check #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python From 438b824ea8a857a55d9632f4e7705e12c2fda3f5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 29 Jul 2017 12:28:29 -0700 Subject: [PATCH 422/766] coordinate: slight modifications for formatting --- renderapi/coordinate.py | 107 +++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 56 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 079bc24e..e16f2819 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -22,8 +22,7 @@ def world_to_local_coordinates(stack, z, x, y, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - """ - maps an world x,y,z coordinate in stack to a local coordinate + """maps an world x,y,z coordinate in stack to a local coordinate Parameters ---------- stack : str @@ -34,8 +33,8 @@ def world_to_local_coordinates(stack, z, x, y, host=None, x coordinate to map y : float y coordinate to map - session : - (Default value = requests.session() + session : requests.session.Session + session object used in request render : renderapi.render.Render render connect object Returns @@ -43,7 +42,7 @@ def world_to_local_coordinates(stack, z, x, y, host=None, json list of dictionaries of local coordinates following this pattern :: - + [ { "tileId": "string", @@ -72,7 +71,7 @@ def local_to_world_coordinates(stack, tileId, x, y, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - """ + """convert coordinate from local to world with webservice request Parameters ---------- @@ -84,16 +83,17 @@ def local_to_world_coordinates(stack, tileId, x, y, x coordinate to map y : float y coordinate to map - session : - (Default value = requests.session() + session : requests.session.Session + session object used in request render : renderapi.render.Render render connect object + Returns ------- dict dictionary of world coordinates following this pattern :: - + { "tileId": "string", "visible": false, @@ -103,7 +103,7 @@ def local_to_world_coordinates(stack, tileId, x, y, ], "error": "string" } - + """ request_url = format_preamble( host, port, owner, project, stack) + \ @@ -123,7 +123,7 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, session=requests.session(), render=None, **kwargs): - """ + """convert coordinate parameters from world to local Parameters ---------- @@ -132,7 +132,7 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, d : list[dict] list of dictionary of world coordinates to map following this schema :: - + [ { "tileId": "string", "world": [ @@ -146,17 +146,19 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, execute_local : boolean (Default value = False) session : requests.session.Session - (Default value = requests.session() + session object used in request render : renderapi.render.Render render connect object + Returns ------- list[list[dict]] list of lists of dictionaries containing local positions - that overlap with this point, following.. + that overlap with this point, (one world point may map + to multiple local points) following.. :: - + [[ { "tileId": "string", "visible": True,False, @@ -185,12 +187,12 @@ def local_to_world_coordinates_batch(stack, d, z, host=None, port=None, owner=None, project=None, session=requests.session(), render=None, **kwargs): - """ + """convert coordinate parameters from local to world Parameters ---------- stack : str - + d : list[dict] list of dictionary of local coordinates to map :: @@ -202,7 +204,7 @@ def local_to_world_coordinates_batch(stack, d, z, host=None, ], "error": "string" }] - + z : float z coordinate to map from session : @@ -241,14 +243,13 @@ def local_to_world_coordinates_batch(stack, d, z, host=None, def package_point_match_data_into_json(dataarray, tileId, local_or_world='local'): - """ - Convert a set of points defined by a numpy array and a tileId to a json + """Convert a set of points defined by a numpy array and a tileId to a json for use in the renderapi Parameters ---------- dataarray : numpy.array - a Nx2 array of points + a Nx2 array of points tileId : str a tileId to package them into @@ -282,18 +283,17 @@ def package_point_match_data_into_json(dataarray, tileId, def unpackage_world_to_local_point_match_from_json(json_answer, tileId): - """ - Converts a dictionary answer from a world>local coordinates call from a dictionary - to numpy array format + """Converts a dictionary answer from a world>local + coordinates call from a dictionary to numpy array format Parameters ---------- json_answer : list[dict] json reponse from a world>local call (N long) - + tileId : str tileId to extract, usually the world tileId passed in - + Returns ------- numpy.array @@ -342,9 +342,8 @@ def unpackage_world_to_local_point_match_from_json(json_answer, tileId): def unpackage_local_to_world_point_match_from_json(json_answer): - """ - converts a local>world call json response into a numpy array - + """converts a local>world call json response into a numpy array + Parameters ---------- json_answer : list[dict] @@ -371,27 +370,26 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, client_script=None, doClientSide=False, number_of_threads=20, session=requests.session(), **kwargs): - """ - map world to local coordinates using numpy array + """map world to local coordinates using numpy array Parameters ---------- stack : str - render stack to map + render stack to map dataarray : numpy.array Nx2 numpy array of points to world points to map tileId : str tileId to map from and to z : float z coordinate to map - render : renderapi.render.RenderClient + render : renderapi.render.Render render connect object doClientSide : boolean (Default value = False) number_of_threads : int (Default value = 20) - session : - (Default value = requests.session() + session : requests.session.Session + session object used in request Returns ------- @@ -452,7 +450,7 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, client_script=None, doClientSide=False, number_of_threads=20, session=requests.session(), **kwargs): - """ + """map local to world coordinates using numpy array Parameters ---------- @@ -471,8 +469,9 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, number_of_threads : int (Default value = 20) session : requests.session.Session - (Default value = requests.session() - render : renderapi.render.RenderClient + session object used in request + render : renderapi.render.Render + render connect object Returns ------- @@ -497,7 +496,7 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, project, client_script, isLocalToWorld=False, store_injson=False, store_outjson=False, number_of_threads=20, memGB='1G'): - """ + """map coordinates using the java client library Parameters ---------- @@ -508,19 +507,21 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, z : float z position to map isLocalToWorld : boolean - (Default value = False) + whether transform is local to world (False implies world to local) store_injson : boolean - (Default value = False) + whether to store input json file (created with tempfile) store_outjson : boolean - (Default value = False) + whether to store output json file (created with tempfile) number_of_threads : int - (Default value = 20) - render : renderapi.render.RenderClient + threads to execute clientside computation + render : renderapi.render.Render render connect object + Returns ------- json - json data as would be returned by client calls of local>world or world>local + json data as would be returned by client calls + of local>world or world>local """ # write point match json to temp file on disk with tempfile.NamedTemporaryFile( @@ -558,9 +559,8 @@ def world_to_local_coordinates_clientside(stack, jsondata, z, host=None, port=None, owner=None, project=None, client_script=None, number_of_threads=20, - session=requests.session(), render=None, **kwargs): - """ + """map_coordinates_clientside for mapping world to local Parameters ---------- @@ -572,11 +572,9 @@ def world_to_local_coordinates_clientside(stack, jsondata, z, z coordinate to map number_of_threads : int number of threads to use when doing parallelization - (Default value = 20) - session : requests.session.Session - (Default value = requests.session() - render : renderapi.render.RenderClient + render : renderapi.render.Render render connect object + Returns ------- json @@ -596,9 +594,8 @@ def local_to_world_coordinates_clientside(stack, jsondata, z, host=None, port=None, owner=None, project=None, client_script=None, number_of_threads=20, - session=requests.session(), render=None, **kwargs): - """ + """map_coordinates_clientside for mapping local to world Parameters ---------- @@ -609,9 +606,7 @@ def local_to_world_coordinates_clientside(stack, jsondata, z, z : float z position to map number_of_threads : int - (Default value = 20) - session : request.session.Session - (Default value = requests.session() + threads for java client script to use during mapping Returns ------- From cc1d6ae0dbb01a8a884b51900bb1e98265ee0358 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 30 Jul 2017 12:36:06 -0700 Subject: [PATCH 423/766] removing flake8 from setup_requires --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index af99635d..47eb4e10 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,5 @@ def run_tests(self): url='https://github.com/fcollman/render-python', packages=['renderapi'], install_requires=required, - setup_requires=['flake8'], tests_require=test_required, cmdclass={'test': PyTest},) From bb78eb5e858ed8b78e15dac044551c5e23517b65 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 30 Jul 2017 13:06:42 -0700 Subject: [PATCH 424/766] added badge and fixed merge --- README.md | 2 ++ docs/index.rst | 10 ---------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 193399bf..e472f78f 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Documentation Status](https://readthedocs.org/projects/render-python/badge/)](http://render-python.readthedocs.io/en/latest/) + # render-python This is a python API client to interact with [render-webservices](https://github.com/saalfeldlab/render) and facilitate python scripting of [tilespec](https://github.com/saalfeldlab/render/blob/master/docs/src/site/markdown/data-model.md) creation diff --git a/docs/index.rst b/docs/index.rst index 2b36c381..656d6784 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,15 +12,6 @@ Welcome to render-python's documentation! :maxdepth: 2 :caption: Contents: -<<<<<<< HEAD -api ---- -.. toctree:: - :maxdepth: 2 - - api/renderapi - -======= API --- @@ -31,7 +22,6 @@ API guide/pointmatchassumptions api/renderapi ->>>>>>> 438b824ea8a857a55d9632f4e7705e12c2fda3f5 Indices and tables ================== From cc907e7b59940234d7fae2a107cc6862f41ad42e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 30 Jul 2017 13:13:50 -0700 Subject: [PATCH 425/766] added documentation to trigger doc rebuild --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index e472f78f..d491a34c 100644 --- a/README.md +++ b/README.md @@ -18,3 +18,7 @@ Render connection objects created with `renderapi.connect()` can default to envi ``` [Usage examples for a development Array Tomography workflow](https://github.com/fcollman/render-python-apps) are available. + +# Documentation + +http://render-python.readthedocs.io/en/latest/ From b7a9d7aeeb6c24be75b2ac25a52fad2c5cd5ce14 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 30 Jul 2017 14:02:08 -0700 Subject: [PATCH 426/766] made base dockerfile --- Dockerfile | 74 +------------------------------------------- Dockerfile.base | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 73 deletions(-) create mode 100644 Dockerfile.base diff --git a/Dockerfile b/Dockerfile index 05eacf88..15253cf2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,78 +1,6 @@ -FROM fcollman/render:latest +FROM fcollman/render-python-base MAINTAINER Forrest Collman (forrest.collman@gmail.com) -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 - -RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ - libglib2.0-0 libxext6 libsm6 libxrender1 \ - git mercurial subversion - -RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ - wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.11-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm ~/miniconda.sh - -RUN apt-get install -y curl grep sed dpkg && \ - TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ - curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ - dpkg -i tini.deb && \ - rm tini.deb && \ - apt-get clean - -# RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ -# libglib2.0-0 libxext6 libsm6 libxrender1 \ -# git mercurial subversion - -# RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ -# wget --quiet https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh -O ~/anaconda.sh && \ -# /bin/bash ~/anaconda.sh -b -p /opt/conda && \ -# rm ~/anaconda.sh - -# RUN apt-get install -y curl grep sed dpkg && \ -# TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ -# curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ -# dpkg -i tini.deb && \ -# rm tini.deb && \ -# apt-get clean - -ENV PATH /opt/conda/bin:$PATH - -#install java -# auto validate license -#RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections -#RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list -#RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list -#RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 -#RUN apt-get update -#RUN apt-get install oracle-java8-installer -y -#ENV JAVA_HOME /www/var/render/deploy/jdkd - -#install pathos,multiprocess with gcc -RUN apt-get update -RUN apt-get install gcc build-essential libgeos-dev imagemagick -y -RUN apt-get install python-setuptools python-dev -y -#RUN apt-get install libblas-dev liblapack-dev -y -RUN apt-get clean - - -#install components for common render-python apps -#jupyter notebook, shapely with geos -# RUN /opt/conda/bin/conda install jupyter -y - - -#RUN easy_install pip -#RUN pip install -U pip setuptools -RUN pip install shapely==1.6b2 -RUN pip install opencv-python -RUN pip install dill==0.2.6 -RUN pip install multiprocess==0.70.5 -RUN pip install pathos==0.2.0 -RUN pip install pillow -RUN conda install scipy -RUN pip install setuptools --upgrade --disable-pip-version-check -#install render python using pip from github -#RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python - RUN mkdir -p /usr/local/render-python COPY . /usr/local/render-python WORKDIR /usr/local/render-python diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 00000000..fb9a4391 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,81 @@ + +FROM fcollman/render:latest +MAINTAINER Forrest Collman (forrest.collman@gmail.com) + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ + libglib2.0-0 libxext6 libsm6 libxrender1 \ + git mercurial subversion + +RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ + wget --quiet https://repo.continuum.io/miniconda/Miniconda2-4.3.11-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh + +RUN apt-get install -y curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb && \ + apt-get clean + +# RUN apt-get update --fix-missing && apt-get install -y wget bzip2 ca-certificates \ +# libglib2.0-0 libxext6 libsm6 libxrender1 \ +# git mercurial subversion + +# RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ +# wget --quiet https://repo.continuum.io/archive/Anaconda2-4.3.1-Linux-x86_64.sh -O ~/anaconda.sh && \ +# /bin/bash ~/anaconda.sh -b -p /opt/conda && \ +# rm ~/anaconda.sh + +# RUN apt-get install -y curl grep sed dpkg && \ +# TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && \ +# curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ +# dpkg -i tini.deb && \ +# rm tini.deb && \ +# apt-get clean + +ENV PATH /opt/conda/bin:$PATH + +#install java +# auto validate license +#RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections +#RUN echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list +#RUN echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list +#RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 +#RUN apt-get update +#RUN apt-get install oracle-java8-installer -y +#ENV JAVA_HOME /www/var/render/deploy/jdkd + +#install pathos,multiprocess with gcc +RUN apt-get update +RUN apt-get install gcc build-essential libgeos-dev imagemagick -y +RUN apt-get install python-setuptools python-dev -y +#RUN apt-get install libblas-dev liblapack-dev -y +RUN apt-get clean + + +#install components for common render-python apps +#jupyter notebook, shapely with geos +# RUN /opt/conda/bin/conda install jupyter -y + + +#RUN easy_install pip +#RUN pip install -U pip setuptools +RUN pip install shapely==1.6b2 +RUN pip install opencv-python +RUN pip install dill==0.2.6 +RUN pip install multiprocess==0.70.5 +RUN pip install pathos==0.2.0 +RUN pip install pillow +RUN conda install scipy +RUN pip install setuptools --upgrade --disable-pip-version-check +RUN mkdir -p /usr/local/render-python +COPY . /usr/local/render-python +WORKDIR /usr/local/render-python +RUN pip install -r requirements.txt +RUN pip install -r ./test/requirements.txt +#install render python using pip from github +#RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python + From 13c7deb30d131445404379ac7072a848df520d59 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 30 Jul 2017 17:11:43 -0700 Subject: [PATCH 427/766] fixed some formatting --- renderapi/client.py | 2 +- renderapi/image.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 7c0332c8..338162be 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -302,7 +302,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, stack to which tilespecs will be added tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` list of tilespecs to import - sharedTransforms :obj:`list` of :obj:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional + sharedTransforms : obj:`list` of :obj:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional list of shared referenced transforms to be ingested poolsize : int degree of parallelism to use diff --git a/renderapi/image.py b/renderapi/image.py index 5a967ef2..2dd8d859 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -217,9 +217,11 @@ def get_section_image(stack, z, scale=1.0, filter=False, Examples -------- - >>>import renderapi - >>>render = renderapi.render.connect('server',8080,'me','myproject') - >>>img = render.run(renderapi.stack.get_section_image,'mystack',3.0) + :: + + >>> import renderapi + >>> render = renderapi.render.connect('server',8080,'me','myproject') + >>> img = render.run(renderapi.stack.get_section_image,'mystack',3.0) """ try: From d3f68155502eb2cca91965818f0145d59ad9fc4e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 30 Jul 2017 17:17:26 -0700 Subject: [PATCH 428/766] fixed minor formatting issues --- renderapi/coordinate.py | 20 ++++++++++---------- renderapi/transform.py | 17 +++++++++-------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index e16f2819..1eeaa3a1 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -133,14 +133,14 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, list of dictionary of world coordinates to map following this schema :: - [ { - "tileId": "string", - "world": [ - [0,0], - [1,0]... - ], - "error": "string" - }] + [ { + "tileId": "string", + "world": [ + [0,0], + [1,0]... + ], + "error": "string" + }] z : float z coordinate to map execute_local : boolean @@ -158,7 +158,6 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, to multiple local points) following.. :: - [[ { "tileId": "string", "visible": True,False, @@ -262,8 +261,9 @@ def package_point_match_data_into_json(dataarray, tileId, dict dictionary representation of those points and tileId following - + :: + { "tileId": "string", "world": [ diff --git a/renderapi/transform.py b/renderapi/transform.py index 2e493965..4f1919c0 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -567,14 +567,15 @@ def estimate(self, A, B, return_params=True, **kwargs): def concatenate(self, model): """concatenate a model to this model -- ported from trakEM2 below: - - final double a00 = m00 * model.m00 + m01 * model.m10; - final double a01 = m00 * model.m01 + m01 * model.m11; - final double a02 = m00 * model.m02 + m01 * model.m12 + m02; - - final double a10 = m10 * model.m00 + m11 * model.m10; - final double a11 = m10 * model.m01 + m11 * model.m11; - final double a12 = m10 * model.m02 + m11 * model.m12 + m12; + :: + + final double a00 = m00 * model.m00 + m01 * model.m10; + final double a01 = m00 * model.m01 + m01 * model.m11; + final double a02 = m00 * model.m02 + m01 * model.m12 + m02; + + final double a10 = m10 * model.m00 + m11 * model.m10; + final double a11 = m10 * model.m01 + m11 * model.m11; + final double a12 = m10 * model.m02 + m11 * model.m12 + m12; Parameters ---------- From e19cc6f3dc1159870473f7c7ffb246a574a9bf29 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 2 Aug 2017 12:22:36 -0700 Subject: [PATCH 429/766] compressing base image pip installs --- Dockerfile.base | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index fb9a4391..a081a614 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -63,12 +63,7 @@ RUN apt-get clean #RUN easy_install pip #RUN pip install -U pip setuptools -RUN pip install shapely==1.6b2 -RUN pip install opencv-python -RUN pip install dill==0.2.6 -RUN pip install multiprocess==0.70.5 -RUN pip install pathos==0.2.0 -RUN pip install pillow +RUN pip install pillow xmltodict pathos==0.2.0 multiprocess==0.70.5 dill==0.2.6 opencv-python shapely==1.6b2 RUN conda install scipy RUN pip install setuptools --upgrade --disable-pip-version-check RUN mkdir -p /usr/local/render-python From 7d4e10092e3277ff4b44e7390ce243071ba2c49e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 14:38:08 -0700 Subject: [PATCH 430/766] added Russel's NonLinearTransform transform class --- renderapi/transform.py | 95 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/renderapi/transform.py b/renderapi/transform.py index 4f1919c0..cbc70be4 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1370,6 +1370,101 @@ def estimate_dstpts(transformlist, src=None): return dstpts +class NonLinearTransform(renderapi.transform.Transform): + """ + render-python class that implements the mpicbg.trakem2.transform.nonLinearTransform class + + Parameters + ---------- + dataString:str or None + data string of transformation + json:dict or NOne + json compatible dictionary representation of the transformation + + Returns + ------- + :class:`NonLinearTransform` + a transform instance + + + """ + + className = 'mpicbg.trakem2.transform.nonLinearTransform' + + def __init__(self, dataString=None, json=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + + def _process_dataString(self, dataString): + # trailing whitespace in string.... for some reason + fields = dataString.split(" ")[:-1] + self.dimension = int(fields[0]) + self.length = int(fields[1]) + + # last 2 fields are width and height + self.width = int(fields[-2]) + self.height = int(fields[-1]) + + # beta is defined in alternating column order + self.beta = numpy.column_stack([ + numpy.array(fields[2:2+self.length*2][::2], dtype='float32'), + numpy.array(fields[3:2+self.length*2][::2], dtype='float32')]) + + # normMean and normVar follow + self.normMean = numpy.array(fields[2+self.length*2:2+self.length*3], dtype='float32') + self.normVar = numpy.array(fields[2+self.length*3:2+self.length*4], dtype='float32') + + def tform(self, src): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + x = src[:, 0] + y = src[:, 1] + + expanded = numpy.zeros([len(x), self.length]) + pidx = 0 + for i in range(1, self.dimension + 1): + for j in range(i, -1, -1): + expanded[:, pidx] = ( + numpy.power(x, j) * numpy.power(y, i - j)) + pidx += 1 + + + expanded[:, :-1] = (expanded[:, :-1] - self.normMean[:-1]) / self.normVar[:-1] + expanded[:, -1] = 100 + + dst = numpy.zeros(src.shape) + for i in range(0, expanded.shape[1]): + dst[:, 0] = dst[:, 0] + expanded[:, i] * self.beta[i][0] + dst[:, 1] = dst[:, 1] + expanded[:, i] * self.beta[i][1] + return dst + + @property + def dataString(self): + shapestring = '{} {}'.format(self.dimension, self.length) + betastring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.beta.ravel()]).replace('e', 'E') + meanstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normMean]).replace('e', 'E') + varstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normVar]).replace('e', 'E') + dimstring = '{} {}'.format(self.height, self.width) + return '{} {} {} {} {} '.format( + shapestring, betastring, meanstring, varstring, dimstring) + + def estimate_transformsum(transformlist, src=None, order=2): """pseudo-composition of transforms in list of transforms using source point transformation and a single estimation. From 069ea4a2155f3fd2805655a0a26f55e522ea6bfc Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 12 Aug 2017 14:59:13 -0700 Subject: [PATCH 431/766] updating docker setup --- .dockerignore | 4 ++++ Dockerfile | 2 +- Dockerfile.base | 1 + docker_base_setup.sh | 5 +++++ docker_setup.sh | 10 +++++----- 5 files changed, 16 insertions(+), 6 deletions(-) create mode 100755 docker_base_setup.sh diff --git a/.dockerignore b/.dockerignore index 6b8710a7..f6154099 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,5 @@ .git +docker_setup.sh +Dockerfile.base +Dockerfile +docker_base_setup.sh diff --git a/Dockerfile b/Dockerfile index 15253cf2..17033990 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM fcollman/render-python-base +FROM fcollman/render-python-base:latest MAINTAINER Forrest Collman (forrest.collman@gmail.com) RUN mkdir -p /usr/local/render-python diff --git a/Dockerfile.base b/Dockerfile.base index a081a614..318e4db1 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -71,6 +71,7 @@ COPY . /usr/local/render-python WORKDIR /usr/local/render-python RUN pip install -r requirements.txt RUN pip install -r ./test/requirements.txt +RUN pip install matplotlib #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python diff --git a/docker_base_setup.sh b/docker_base_setup.sh new file mode 100755 index 00000000..19599ecf --- /dev/null +++ b/docker_base_setup.sh @@ -0,0 +1,5 @@ +docker pull atbigdawg:5000/fcollman/render:latest +docker tag atbigdawg:5000/fcollman/render:latest fcollman/render:latest +docker build -t fcollman/render-python-base:latest -f Dockerfile.base . +docker tag fcollman/render-python-base:latest atbigdawg:5000/fcollman/render-python-base:latest +docker push atbigdawg:5000/fcollman/render-python-base:latest diff --git a/docker_setup.sh b/docker_setup.sh index 0d59f751..2796200a 100755 --- a/docker_setup.sh +++ b/docker_setup.sh @@ -1,5 +1,5 @@ -docker pull atbigdawg:5000/fcollman/render -docker tag atbigdawg:5000/fcollman/render fcollman/render -docker build -t fcollman/render-python . -docker tag fcollman/render-python atbigdawg:5000/fcollman/render-python -docker push atbigdawg:5000/fcollman/render-python +docker pull atbigdawg:5000/fcollman/render-python-base:latest +docker tag atbigdawg:5000/fcollman/render-python-base:latest fcollman/render-python-base:latest +docker build -t fcollman/render-python:latest . +docker tag fcollman/render-python:latest atbigdawg:5000/fcollman/render-python:latest +docker push atbigdawg:5000/fcollman/render-python:latest From 42ad54d1762789904830fc6982e6980e963d057e Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 14:59:42 -0700 Subject: [PATCH 432/766] working on testing --- renderapi/transform.py | 21 ++++++----- test/test_transform.py | 86 +++++++++++++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 31 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index cbc70be4..24234277 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1370,7 +1370,7 @@ def estimate_dstpts(transformlist, src=None): return dstpts -class NonLinearTransform(renderapi.transform.Transform): +class NonLinearTransform(Transform): """ render-python class that implements the mpicbg.trakem2.transform.nonLinearTransform class @@ -1391,12 +1391,13 @@ class NonLinearTransform(renderapi.transform.Transform): className = 'mpicbg.trakem2.transform.nonLinearTransform' - def __init__(self, dataString=None, json=None): + def __init__(self, dataString=None, json=None,transformId=None): if json is not None: self.from_dict(json) else: if dataString is not None: self._process_dataString(dataString) + self.transformId = transformId def _process_dataString(self, dataString): # trailing whitespace in string.... for some reason @@ -1409,13 +1410,13 @@ def _process_dataString(self, dataString): self.height = int(fields[-1]) # beta is defined in alternating column order - self.beta = numpy.column_stack([ - numpy.array(fields[2:2+self.length*2][::2], dtype='float32'), - numpy.array(fields[3:2+self.length*2][::2], dtype='float32')]) + self.beta = np.column_stack([ + np.array(fields[2:2+self.length*2][::2], dtype='float32'), + np.array(fields[3:2+self.length*2][::2], dtype='float32')]) # normMean and normVar follow - self.normMean = numpy.array(fields[2+self.length*2:2+self.length*3], dtype='float32') - self.normVar = numpy.array(fields[2+self.length*3:2+self.length*4], dtype='float32') + self.normMean = np.array(fields[2+self.length*2:2+self.length*3], dtype='float32') + self.normVar = np.array(fields[2+self.length*3:2+self.length*4], dtype='float32') def tform(self, src): """transform a set of points through this transformation @@ -1433,19 +1434,19 @@ def tform(self, src): x = src[:, 0] y = src[:, 1] - expanded = numpy.zeros([len(x), self.length]) + expanded = np.zeros([len(x), self.length]) pidx = 0 for i in range(1, self.dimension + 1): for j in range(i, -1, -1): expanded[:, pidx] = ( - numpy.power(x, j) * numpy.power(y, i - j)) + np.power(x, j) * np.power(y, i - j)) pidx += 1 expanded[:, :-1] = (expanded[:, :-1] - self.normMean[:-1]) / self.normVar[:-1] expanded[:, -1] = 100 - dst = numpy.zeros(src.shape) + dst = np.zeros(src.shape) for i in range(0, expanded.shape[1]): dst[:, 0] = dst[:, 0] + expanded[:, i] * self.beta[i][0] dst[:, 1] = dst[:, 1] + expanded[:, i] * self.beta[i][1] diff --git a/test/test_transform.py b/test/test_transform.py index 99a0a288..eab994fe 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -12,9 +12,9 @@ def test_affine_rot_90(): points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float) am.estimate(points_in, points_out) - assert(np.abs(am.scale[0]-1.0) < .00001) - assert(np.abs(am.scale[1]-1.0) < .00001) - assert(np.abs(am.rotation + np.pi/2) < .000001) + assert(np.abs(am.scale[0] - 1.0) < .00001) + assert(np.abs(am.scale[1] - 1.0) < .00001) + assert(np.abs(am.rotation + np.pi / 2) < .000001) assert(np.abs(am.translation[0]) < .000001) assert(np.abs(am.translation[1]) < .000001) assert(np.abs(am.shear) < .000001) @@ -23,14 +23,14 @@ def test_affine_rot_90(): new_points = am.tform(points) old_points = am.inverse_tform(new_points) - assert(np.sum(np.abs(points-old_points)) < (.0001*len(points.ravel()))) + assert(np.sum(np.abs(points - old_points)) < (.0001 * len(points.ravel()))) am_inverse = renderapi.transform.AffineModel() am_inverse.estimate(points_out, points_in) identity = am.concatenate(am_inverse) - assert(np.abs(identity.scale[0]-1.0) < .00001) - assert(np.abs(identity.scale[1]-1.0) < .00001) + assert(np.abs(identity.scale[0] - 1.0) < .00001) + assert(np.abs(identity.scale[1] - 1.0) < .00001) assert(np.abs(identity.rotation) < .000001) assert(np.abs(identity.translation[0]) < .000001) assert(np.abs(identity.translation[1]) < .000001) @@ -39,20 +39,20 @@ def test_affine_rot_90(): def test_affine_random(): - am = renderapi.transform.AffineModel(M00=.9, - M10=-0.2, - M01=0.3, - M11=.85, - B0=245.3, - B1=-234.1) + am = renderapi.transform.AffineModel(M00=.9, + M10=-0.2, + M01=0.3, + M11=.85, + B0=245.3, + B1=-234.1) - points_in = np.random.rand(10, 2) - points_out = am.tform(points_in) + points_in = np.random.rand(10, 2) + points_out = am.tform(points_in) - am_fit = renderapi.transform.AffineModel() - am_fit.estimate(points_in, points_out) + am_fit = renderapi.transform.AffineModel() + am_fit.estimate(points_in, points_out) - assert(np.sum(np.abs(am.M.ravel()-am_fit.M.ravel())) < (.001*6)) + assert(np.sum(np.abs(am.M.ravel() - am_fit.M.ravel())) < (.001 * 6)) def test_invert_Affine(): @@ -231,7 +231,7 @@ def test_reference_transform(): ts = renderapi.tilespec.TileSpec(json=j) ref_ts = [tform for tform in ts.tforms if isinstance( - tform, renderapi.transform.ReferenceTransform)][0] + tform, renderapi.transform.ReferenceTransform)][0] ref_args = renderapi.transform.ReferenceTransform(ref_ts.refId) ref_dd = renderapi.transform.ReferenceTransform(json=ref_ts.to_dict()) @@ -308,17 +308,17 @@ def estimate_homography_transform( scale = np.random.rand() random_scale = (renderapi.transform.AffineModel( M00=scale, M11=scale) - if do_scale else renderapi.transform.AffineModel()) + if do_scale else renderapi.transform.AffineModel()) random_translate = (renderapi.transform.AffineModel( B0=np.random.rand(), B1=np.random.rand()) - if do_translate else renderapi.transform.AffineModel()) + if do_translate else renderapi.transform.AffineModel()) theta = np.random.rand() * 2 * np.pi random_rotate = (renderapi.transform.AffineModel( M00=np.cos(theta), M01=-np.sin(theta), M10=np.sin(theta), M11=np.cos(theta)) - if do_rotate else renderapi.transform.AffineModel()) + if do_rotate else renderapi.transform.AffineModel()) target_tform = random_translate.concatenate( random_rotate.concatenate(random_scale)) @@ -354,3 +354,47 @@ def test_estimate_translation_transform(): estimate_homography_transform( do_scale=False, do_rotate=False, transformclass=renderapi.transform.TranslationModel) + + +def test_non_linear_transform(): + lens_tform = renderapi.transform.NonLinearTransform(dataString=\ + ("6 28 535.9261337198133 -0.07156495284083819 " + "1.5429761616475304 482.5391851962856 -1.3964665434772456 " + "-6.442573400985017 -2.9852709783360574 4.123111145658342 " + "-11.540309826482599 5.014980246618293 13.616409494909078 " + "6.8938038217739255 -2.580163142352532 -5.848702515778433 " + "11.520166623181623 -0.26414636893418164 17.725654626324285 " + "-3.6910106603259862 -5.747999298096374 6.850067634843171 " + "16.792611164648633 7.81339608279572 -1.6748576247746882 " + "7.294185157592906 -8.213479728551924 -5.181177837004834 " + "-2.363460931085683 2.9307897710181123 -19.60977878575318 " + "-5.626357722731385 -3.8857454141075323 -15.21120604177655 " + "-21.23967741658828 -7.433492911261084 5.986054967656173 " + "-0.2472470831304605 -4.252500955671167 6.285397956497022 " + "-10.087631670227893 -2.936382384586807 13.850132289055523 " + "-1.6230114467674603 -6.137660325940608 12.358797331540874 " + "15.361852639613971 -7.395509537102164 -3.4680250777444144 " + "11.543534617091353 2.1153010560743724 -6.1108581863128535 " + "2.4532196427825284 -1.6047644982741716 5.190795110418094 " + "0.40139303958456196 11.96302944378589 12.660577579061767 " + "1196.3041637890071 1266.0596612286872 1719846.5445062846 " + "1485783.6202875 1838252.7229766874 2.714668570418218E9 " + "2.106666890772152E9 2.1421214796751451E9 2.8349699962925887E9 " + "4.528468416208078E12 3.2949497103892886E12 3.0161488655484473E12 " + "3.2930355498080786E12 4.550261299826646E12 7.824367950003595E15 " + "5.464560367850104E15 4.686502506566912E15 4.616510470870998E15 " + "5.276444951800235E15 7.530553575860649E15 1.3844135530148071E19 " + "9.408014352057864E18 7.7275759384866017E18 7.1369460697416591E18 " + "7.373674200122966E18 8.7221483757320745E18 1.277368891891578E19 " + "100.0 537.4423453941299 485.2431449404924 1253541.1543424227 " + "899439.7598253534 1082433.2436210443 2.5451681382231455E9 " + "1.814150348079708E9 1.6692374367919033E9 2.1769137889095693E9 " + "4.992425732284977E12 3.5258872445513765E12 3.074020072484778E12 " + "3.1163128442286265E12 4.266956980967905E12 9.700243843041522E15 " + "6.781968704541316E15 5.745611297869626E15 5.443791159762913E15 " + "5.856309886603154E15 8.280136088463922E15 1.8821205899653665E19 " + "1.3042150458228838E19 1.0829624014456685E19 9.899910593314652E18 " + "9.885988268659214E18 1.1051253229808925E19 1.6002173610912907E19 " + "0.0 2048 2048 "), + transformId="testing") + From fdb3606083730ac2f76c9ac59da65c9e08257266 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 16:33:50 -0700 Subject: [PATCH 433/766] refactored and added a test --- renderapi/transform.py | 60 ++++++++++++++++++++++++++---------------- test/test_transform.py | 10 +++++++ 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 24234277..2bdfbad9 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1391,6 +1391,9 @@ class NonLinearTransform(Transform): className = 'mpicbg.trakem2.transform.nonLinearTransform' + + + def __init__(self, dataString=None, json=None,transformId=None): if json is not None: self.from_dict(json) @@ -1402,6 +1405,7 @@ def __init__(self, dataString=None, json=None,transformId=None): def _process_dataString(self, dataString): # trailing whitespace in string.... for some reason fields = dataString.split(" ")[:-1] + self.dimension = int(fields[0]) self.length = int(fields[1]) @@ -1409,28 +1413,18 @@ def _process_dataString(self, dataString): self.width = int(fields[-2]) self.height = int(fields[-1]) - # beta is defined in alternating column order - self.beta = np.column_stack([ - np.array(fields[2:2+self.length*2][::2], dtype='float32'), - np.array(fields[3:2+self.length*2][::2], dtype='float32')]) + data = np.array(fields[2:-2],dtype='float32') + self.beta=data[0:2*self.length].reshape(self.length,2) + assert(self.beta.shape[0]==self.length) # normMean and normVar follow - self.normMean = np.array(fields[2+self.length*2:2+self.length*3], dtype='float32') - self.normVar = np.array(fields[2+self.length*3:2+self.length*4], dtype='float32') + self.normMean = data[self.length*2:self.length*3] + self.normVar = data[self.length*3:self.length*4] + assert(self.normMean.shape[0]==self.length) + assert(self.normVar.shape[0]==self.length) - def tform(self, src): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points + def kernelExpand(self,src): - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ x = src[:, 0] y = src[:, 1] @@ -1444,13 +1438,33 @@ def tform(self, src): expanded[:, :-1] = (expanded[:, :-1] - self.normMean[:-1]) / self.normVar[:-1] - expanded[:, -1] = 100 + expanded[:, -1] = 100.0 + return expanded + + def tform(self, src): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + + # final double[] featureVector = kernelExpand(position); + # return multiply(beta, featureVector); + nsrc = np.array(src,dtype=np.float64) + featureVector = self.kernelExpand(nsrc) dst = np.zeros(src.shape) - for i in range(0, expanded.shape[1]): - dst[:, 0] = dst[:, 0] + expanded[:, i] * self.beta[i][0] - dst[:, 1] = dst[:, 1] + expanded[:, i] * self.beta[i][1] - return dst + for i in range(0, featureVector.shape[1]): + dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i,0]) + dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i,1]) + return np.array(dst,dtype=src.dtype) @property def dataString(self): diff --git a/test/test_transform.py b/test/test_transform.py index eab994fe..2435f6cd 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -398,3 +398,13 @@ def test_non_linear_transform(): "0.0 2048 2048 "), transformId="testing") + ticks = np.arange(0,2048,64,np.float) + xx,yy = np.meshgrid(ticks,ticks) + x = np.ravel(xx).T + y = np.ravel(yy).T + xy = np.vstack((x,y)).T + + xyp=lens_tform.tform(xy) + dv = xyp-xy + mean_disp= np.mean(np.sqrt(np.sum(dv**2,axis=1))) + assert((mean_disp-0.7570507)<.01) \ No newline at end of file From 573c5221697f41d600fa9003a48eba1009e2671f Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 16:36:04 -0700 Subject: [PATCH 434/766] added documentation --- renderapi/transform.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/renderapi/transform.py b/renderapi/transform.py index 2bdfbad9..271d48f1 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1424,7 +1424,18 @@ def _process_dataString(self, dataString): assert(self.normVar.shape[0]==self.length) def kernelExpand(self,src): + """creates an expanded representation of the x,y src points in a polynomial form + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a (N x self.length) array of coefficents + """ x = src[:, 0] y = src[:, 1] From 4d133440b118a894c41003571276c7eaddbcefbe Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 17:00:08 -0700 Subject: [PATCH 435/766] adding loader handler for non linear --- renderapi/transform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 271d48f1..7f9b25a9 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -161,7 +161,8 @@ def load_leaf_json(d): lambda x: Polynomial2DTransform(json=x), TranslationModel.className: lambda x: TranslationModel(json=x), RigidModel.className: lambda x: RigidModel(json=x), - SimilarityModel.className: lambda x: SimilarityModel(json=x)} + SimilarityModel.className: lambda x: SimilarityModel(json=x), + NonLinearTransform.className lambda x: NonLinearTransform(json=x)} tform_type = d.get('type', 'leaf') if tform_type != 'leaf': From 97cde432df32e03a96a1b8f82c751ec25bdbdfa0 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 17:04:49 -0700 Subject: [PATCH 436/766] changed to raising RenderErrors when coefficents are wrong --- renderapi/transform.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 7f9b25a9..a713ffb2 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1416,13 +1416,16 @@ def _process_dataString(self, dataString): data = np.array(fields[2:-2],dtype='float32') self.beta=data[0:2*self.length].reshape(self.length,2) - assert(self.beta.shape[0]==self.length) + if not (self.beta.shape[0]==self.length): + raise RenderError("not correct number of coefficents") # normMean and normVar follow self.normMean = data[self.length*2:self.length*3] self.normVar = data[self.length*3:self.length*4] - assert(self.normMean.shape[0]==self.length) - assert(self.normVar.shape[0]==self.length) + if not (self.normMean.shape[0]==self.length): + raise RenderError("incorrect number of normMean coefficents") + if not (self.normVar.shape[0]==self.length): + raise RenderError("incorrect number of normVar coefficents") def kernelExpand(self,src): """creates an expanded representation of the x,y src points in a polynomial form From ffe0ca1be736e115a6e8bead5a4c3022acf5b5bb Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 17:10:30 -0700 Subject: [PATCH 437/766] fixed bug in serialization --- renderapi/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index a713ffb2..63e375b0 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -359,7 +359,7 @@ def to_dict(self): d['className'] = self.className d['dataString'] = self.dataString if self.transformId is not None: - d['transformId'] = self.transformId + d['id'] = self.transformId return d def from_dict(self, d): @@ -371,7 +371,7 @@ def from_dict(self, d): json compatible representation of this transform """ self.className = d['className'] - self.transformId = d.get('transformId', None) + self.transformId = d.get('id', None) self._process_dataString(d['dataString']) def _process_dataString(self, datastring): From 149d845a8dd93bb3ec5b585f37ee4a5f79df7340 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 17:12:04 -0700 Subject: [PATCH 438/766] typo --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 63e375b0..4208f561 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -162,7 +162,7 @@ def load_leaf_json(d): TranslationModel.className: lambda x: TranslationModel(json=x), RigidModel.className: lambda x: RigidModel(json=x), SimilarityModel.className: lambda x: SimilarityModel(json=x), - NonLinearTransform.className lambda x: NonLinearTransform(json=x)} + NonLinearTransform.className: lambda x: NonLinearTransform(json=x)} tform_type = d.get('type', 'leaf') if tform_type != 'leaf': From bbd8ba8c6d5c3b70b161ba81d6b0a22c7be70bea Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 17:30:01 -0700 Subject: [PATCH 439/766] added Lenscorrection class --- renderapi/transform.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 4208f561..9428196c 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -162,7 +162,8 @@ def load_leaf_json(d): TranslationModel.className: lambda x: TranslationModel(json=x), RigidModel.className: lambda x: RigidModel(json=x), SimilarityModel.className: lambda x: SimilarityModel(json=x), - NonLinearTransform.className: lambda x: NonLinearTransform(json=x)} + NonLinearTransform.className: lambda x: NonLinearTransform(json=x), + LensCorrection.className: lambda x: LensCorrection(json=x)} tform_type = d.get('type', 'leaf') if tform_type != 'leaf': @@ -1371,6 +1372,8 @@ def estimate_dstpts(transformlist, src=None): return dstpts + + class NonLinearTransform(Transform): """ render-python class that implements the mpicbg.trakem2.transform.nonLinearTransform class @@ -1393,8 +1396,6 @@ class NonLinearTransform(Transform): className = 'mpicbg.trakem2.transform.nonLinearTransform' - - def __init__(self, dataString=None, json=None,transformId=None): if json is not None: self.from_dict(json) @@ -1495,6 +1496,14 @@ def dataString(self): shapestring, betastring, meanstring, varstring, dimstring) +class LensCorrection(Transform): + """ + a placeholder for the lenscorrection transform, same as NonLinearTransform + for now + """ + className = 'lenscorrection.NonLinearTransform' + + def estimate_transformsum(transformlist, src=None, order=2): """pseudo-composition of transforms in list of transforms using source point transformation and a single estimation. From 2052308a740bc5e93480c81d56a24fed7efda2f3 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 17:35:25 -0700 Subject: [PATCH 440/766] wrong subclass --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 9428196c..5857a92c 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1496,7 +1496,7 @@ def dataString(self): shapestring, betastring, meanstring, varstring, dimstring) -class LensCorrection(Transform): +class LensCorrection(NonLinearTransform): """ a placeholder for the lenscorrection transform, same as NonLinearTransform for now From d5d77b291e8a18e521ff5734b65873f4420c08ef Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 18:13:52 -0700 Subject: [PATCH 441/766] changing className to reflect desired naming --- renderapi/transform.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 5857a92c..ad8b30ad 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -163,7 +163,8 @@ def load_leaf_json(d): RigidModel.className: lambda x: RigidModel(json=x), SimilarityModel.className: lambda x: SimilarityModel(json=x), NonLinearTransform.className: lambda x: NonLinearTransform(json=x), - LensCorrection.className: lambda x: LensCorrection(json=x)} + LensCorrection.className: lambda x: LensCorrection(json=x), + NonLinearCoordinateTransform.className: lambda x: NonLinearCoordinateTransform(json=x)} tform_type = d.get('type', 'leaf') if tform_type != 'leaf': @@ -1374,9 +1375,9 @@ def estimate_dstpts(transformlist, src=None): -class NonLinearTransform(Transform): +class NonLinearCoordinateTransform(Transform): """ - render-python class that implements the mpicbg.trakem2.transform.nonLinearTransform class + render-python class that implements the mpicbg.trakem2.transform.NonLinearCoordinateTransform class Parameters ---------- @@ -1393,8 +1394,7 @@ class NonLinearTransform(Transform): """ - className = 'mpicbg.trakem2.transform.nonLinearTransform' - + className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' def __init__(self, dataString=None, json=None,transformId=None): if json is not None: @@ -1495,8 +1495,10 @@ def dataString(self): return '{} {} {} {} {} '.format( shapestring, betastring, meanstring, varstring, dimstring) +class NonLinearTransform(NonLinearTransform): + classname = 'mpicbg.trakem2.transform.nonLinearTransform' -class LensCorrection(NonLinearTransform): +class LensCorrection(NonLinearCoordinateTransform): """ a placeholder for the lenscorrection transform, same as NonLinearTransform for now From 0f01c844ed1d6f95e7c04b4b641d400a6bd83f17 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 18:14:33 -0700 Subject: [PATCH 442/766] typo --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index ad8b30ad..b283bcb0 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1496,7 +1496,7 @@ def dataString(self): shapestring, betastring, meanstring, varstring, dimstring) class NonLinearTransform(NonLinearTransform): - classname = 'mpicbg.trakem2.transform.nonLinearTransform' + className = 'mpicbg.trakem2.transform.nonLinearTransform' class LensCorrection(NonLinearCoordinateTransform): """ From 716d7b8b1b4cba19ae3793638476afe1b9c959be Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 23:51:48 -0700 Subject: [PATCH 443/766] fixed bug so whitespace is optional or not --- renderapi/transform.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index b283bcb0..67332db9 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1405,12 +1405,14 @@ def __init__(self, dataString=None, json=None,transformId=None): self.transformId = transformId def _process_dataString(self, dataString): - # trailing whitespace in string.... for some reason - fields = dataString.split(" ")[:-1] + + fields = dataString.split(" ") self.dimension = int(fields[0]) self.length = int(fields[1]) + #cutoff whitespace if there + fields=fields[0:2+4*length+2] # last 2 fields are width and height self.width = int(fields[-2]) self.height = int(fields[-1]) From c0cf985b2f2c1710bd2738309e51f2cb926447f0 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 12 Aug 2017 23:54:31 -0700 Subject: [PATCH 444/766] typo --- renderapi/transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 67332db9..7abd6dc0 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1497,7 +1497,7 @@ def dataString(self): return '{} {} {} {} {} '.format( shapestring, betastring, meanstring, varstring, dimstring) -class NonLinearTransform(NonLinearTransform): +class NonLinearTransform(NonLinearCoordinateTransform): className = 'mpicbg.trakem2.transform.nonLinearTransform' class LensCorrection(NonLinearCoordinateTransform): From c5ac9e142e76d4ff04c99ad60ad148f26a1b9b4d Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sun, 13 Aug 2017 00:13:11 -0700 Subject: [PATCH 445/766] bug fix --- renderapi/transform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 7abd6dc0..cbd352c7 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1403,6 +1403,7 @@ def __init__(self, dataString=None, json=None,transformId=None): if dataString is not None: self._process_dataString(dataString) self.transformId = transformId + self.className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' def _process_dataString(self, dataString): @@ -1412,7 +1413,7 @@ def _process_dataString(self, dataString): self.length = int(fields[1]) #cutoff whitespace if there - fields=fields[0:2+4*length+2] + fields=fields[0:2+4*self.length+2] # last 2 fields are width and height self.width = int(fields[-2]) self.height = int(fields[-1]) From 70118be8d60aad3a7766d1d825c9ff1a4449ef96 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 15 Aug 2017 17:22:36 -0700 Subject: [PATCH 446/766] PEP8 and render formatting --- renderapi/render.py | 2 +- renderapi/transform.py | 84 ++++++++++++++++++++++-------------------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 11b7118a..ad08f8ef 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -1,11 +1,11 @@ #!/usr/bin/env python import logging import os -from functools import wraps import requests from .utils import defaultifNone, NullHandler, fitargspec from .errors import ClientScriptError, RenderError from decorator import decorator + logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) diff --git a/renderapi/transform.py b/renderapi/transform.py index cbd352c7..db623480 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -164,7 +164,8 @@ def load_leaf_json(d): SimilarityModel.className: lambda x: SimilarityModel(json=x), NonLinearTransform.className: lambda x: NonLinearTransform(json=x), LensCorrection.className: lambda x: LensCorrection(json=x), - NonLinearCoordinateTransform.className: lambda x: NonLinearCoordinateTransform(json=x)} + NonLinearCoordinateTransform.className: + lambda x: NonLinearCoordinateTransform(json=x)} tform_type = d.get('type', 'leaf') if tform_type != 'leaf': @@ -571,7 +572,7 @@ def estimate(self, A, B, return_params=True, **kwargs): def concatenate(self, model): """concatenate a model to this model -- ported from trakEM2 below: :: - + final double a00 = m00 * model.m00 + m01 * model.m10; final double a01 = m00 * model.m01 + m01 * model.m11; final double a02 = m00 * model.m02 + m01 * model.m12 + m02; @@ -752,8 +753,6 @@ class TranslationModel(AffineModel): def __init__(self, *args, **kwargs): super(TranslationModel, self).__init__(*args, **kwargs) - # raise NotImplementedError( - # 'TranslationModel not implemented. please use Affine') def _process_dataString(self, dataString): """expected dataString is 'tx ty'""" @@ -843,8 +842,6 @@ class RigidModel(AffineModel): def __init__(self, *args, **kwargs): super(RigidModel, self).__init__(*args, **kwargs) - # raise NotImplementedError( - # 'RigidModel not implemented. please use Affine') def _process_dataString(self, dataString): """expected datastring is 'theta tx ty'""" @@ -972,8 +969,6 @@ class SimilarityModel(RigidModel): def __init__(self, *args, **kwargs): super(SimilarityModel, self).__init__(*args, **kwargs) - # raise NotImplementedError( - # 'SimilarityModel not implemented. please use Affine') def _process_dataString(self, dataString): """expected datastring is 's theta tx ty'""" @@ -1373,17 +1368,16 @@ def estimate_dstpts(transformlist, src=None): return dstpts - - class NonLinearCoordinateTransform(Transform): """ - render-python class that implements the mpicbg.trakem2.transform.NonLinearCoordinateTransform class - + render-python class that implements the + mpicbg.trakem2.transform.NonLinearCoordinateTransform class + Parameters ---------- - dataString:str or None + dataString: str or None data string of transformation - json:dict or NOne + json: dict or None json compatible dictionary representation of the transformation Returns @@ -1396,7 +1390,7 @@ class NonLinearCoordinateTransform(Transform): className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' - def __init__(self, dataString=None, json=None,transformId=None): + def __init__(self, dataString=None, json=None, transformId=None): if json is not None: self.from_dict(json) else: @@ -1408,31 +1402,41 @@ def __init__(self, dataString=None, json=None,transformId=None): def _process_dataString(self, dataString): fields = dataString.split(" ") - + self.dimension = int(fields[0]) self.length = int(fields[1]) - #cutoff whitespace if there - fields=fields[0:2+4*self.length+2] + # cutoff whitespace if there + fields = fields[0:2+4*self.length+2] # last 2 fields are width and height self.width = int(fields[-2]) self.height = int(fields[-1]) - - data = np.array(fields[2:-2],dtype='float32') - self.beta=data[0:2*self.length].reshape(self.length,2) - if not (self.beta.shape[0]==self.length): + + data = np.array(fields[2:-2], dtype='float32') + try: + self.beta = data[0:2*self.length].reshape(self.length, 2) + except ValueError as e: + raise RenderError( + 'Incorrect number of coefficients in ' + 'NonLinearCoordinateTransform. msg: {}'.format(e)) + if not (self.beta.shape[0] == self.length): raise RenderError("not correct number of coefficents") # normMean and normVar follow self.normMean = data[self.length*2:self.length*3] self.normVar = data[self.length*3:self.length*4] - if not (self.normMean.shape[0]==self.length): - raise RenderError("incorrect number of normMean coefficents") - if not (self.normVar.shape[0]==self.length): - raise RenderError("incorrect number of normVar coefficents") - - def kernelExpand(self,src): - """creates an expanded representation of the x,y src points in a polynomial form + if not (self.normMean.shape[0] == self.length): + raise RenderError( + "incorrect number of normMean coefficents " + "{} != length {}".format(self.normMean.shape[0], self.length)) + if not (self.normVar.shape[0] == self.length): + raise RenderError( + "incorrect number of normVar coefficents " + "{} != {}".format(self.normVar.shape[0], self.length)) + + def kernelExpand(self, src): + """creates an expanded representation of the x,y + src points in a polynomial form Parameters ---------- @@ -1446,7 +1450,7 @@ def kernelExpand(self,src): """ x = src[:, 0] y = src[:, 1] - + expanded = np.zeros([len(x), self.length]) pidx = 0 for i in range(1, self.dimension + 1): @@ -1455,8 +1459,8 @@ def kernelExpand(self,src): np.power(x, j) * np.power(y, i - j)) pidx += 1 - - expanded[:, :-1] = (expanded[:, :-1] - self.normMean[:-1]) / self.normVar[:-1] + expanded[:, :-1] = ((expanded[:, :-1] - self.normMean[:-1]) / + self.normVar[:-1]) expanded[:, -1] = 100.0 return expanded @@ -1476,31 +1480,33 @@ def tform(self, src): # final double[] featureVector = kernelExpand(position); # return multiply(beta, featureVector); - nsrc = np.array(src,dtype=np.float64) + nsrc = np.array(src, dtype=np.float64) featureVector = self.kernelExpand(nsrc) dst = np.zeros(src.shape) for i in range(0, featureVector.shape[1]): - dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i,0]) - dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i,1]) - return np.array(dst,dtype=src.dtype) - + dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i, 0]) + dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i, 1]) + return np.array(dst, dtype=src.dtype) + @property def dataString(self): shapestring = '{} {}'.format(self.dimension, self.length) betastring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.beta.ravel()]).replace('e', 'E') + for i in self.beta.ravel()]).replace('e', 'E') meanstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.normMean]).replace('e', 'E') + for i in self.normMean]).replace('e', 'E') varstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') for i in self.normVar]).replace('e', 'E') dimstring = '{} {}'.format(self.height, self.width) return '{} {} {} {} {} '.format( shapestring, betastring, meanstring, varstring, dimstring) + class NonLinearTransform(NonLinearCoordinateTransform): className = 'mpicbg.trakem2.transform.nonLinearTransform' + class LensCorrection(NonLinearCoordinateTransform): """ a placeholder for the lenscorrection transform, same as NonLinearTransform From 3d70f0f92910942d1ae170baa25bc40a1ab96092 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 15 Aug 2017 18:13:19 -0700 Subject: [PATCH 447/766] test: tilespecs are more synthetic, contain multiple mipmaplevels and do not compare non-Affine datastring matching (change related to #21) --- test/test_files/tilespecs.json | 43 ++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/test/test_files/tilespecs.json b/test/test_files/tilespecs.json index bd6eaea7..f990337a 100644 --- a/test/test_files/tilespecs.json +++ b/test/test_files/tilespecs.json @@ -15,17 +15,19 @@ "maxIntensity": 65535, "mipmapLevels": { "0": { - "imageUrl": "file:/data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + "imageUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8.tif" + }, + "1": { + "imageUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8_mmL1.tif.tif", + "maskUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8_mmL1_mask.tif.tif" + }, + "2": { + "imageUrl": "file:///data/20160710195316413_243774_7R_SID_01_redo_0_11_7_3_15_8_mmL2.tif.tif" } }, "transforms": { "type": "list", "specList": [ - { - "type": "leaf", - "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", - "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " - }, { "type": "leaf", "className": "mpicbg.trakem2.transform.AffineModel2D", @@ -57,16 +59,17 @@ "mipmapLevels": { "0": { "imageUrl": "file:/data/20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8.tif" + }, + "1": { + "imageUrl": "file:/data/20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8_mmL1.tif.tif" + }, + "2": { + "imageUrl": "file:/data/20160710195316873_243774_7R_SID_01_redo_0_11_7_3_16_8_mmL2.tif.tif" } }, "transforms": { "type": "list", "specList": [ - { - "type": "leaf", - "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", - "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " - }, { "type": "leaf", "className": "mpicbg.trakem2.transform.AffineModel2D", @@ -94,20 +97,24 @@ "width": 3840, "height": 3840, "minIntensity": 0, - "maxIntensity": 65535, + "maxIntensity": 255, "mipmapLevels": { "0": { - "imageUrl": "file:/data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8.tif" + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8.tif" + }, + "1": { + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8_mmL1.tif.tif" + }, + "2": { + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8_mmL2.tif.tif" + }, + "5": { + "imageUrl": "file:///data/20160710195317333_243774_7R_SID_01_redo_0_11_7_3_17_8_mmL5.tif.tif" } }, "transforms": { "type": "list", "specList": [ - { - "type": "leaf", - "className": "mpicbg.trakem2.transform.NonLinearCoordinateTransform", - "dataString": "5 21 1103.117269905359 -5.606757285805906 5.321142212984271 1041.3480932107918 -10.43426929509782 7.428382306562735 -14.137269334106124 7.299563138046944 -15.904206050362355 18.284978789358718 3.4106705605908587 -23.24346650864392 -22.18066749731861 -3.583245180840507 3.2706525226745384 27.117126387399466 20.96651792381158 -1.2219610254066424 -10.531856407305256 0.3306037423046746 2.054811912746283 28.849184526610635 34.799030853758985 -7.4227129043141495 -8.211918715339516 -27.434469512200955 -3.1066643785881 3.8503170392714026 1.7124046587349628 1.8506627747180158 4.886666182237065 -2.949889582798017 -4.5150121503501515 -11.363818954372583 -9.8507389567363 8.672520277073502 5.89027776049669 4.924363838689751 -1.4446630732848895 -2.3876262067533727 19.9189118558668 21.72003155506043 1991.9406329917294 2171.99091870329 5136325.014398676 4300827.048910938 5862798.046430213 1.4793145755295639E10 1.1081068898569233E10 1.1535665428036394E10 1.737179414476557E10 4.5383202598685734E13 3.1913868517100258E13 2.9612848039584566E13 3.403565936230745E13 5.43037094200083E13 1.45059294093077376E17 9.790266764463008E16 8.5058013431605168E16 8.7104768756103424E16 1.06057724033609104E17 1.75771529104524608E17 100.0 1080.9907552180462 1070.1850841521746 4359136.9617994055 3334088.839525756 4464532.472685248 1.6068544862341637E10 1.1728566370432955E10 1.1733314229813484E10 1.6768452787562666E10 5.830517220342475E13 4.128824258570168E13 3.8519115805773336E13 4.149420039803858E13 6.153259860972069E13 2.11108412972822656E17 1.46312985770820672E17 1.31484322773016992E17 1.31813167980595536E17 1.47560207595069728E17 2.24412785271596352E17 0.0 3840 3840 " - }, { "type": "leaf", "className": "mpicbg.trakem2.transform.AffineModel2D", From eb34e3004799bd706df951ebe96adc6c68a4f4d1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 24 Aug 2017 10:44:52 -0700 Subject: [PATCH 448/766] adding pandas and jupyter to base image --- Dockerfile.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.base b/Dockerfile.base index 318e4db1..2cc4bbdf 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -71,7 +71,7 @@ COPY . /usr/local/render-python WORKDIR /usr/local/render-python RUN pip install -r requirements.txt RUN pip install -r ./test/requirements.txt -RUN pip install matplotlib +RUN pip install matplotlib pandas jupyter #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python From 3b8d64cabfd3a47842eef3363d1479b193aaf355 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 25 Aug 2017 10:36:39 -0700 Subject: [PATCH 449/766] moved test again --- Dockerfile.base | 2 +- setup.py | 2 +- test_requirements.txt | 10 ++++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 test_requirements.txt diff --git a/Dockerfile.base b/Dockerfile.base index 2cc4bbdf..e8a9f1e6 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -70,7 +70,7 @@ RUN mkdir -p /usr/local/render-python COPY . /usr/local/render-python WORKDIR /usr/local/render-python RUN pip install -r requirements.txt -RUN pip install -r ./test/requirements.txt +RUN pip install -r test_requirements.txt RUN pip install matplotlib pandas jupyter #install render python using pip from github #RUN pip install -e git+https://github.com/fcollman/render-python.git@master#egg=render-python diff --git a/setup.py b/setup.py index 47eb4e10..5facda2f 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def run_tests(self): sys.exit(errno) -with open('test/requirements.txt', 'r') as f: +with open('test_requirements.txt', 'r') as f: test_required = f.read().splitlines() with open('requirements.txt', 'r') as f: diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..18650732 --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,10 @@ +coverage>=4.1 +mock>=2.0.0 +pep8>=1.7.0 +pytest>=3.0.5 +pytest-cov>=2.2.1 +pytest-pep8>=1.0.6 +pytest-xdist>=1.14 +flake8>=3.0.4 +pylint>=1.5.4 +scipy \ No newline at end of file From 0ab3effe13a951669f70fac4528948ed1a29dbf6 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Fri, 25 Aug 2017 10:44:59 -0700 Subject: [PATCH 450/766] whoops removing old test requirements --- test/requirements.txt | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 test/requirements.txt diff --git a/test/requirements.txt b/test/requirements.txt deleted file mode 100644 index 18650732..00000000 --- a/test/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -coverage>=4.1 -mock>=2.0.0 -pep8>=1.7.0 -pytest>=3.0.5 -pytest-cov>=2.2.1 -pytest-pep8>=1.0.6 -pytest-xdist>=1.14 -flake8>=3.0.4 -pylint>=1.5.4 -scipy \ No newline at end of file From b7e124cacf2fd71235fba76cb1439de24ecb2055 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 26 Aug 2017 10:56:44 -0700 Subject: [PATCH 451/766] increment version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5facda2f..7f1963f7 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.0', + version='1.0.1', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From 7a162c9a1a67445979e1a307bda82622468e5566 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Sat, 26 Aug 2017 12:05:56 -0700 Subject: [PATCH 452/766] made the integration test sources configurable by environment variables --- integration_tests/test_data.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py index 2e82500a..a332d8c3 100644 --- a/integration_tests/test_data.py +++ b/integration_tests/test_data.py @@ -1,9 +1,10 @@ +import os -render_host = 'renderservice' -render_port = 8080 -client_script_location = ('/var/www/render/render-ws-java-client/' - 'src/main/scripts/') -tilespec_file = ('/var/www/render/examples/' - 'example_1/cycle1_step1_acquire_tiles.json') -tform_file = ('/var/www/render/examples/' - 'example_1/cycle1_step1_acquire_transforms.json') +render_host = os.environ.get('RENDER_HOST','renderservice') +render_port = os.environ.get('RENDER_PORT',8080) +client_script_location = os.environ.get('RENDER_CLIENT_SCRIPTS', + ('/var/www/render/render-ws-java-client/' + 'src/main/scripts/')) +example_dir = os.environ('RENDER_EXAMPLE_DATA','/var/www/render/examples/') +tilespec_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_tiles.json') +tform_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_transforms.json') From c505a7fac03b04a5bbb2121dc74ce3eddb7c56cd Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 00:11:58 -0700 Subject: [PATCH 453/766] mixxed a get call --- integration_tests/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py index a332d8c3..6b7c6bce 100644 --- a/integration_tests/test_data.py +++ b/integration_tests/test_data.py @@ -5,6 +5,6 @@ client_script_location = os.environ.get('RENDER_CLIENT_SCRIPTS', ('/var/www/render/render-ws-java-client/' 'src/main/scripts/')) -example_dir = os.environ('RENDER_EXAMPLE_DATA','/var/www/render/examples/') +example_dir = os.environ.get('RENDER_EXAMPLE_DATA','/var/www/render/examples/') tilespec_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_tiles.json') tform_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_transforms.json') From aa440d2d761a83335a13f3b62b53b9fa3d5333fb Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 00:21:51 -0700 Subject: [PATCH 454/766] changed docker file to /shared --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 17033990..f632733f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ FROM fcollman/render-python-base:latest MAINTAINER Forrest Collman (forrest.collman@gmail.com) -RUN mkdir -p /usr/local/render-python -COPY . /usr/local/render-python -WORKDIR /usr/local/render-python +RUN mkdir -p /shared/render-python +COPY . /shared/render-python +WORKDIR /shared/render-python RUN python setup.py install ENTRYPOINT [ "/usr/bin/tini", "--" ] From b34c57267478df58eef2394e91d4aece2df61e88 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 00:23:24 -0700 Subject: [PATCH 455/766] changed install to develop --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f632733f..0d10f45b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ MAINTAINER Forrest Collman (forrest.collman@gmail.com) RUN mkdir -p /shared/render-python COPY . /shared/render-python WORKDIR /shared/render-python -RUN python setup.py install +RUN python setup.py develop ENTRYPOINT [ "/usr/bin/tini", "--" ] CMD [ "/bin/bash" ] From 0e1c4a881e54ee0b0ccf7d1ad262d11f28590d0c Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 20:14:57 -0700 Subject: [PATCH 456/766] trying to debug the integration test --- integration_tests/test_client_integrated.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index e5261d67..ac72d024 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -177,9 +177,13 @@ def test_importTransformChangesClient(render, teststack): teststack, deststack, TCCjson, changeMode='APPEND', render=render) renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) os.remove(TCCjson) + + output_ts = renderapi.tilespec.get_tile_specs_from_stack( + deststack, render=render) + print output_ts[0].tforms[-1] + print output_ts[-1].tforms[-1] assert all([ts.tforms[-1].to_dict() == tform_to_append.to_dict() - for ts in renderapi.tilespec.get_tile_specs_from_stack( - deststack, render=render)]) + for ts in output_ts]) renderapi.stack.delete_stack(deststack, render=render) From f4e699a50f9cdf791d7ee96ce366a02e43ea1202 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 22:53:30 -0700 Subject: [PATCH 457/766] using pip install to install instead --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0d10f45b..78249924 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,8 @@ MAINTAINER Forrest Collman (forrest.collman@gmail.com) RUN mkdir -p /shared/render-python COPY . /shared/render-python +RUN pip install -e /shared/render-python WORKDIR /shared/render-python -RUN python setup.py develop ENTRYPOINT [ "/usr/bin/tini", "--" ] CMD [ "/bin/bash" ] From df4449e75334812e4311eee3bc78182936895702 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 23:01:18 -0700 Subject: [PATCH 458/766] adding intentional bad test to test CI --- test/test_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_client.py b/test/test_client.py index b4acad94..17a61d57 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -61,4 +61,7 @@ def test_decorator(): (owner,host,port,project,client_scripts)=my_decorated(5,render=r) assert(owner == args['owner']) (owner,host,port,project,client_scripts)=my_decorated(5,owner='newowner',render=r) - assert(owner == 'newowner') \ No newline at end of file + assert(owner == 'newowner') + +def test_bad(): + assert False \ No newline at end of file From ebbb7562b4d1c6867a7b95c4a881d3f1954c2317 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 23:05:16 -0700 Subject: [PATCH 459/766] fixing bad test --- test/test_client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/test_client.py b/test/test_client.py index 17a61d57..5562657e 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -62,6 +62,3 @@ def test_decorator(): assert(owner == args['owner']) (owner,host,port,project,client_scripts)=my_decorated(5,owner='newowner',render=r) assert(owner == 'newowner') - -def test_bad(): - assert False \ No newline at end of file From 75c96fff83570c5d5bfee640c9653d84f5552b8b Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 23:19:13 -0700 Subject: [PATCH 460/766] changed debug step --- integration_tests/test_client_integrated.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index ac72d024..8c6982de 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -180,8 +180,7 @@ def test_importTransformChangesClient(render, teststack): output_ts = renderapi.tilespec.get_tile_specs_from_stack( deststack, render=render) - print output_ts[0].tforms[-1] - print output_ts[-1].tforms[-1] + assert all([ts.tforms[-1].to_dict() == tform_to_append.to_dict() for ts in output_ts]) renderapi.stack.delete_stack(deststack, render=render) @@ -199,7 +198,12 @@ def test_transformSectionClient(render, teststack, tform.dataString.replace(" ", ","), zvalues, targetStack=deststack, render=render) renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) + + = renderapi.tilespec.get_tile_specs_from_stack( + deststack, render=render) + print output_ts[0].tforms[-1] + print output_ts[-1].tforms[-1] + assert all([ts.tforms[-1].to_dict() == tform.to_dict() - for ts in renderapi.tilespec.get_tile_specs_from_stack( - deststack, render=render)]) + for ts in output_ts]) renderapi.stack.delete_stack(deststack, render=render) From c418222abf4bb07a69f13e8b1f63b564785fe96f Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 23:19:34 -0700 Subject: [PATCH 461/766] tpyo --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 8c6982de..b3610946 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -199,7 +199,7 @@ def test_transformSectionClient(render, teststack, render=render) renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) - = renderapi.tilespec.get_tile_specs_from_stack( + output_ts = renderapi.tilespec.get_tile_specs_from_stack( deststack, render=render) print output_ts[0].tforms[-1] print output_ts[-1].tforms[-1] From b9b602ebac316ba3dec5a84ec27740645c4868f1 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 23:26:26 -0700 Subject: [PATCH 462/766] improved debug statements --- integration_tests/test_client_integrated.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index b3610946..2aa1fe68 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -201,9 +201,9 @@ def test_transformSectionClient(render, teststack, output_ts = renderapi.tilespec.get_tile_specs_from_stack( deststack, render=render) - print output_ts[0].tforms[-1] - print output_ts[-1].tforms[-1] - + root.debug(output_ts[0].tforms[0].to_dict()) + root.debug(output_ts[-1].tforms[-1].to_dict()) + root.debug(tform.to_dict()) assert all([ts.tforms[-1].to_dict() == tform.to_dict() for ts in output_ts]) renderapi.stack.delete_stack(deststack, render=render) From a5f02abcd07830d9df2b3f102ab754b95e1a2446 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 30 Aug 2017 23:31:35 -0700 Subject: [PATCH 463/766] added ID to fix test --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 2aa1fe68..dc183148 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -191,7 +191,7 @@ def test_transformSectionClient(render, teststack, deststack = 'test_stack_TSC' transformId = 'TSC_testtransform' zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) - tform = renderapi.transform.AffineModel() + tform = renderapi.transform.AffineModel(transformId=transformId) renderapi.client.transformSectionClient( teststack, transformId, tform.className, @@ -201,7 +201,7 @@ def test_transformSectionClient(render, teststack, output_ts = renderapi.tilespec.get_tile_specs_from_stack( deststack, render=render) - root.debug(output_ts[0].tforms[0].to_dict()) + root.debug(output_ts[0].tforms[-1].to_dict()) root.debug(output_ts[-1].tforms[-1].to_dict()) root.debug(tform.to_dict()) assert all([ts.tforms[-1].to_dict() == tform.to_dict() From e13123c18f442910c2c035f5a494b4e227c79df8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 31 Aug 2017 15:11:14 -0700 Subject: [PATCH 464/766] bumping version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f1963f7..97490b92 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.0.1', + version='1.0.2', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From 3784538fdb8bae7de240a1426a4a386f60281df8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 12 Sep 2017 12:41:04 -0700 Subject: [PATCH 465/766] added a resolvedtiles feature (#27) * resolvedtiles v1 * adding resolvedtiles to init * fixed transform documentation * adding more testing files to unit tests reference * fixed test of resolved tiles * fixed from_dict * added more tests of resolved_tiles * added integration test of get_resolvedtiles_from_z * trying to debug an integration test * fixed resolvedtiles to work with actual server schema * trying to fix test with actual schema.... * removing print debug statement * bumping version number * implementing emtpy list initialization and removing numpy * added a prototype with a stub for a function to return derefereced tilespecs * resolvedtiles: import cleanup, PEP8 --- integration_tests/test_stack_integrated.py | 11 + renderapi/__init__.py | 3 +- renderapi/resolvedtiles.py | 92 +++++++ renderapi/transform.py | 8 +- setup.py | 2 +- test/rendersettings.py | 16 +- test/test_files/tilespec_references.json | 283 +++++++++++++++++++++ test/test_files/transform_references.json | 51 ++++ test/test_resolvedtiles.py | 30 +++ 9 files changed, 484 insertions(+), 12 deletions(-) create mode 100644 renderapi/resolvedtiles.py create mode 100644 test/test_files/tilespec_references.json create mode 100644 test/test_files/transform_references.json create mode 100644 test/test_resolvedtiles.py diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index c22c210b..f251ccf3 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -419,3 +419,14 @@ def test_get_sectionId_for_z(render, teststack, render_example_tilespec_and_tran (tilespecs, tforms) = render_example_tilespec_and_transforms sectionId = render.run(renderapi.stack.get_sectionId_for_z, teststack, tilespecs[0].z) assert (sectionId == tilespecs[0].layout.sectionId) + +def test_get_resolvedtiles_from_z(render, teststack, + render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + resolved_tiles = renderapi.resolvedtiles.get_resolved_tiles_from_z(teststack, + tilespecs[0].z, + render=render) + tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] + assert(len(tsz)==len(resolved_tiles.tilespecs)) + matching_ts = next(ts for ts in resolved_tiles.tilespecs if ts.tileId == tsz[0].tileId) + assert (len(matching_ts.tforms)==len(tsz[0].tforms)) diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 3a26a5cb..25146c65 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -9,9 +9,10 @@ from . import transform from . import pointmatch from . import coordinate +from . import resolvedtiles from .render import connect from .render import Render __all__ = ['render', 'client', 'tilespec', 'errors', 'stack', 'image', 'pointmatch', 'coordinate', - 'connect', 'transform', 'Render'] + 'connect', 'transform', 'resolvedtiles','Render'] diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py new file mode 100644 index 00000000..64893a30 --- /dev/null +++ b/renderapi/resolvedtiles.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +from .tilespec import TileSpec +from .transform import load_transform_json +from .utils import NullHandler +from .render import format_preamble, renderaccess +from .errors import RenderError +import logging +import requests + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class ResolvedTiles: + def __init__(self, tilespecs=None, transformList=None, json=None): + if json is None: + if tilespecs is None: + self.tilespecs = [] + else: + self.tilespecs = tilespecs + if transformList is None: + self.transforms = [] + else: + self.transforms = transformList + else: + self.from_dict(json) + + def to_dict(self): + d = { + 'transformIdToSpecMap': {tf.transformId: tf.to_dict() + for tf in self.transforms}, + 'tileIdToSpecMap': {ts.tileId: ts.to_dict() + for ts in self.tilespecs} + } + return d + + def from_dict(self, d): + self.tilespecs = [] + self.transforms = [] + for ts in d['tileIdToSpecMap'].values(): + self.tilespecs.append(TileSpec(json=ts)) + for transformId, tform_json in d['transformIdToSpecMap'].iteritems(): + tform_json['transformId'] = transformId + self.transforms.append(load_transform_json(tform_json)) + + #def get_tilespecs(): + """return a set of TileSpecs that include resolved tilespecs + + Returns + ------- + List(renderapi.tilespec.TileSpec) + A list of tilespecs stored in this ResolvedTiles with the transformations dereferenced + """ + + +@renderaccess +def get_resolved_tiles_from_z(stack, z, host=None, port=None, + owner=None, project=None, + session=requests.session(), + render=None, **kwargs): + """Get a set of ResolvedTiles from a specific z value. + Returns a tuple of tilespecs and referenced transforms. + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack + z : float + render z + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + sessions object to connect with + + Returns + ------- + :obj:`ResolvedTiles` + ResolvedTiles object containing tilespecs and transforms + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/z/%f/resolvedTiles' % (z) + logger.debug(request_url) + r = session.get(request_url) + try: + d = r.json() + except Exception as e: + logger.error(e) + logger.error(r.text) + raise RenderError(r.text) + return ResolvedTiles(json=d) diff --git a/renderapi/transform.py b/renderapi/transform.py index db623480..3e0689a1 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -117,8 +117,8 @@ def load_transform_json(d, default_type='leaf'): Returns ------- - func - proper function to deserialize this transformation + renderapi.transform.Transform + deserialized transformation using the most appropriate class Raises ------ @@ -146,8 +146,8 @@ def load_leaf_json(d): Returns ------- - func - proper function to deserialize this transformation + renderapi.transform.Transform + deserialized transformation Raises ------ diff --git a/setup.py b/setup.py index 97490b92..7342a3fd 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.0.2', + version='1.0.3', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', diff --git a/test/rendersettings.py b/test/rendersettings.py index 92409a08..19e9a035 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -28,14 +28,18 @@ 'RENDER_CLIENT_SCRIPT': DEFAULT_RENDER_CLIENT['client_script'], 'RENDER_CLIENT_HEAP': DEFAULT_RENDER_CLIENT['memGB']}) -TEST_TILESPECS_FILE = os.path.join(os.path.dirname( - __file__), 'test_files', 'tilespecs.json') +TEST_FILES_DIR = os.path.join(os.path.dirname( + __file__), 'test_files') -INTERPOLATED_TRANSFORM_TILESPEC = os.path.join(os.path.dirname( - __file__), 'test_files', 'tilespec_interpolated.json') +TEST_TILESPECS_FILE = os.path.join(TEST_FILES_DIR, 'tilespecs.json') -REFERENCE_TRANSFORM_TILESPEC = os.path.join(os.path.dirname( - __file__), 'test_files', 'tilespec_ref.json') +INTERPOLATED_TRANSFORM_TILESPEC = os.path.join(TEST_FILES_DIR, 'tilespec_interpolated.json') + +REFERENCE_TRANSFORM_TILESPEC = os.path.join(TEST_FILES_DIR, 'tilespec_ref.json') + +REFERENCE_TRANSFORM_TILESPECS = os.path.join(TEST_FILES_DIR, 'tilespec_references.json') + +REFERENCE_TRANSFORM_SPECS = os.path.join(TEST_FILES_DIR,'transform_references.json') NONLINEAR_TRANSFORM_KWARGS = { 'className': "mpicbg.trakem2.transform.NonLinearCoordinateTransform", diff --git a/test/test_files/tilespec_references.json b/test/test_files/tilespec_references.json new file mode 100644 index 00000000..18d5a50b --- /dev/null +++ b/test/test_files/tilespec_references.json @@ -0,0 +1,283 @@ +[ + { + "tileId": "150304141121052104.3407.0", + "z": 3407, + "width": 2560, + "height": 2160, + "layout": { + "sectionId": "3407.0", + "temca": "7", + "camera": "0", + "imageRow": 104, + "imageCol": 52, + "stageX": 97500, + "stageY": 189800, + "rotation": 180 + }, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3407/col0052_row0104_cam0.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera0" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.988634000000 0.036349000000 -0.034204000000 0.948211000000 222.545418368929 100.738559261896" + } + ] + } + }, + { + "tileId": "150304141121052105.3407.0", + "z": 3407, + "width": 2560, + "height": 2160, + "layout": { + "sectionId": "3407.0", + "temca": "7", + "camera": "0", + "imageRow": 105, + "imageCol": 52, + "stageX": 97500, + "stageY": 191650, + "rotation": 180 + }, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3407/col0052_row0105_cam0.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera0" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.988756000000 0.038429000000 -0.033347000000 0.946228000000 115.792562368937 1965.888095261878" + } + ] + } + }, + { + "tileId" : "150304141121053104.3407.0", + "z" : 3407.0, + "width" : 2560.0, + "height" : 2160.0, + "layout" : { + "sectionId" : "3407.0", + "temca" : "7", + "camera" : "0", + "imageRow" : 104, + "imageCol" : 53, + "stageX" : 99450.0, + "stageY" : 189800.0, + "rotation" : 180.0 + }, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3407/col0053_row0104_cam0.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera0" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.988139000000 0.037567000000 -0.031542000000 0.948947000000 2195.971461368928 368.161869261879" + } + ] + } + }, + { + "tileId" : "150304141121053105.3407.0", + "layout" : { + "sectionId" : "3407.0", + "temca" : "7", + "camera" : "0", + "imageRow" : 105, + "imageCol" : 53, + "stageX" : 99450.0, + "stageY" : 191650.0, + "rotation" : 180.0 + }, + "z" : 3407.0, + "width" : 2560.0, + "height" : 2160.0, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3407/col0053_row0105_cam0.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam0.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera0" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.987798000000 0.038469000000 -0.035973000000 0.947612000000 2098.872003368931 2199.856228261895" + } + ] + } + }, + { + "tileId": "150304141121055030.3408.1", + "layout": { + "sectionId": "3408.1", + "temca": "7", + "camera": "3", + "imageRow": 30, + "imageCol": 55, + "stageX": 103350, + "stageY": 54800, + "rotation": 180 + }, + "z": 3408, + "width": 2560, + "height": 2160, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3408/col0055_row0030_cam3.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam3.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera3" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.982457000000 0.035841000000 -0.042949000000 0.944933000000 702.755960368930 1277.236107261851" + } + ] + } + }, + { + "tileId": "150304141121055031.3408.1", + "layout": { + "sectionId": "3408.1", + "temca": "7", + "camera": "3", + "imageRow": 31, + "imageCol": 55, + "stageX": 103350, + "stageY": 56650, + "rotation": 180 + }, + "z": 3408, + "width": 2560, + "height": 2160, + "mipmapLevels": { + "0": { + "imageUrl": "file:/tmp/example_1/image_data/s_3408/col0055_row0031_cam3.tif", + "maskUrl": "file:/tmp/example_1/image_masks/temca7_cam3.png" + } + }, + "transforms": { + "type": "list", + "specList": [ + { + "type": "ref", + "refId": "LENS_CORRECTION_temca7_camera3" + }, + { + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "0.982235000000 0.035777000000 -0.042126000000 0.943634000000 590.676579368926 3122.673506261839" + } + ] + } + }, + { + "tileId" : "150304141121056030.3408.1", + "z" : 3408.0, + "width" : 2560.0, + "height" : 2160.0, + "layout" : { + "sectionId" : "3408.1", + "temca" : "7", + "camera" : "2", + "imageRow" : 30, + "imageCol" : 56, + "stageX" : 105000.0, + "stageY" : 54800.0, + "rotation" : 180.0 + }, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3408/col0056_row0030_cam2.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam2.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera2" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.982463000000 0.042319000000 -0.038484000000 0.945792000000 2502.850133368935 1376.114864261879" + } + ] + } + }, + { + "tileId" : "150304141121056031.3408.1", + "z" : 3408.0, + "width" : 2560.0, + "height" : 2160.0, + "layout" : { + "sectionId" : "3408.1", + "temca" : "7", + "camera" : "2", + "imageRow" : 31, + "imageCol" : 56, + "stageX" : 105000.0, + "stageY" : 56650.0, + "rotation" : 180.0 + }, + "mipmapLevels" : { + "0" : { + "imageUrl" : "file:/tmp/example_1/image_data/s_3408/col0056_row0031_cam2.tif", + "maskUrl" : "file:/tmp/example_1/image_masks/temca7_cam2.png" + } + }, + "transforms" : { + "type" : "list", + "specList" : [ + { + "type" : "ref", + "refId" : "LENS_CORRECTION_temca7_camera2" + }, + { + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.982244000000 0.041618000000 -0.037589000000 0.944900000000 2387.030142368923 3217.084657261847" + } + ] + } + } + ] + \ No newline at end of file diff --git a/test/test_files/transform_references.json b/test/test_files/transform_references.json new file mode 100644 index 00000000..493c9afb --- /dev/null +++ b/test/test_files/transform_references.json @@ -0,0 +1,51 @@ +[ + { + "type" : "list", + "id" : "LENS_CORRECTION_temca7_camera0", + "specList" : [ + { + "type" : "leaf", + "className" : "lenscorrection.NonLinearTransform", + "dataString" : "5 21 700.707955266298 -2.2001080559868793 13.873040384630464 627.0743514702998 -13.529483853406793 7.228735711439736 -10.839669176402502 -6.139021966264066 -11.72071825781854 -10.33928052302308 4.278978527660939 -3.4594411956970808 -2.5020177814252778 0.8912979016273739 3.35982841409578 -8.64411101181073 0.09093625547696149 8.332230284245867 7.479082855684247 -1.284703107571685 1.8807143771547103 5.941259214658612 4.209581504855379 5.257758911157933 2.3978334920077202 3.754133480575426 -0.5929877413140048 -0.7779841026367293 -2.944571086212573 1.1933155763290975 0.4661295617851611 -2.412931393336346 -2.6955605399934246 -0.22775323647894358 0.7122079198088755 -2.648021400179547 -1.6624574262499792 -0.02708163303879585 1.1475940290759183 0.23485421235502013 11.471758011138729 11.048994469455984 1147.1734708175197 1104.8993194529953 1794114.7301473576 1265879.0975329224 1604717.7004822914 3.2387227602706795E9 1.9734583414478405E9 1.8483776775925295E9 2.6122108229352436E9 6.344304427022081E12 3.546610513583933E12 2.87805285556924E12 3.0253326094556655E12 4.532508846629051E12 1.3094445313371616E16 6.91662907507397E15 5.157572930400959E15 4.70862651090562E15 5.276554689026626E15 8.193085649459779E15 100.0 691.4672425752566 619.6213386843245 1767931.6182683974 1129448.610690605 1399095.6047029237 4.185162060049768E9 2.476443778259326E9 2.1902776293974566E9 2.899873921130305E9 9.847375063509006E12 5.576800271136418E12 4.489070210105462E12 4.3313999043551543E12 5.925923866832138E12 2.3253941132175516E16 1.2802457301674196E16 9.817920026167838E15 8.617375639277423E15 8.677623871959977E15 1.2087680046683428E16 0.0 2560 2160 " + }, + { + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "1.0225774 0.004573957 -0.008069136 1.0538727 130.0 30.0" + } + ] + }, + { + "id" : "LENS_CORRECTION_temca7_camera2", + "type" : "list", + "specList" : [ + { + "type" : "leaf", + "className" : "lenscorrection.NonLinearTransform", + "dataString" : "5 21 711.4721917133976 -5.589914769754155 -1.1212879916039302 611.9788313567354 0.9060213772013888 17.225578046304864 -3.375415049826092 -13.228357077450797 -4.0000251672817235 -9.45662859209138 -1.634495499587608 -10.304113753791569 -1.213393557789903 3.280829150417878 3.806378900638064 -3.504968777660551 7.201798010176688 4.68489990495069 6.5940053140106905 5.003072665329844 2.207293734167834 2.8728531024175634 -0.05073765949401299 3.623864247802219 2.95885160398298 0.8934478162984338 -4.8060384709988995 -0.13469418887316031 -2.0248910123223816 -1.208647910444716 -0.6461420209825857 -1.1353187792931874 -0.33149925100723077 -1.1122145006790851 -0.5921228750450704 -1.0352160750358692 -0.9975754041991061 0.5723447761009257 1.4718233160631549 -0.4660362839593981 13.271729787926228 11.36085817904796 1327.1706044725697 1136.0903783862736 2272546.952672654 1506473.4529777775 1650604.9178121507 4.337314511226062E9 2.5744385721076655E9 2.194297853083332E9 2.663116597892426E9 8.809638594278018E12 4.903814127489643E12 3.748933696078432E12 3.551527713373412E12 4.561781316801743E12 1.864076900958326E16 9.938324654099888E15 7.134639652469553E15 6.067832412917443E15 6.103406845480805E15 8.129116449151267E15 100.0 714.9731259974685 599.9322125157199 1909272.5769361807 1216360.74472006 1355493.1773583877 4.666723318413752E9 2.795306737181569E9 2.369412143315229E9 2.7942938019494762E9 1.1235656814375736E13 6.477705273329607E12 5.098545422604573E12 4.656013779749117E12 5.679481768816695E12 2.6992898463066668E16 1.51679732641352E16 1.1460688242542052E16 9.727988954220464E15 9.253755629882418E15 1.1537151295307158E16 0.0 2560 2160 " + }, + { + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "0.98033196 -0.011563861 -0.007845614 1.0093758 130.0 30.0" + } + ] + }, + { + "id" : "LENS_CORRECTION_temca7_camera3", + "type" : "list", + "specList" : [ + { + "type" : "leaf", + "className" : "lenscorrection.NonLinearTransform", + "dataString" : "5 21 716.6873371364047 6.053095456716258 -7.33011813717707 615.9404061487677 -10.10392129154934 -3.178743380467174 -0.9080793854570661 -7.555073092930774 6.836976455645333 -10.919574061891632 6.010802574254718 3.4312960837342352 1.9319897093811633 2.278773522479664 4.261230047320696 -3.3766138130897687 -10.037252509049736 8.879376829985034 0.2511892646304901 -7.474464984035155 -0.12651470010799137 3.018706620210251 -0.6984470933503166 1.997670980472821 -0.33445113401323834 1.8050086636786895 10.515312559392513 -6.9198312805652265 -0.03371009293810445 2.8556473768134767 -0.6863440894714427 -0.8946743620783042 0.6330640226897923 0.1726669199954567 0.6353096977003143 -0.8782956379063966 0.3367041976112679 -0.6744341635057034 -3.6657658385583005 2.8912392853921967 12.576628285752045 9.997393590654209 1257.663252302061 999.7429242926293 2093303.624799589 1245004.0742310893 1366319.5248503662 3.9345304836190944E9 2.0606718153677976E9 1.6892732155723114E9 2.1396317407128017E9 7.919490965364902E12 3.8544763741048306E12 2.7808203804989346E12 2.632847197055338E12 3.6107639405689927E12 1.6662293914167926E16 7.721975010374804E15 5.17652292334484E15 4.3144423730759265E15 4.428686756822473E15 6.384364622813321E15 100.0 715.2689070153682 605.6814246284501 1880885.1217286212 1109432.3712253342 1320610.7518000966 4.546331733157403E9 2.4653417254374003E9 2.0918157620442765E9 2.6594039686218805E9 1.0867868593386514E13 5.60885189992376E12 4.3212574170000156E12 4.01855921622566E12 5.314489633143806E12 2.5987814164131128E16 1.297613219424449E16 9.483497223595874E15 8.031119671564667E15 7.84530320604815E15 1.0658513461698616E16 0.0 2560 2160 " + }, + { + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.AffineModel2D", + "dataString" : "1.0059261 0.0090332 0.002712006 0.99720263 130.0 30.0" + } + ] + } + ] + \ No newline at end of file diff --git a/test/test_resolvedtiles.py b/test/test_resolvedtiles.py new file mode 100644 index 00000000..f932eb84 --- /dev/null +++ b/test/test_resolvedtiles.py @@ -0,0 +1,30 @@ +import renderapi +import pytest +import json +import rendersettings + + +@pytest.fixture(scope='module') +def referenced_tilespecs_and_transforms(): + with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS,'r') as fp: + ds = json.load(fp) + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ds] + + with open(rendersettings.REFERENCE_TRANSFORM_SPECS,'r') as fp: + ds = json.load(fp) + transforms = [renderapi.transform.load_transform_json(tf) for tf in ds] + return tilespecs,transforms + +@pytest.fixture(scope='module') +def resolvedtiles_object(referenced_tilespecs_and_transforms): + tilespecs,transforms = referenced_tilespecs_and_transforms + resolved_tiles = renderapi.resolvedtiles.ResolvedTiles(tilespecs = tilespecs, + transformList=transforms) + return resolved_tiles + +def test_resolvedtiles_from_dict(resolvedtiles_object,referenced_tilespecs_and_transforms): + tilespecs,transforms = referenced_tilespecs_and_transforms + d=resolvedtiles_object.to_dict() + resolved_tiles = renderapi.resolvedtiles.ResolvedTiles(json=d) + assert(len(tilespecs)==len(resolved_tiles.tilespecs)) + assert(len(transforms)==len(resolved_tiles.transforms)) From 769e8f3d0c5b5bd4fe7fb100aceb2c9b859190cb Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 13 Sep 2017 14:43:16 -0700 Subject: [PATCH 466/766] fixing missing poolsize constraint for test --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index dc183148..583df3da 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -99,7 +99,7 @@ def test_import_jsonfiles_parallel_multiple( with renderapi.client.WithPool(poolsize) as pool: results = pool.map(lambda x: x**2, mylist) test_import_jsonfiles_parallel( - render, render_example_tilespec_and_transforms, stack, poolsize) + render, render_example_tilespec_and_transforms, stack, poolsize=3) def test_import_tilespecs_parallel(render, From edd1b9a75c9d954e8b526e058f3e9c53d6ad0697 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 13 Sep 2017 14:43:46 -0700 Subject: [PATCH 467/766] reducing poolsize on multiple --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 583df3da..8f2b9575 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -92,14 +92,14 @@ def test_import_jsonfiles_parallel( def test_import_jsonfiles_parallel_multiple( - render, render_example_tilespec_and_transforms, poolsize=5): + render, render_example_tilespec_and_transforms, poolsize=3): stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] mylist = range(10) for stack in stacks: with renderapi.client.WithPool(poolsize) as pool: results = pool.map(lambda x: x**2, mylist) test_import_jsonfiles_parallel( - render, render_example_tilespec_and_transforms, stack, poolsize=3) + render, render_example_tilespec_and_transforms, stack, poolsize=poolsize) def test_import_tilespecs_parallel(render, From dc96670a71dfaaffebbbca344030859b93c9a3d4 Mon Sep 17 00:00:00 2001 From: forrest collman Date: Wed, 13 Sep 2017 16:15:11 -0700 Subject: [PATCH 468/766] made pool size configurable and unified in test_data --- integration_tests/test_client_integrated.py | 10 +++++----- integration_tests/test_data.py | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 8f2b9575..0cbbe8f8 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -8,7 +8,7 @@ import numpy as np import dill from test_data import (render_host, render_port, - client_script_location, tilespec_file, tform_file) + client_script_location, tilespec_file, tform_file, test_pool_size) from pathos.multiprocessing import ProcessingPool as Pool root = logging.getLogger() @@ -79,7 +79,7 @@ def test_import_jsonfiles_validate_client( def test_import_jsonfiles_parallel( render, render_example_tilespec_and_transforms, - stack='test_import_jsonfiles_parallel', poolsize=5): + stack='test_import_jsonfiles_parallel', poolsize=test_pool_size): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files( @@ -92,7 +92,7 @@ def test_import_jsonfiles_parallel( def test_import_jsonfiles_parallel_multiple( - render, render_example_tilespec_and_transforms, poolsize=3): + render, render_example_tilespec_and_transforms, poolsize=test_pool_size): stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] mylist = range(10) for stack in stacks: @@ -109,7 +109,7 @@ def test_import_tilespecs_parallel(render, (tilespecs, tforms) = render_example_tilespec_and_transforms renderapi.client.import_tilespecs_parallel( stack, tilespecs, sharedTransforms=tforms, - poolsize=3, render=render) + poolsize=test_pool_size, render=render) validate_stack_import(render, stack, tilespecs) @@ -121,7 +121,7 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles( - stack, tfiles, transformFile=transformFile, poolsize=3, render=render) + stack, tfiles, transformFile=transformFile, poolsize=test_pool_size, render=render) validate_stack_import(render, stack, tilespecs) diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py index 6b7c6bce..ba2a97ad 100644 --- a/integration_tests/test_data.py +++ b/integration_tests/test_data.py @@ -8,3 +8,4 @@ example_dir = os.environ.get('RENDER_EXAMPLE_DATA','/var/www/render/examples/') tilespec_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_tiles.json') tform_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_transforms.json') +test_pool_size = os.environ.get('RENDER_PYTHON_TEST_POOL_SIZE',3) \ No newline at end of file From 18a4884d9d9fd18c29c729b096459a54c99852ff Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 14 Sep 2017 11:35:13 -0700 Subject: [PATCH 469/766] Rst readme (#28) * changed to restructured text to facilitate pypi deployment * converted readme to restructured text * trying different linking in rst * trying again * how about this? * another try * yet another try to make github render links * fixed rest of links * one more fix --- README.md | 24 ------------------------ README.rst | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 24 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/README.md b/README.md deleted file mode 100644 index d491a34c..00000000 --- a/README.md +++ /dev/null @@ -1,24 +0,0 @@ -[![Documentation Status](https://readthedocs.org/projects/render-python/badge/)](http://render-python.readthedocs.io/en/latest/) - -# render-python - -This is a python API client to interact with [render-webservices](https://github.com/saalfeldlab/render) and facilitate python scripting of [tilespec](https://github.com/saalfeldlab/render/blob/master/docs/src/site/markdown/data-model.md) creation - -it presently interacts with render via a web-api, though the [client module](renderapi/client.py) aims to interface by calling java client scripts to avoid server-side processing. - -Render connection objects created with `renderapi.connect()` can default to environment variables. Below is an example of the variables which can be sourced and added to, e.g., ~/.bashrc or ~/.bash_profile. -``` - export RENDER_HOST="localhost" - export RENDER_PORT="8080" - export RENDER_PROJECT="YOURPROJECT" - export RENDER_OWNER="YOURNAME" - export RENDER_CLIENT_SCRIPTS=".../render/render-ws-java-client/src/main/scripts" - export RENDER_CLIENT_SCRIPT="$RENDER_CLIENT_SCRIPTS/run_ws_client.sh" - export RENDER_CLIENT_HEAP="1G" -``` - -[Usage examples for a development Array Tomography workflow](https://github.com/fcollman/render-python-apps) are available. - -# Documentation - -http://render-python.readthedocs.io/en/latest/ diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..41601a9e --- /dev/null +++ b/README.rst @@ -0,0 +1,30 @@ +.. image:: https://readthedocs.org/projects/render-python/badge/ + :target: http://render-python.readthedocs.io/en/latest/ + :alt: Documentation Status + +render-python +############# + +This is a python API client to interact with `render `_ and facilitate python scripting of `tilespec `_ creation + +it presently interacts with render via a web-api, though the `client module `_ aims to interface by calling java client scripts to avoid server-side processing. + +Render connection objects created with `renderapi.connect()` can default to environment variables. Below is an example of the variables which can be sourced and added to, e.g., ~/.bashrc or ~/.bash_profile. +:: + + export RENDER_HOST="localhost" + export RENDER_PORT="8080" + export RENDER_PROJECT="YOURPROJECT" + export RENDER_OWNER="YOURNAME" + export RENDER_CLIENT_SCRIPTS=".../render/render-ws-java-client/src/main/scripts" + export RENDER_CLIENT_SCRIPT="$RENDER_CLIENT_SCRIPTS/run_ws_client.sh" + export RENDER_CLIENT_HEAP="1G" + + +`Usage examples for a development Array Tomography workflow `_ are available. + +Documentation +############# +http://render-python.readthedocs.io/en/latest/ + +.. _render : \ No newline at end of file From df4c15c1b1f707385aa352f83c550a661bc9cdd8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 7 Oct 2017 10:05:42 -0700 Subject: [PATCH 470/766] added environment variables to test config --- test/rendersettings.py | 6 +++--- test/test_client.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/rendersettings.py b/test/rendersettings.py index 19e9a035..44590eaf 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -4,11 +4,11 @@ import os DEFAULT_RENDER = { - 'host': 'http://renderhost', - 'port': 8080, + 'host': os.environ.get('RENDER_HOST','http://renderhost'), + 'port': os.environ.get('RENDER_PORT',8080), 'owner': 'renderowner', 'project': 'renderproject', - 'client_scripts': '/path/to/client_scripts' + 'client_scripts': os.environ.get('RENDER_CLIENT_SCRIPTS','/path/to/client_scripts') } DEFAULT_RENDER_CLIENT = dict(DEFAULT_RENDER, **{ diff --git a/test/test_client.py b/test/test_client.py index 5562657e..84aaf4af 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -3,11 +3,11 @@ import rendersettings args = { - 'host': 'renderhost', - 'port': 8080, + 'host': os.environ.get('RENDER_HOST','renderhost'), + 'port': os.environ.get('RENDER_PORT',8080), 'owner': 'renderowner', 'project': 'renderproject', - 'client_scripts': '/path/to/client_scripts' + 'client_scripts': os.environ.get('RENDER_CLIENT_SCRIPTS',/path/to/client_scripts' } def test_render_client(): r = renderapi.render.connect(**args) From 052e15bad95313a1d1c2446d82cb69f8a82b9a5e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 7 Oct 2017 10:05:48 -0700 Subject: [PATCH 471/766] added tox --- .gitignore | 1 + MANIFEST.in | 2 ++ tox.ini | 13 +++++++++++++ 3 files changed, 16 insertions(+) create mode 100644 MANIFEST.in create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 2d958883..290b7718 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ test-reports/ docs/_build/ htmlcov/ .cache +.tox diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..768217ce --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include requirements.txt +include test_requirements.txt \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..72655f4c --- /dev/null +++ b/tox.ini @@ -0,0 +1,13 @@ +# Tox (https://tox.readthedocs.io/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + + +[tox] +envlist = py27, py36 + +[testenv] +commands = pytest +deps = -rrequirements.txt + -rtest_requirements.txt \ No newline at end of file From 9c6044588d4a06dc8bb68b144e33abcc5531afe8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 7 Oct 2017 10:08:17 -0700 Subject: [PATCH 472/766] fixed bug in test_client and configured tox to only run unit tests --- test/test_client.py | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_client.py b/test/test_client.py index 84aaf4af..cd79c720 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -7,7 +7,7 @@ 'port': os.environ.get('RENDER_PORT',8080), 'owner': 'renderowner', 'project': 'renderproject', - 'client_scripts': os.environ.get('RENDER_CLIENT_SCRIPTS',/path/to/client_scripts' + 'client_scripts': os.environ.get('RENDER_CLIENT_SCRIPTS','/path/to/client_scripts') } def test_render_client(): r = renderapi.render.connect(**args) diff --git a/tox.ini b/tox.ini index 72655f4c..33ea9451 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,6 @@ envlist = py27, py36 [testenv] -commands = pytest +commands = pytest test deps = -rrequirements.txt -rtest_requirements.txt \ No newline at end of file From 8aa6fc71b12de2541307fcbcdb5ab4e37cd060d4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 7 Oct 2017 10:31:59 -0700 Subject: [PATCH 473/766] removed env variables --- test/rendersettings.py | 6 +++--- test/test_client.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/rendersettings.py b/test/rendersettings.py index 44590eaf..19e9a035 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -4,11 +4,11 @@ import os DEFAULT_RENDER = { - 'host': os.environ.get('RENDER_HOST','http://renderhost'), - 'port': os.environ.get('RENDER_PORT',8080), + 'host': 'http://renderhost', + 'port': 8080, 'owner': 'renderowner', 'project': 'renderproject', - 'client_scripts': os.environ.get('RENDER_CLIENT_SCRIPTS','/path/to/client_scripts') + 'client_scripts': '/path/to/client_scripts' } DEFAULT_RENDER_CLIENT = dict(DEFAULT_RENDER, **{ diff --git a/test/test_client.py b/test/test_client.py index cd79c720..5562657e 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -3,11 +3,11 @@ import rendersettings args = { - 'host': os.environ.get('RENDER_HOST','renderhost'), - 'port': os.environ.get('RENDER_PORT',8080), + 'host': 'renderhost', + 'port': 8080, 'owner': 'renderowner', 'project': 'renderproject', - 'client_scripts': os.environ.get('RENDER_CLIENT_SCRIPTS','/path/to/client_scripts') + 'client_scripts': '/path/to/client_scripts' } def test_render_client(): r = renderapi.render.connect(**args) From 007e1fbcf2980c00c2435a1e4b890378ce0b8241 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sun, 8 Oct 2017 09:29:47 -0700 Subject: [PATCH 474/766] made python 3 compatible with unit tests --- renderapi/resolvedtiles.py | 2 +- renderapi/transform.py | 20 ++++++++++++-------- test/test_transform.py | 12 ++++++++++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py index 64893a30..d76f8e37 100644 --- a/renderapi/resolvedtiles.py +++ b/renderapi/resolvedtiles.py @@ -39,7 +39,7 @@ def from_dict(self, d): self.transforms = [] for ts in d['tileIdToSpecMap'].values(): self.tilespecs.append(TileSpec(json=ts)) - for transformId, tform_json in d['transformIdToSpecMap'].iteritems(): + for transformId, tform_json in d['transformIdToSpecMap'].items(): tform_json['transformId'] = transformId self.transforms.append(load_transform_json(tform_json)) diff --git a/renderapi/transform.py b/renderapi/transform.py index 3e0689a1..0f79dd29 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1240,10 +1240,10 @@ def _format_raveled_params(raveled_params): a (2,K/2) matrix of parameters, with first row for x and 2nd row for y """ - + halfway = int (len(raveled_params) / 2) return np.array( - [[float(d) for d in raveled_params[:len(raveled_params) / 2]], - [float(d) for d in raveled_params[len(raveled_params) / 2:]]]) + [[float(d) for d in raveled_params[:halfway]], + [float(d) for d in raveled_params[halfway:]]]) def tform(self, points): """transform a set of points through this transformation @@ -1538,16 +1538,20 @@ def estimate_transformsum(transformlist, src=None, order=2): def flatten(l): """generator-iterator to flatten deep lists of lists""" for i in l: - if (isinstance(i, Iterable) and not - isinstance(i, basestring)): - for sub in flatten(i): - yield sub + if isinstance(i, Iterable): + try: + notstring = isinstance(i, basestring) + except NameError as e: + notstring = isinstance(i, str) + if notstring: + for sub in flatten(i): + yield sub else: yield i dstpts = estimate_dstpts(transformlist, src) tforms = flatten(transformlist) - if all([tform.className == AffineModel.className + if all([(tform.className == AffineModel.className) for tform in tforms]): am = AffineModel() am.estimate(A=src, B=dstpts, return_params=False) diff --git a/test/test_transform.py b/test/test_transform.py index 2435f6cd..b266de54 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -3,7 +3,13 @@ import numpy as np import scipy.linalg import rendersettings +import importlib +def cross_py23_reload(module): + try: + reload(module) + except NameError as e: + importlib.reload(module) def test_affine_rot_90(): am = renderapi.transform.AffineModel() @@ -81,7 +87,9 @@ def noscipy_import(name, globals=None, locals=None, raise ImportError return realimport(name, globals, locals, fromlist, level) builtins.__import__ = noscipy_import - reload(renderapi.transform) + + cross_py23_reload(renderapi.transform) + assert(renderapi.transform.svd is np.linalg.svd if use_numpy else renderapi.transform.svd is scipy.linalg.svd) @@ -99,7 +107,7 @@ def noscipy_import(name, globals=None, locals=None, if use_numpy: builtins.__import__ = realimport - reload(renderapi.transform) + cross_py23_reload(renderapi.transform) assert(renderapi.transform.svd is scipy.linalg.svd) From 00b5cc7c2503a9cefa86da496cf57807b6d20b8e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 9 Oct 2017 10:18:52 -0700 Subject: [PATCH 475/766] made changes to allow python3 compatibility --- integration_tests/test_client_integrated.py | 2 +- integration_tests/test_stack_integrated.py | 2 +- renderapi/client.py | 2 +- renderapi/coordinate.py | 8 ++++---- renderapi/utils.py | 3 ++- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 0cbbe8f8..5da29ec8 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -42,7 +42,7 @@ def render_example_tilespec_and_transforms(): tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] - print tforms + root.debug(tforms) return (tilespecs, tforms) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index f251ccf3..fcc5e5fe 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -62,7 +62,7 @@ def render_example_tilespec_and_transforms(): tilespecs = [renderapi.tilespec.TileSpec(json=ts) for ts in ts_json] tforms = [renderapi.transform.load_transform_json(td) for td in tform_json] - print tforms + root.debug(tforms) return (tilespecs, tforms) diff --git a/renderapi/client.py b/renderapi/client.py index 338162be..db8c15de 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -321,7 +321,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, memGB=memGB, **kwargs) # TODO this is a weird way to do splits.... is that okay? - tilespec_groups = [tilespecs[i::poolsize] for i in xrange(poolsize)] + tilespec_groups = [tilespecs[i::poolsize] for i in range(poolsize)] with WithPool(poolsize) as pool: pool.map(partial_import, tilespec_groups) if close_stack: diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 1eeaa3a1..5d1b6178 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -3,7 +3,7 @@ coordinate mapping functions for render api ''' from .render import format_preamble, renderaccess -from .utils import NullHandler +from .utils import NullHandler, renderdumps, renderdump from .client import coordinateClient from .errors import RenderError import requests @@ -176,7 +176,7 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%s/world-to-local-coordinates" % (str(z)) - r = session.put(request_url, data=json.dumps(d), + r = session.put(request_url, data=renderdumps(d), headers={"content-type": "application/json"}) return r.json() @@ -230,7 +230,7 @@ def local_to_world_coordinates_batch(stack, d, z, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%s/local-to-world-coordinates" % (str(z)) - r = session.put(request_url, data=json.dumps(d), + r = session.put(request_url, data=renderdumps(d), headers={"content-type": "application/json"}) try: return r.json() @@ -529,7 +529,7 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, mode='w', delete=False) as f: logger.debug('jsondata:{}'.format(jsondata)) json_inpath = f.name - json.dump(jsondata, f) + renderdump(jsondata, f) # get a temporary location for the output with tempfile.NamedTemporaryFile( diff --git a/renderapi/utils.py b/renderapi/utils.py index afa2a795..8eebdbad 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -8,7 +8,7 @@ import copy import json from .errors import RenderError - +import numpy class NullHandler(logging.Handler): """handler to avoid logging errors for, e.g., missing logger setup""" @@ -41,6 +41,7 @@ def default(self, obj): json encodable datatype """ + if isinstance(obj, numpy.integer): return int(obj) to_dict = getattr(obj, "to_dict", None) if callable(to_dict): return obj.to_dict() From e29ba5b8a52e581e34ec83765d555fe64c4b2a57 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 09:32:31 -0700 Subject: [PATCH 476/766] bumping version to denote python 3 compatability --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7342a3fd..a7290c65 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.0.3', + version='1.2.0', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From 95128fdb4633663fe495a5c249b5de882e7b9e85 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:10:14 -0700 Subject: [PATCH 477/766] add travis --- .travis.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..a4b1fbfd --- /dev/null +++ b/.travis.yml @@ -0,0 +1,31 @@ +language: python +python: + - "2.7" + - "3.5" + - "3.5-dev" # 3.5 development branch + - "3.6" + - "3.6-dev" # 3.6 development branch +services: + - docker +# command to install dependencies +addons: + apt: + packages: + - libblas-dev + - liblapack-dev + - libatlas-base-dev + - libopenblas-base + - libopenblas-dev + - gfortran + - oracle-java8-set-default +install: + - pip install -r requirements.txt + - pip install -r test_requirements.txt + - cp -R example_1 /tmp/ +before_install: + - docker-compose up -d +env: + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=/tmp RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render-client-jar +# command to run tests +script: + - python setup.py test # or py.test for Python versions 3.5 and below From 2437bb624f4a48ebd54b1392cf3fa297539839c4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:17:19 -0700 Subject: [PATCH 478/766] changed travis config --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4b1fbfd..1ff18032 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,14 +18,16 @@ addons: - libopenblas-dev - gfortran - oracle-java8-set-default + - maven install: - pip install -r requirements.txt - pip install -r test_requirements.txt - - cp -R example_1 /tmp/ + - git clone https://github.com/saalfeldlab/render.git + - mvn package before_install: - docker-compose up -d env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=/tmp RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render-client-jar + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From f019834cdd7befed5c5a449d007fba4e3e2aa488 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:18:43 -0700 Subject: [PATCH 479/766] added docker-compose --- docker-compose.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..a6e92280 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: "2.0" +services: + renderservice: + image: fcollman/render-ws + ports: + - "8080:8080" + links: + - mongo + volumes: + - $RENDER_EXAMPLE_DATA/example_1:/tmp/example_1:ro + mongo: + image: mongo:3.4.2 + ports: + - "27017" + security_opt: + - seccomp:unconfined \ No newline at end of file From f50e0693cb15f0cb223bd88d376fbbdc3cf69d5b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:19:49 -0700 Subject: [PATCH 480/766] skipping tests on building --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1ff18032..c7070b55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ install: - pip install -r requirements.txt - pip install -r test_requirements.txt - git clone https://github.com/saalfeldlab/render.git - - mvn package + - mvn package -DskipTests before_install: - docker-compose up -d env: From 20a45256f6b465c1e1067b485cea61403969baee Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:23:32 -0700 Subject: [PATCH 481/766] travis update --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7070b55..1fbf46b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,12 @@ addons: install: - pip install -r requirements.txt - pip install -r test_requirements.txt - - git clone https://github.com/saalfeldlab/render.git - - mvn package -DskipTests + - git clone https://github.com/saalfeldlab/render.git fcollman/render + - cd fcollman/render && mvn package -DskipTests before_install: - docker-compose up -d env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From 78bc87d24fa53c1f5b79a323d0039355fc8be570 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:32:51 -0700 Subject: [PATCH 482/766] trying to move git clone step --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1fbf46b7..37216f22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,10 @@ addons: install: - pip install -r requirements.txt - pip install -r test_requirements.txt + +before_install: - git clone https://github.com/saalfeldlab/render.git fcollman/render - cd fcollman/render && mvn package -DskipTests -before_install: - docker-compose up -d env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/scripts From 46be4963e840fd47fbb9455d6fa152ee2c0f646a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:43:19 -0700 Subject: [PATCH 483/766] modified build --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 37216f22..3f2d6ef9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,10 +22,9 @@ addons: install: - pip install -r requirements.txt - pip install -r test_requirements.txt - before_install: - git clone https://github.com/saalfeldlab/render.git fcollman/render - - cd fcollman/render && mvn package -DskipTests + - mvn package -DskipTests -f fcollman/render/pom.xml - docker-compose up -d env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/scripts From 6308bed886adefb810202fdac444ef0a103bd6a3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 16:52:00 -0700 Subject: [PATCH 484/766] only build java client --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3f2d6ef9..74dcfb42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: - pip install -r test_requirements.txt before_install: - git clone https://github.com/saalfeldlab/render.git fcollman/render - - mvn package -DskipTests -f fcollman/render/pom.xml + - mvn package -pl render-ws-java-client -am -DskipTests -f fcollman/render/pom.xml - docker-compose up -d env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/scripts From b2a0e73a01b97f0ce5dddc2680099def9563d4d3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 17:15:37 -0700 Subject: [PATCH 485/766] remove references to import_json.sh --- renderapi/client.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index db8c15de..09a4e19d 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -44,7 +44,7 @@ def __exit__(self, *args, **kwargs): @renderaccess def import_single_json_file(stack, jsonfile, transformFile=None, - client_scripts=None, host=None, port=None, + client_script=None, memGB=None, host=None, port=None, owner=None, project=None, render=None, **kwargs): """calls client script to import given jsonfile @@ -67,14 +67,9 @@ def import_single_json_file(stack, jsonfile, transformFile=None, my_env = os.environ.copy() stack_params = make_stack_params( host, port, owner, project, stack) - cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - [jsonfile] - logger.debug(cmd) - proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - proc.wait() - logger.debug(proc.stdout.read()) + call_run_ws_client('org.janelia.render.client.ImportJsonClient', + stack_params + transform_params + [jsonfile], + client_script=client_script,memGB=memGB) @renderaccess @@ -159,7 +154,7 @@ def import_jsonfiles_parallel( @renderaccess def import_jsonfiles(stack, jsonfiles, transformFile=None, - client_scripts=None, host=None, port=None, + client_script=None, memGB=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): """import jsons using client script serially @@ -186,14 +181,9 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, my_env = os.environ.copy() stack_params = make_stack_params( host, port, owner, project, stack) - cmd = [os.path.join(client_scripts, 'import_json.sh')] + \ - stack_params + \ - transform_params + \ - jsonfiles - logger.debug(cmd) - proc = subprocess.Popen(cmd, env=my_env, stdout=subprocess.PIPE) - proc.wait() - logger.debug(proc.stdout.read()) + call_run_ws_client('org.janelia.render.client.ImportJsonClient', + stack_params + transform_params + jsonfiles, + client_script=client_script,memGB=memGB) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From b252764374b5f02d5c66aa739c70ffdbe24d6bdb Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 17:30:58 -0700 Subject: [PATCH 486/766] further cuts --- renderapi/client.py | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 09a4e19d..7258c452 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -43,7 +43,8 @@ def __exit__(self, *args, **kwargs): @renderaccess -def import_single_json_file(stack, jsonfile, transformFile=None, +def import_single_json_file(stack, jsonfile, transformFile=None, + subprocess_mode=None, client_script=None, memGB=None, host=None, port=None, owner=None, project=None, render=None, **kwargs): """calls client script to import given jsonfile @@ -64,12 +65,11 @@ def import_single_json_file(stack, jsonfile, transformFile=None, transform_params = [] else: transform_params = ['--transformFile', transformFile] - my_env = os.environ.copy() stack_params = make_stack_params( host, port, owner, project, stack) call_run_ws_client('org.janelia.render.client.ImportJsonClient', stack_params + transform_params + [jsonfile], - client_script=client_script,memGB=memGB) + client_script=client_script,memGB=memGB,subprocess_mode=subprocess_mode) @renderaccess @@ -153,7 +153,7 @@ def import_jsonfiles_parallel( @renderaccess -def import_jsonfiles(stack, jsonfiles, transformFile=None, +def import_jsonfiles(stack, jsonfiles, transformFile=None, subprocess_mode=None, client_script=None, memGB=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): @@ -178,22 +178,22 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, transform_params = [] else: transform_params = ['--transformFile', transformFile] - my_env = os.environ.copy() stack_params = make_stack_params( host, port, owner, project, stack) call_run_ws_client('org.janelia.render.client.ImportJsonClient', stack_params + transform_params + jsonfiles, - client_script=client_script,memGB=memGB) + client_script=client_script,memGB=memGB,subprocess_mode=subprocess_mode) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @renderaccess def import_jsonfiles_validate_client(stack, jsonfiles, - transformFile=None, client_scripts=None, + transformFile=None, client_script=None, host=None, port=None, owner=None, project=None, close_stack=True, mem=6, - validator=None, + validator=None, subprocess_mode=None, + memGB=None, render=None, **kwargs): """Uses java client for parallelization and validation @@ -222,18 +222,14 @@ def import_jsonfiles_validate_client(stack, jsonfiles, my_env = os.environ.copy() stack_params = make_stack_params(host, port, owner, project, stack) - cmd = [os.path.join(client_scripts, 'run_ws_client.sh')] + \ - ['{}G'.format(str(int(mem))), - 'org.janelia.render.client.ImportJsonClient'] + \ + set_stack_state(stack, 'LOADING', host, port, owner, project) + + call_run_ws_client('org.janelia.render.client.ImportJsonClient', stack_params + \ validator_params + \ transform_params + \ - jsonfiles - - set_stack_state(stack, 'LOADING', host, port, owner, project) - logger.debug(cmd) - - subprocess.call(cmd, env=my_env) + jsonfiles,client_script=client_script, + memGB=memGB,subprocess_mode=subprocess_mode) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) From 2e385fce4adb1773f64b3fdbc2acde574e484086 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 17:42:33 -0700 Subject: [PATCH 487/766] added scripts to repo --- scripts/run_ws_client.sh | 31 +++++++++++++++++++++++++++++++ scripts/setup_java_env.sh | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 scripts/run_ws_client.sh create mode 100644 scripts/setup_java_env.sh diff --git a/scripts/run_ws_client.sh b/scripts/run_ws_client.sh new file mode 100644 index 00000000..44dc9991 --- /dev/null +++ b/scripts/run_ws_client.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +######################################### +# +# Generic shell script wrapper for running Java clients +# that use the render web services. +# +# USAGE: $0 [client-arg-0] ... [client-arg-n] +# +######################################### + +if (( $# < 2 )); then + echo """ +ERROR: missing memory and/or main class parameters + +USAGE: $0 [client-arg-0] ... [client-arg-n] + +EXAMPLE: $0 1G org.janelia.render.client.ImportJsonClient --baseDataUrl http://localhost:8080/render-ws/v1 --owner demo ... +""" + exit 1 +fi + +MEMORY="$1" +MAIN_CLASS="$2" +shift 2 + +ABSOLUTE_SCRIPT=`readlink -m $0` +SCRIPTS_DIR=`dirname ${ABSOLUTE_SCRIPT}` +. ${SCRIPTS_DIR}/setup_java_env.sh ${MEMORY} + +runJavaCommandAndExit ${MAIN_CLASS} $* \ No newline at end of file diff --git a/scripts/setup_java_env.sh b/scripts/setup_java_env.sh new file mode 100644 index 00000000..40287e05 --- /dev/null +++ b/scripts/setup_java_env.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +ABSOLUTE_ENV_SCRIPT=`readlink -m $0` + +export SCRIPTS_DIR=`dirname ${ABSOLUTE_ENV_SCRIPT}` +export REPO_DIR=`readlink -m ${SCRIPTS_DIR}/../..` +export INSTALL_DIR=`readlink -m ${REPO_DIR}/deploy` + +if [ -z "$RENDER_CLIENT_JAR" ] +then + export RENDER_CLIENT_JAR=`readlink -m ${REPO_DIR}/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` +fi +if [ -z "$JAVA_HOME" ] +then + export JAVA_HOME=`readlink -m ${REPO_DIR}/deploy/jdk*` +fi + +export BASE_JAVA_COMMAND="${JAVA_HOME}/bin/java -cp ${RENDER_CLIENT_JAR}" + +# request memory up-front and use serial garbage collector to keep GC threads from taking over cluster node +export JAVA_MEMORY="${1:-1G}" +export JAVA_OPTS="-Xms${JAVA_MEMORY} -Xmx${JAVA_MEMORY} -Djava.awt.headless=true -XX:+UseSerialGC" + +function runJavaCommandAndExit { + COMMAND="${BASE_JAVA_COMMAND} ${JAVA_OPTS} $*" + echo """ + Running: ${COMMAND} + +""" + ${COMMAND} + exit $? +} From 60911e0430e7f1c0ef1ce7ea783574dab85a0c6f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 17:46:21 -0700 Subject: [PATCH 488/766] edited travis to use new scripts --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 74dcfb42..7d12c866 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,9 +25,10 @@ install: before_install: - git clone https://github.com/saalfeldlab/render.git fcollman/render - mvn package -pl render-ws-java-client -am -DskipTests -f fcollman/render/pom.xml + - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - docker-compose up -d env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/scripts + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From 718486b03380be05c1090167166d2209b18206e4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 10 Oct 2017 17:59:37 -0700 Subject: [PATCH 489/766] changed location of scripts --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d12c866..a8282437 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ before_install: - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - docker-compose up -d env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render-python/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From af97c34079eaf8125d6bf89616a63d3977e679bf Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 06:23:16 -0700 Subject: [PATCH 490/766] moving client_scripts --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a8282437..7d12c866 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ before_install: - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - docker-compose up -d env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render-python/scripts + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From 40a00630f906a90f50c69179cad434eb336930b5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 06:29:56 -0700 Subject: [PATCH 491/766] changed permissions --- scripts/run_ws_client.sh | 0 scripts/setup_java_env.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/run_ws_client.sh mode change 100644 => 100755 scripts/setup_java_env.sh diff --git a/scripts/run_ws_client.sh b/scripts/run_ws_client.sh old mode 100644 new mode 100755 diff --git a/scripts/setup_java_env.sh b/scripts/setup_java_env.sh old mode 100644 new mode 100755 From 94fc397abdb6e083cbae42a43c2a118718f15c25 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 06:37:36 -0700 Subject: [PATCH 492/766] moving example data to before_install --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7d12c866..aa1a4b83 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,9 +26,10 @@ before_install: - git clone https://github.com/saalfeldlab/render.git fcollman/render - mvn package -pl render-ws-java-client -am -DskipTests -f fcollman/render/pom.xml - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` + - export RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources - docker-compose up -d env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From d694cd9d36b12ce4e4865b52c807f088828aef1f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 06:45:38 -0700 Subject: [PATCH 493/766] changing example data --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a6e92280..9b9a3fa0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: links: - mongo volumes: - - $RENDER_EXAMPLE_DATA/example_1:/tmp/example_1:ro + - ${RENDER_EXAMPLE_DATA}/example_1:/tmp/example_1:ro mongo: image: mongo:3.4.2 ports: From 78c1437f5cdeb59defd50949fab8a7e0397d1cec Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 06:51:05 -0700 Subject: [PATCH 494/766] trying quotes --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9b9a3fa0..4f6b04bb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: links: - mongo volumes: - - ${RENDER_EXAMPLE_DATA}/example_1:/tmp/example_1:ro + - "${RENDER_EXAMPLE_DATA}/example_1:/tmp/example_1:ro" mongo: image: mongo:3.4.2 ports: From 794e6cdf731d3e483b57f5ccc11d39ce13f119f3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:00:19 -0700 Subject: [PATCH 495/766] adding readlink to env definition --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index aa1a4b83..834f05f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - git clone https://github.com/saalfeldlab/render.git fcollman/render - mvn package -pl render-ws-java-client -am -DskipTests -f fcollman/render/pom.xml - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - - export RENDER_EXAMPLE_DATA=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources + - export RENDER_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources` - docker-compose up -d env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts From 1eddca383c971e95dba76ca00a0baac7168b40a1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:03:52 -0700 Subject: [PATCH 496/766] adding maven caching --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 834f05f3..ece670cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,3 +33,6 @@ env: # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below +cache: + directories: + - $HOME/.m2 \ No newline at end of file From f78a78fa1523eb70524cb4c33a97becff53a7824 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:07:06 -0700 Subject: [PATCH 497/766] trying to debug --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ece670cc..98bfc030 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ before_install: - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - export RENDER_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources` - docker-compose up -d + - ls $RENDER_EXAMPLE_DATA/example_1 env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts # command to run tests From a476f6104e2a1142c79d1f8f62e3980996e57955 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:17:22 -0700 Subject: [PATCH 498/766] copying image data to tmp --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 98bfc030..f49634e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,7 @@ before_install: - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - export RENDER_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources` - docker-compose up -d - - ls $RENDER_EXAMPLE_DATA/example_1 + - mkdir -p /tmp/example_1 && cp -R $RENDER_EXAMPLE_DATA/example_1 /tmp/. env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts # command to run tests From a8555259c4c72571a6359b6ace875c687baba038 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:20:32 -0700 Subject: [PATCH 499/766] adding pip cache --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f49634e0..91f21eee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,7 @@ env: # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below -cache: +cache: + - pip directories: - $HOME/.m2 \ No newline at end of file From db268e6ca4d1f3cf775b5268ddb5954c0909ca24 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:21:33 -0700 Subject: [PATCH 500/766] trying to cache pip --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 91f21eee..a1ecdff1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,6 @@ env: script: - python setup.py test # or py.test for Python versions 3.5 and below cache: - - pip + pip: true directories: - $HOME/.m2 \ No newline at end of file From a5f47303d742d199e8b139b83a1df9b2127ef43e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:26:06 -0700 Subject: [PATCH 501/766] switching the shallow git clone of only master --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a1ecdff1..ff39f8db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ install: - pip install -r requirements.txt - pip install -r test_requirements.txt before_install: - - git clone https://github.com/saalfeldlab/render.git fcollman/render + - git clone --depth 1 https://github.com/saalfeldlab/render.git fcollman/render -b master - mvn package -pl render-ws-java-client -am -DskipTests -f fcollman/render/pom.xml - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - export RENDER_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources` From ea859eb1ea2bdf88b393840211365e61ebb3bf7e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:44:01 -0700 Subject: [PATCH 502/766] moving travis and adding automerging --- .travis.yml => travis/.travis.yml | 5 ++++- travis/merge_script.sh | 34 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) rename .travis.yml => travis/.travis.yml (63%) create mode 100755 travis/merge_script.sh diff --git a/.travis.yml b/travis/.travis.yml similarity index 63% rename from .travis.yml rename to travis/.travis.yml index ff39f8db..15ceb482 100644 --- a/.travis.yml +++ b/travis/.travis.yml @@ -37,4 +37,7 @@ script: cache: pip: true directories: - - $HOME/.m2 \ No newline at end of file + - $HOME/.m2 +secure: "x3rK0ICjrT1yAguEgq7/UyNDDCbbr46j7DsHLzEEgoM4l5crRECfdclgws4s4Z7RH6TRrW3+CKvhJuq/71QRIWBUN/ZUK9REEvgC13mOFFN7PQlhQGHTQPSiJ1oUTtXG5KenNBOOgz9q4vJNFnahn+L+GuN2TiAYRJGbyE8G5A5uFLIhXHVb+Xo295XkarFtX8EFKQmJJzdyFgqU+NsKW2gbq0hIASmQi3swJZhmbzQXhaj0gCuLjQfnDR+3qHzE+v5mQfObk4v6FRf3mdnZRVqV4S67yKONDK5LCkuI69C/1EKPiAfEEg5RKcRIzQlDjApgYVFf+jFjrVy6RLU9xOp2dstSFmynL96N+K4HoRAw53F08WokBih5hbsEwGAb/Fat1fLVy1hqmboF4d5Fy42TXrPmHgkqwlMXABPVqKwoWQTo0sANQdOVNWaF6NyMDTkUwtAzD9IG+Qwu/9v1zleT0VQ92Uk0s8wTDlkVny+8XGv4Pi2sNBcgG+huECNAOQKCeFbMn0LfMoGaKDXNCW4OHp59wXXXJQHAaN8xujwGpwQxJKb3iwf6uY9wnBpfKtaaCqOcP3Y/WhuiuKf4477bZPoQXD+2DFAPlC5nVjn+LzzN1HigdPCWn8F+SianZsL/iuxgEqmiW6OFGGzpqD4zHFNACky3fTS7yBLnd+Q=" +after_success: + - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=cdown/srt travis/merge_script.sh" \ No newline at end of file diff --git a/travis/merge_script.sh b/travis/merge_script.sh new file mode 100755 index 00000000..c172cbce --- /dev/null +++ b/travis/merge_script.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e + +: "${BRANCHES_TO_MERGE_REGEX?}" "${BRANCH_TO_MERGE_INTO?}" +: "${GITHUB_SECRET_TOKEN?}" "${GITHUB_REPO?}" + +export GIT_COMMITTER_EMAIL='travis@travis' +export GIT_COMMITTER_NAME='Travis CI' + +if ! grep -q "$BRANCHES_TO_MERGE_REGEX" <<< "$TRAVIS_BRANCH"; then + printf "Current branch %s doesn't match regex %s, exiting\\n" \ + "$TRAVIS_BRANCH" "$BRANCHES_TO_MERGE_REGEX" >&2 + exit 0 +fi + +# Since Travis does a partial checkout, we need to get the whole thing +repo_temp=$(mktemp -d) +git clone "https://github.com/$GITHUB_REPO" "$repo_temp" + +# shellcheck disable=SC2164 +cd "$repo_temp" + +printf 'Checking out %s\n' "$BRANCH_TO_MERGE_INTO" >&2 +git checkout "$BRANCH_TO_MERGE_INTO" + +printf 'Merging %s\n' "$TRAVIS_COMMIT" >&2 +git merge --ff-only "$TRAVIS_COMMIT" + +printf 'Pushing to %s\n' "$GITHUB_REPO" >&2 + +push_uri="https://$GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO" + +# Redirect to /dev/null to avoid secret leakage +git push "$push_uri" "$BRANCH_TO_MERGE_INTO" >/dev/null 2>&1 +git push "$push_uri" :"$TRAVIS_BRANCH" >/dev/null 2>&1 From 74796748032e8df81ccd03c8371978e2cdad3d1b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:45:09 -0700 Subject: [PATCH 503/766] moved travis file --- travis/{.travis.yml => travis.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename travis/{.travis.yml => travis.yml} (100%) diff --git a/travis/.travis.yml b/travis/travis.yml similarity index 100% rename from travis/.travis.yml rename to travis/travis.yml From 719cee53334f511dea60a40d9fcb1089ae873bde Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:45:43 -0700 Subject: [PATCH 504/766] moving travis again --- {travis => .travis}/merge_script.sh | 0 {travis => .travis}/travis.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {travis => .travis}/merge_script.sh (100%) rename {travis => .travis}/travis.yml (100%) diff --git a/travis/merge_script.sh b/.travis/merge_script.sh similarity index 100% rename from travis/merge_script.sh rename to .travis/merge_script.sh diff --git a/travis/travis.yml b/.travis/travis.yml similarity index 100% rename from travis/travis.yml rename to .travis/travis.yml From 8423b75c4066bab4138b6cab9ba19a041d78c317 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 07:47:01 -0700 Subject: [PATCH 505/766] moving travis components again! --- .travis/travis.yml => .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .travis/travis.yml => .travis.yml (97%) diff --git a/.travis/travis.yml b/.travis.yml similarity index 97% rename from .travis/travis.yml rename to .travis.yml index 15ceb482..cd72a8e0 100644 --- a/.travis/travis.yml +++ b/.travis.yml @@ -40,4 +40,4 @@ cache: - $HOME/.m2 secure: "x3rK0ICjrT1yAguEgq7/UyNDDCbbr46j7DsHLzEEgoM4l5crRECfdclgws4s4Z7RH6TRrW3+CKvhJuq/71QRIWBUN/ZUK9REEvgC13mOFFN7PQlhQGHTQPSiJ1oUTtXG5KenNBOOgz9q4vJNFnahn+L+GuN2TiAYRJGbyE8G5A5uFLIhXHVb+Xo295XkarFtX8EFKQmJJzdyFgqU+NsKW2gbq0hIASmQi3swJZhmbzQXhaj0gCuLjQfnDR+3qHzE+v5mQfObk4v6FRf3mdnZRVqV4S67yKONDK5LCkuI69C/1EKPiAfEEg5RKcRIzQlDjApgYVFf+jFjrVy6RLU9xOp2dstSFmynL96N+K4HoRAw53F08WokBih5hbsEwGAb/Fat1fLVy1hqmboF4d5Fy42TXrPmHgkqwlMXABPVqKwoWQTo0sANQdOVNWaF6NyMDTkUwtAzD9IG+Qwu/9v1zleT0VQ92Uk0s8wTDlkVny+8XGv4Pi2sNBcgG+huECNAOQKCeFbMn0LfMoGaKDXNCW4OHp59wXXXJQHAaN8xujwGpwQxJKb3iwf6uY9wnBpfKtaaCqOcP3Y/WhuiuKf4477bZPoQXD+2DFAPlC5nVjn+LzzN1HigdPCWn8F+SianZsL/iuxgEqmiW6OFGGzpqD4zHFNACky3fTS7yBLnd+Q=" after_success: - - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=cdown/srt travis/merge_script.sh" \ No newline at end of file + - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=cdown/srt .travis/merge_script.sh" \ No newline at end of file From 2435c55d2c3d0d59e10f3b903a03dd7cdb0253e5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 10:38:24 -0700 Subject: [PATCH 506/766] changing java_env to RENDER_JAVA_HOME --- .travis.yml | 1 + scripts/setup_java_env.sh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd72a8e0..52c4aced 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ before_install: - mvn package -pl render-ws-java-client -am -DskipTests -f fcollman/render/pom.xml - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - export RENDER_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources` + - export RENDER_JAVA_HOME=$JAVA_HOME - docker-compose up -d - mkdir -p /tmp/example_1 && cp -R $RENDER_EXAMPLE_DATA/example_1 /tmp/. env: diff --git a/scripts/setup_java_env.sh b/scripts/setup_java_env.sh index 40287e05..76cdedb3 100755 --- a/scripts/setup_java_env.sh +++ b/scripts/setup_java_env.sh @@ -10,9 +10,9 @@ if [ -z "$RENDER_CLIENT_JAR" ] then export RENDER_CLIENT_JAR=`readlink -m ${REPO_DIR}/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` fi -if [ -z "$JAVA_HOME" ] +if [ -z "$RENDER_JAVA_HOME" ] then - export JAVA_HOME=`readlink -m ${REPO_DIR}/deploy/jdk*` + export RENDER_JAVA_HOME=`readlink -m ${REPO_DIR}/deploy/jdk*` fi export BASE_JAVA_COMMAND="${JAVA_HOME}/bin/java -cp ${RENDER_CLIENT_JAR}" From 13a06b0fa3d7731ec5a4aaef02e90e6aaa1eefa4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 12 Oct 2017 16:11:25 -0700 Subject: [PATCH 507/766] implemented stack bounds in sectionClient --- integration_tests/test_client_integrated.py | 26 +++++++++++++++++++++ renderapi/client.py | 20 ++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 5da29ec8..a16e0d28 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -10,6 +10,7 @@ from test_data import (render_host, render_port, client_script_location, tilespec_file, tform_file, test_pool_size) from pathos.multiprocessing import ProcessingPool as Pool +import PIL root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -163,6 +164,31 @@ def test_renderSectionClient(render, teststack): pngfiles += [f for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) +def test_renderSectionClient_bounded(render,teststack): + bounds = renderapi.stack.get_stack_bounds(teststack,render=render) + + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + root_directory = tempfile.mkdtemp() + root.debug('section_directory:{}'.format(root_directory)) + renderapi.client.renderSectionClient(teststack, + root_directory, + zvalues[0], + scale=.05, + render=render, + bounds=bounds, + format='png') + + section_directory = os.path.join( + root_directory, 'test_project', teststack, 'sections_at_0.05') + pngfiles = [] + for (dirpath, dirname, filenames) in os.walk(section_directory): + pngfiles += [f for f in filenames if f.endswith('png')] + assert len(pngfiles) == 1 + img=PIL.Image.open(pngfiles[0]) + width,height = img.size + assert(width == bounds['maxX']-bounds['minX']) + assert(height == bounds['maxY']-bounds['minY']) + def test_importTransformChangesClient(render, teststack): deststack = 'test_stack_TCC' diff --git a/renderapi/client.py b/renderapi/client.py index 7258c452..9cf129a5 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -676,7 +676,8 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, @renderaccess def renderSectionClient(stack, rootDirectory, zs, scale=None, - maxIntensity=None, minIntensity=None, format=None, + maxIntensity=None, minIntensity=None, bounds=None, + format=None, doFilter=None, fillWithNoise=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, @@ -698,6 +699,8 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, value todisplay as white on a linear colormap minIntensity : int value to display as black on a linear colormap + bounds: dict + dictionary with keys of minX maxX minY maxY format : str output image format in 'PNG', 'TIFF', 'JPEG' doFilter : str @@ -708,13 +711,26 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, image values with uniform noise """ + if bounds is not None: + if bounds['maxX'] Date: Thu, 12 Oct 2017 16:53:22 -0700 Subject: [PATCH 508/766] fixing bug to make it a list of str of integers for CLI --- renderapi/client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 9cf129a5..a4a16b3d 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -717,7 +717,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, if bounds['maxY'] Date: Fri, 13 Oct 2017 07:03:06 -0700 Subject: [PATCH 509/766] fixed test --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index a16e0d28..4b60263b 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -172,7 +172,7 @@ def test_renderSectionClient_bounded(render,teststack): root.debug('section_directory:{}'.format(root_directory)) renderapi.client.renderSectionClient(teststack, root_directory, - zvalues[0], + [zvalues[0]], scale=.05, render=render, bounds=bounds, From 00e96a3f5e107523cdeb52c3f27cff1235be5578 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:03:51 -0700 Subject: [PATCH 510/766] fixed scale of output --- integration_tests/test_client_integrated.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 4b60263b..a3649c91 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -164,7 +164,7 @@ def test_renderSectionClient(render, teststack): pngfiles += [f for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) -def test_renderSectionClient_bounded(render,teststack): +def test_renderSectionClient_bounded(render,teststack,scale=.05): bounds = renderapi.stack.get_stack_bounds(teststack,render=render) zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) @@ -173,7 +173,7 @@ def test_renderSectionClient_bounded(render,teststack): renderapi.client.renderSectionClient(teststack, root_directory, [zvalues[0]], - scale=.05, + scale=scale, render=render, bounds=bounds, format='png') @@ -186,8 +186,8 @@ def test_renderSectionClient_bounded(render,teststack): assert len(pngfiles) == 1 img=PIL.Image.open(pngfiles[0]) width,height = img.size - assert(width == bounds['maxX']-bounds['minX']) - assert(height == bounds['maxY']-bounds['minY']) + assert(width == (bounds['maxX']-bounds['minX'])*scale) + assert(height == (bounds['maxY']-bounds['minY'])*scale) def test_importTransformChangesClient(render, teststack): From eb5501eb3f7399eefb0af855b6d83245a22df940 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:16:20 -0700 Subject: [PATCH 511/766] made bounds a comma sep thing --- renderapi/client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index a4a16b3d..c651d16c 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -717,11 +717,9 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, if bounds['maxY'] Date: Fri, 13 Oct 2017 07:17:31 -0700 Subject: [PATCH 512/766] trying to use new render scripts --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52c4aced..db45e12a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ before_install: - docker-compose up -d - mkdir -p /tmp/example_1 && cp -R $RENDER_EXAMPLE_DATA/example_1 /tmp/. env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/scripts + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From 2045c6c094b56e8548dad5b63ad835664c59640d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:19:42 -0700 Subject: [PATCH 513/766] typo --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index c651d16c..8bbca593 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -718,7 +718,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, raise ClientScriptError('maxY:{} is less than minY:{}'.format(bounds['maxY'],bounds['minY'])) try: bound_list= ','.join(map(lambda x: str(int(x)), - [bounds['minX'],bounds['maxX'],bounds['minY'],bounds['maxY'])) + [bounds['minX'],bounds['maxX'],bounds['minY'],bounds['maxY']])) bound_param = ['--bounds', bound_list] except KeyError as e: raise ClientScriptError('bounds does not contain correct keys {}'.format(bounds)) From d32ff5224b23fb24eec3d7bbf1c5b02a0336a36f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:28:46 -0700 Subject: [PATCH 514/766] fixed full path in test --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index a3649c91..12f14a84 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -182,7 +182,7 @@ def test_renderSectionClient_bounded(render,teststack,scale=.05): root_directory, 'test_project', teststack, 'sections_at_0.05') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(section_directory): - pngfiles += [f for f in filenames if f.endswith('png')] + pngfiles += [os.path.join(dirpath,f) for f in filenames if f.endswith('png')] assert len(pngfiles) == 1 img=PIL.Image.open(pngfiles[0]) width,height = img.size From 57578f278e4bf141a12f9958c6a3c4ed2f00f265 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:36:46 -0700 Subject: [PATCH 515/766] changing test assertion to be within 1 pixel of expected --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 12f14a84..b4f4d5d9 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -186,8 +186,8 @@ def test_renderSectionClient_bounded(render,teststack,scale=.05): assert len(pngfiles) == 1 img=PIL.Image.open(pngfiles[0]) width,height = img.size - assert(width == (bounds['maxX']-bounds['minX'])*scale) - assert(height == (bounds['maxY']-bounds['minY'])*scale) + assert(np.abs(width - (bounds['maxX']-bounds['minX'])*scale)<1) + assert(np.abs(height - (bounds['maxY']-bounds['minY'])*scale)<1) def test_importTransformChangesClient(render, teststack): From 29761ef7419256bc2301fb12611d4cfe62bdced7 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:47:02 -0700 Subject: [PATCH 516/766] added tests for bad bounds --- integration_tests/test_client_integrated.py | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index b4f4d5d9..33d5177b 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -164,6 +164,7 @@ def test_renderSectionClient(render, teststack): pngfiles += [f for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) + def test_renderSectionClient_bounded(render,teststack,scale=.05): bounds = renderapi.stack.get_stack_bounds(teststack,render=render) @@ -189,6 +190,36 @@ def test_renderSectionClient_bounded(render,teststack,scale=.05): assert(np.abs(width - (bounds['maxX']-bounds['minX'])*scale)<1) assert(np.abs(height - (bounds['maxY']-bounds['minY'])*scale)<1) +def test_renderSectionClient_badBounds(teststack,scale=.05): + root_directory = tempfile.mkdtemp() + root.debug('section_directory:{}'.format(root_directory)) + with pytest.raises(renderapi.client.ClientScriptError) as e: + bounds = {} + renderapi.client.renderSectionClient(teststack, + root_directory, + [0], + scale=scale, + render=render, + bounds=bounds, + format='png') + with pytest.raises(renderapi.client.ClientScriptError) as e: + bounds = {'maxX':1000,'minX':2000,'minY':1000,'maxY':2000} + renderapi.client.renderSectionClient(teststack, + root_directory, + [0], + scale=scale, + render=render, + bounds=bounds, + format='png') + with pytest.raises(renderapi.client.ClientScriptError) as e: + bounds = {'maxX':2000,'minX':1000,'minY':2000,'maxY':1000} + renderapi.client.renderSectionClient(teststack, + root_directory, + [0], + scale=scale, + render=render, + bounds=bounds, + format='png') def test_importTransformChangesClient(render, teststack): deststack = 'test_stack_TCC' From 49d80d73fed4e1ab10c488b3c7253d5af49bc328 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:56:59 -0700 Subject: [PATCH 517/766] parameterize tests --- integration_tests/test_client_integrated.py | 114 +++++++------------- 1 file changed, 41 insertions(+), 73 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 33d5177b..3025efd6 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -146,80 +146,48 @@ def test_tile_pair_client(render, teststack, **kwargs): assert len(tilepairjson['neighborPairs']) > 3 -def test_renderSectionClient(render, teststack): - zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) +@pytest.mark.parametrize("bounds,raises", [ + ({}, True), + ({'maxX': 1000, 'minX': 2000, 'minY': 1000, 'maxY': 2000}, True), + ({'maxX': 2000, 'minX': 1000, 'minY': 2000, 'maxY': 1000}, True), + ({'maxX': 2000, 'minX': 1000, 'minY': 1000, 'maxY': 2000}, False), + (None, False) +]) +def test_renderSectionClient(render,teststack, bounds, raises, scale=.05): root_directory = tempfile.mkdtemp() root.debug('section_directory:{}'.format(root_directory)) - renderapi.client.renderSectionClient(teststack, - root_directory, - zvalues, - scale=.05, - render=render, - format='png') - - section_directory = os.path.join( - root_directory, 'test_project', teststack, 'sections_at_0.05') - pngfiles = [] - for (dirpath, dirname, filenames) in os.walk(section_directory): - pngfiles += [f for f in filenames if f.endswith('png')] - assert len(pngfiles) == len(zvalues) - - -def test_renderSectionClient_bounded(render,teststack,scale=.05): - bounds = renderapi.stack.get_stack_bounds(teststack,render=render) - zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) - root_directory = tempfile.mkdtemp() - root.debug('section_directory:{}'.format(root_directory)) - renderapi.client.renderSectionClient(teststack, - root_directory, - [zvalues[0]], - scale=scale, - render=render, - bounds=bounds, - format='png') - - section_directory = os.path.join( - root_directory, 'test_project', teststack, 'sections_at_0.05') - pngfiles = [] - for (dirpath, dirname, filenames) in os.walk(section_directory): - pngfiles += [os.path.join(dirpath,f) for f in filenames if f.endswith('png')] - assert len(pngfiles) == 1 - img=PIL.Image.open(pngfiles[0]) - width,height = img.size - assert(np.abs(width - (bounds['maxX']-bounds['minX'])*scale)<1) - assert(np.abs(height - (bounds['maxY']-bounds['minY'])*scale)<1) - -def test_renderSectionClient_badBounds(teststack,scale=.05): - root_directory = tempfile.mkdtemp() - root.debug('section_directory:{}'.format(root_directory)) - with pytest.raises(renderapi.client.ClientScriptError) as e: - bounds = {} - renderapi.client.renderSectionClient(teststack, - root_directory, - [0], - scale=scale, - render=render, - bounds=bounds, - format='png') - with pytest.raises(renderapi.client.ClientScriptError) as e: - bounds = {'maxX':1000,'minX':2000,'minY':1000,'maxY':2000} - renderapi.client.renderSectionClient(teststack, - root_directory, - [0], - scale=scale, - render=render, - bounds=bounds, - format='png') - with pytest.raises(renderapi.client.ClientScriptError) as e: - bounds = {'maxX':2000,'minX':1000,'minY':2000,'maxY':1000} + + if raises: + with pytest.raises(renderapi.client.ClientScriptError) as e: + renderapi.client.renderSectionClient(teststack, + root_directory, + zvalues, + scale=scale, + render=render, + bounds=bounds, + format='png') + else: renderapi.client.renderSectionClient(teststack, - root_directory, - [0], - scale=scale, - render=render, - bounds=bounds, - format='png') + root_directory, + zvalues, + scale=scale, + render=render, + bounds=bounds, + format='png') + pngfiles = [] + for (dirpath, dirname, filenames) in os.walk(root_directory): + pngfiles += [f for f in filenames if f.endswith('png')] + assert len(pngfiles) == len(zvalues) + if bounds is not None: + for f in pngfiles: + img = PIL.Image.open(f) + width, height = img.size + assert( + np.abs(width - (bounds['maxX'] - bounds['minX']) * scale) < 1) + assert( + np.abs(height - (bounds['maxY'] - bounds['minY']) * scale) < 1) + def test_importTransformChangesClient(render, teststack): deststack = 'test_stack_TCC' @@ -234,9 +202,9 @@ def test_importTransformChangesClient(render, teststack): teststack, deststack, TCCjson, changeMode='APPEND', render=render) renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) os.remove(TCCjson) - + output_ts = renderapi.tilespec.get_tile_specs_from_stack( - deststack, render=render) + deststack, render=render) assert all([ts.tforms[-1].to_dict() == tform_to_append.to_dict() for ts in output_ts]) @@ -257,7 +225,7 @@ def test_transformSectionClient(render, teststack, renderapi.stack.set_stack_state(deststack, 'COMPLETE', render=render) output_ts = renderapi.tilespec.get_tile_specs_from_stack( - deststack, render=render) + deststack, render=render) root.debug(output_ts[0].tforms[-1].to_dict()) root.debug(output_ts[-1].tforms[-1].to_dict()) root.debug(tform.to_dict()) From de4ed56bd6022e4cde0175687f4eac10f70ec0c1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 07:59:34 -0700 Subject: [PATCH 518/766] removing scripts because they are now in mainline render --- scripts/run_ws_client.sh | 31 ------------------------------- scripts/setup_java_env.sh | 32 -------------------------------- 2 files changed, 63 deletions(-) delete mode 100755 scripts/run_ws_client.sh delete mode 100755 scripts/setup_java_env.sh diff --git a/scripts/run_ws_client.sh b/scripts/run_ws_client.sh deleted file mode 100755 index 44dc9991..00000000 --- a/scripts/run_ws_client.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -######################################### -# -# Generic shell script wrapper for running Java clients -# that use the render web services. -# -# USAGE: $0 [client-arg-0] ... [client-arg-n] -# -######################################### - -if (( $# < 2 )); then - echo """ -ERROR: missing memory and/or main class parameters - -USAGE: $0 [client-arg-0] ... [client-arg-n] - -EXAMPLE: $0 1G org.janelia.render.client.ImportJsonClient --baseDataUrl http://localhost:8080/render-ws/v1 --owner demo ... -""" - exit 1 -fi - -MEMORY="$1" -MAIN_CLASS="$2" -shift 2 - -ABSOLUTE_SCRIPT=`readlink -m $0` -SCRIPTS_DIR=`dirname ${ABSOLUTE_SCRIPT}` -. ${SCRIPTS_DIR}/setup_java_env.sh ${MEMORY} - -runJavaCommandAndExit ${MAIN_CLASS} $* \ No newline at end of file diff --git a/scripts/setup_java_env.sh b/scripts/setup_java_env.sh deleted file mode 100755 index 76cdedb3..00000000 --- a/scripts/setup_java_env.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -ABSOLUTE_ENV_SCRIPT=`readlink -m $0` - -export SCRIPTS_DIR=`dirname ${ABSOLUTE_ENV_SCRIPT}` -export REPO_DIR=`readlink -m ${SCRIPTS_DIR}/../..` -export INSTALL_DIR=`readlink -m ${REPO_DIR}/deploy` - -if [ -z "$RENDER_CLIENT_JAR" ] -then - export RENDER_CLIENT_JAR=`readlink -m ${REPO_DIR}/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` -fi -if [ -z "$RENDER_JAVA_HOME" ] -then - export RENDER_JAVA_HOME=`readlink -m ${REPO_DIR}/deploy/jdk*` -fi - -export BASE_JAVA_COMMAND="${JAVA_HOME}/bin/java -cp ${RENDER_CLIENT_JAR}" - -# request memory up-front and use serial garbage collector to keep GC threads from taking over cluster node -export JAVA_MEMORY="${1:-1G}" -export JAVA_OPTS="-Xms${JAVA_MEMORY} -Xmx${JAVA_MEMORY} -Djava.awt.headless=true -XX:+UseSerialGC" - -function runJavaCommandAndExit { - COMMAND="${BASE_JAVA_COMMAND} ${JAVA_OPTS} $*" - echo """ - Running: ${COMMAND} - -""" - ${COMMAND} - exit $? -} From 611e325337dcfceddaa9f2dedfcc20e2a7b370a4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 08:02:23 -0700 Subject: [PATCH 519/766] fixed error catching --- renderapi/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 8bbca593..ea9d6b67 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -712,11 +712,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, """ if bounds is not None: - if bounds['maxX'] Date: Fri, 13 Oct 2017 08:05:41 -0700 Subject: [PATCH 520/766] bumping version number with new feature --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a7290c65..79aaa6da 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.2.0', + version='1.3.0', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From 52cfc470947f3a6fb1b2ad56e5bd5f88fa0be9b6 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 13 Oct 2017 08:06:58 -0700 Subject: [PATCH 521/766] fixing paths in tests again --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 3025efd6..459b960c 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -177,7 +177,7 @@ def test_renderSectionClient(render,teststack, bounds, raises, scale=.05): format='png') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(root_directory): - pngfiles += [f for f in filenames if f.endswith('png')] + pngfiles += [os.path.join(dirpath,f) for f in filenames if f.endswith('png')] assert len(pngfiles) == len(zvalues) if bounds is not None: for f in pngfiles: From c373bbed49c642ee88fafb1266e21c4040bc24a1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 16 Oct 2017 09:38:05 -0700 Subject: [PATCH 522/766] added dynamic mongo config env variable --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4f6b04bb..ebae016f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,9 +8,11 @@ services: - mongo volumes: - "${RENDER_EXAMPLE_DATA}/example_1:/tmp/example_1:ro" + environment: + - MONGO_HOST=mongo mongo: image: mongo:3.4.2 - ports: + expose: - "27017" security_opt: - seccomp:unconfined \ No newline at end of file From 8372544c438d42725798e10e584f1433692e007e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 27 Oct 2017 12:20:21 -0400 Subject: [PATCH 523/766] fixing bounds for new version of render --- integration_tests/test_stack_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index fcc5e5fe..018963d9 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -225,7 +225,7 @@ def teststack(request, render, render_example_tilespec_and_transforms): def test_stack_bounds(render, teststack): # check the stack bounds stack_bounds = render.run(renderapi.stack.get_stack_bounds, teststack) - expected_bounds = {u'maxZ': 3408.0, u'maxX': 5102.0, u'maxY': 5385.0, + expected_bounds = {u'maxZ': 3408.0, u'maxX': 5103.0, u'maxY': 5385.0, u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} for key in stack_bounds.keys(): @@ -238,7 +238,7 @@ def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): zbounds = render.run(renderapi.stack.get_bounds_from_z, teststack, tilespecs[0].z) - expected_bounds = {u'maxZ': 3407.0, u'maxX': 4917.0, u'maxY': 4506.0, + expected_bounds = {u'maxZ': 3407.0, u'maxX': 4918.0, u'maxY': 4506.0, u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} for key in zbounds.keys(): assert np.abs(zbounds[key]-expected_bounds[key]) < 1.0 From d87b418f00219bbc3540735108565e51e64c31cd Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 27 Oct 2017 12:21:04 -0400 Subject: [PATCH 524/766] bumping version to indicate needing new render to have tests pass --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 79aaa6da..0d6914f8 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.3.0', + version='1.4.0', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From eb59bdd5bc029cadf164ee6d0bccc9158e3a6595 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 27 Oct 2017 12:35:20 -0400 Subject: [PATCH 525/766] more bounds fix --- integration_tests/test_stack_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 018963d9..5449f175 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -238,7 +238,7 @@ def test_z_bounds(render, teststack, render_example_tilespec_and_transforms): zbounds = render.run(renderapi.stack.get_bounds_from_z, teststack, tilespecs[0].z) - expected_bounds = {u'maxZ': 3407.0, u'maxX': 4918.0, u'maxY': 4506.0, + expected_bounds = {u'maxZ': 3407.0, u'maxX': 4918.0, u'maxY': 4507.0, u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} for key in zbounds.keys(): assert np.abs(zbounds[key]-expected_bounds[key]) < 1.0 From 72f54d93475b57e4aa60d303c253c1976ad2b766 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 27 Oct 2017 12:39:14 -0400 Subject: [PATCH 526/766] one more bound --- integration_tests/test_stack_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 5449f175..9b9a8b12 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -225,7 +225,7 @@ def teststack(request, render, render_example_tilespec_and_transforms): def test_stack_bounds(render, teststack): # check the stack bounds stack_bounds = render.run(renderapi.stack.get_stack_bounds, teststack) - expected_bounds = {u'maxZ': 3408.0, u'maxX': 5103.0, u'maxY': 5385.0, + expected_bounds = {u'maxZ': 3408.0, u'maxX': 5103.0, u'maxY': 5386.0, u'minX': 149.0, u'minY': 130.0, u'minZ': 3407.0} for key in stack_bounds.keys(): From 7e84547facaa5bcb912001daf5e2ccd53d0ff75d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 31 Oct 2017 09:30:06 -0700 Subject: [PATCH 527/766] raised exception if client script fails --- renderapi/client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index ea9d6b67..c6048459 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -420,9 +420,13 @@ def call_run_ws_client(className, add_args=[], renderclient=None, logger.warning( 'Unknown subprocess mode {} specified -- ' 'using default subprocess.call'.format(subprocess_mode)) - return subprocess_modes.get( - subprocess_mode, subprocess.call)( - map(str, [client_script, memGB, className] + add_args)) + args = map(str, [client_script, memGB, className] + add_args) + ret_val= subprocess_modes.get( + subprocess_mode, subprocess.call)(args) + if (ret_val != 0): + raise ClientScriptError('client_script call {} failed'.format(args)) + else: + return ret_val def get_param(var, flag): From 127938e8bcb497f200d87b5b775f01902f3ec06d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 31 Oct 2017 10:24:14 -0700 Subject: [PATCH 528/766] added changes to tox --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 33ea9451..143a2850 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py27, py36 [testenv] -commands = pytest test +commands = pytest deps = -rrequirements.txt - -rtest_requirements.txt \ No newline at end of file + -rtest_requirements.txt +passenv = RENDER_HOST RENDER_PORT RENDER_EXAMPLE_DATA RENDER_CLIENT_SCRIPTS RENDER_PYTHON_TEST_POOL_SIZE RENDER_JAVA_HOME From a489c4da965bb4e943694e207c76e7800b108d15 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 31 Oct 2017 16:31:58 -0700 Subject: [PATCH 529/766] patch exception raising to work on subprocess_calls --- renderapi/client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index c6048459..e56d3597 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -421,9 +421,11 @@ def call_run_ws_client(className, add_args=[], renderclient=None, 'Unknown subprocess mode {} specified -- ' 'using default subprocess.call'.format(subprocess_mode)) args = map(str, [client_script, memGB, className] + add_args) - ret_val= subprocess_modes.get( - subprocess_mode, subprocess.call)(args) - if (ret_val != 0): + sub_mode = subprocess_modes.get(subprocess_mode, subprocess.call) + ret_val= sub_mode(args) + #if you are using call, then returning 0 means the subprocess failed + #and we should raise an exception + if (ret_val != 0) and (sub_mode == subprocess.call): raise ClientScriptError('client_script call {} failed'.format(args)) else: return ret_val From 25ef16af1c9ebfe97d2e3f9fa3bd3df6fa869e25 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 31 Oct 2017 17:05:04 -0700 Subject: [PATCH 530/766] added tests for all call modes and captured exceptions --- integration_tests/test_client_integrated.py | 15 ++++-- renderapi/client.py | 58 ++++++++++++--------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 459b960c..fd20ca4c 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -64,19 +64,28 @@ def validate_stack_import(render, stack, tilespecs): ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) - +@pytest.mark.parameterize('call_mode',('call','check_call','check_output')) def test_import_jsonfiles_validate_client( - render, render_example_tilespec_and_transforms): + render, render_example_tilespec_and_transforms,call_mode): stack = 'test_import_jsonfiles_validate_client' renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files( render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles_validate_client( - stack, tfiles, transformFile=transformFile, render=render) + stack, tfiles, transformFile=transformFile, render=render, + subprocess_mode=call_mode) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) +@pytest.mark.parameterize('call_mode',('call','check_call','check_output')) +def test_failed_jsonfiles_validate_client( + render, render_example_tilespec_and_transforms,call_mode): + stack = 'test_failed_import_jsonfiles_validate_client' + with pytest.raises(renderapi.errors.ClientScriptError): + renderapi.client.import_jsonfiles_validate_client( + stack, ['not_a_file'], render=render, + subprocess_mode=call_mode) def test_import_jsonfiles_parallel( render, render_example_tilespec_and_transforms, diff --git a/renderapi/client.py b/renderapi/client.py index e56d3597..46b9c3e7 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -35,6 +35,7 @@ class WithPool(Pool): >>> with WithPool(number_processes) as pool: >>> pool.map(myfunc, myInput) """ + def __init__(self, *args, **kwargs): super(WithPool, self).__init__(*args, **kwargs) @@ -43,7 +44,7 @@ def __exit__(self, *args, **kwargs): @renderaccess -def import_single_json_file(stack, jsonfile, transformFile=None, +def import_single_json_file(stack, jsonfile, transformFile=None, subprocess_mode=None, client_script=None, memGB=None, host=None, port=None, owner=None, project=None, render=None, **kwargs): @@ -68,8 +69,8 @@ def import_single_json_file(stack, jsonfile, transformFile=None, stack_params = make_stack_params( host, port, owner, project, stack) call_run_ws_client('org.janelia.render.client.ImportJsonClient', - stack_params + transform_params + [jsonfile], - client_script=client_script,memGB=memGB,subprocess_mode=subprocess_mode) + stack_params + transform_params + [jsonfile], + client_script=client_script, memGB=memGB, subprocess_mode=subprocess_mode) @renderaccess @@ -181,8 +182,8 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, subprocess_mode=None, stack_params = make_stack_params( host, port, owner, project, stack) call_run_ws_client('org.janelia.render.client.ImportJsonClient', - stack_params + transform_params + jsonfiles, - client_script=client_script,memGB=memGB,subprocess_mode=subprocess_mode) + stack_params + transform_params + jsonfiles, + client_script=client_script, memGB=memGB, subprocess_mode=subprocess_mode) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -225,11 +226,11 @@ def import_jsonfiles_validate_client(stack, jsonfiles, set_stack_state(stack, 'LOADING', host, port, owner, project) call_run_ws_client('org.janelia.render.client.ImportJsonClient', - stack_params + \ - validator_params + \ - transform_params + \ - jsonfiles,client_script=client_script, - memGB=memGB,subprocess_mode=subprocess_mode) + stack_params + + validator_params + + transform_params + + jsonfiles, client_script=client_script, + memGB=memGB, subprocess_mode=subprocess_mode) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -261,10 +262,10 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, trjson = renderdump_temp(sharedTransforms) importJsonClient(stack, tileFiles=[tsjson], transformFile=( - trjson if sharedTransforms is not None else None), - subprocess_mode=subprocess_mode, host=host, port=port, - owner=owner, project=project, - client_script=client_script, memGB=memGB) + trjson if sharedTransforms is not None else None), + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, + client_script=client_script, memGB=memGB) os.remove(tsjson) if sharedTransforms is not None: @@ -422,9 +423,13 @@ def call_run_ws_client(className, add_args=[], renderclient=None, 'using default subprocess.call'.format(subprocess_mode)) args = map(str, [client_script, memGB, className] + add_args) sub_mode = subprocess_modes.get(subprocess_mode, subprocess.call) - ret_val= sub_mode(args) - #if you are using call, then returning 0 means the subprocess failed - #and we should raise an exception + try: + ret_val = sub_mode(args) + except subprocess.CalledProcessError as e: + raise ClientScriptError('client_script call {} failed'.format(args)) + + # if you are using call, then returning 0 means the subprocess failed + # and we should raise an exception if (ret_val != 0) and (sub_mode == subprocess.call): raise ClientScriptError('client_script call {} failed'.format(args)) else: @@ -719,15 +724,18 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, """ if bounds is not None: try: - if bounds['maxX'] Date: Tue, 31 Oct 2017 17:13:47 -0700 Subject: [PATCH 531/766] spelling mistake --- integration_tests/test_client_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index fd20ca4c..76d6511c 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -64,7 +64,7 @@ def validate_stack_import(render, stack, tilespecs): ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) -@pytest.mark.parameterize('call_mode',('call','check_call','check_output')) +@pytest.mark.parametrize('call_mode',('call','check_call','check_output')) def test_import_jsonfiles_validate_client( render, render_example_tilespec_and_transforms,call_mode): stack = 'test_import_jsonfiles_validate_client' @@ -78,7 +78,7 @@ def test_import_jsonfiles_validate_client( validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) -@pytest.mark.parameterize('call_mode',('call','check_call','check_output')) +@pytest.mark.parametrize('call_mode',('call','check_call','check_output')) def test_failed_jsonfiles_validate_client( render, render_example_tilespec_and_transforms,call_mode): stack = 'test_failed_import_jsonfiles_validate_client' From f44431a27cdc228a1f9e04c3ac11ffa6741a50c9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 31 Oct 2017 17:18:08 -0700 Subject: [PATCH 532/766] making stack so tests pass --- integration_tests/test_client_integrated.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 76d6511c..4f915a87 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -82,6 +82,7 @@ def test_import_jsonfiles_validate_client( def test_failed_jsonfiles_validate_client( render, render_example_tilespec_and_transforms,call_mode): stack = 'test_failed_import_jsonfiles_validate_client' + renderapi.stack.create_stack(stack, render=render) with pytest.raises(renderapi.errors.ClientScriptError): renderapi.client.import_jsonfiles_validate_client( stack, ['not_a_file'], render=render, From fc594e362f276c35c87b91ee72e2dd6691f72805 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Oct 2017 17:20:05 -0700 Subject: [PATCH 533/766] added first version of channel support initial channel implementation fixed docker compose and tox fixed import fixed mutichannel calls removing autofailed test fixed tests changed travis testing framework for render-app removing unneccesary environment variable added simple channel unit test fixed test really fixed test fixed travis bug travis fix again.. --- .travis.yml | 15 +- docker-compose.yml | 1 + integration_tests/test_data.py | 32 ++- .../test_files/test_2_channels.json | 92 ++++++ integration_tests/test_multichannel.py | 36 +++ integration_tests/test_stack_integrated.py | 6 +- renderapi/channel.py | 56 ++++ renderapi/image.py | 19 +- renderapi/image_pyramid.py | 125 +++++++++ renderapi/layout.py | 107 +++++++ renderapi/tilespec.py | 262 ++---------------- test/test_channel.py | 29 ++ test_requirements.txt | 3 +- tox.ini | 5 +- 14 files changed, 532 insertions(+), 256 deletions(-) create mode 100755 integration_tests/test_files/test_2_channels.json create mode 100644 integration_tests/test_multichannel.py create mode 100644 renderapi/channel.py create mode 100644 renderapi/image_pyramid.py create mode 100644 renderapi/layout.py create mode 100644 test/test_channel.py diff --git a/.travis.yml b/.travis.yml index db45e12a..edf871ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,15 +23,18 @@ install: - pip install -r requirements.txt - pip install -r test_requirements.txt before_install: - - git clone --depth 1 https://github.com/saalfeldlab/render.git fcollman/render -b master - - mvn package -pl render-ws-java-client -am -DskipTests -f fcollman/render/pom.xml - - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - - export RENDER_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/resources` + - git clone --depth 1 https://github.com/saalfeldlab/render.git render -b master + - mvn package -pl render-ws-java-client -am -DskipTests -f render/pom.xml + - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` + - export RENDER_WS_JAVA_CLIENT_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/resources` + - export RENDER_APP_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/render/render-app/src/test/resources` - export RENDER_JAVA_HOME=$JAVA_HOME + - mkdir -p /tmp/example_1 && cp -R $RENDER_WS_JAVA_CLIENT_EXAMPLE_DATA/example_1 /tmp/. + - cp -R $RENDER_APP_EXAMPLE_DATA/* /tmp/. + - export RENDER_EXAMPLE_DATA=/tmp - docker-compose up -d - - mkdir -p /tmp/example_1 && cp -R $RENDER_EXAMPLE_DATA/example_1 /tmp/. env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/fcollman/render/render-ws-java-client/src/main/scripts + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below diff --git a/docker-compose.yml b/docker-compose.yml index ebae016f..8f52a483 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: - mongo volumes: - "${RENDER_EXAMPLE_DATA}/example_1:/tmp/example_1:ro" + - "${RENDER_EXAMPLE_DATA}/multichannel-test:/tmp/multichannel-test:ro" environment: - MONGO_HOST=mongo mongo: diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py index ba2a97ad..9ed3ae47 100644 --- a/integration_tests/test_data.py +++ b/integration_tests/test_data.py @@ -1,11 +1,39 @@ import os +from jinja2 import Environment, FileSystemLoader +import json + +def render_json_template(env, template_file, **kwargs): + template = env.get_template(template_file) + d = json.loads(template.render(**kwargs)) + return d + +example_dir = os.environ.get('RENDER_EXAMPLE_DATA','/var/www/render/examples/') + +test_files_dir = os.path.join(os.path.dirname(__file__), 'test_files') +example_env = Environment(loader=FileSystemLoader(test_files_dir)) + render_host = os.environ.get('RENDER_HOST','renderservice') render_port = os.environ.get('RENDER_PORT',8080) client_script_location = os.environ.get('RENDER_CLIENT_SCRIPTS', ('/var/www/render/render-ws-java-client/' 'src/main/scripts/')) -example_dir = os.environ.get('RENDER_EXAMPLE_DATA','/var/www/render/examples/') + +render_params = { + 'host':render_host, + 'port':render_port, + 'owner':'test', + 'project':'test_project', + 'client_scripts':client_script_location +} + tilespec_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_tiles.json') tform_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_transforms.json') -test_pool_size = os.environ.get('RENDER_PYTHON_TEST_POOL_SIZE',3) \ No newline at end of file +test_pool_size = os.environ.get('RENDER_PYTHON_TEST_POOL_SIZE',3) + +multi_channel_dir = os.path.join(example_dir,'multichannel-test') + +test_2_channels_d = render_json_template(example_env, + 'test_2_channels.json', + multi_channel_example_dir=multi_channel_dir) + diff --git a/integration_tests/test_files/test_2_channels.json b/integration_tests/test_files/test_2_channels.json new file mode 100755 index 00000000..b00cce8c --- /dev/null +++ b/integration_tests/test_files/test_2_channels.json @@ -0,0 +1,92 @@ +[ + { + "tileId": "100000001003000", + "z": 1.0, + "minX": 662.0, + "minY": 1659.0, + "maxX": 2709.0, + "maxY": 3707.0, + "width": 2048.0, + "height": 2048.0, + "channels": [ + { + "name": "DAPI", + "minIntensity":100.0, + "maxIntensity":6000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/DAPI_1_S0001_F0003_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + }, + { + "name": "TdTomato", + "minIntensity": 0.0, + "maxIntensity": 10000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/TdTomato_S0001_F0003_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + } + ], + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.000000 0.000000 0.000000 1.000000 662.6192164270324 1659.706576042783 " + } + ] + }, + "meshCellSize": 64.0 + }, + { + "tileId": "100000001004000", + "z": 1.0, + "minX":2296.0, + "minY":1648.0, + "maxX":4343.0, + "maxY":3695.0, + "width": 2048.0, + "height": 2048.0, + "channels": [ + { + "name": "DAPI", + "minIntensity":100.0, + "maxIntensity":6000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/DAPI_1_S0001_F0004_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + }, + { + "name": "TdTomato", + "minIntensity": 0.0, + "maxIntensity": 10000.0, + "mipmapLevels": { + "0": { + "imageUrl": "{{multi_channel_example_dir}}/TdTomato_S0001_F0004_Z00.tif", + "maskUrl": "{{multi_channel_example_dir}}/fullcmos.tif" + } + } + } + ], + "transforms": { + "type": "list", + "specList": [ + { + "type": "leaf", + "className": "mpicbg.trakem2.transform.AffineModel2D", + "dataString": "1.000000 0.000000 0.000000 1.000000 2296.8235136181947 1648.2548941942687 " + } + ] + }, + "meshCellSize": 64.0 + } +] diff --git a/integration_tests/test_multichannel.py b/integration_tests/test_multichannel.py new file mode 100644 index 00000000..e30c5e28 --- /dev/null +++ b/integration_tests/test_multichannel.py @@ -0,0 +1,36 @@ +import renderapi +from test_data import render_params, test_2_channels_d +import pytest +import logging +import sys + +root = logging.getLogger() +root.setLevel(logging.DEBUG) + +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') +ch.setFormatter(formatter) +root.addHandler(ch) + +render_params['project']='multi_channel_test' + +@pytest.fixture(scope='module') +def render(): + return renderapi.connect(**render_params) + +@pytest.fixture(scope='module') +def multichannel_test_stack(render): + stack = 'multichannel_test' + tilespecs = [renderapi.tilespec.TileSpec(json = d) for d in test_2_channels_d] + renderapi.stack.create_stack(stack,render=render) + renderapi.client.import_tilespecs(stack,tilespecs, render=render) + renderapi.stack.set_stack_state(stack, 'COMPLETE',render=render) + return stack + +def test_section_image_channels(render,multichannel_test_stack): + section_image = renderapi.image.get_section_image(multichannel_test_stack, + 1.0,channel='DAPI', + render=render) + print(section_image.shape) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 9b9a8b12..6215048d 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -7,7 +7,8 @@ import json import numpy as np from test_data import (render_host, render_port, - client_script_location, tilespec_file, tform_file) + client_script_location, tilespec_file, + tform_file, test_2_channels_d) root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -430,3 +431,6 @@ def test_get_resolvedtiles_from_z(render, teststack, assert(len(tsz)==len(resolved_tiles.tilespecs)) matching_ts = next(ts for ts in resolved_tiles.tilespecs if ts.tileId == tsz[0].tileId) assert (len(matching_ts.tforms)==len(tsz[0].tforms)) + + + diff --git a/renderapi/channel.py b/renderapi/channel.py new file mode 100644 index 00000000..8509c9e1 --- /dev/null +++ b/renderapi/channel.py @@ -0,0 +1,56 @@ +from .image_pyramid import ImagePyramid, MipMapLevel + +class Channel: + '''class for storing channels of different mipmapsources''' + + def __init__(self,name=None,maxIntensity=None,minIntensity=None,ip=None,json=None): + ''' + Parameters + ========== + name: str + name of channel + maxIntensity: int + maximum intensity to display (optional) + minIntesnity: int + minimum default intensity to display (optional) + ip: ImagePyramid + set of mipmaplevel images for this channel + json: dict + json representation of this channel + ''' + + if json is not None: + self.from_dict(json) + else: + self.name=name + self.maxIntensity=maxIntensity + self.minIntensity=minIntensity + self.ip = ip + + def to_dict(self): + ''' method for serializing this class to a json compatible dictionary''' + d = {} + d['name']=self.name + if self.minIntensity is not None: + d['minIntensity']=self.minIntensity + if self.maxIntensity is not None: + d['maxIntensity']=self.maxIntensity + d['mipmapLevels'] = self.ip.to_ordered_dict() + return d + + def from_dict(self,d): + ''' method for deserializing this class from a json compatible dictionary + + Parameters + ========== + d: dict + json compatible dictionary representation of this channel + ''' + self.name=d['name'] + self.minIntensity=d['minIntensity'] + self.maxIntensity=d['maxIntensity'] + self.ip = ImagePyramid(mipMapLevels=[ + MipMapLevel( + int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) + for l, v in d['mipmapLevels'].items()]) + \ No newline at end of file diff --git a/renderapi/image.py b/renderapi/image.py index 2dd8d859..c5e07add 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -26,6 +26,7 @@ @renderaccess def get_bb_image(stack, z, x, y, width, height, scale=1.0, + channel=None, minIntensity=None, maxIntensity=None, binaryMask=None, filter=None, maxTileSpecsToRender=None, host=None, port=None, owner=None, project=None, @@ -51,6 +52,8 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, number of units @scale=1.0 down (+y) of bounding box to render scale : float scale to render image at (default 1.0) + channel : str + channel name to render binaryMask : bool whether to treat maskimage as binary maxTileSpecsToRender : int @@ -91,6 +94,8 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, qparams['filter'] = jbool(filter) if maxTileSpecsToRender is not None: qparams['maxTileSpecsToRender'] = maxTileSpecsToRender + if channel is not None: + qparams.update({'channels':channel}) r = session.get(request_url, params=qparams) try: @@ -103,7 +108,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, @renderaccess -def get_tile_image_data(stack, tileId, normalizeForMatching=True, +def get_tile_image_data(stack, tileId, channel=None,normalizeForMatching=True, removeAllOption=False, scale=None, filter=None, host=None, port=None, owner=None, project=None, img_format=None, @@ -118,6 +123,8 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, name of render stack to get tile from tileId : str tileId of tile to render + channel : str + channel name to render normalizeForMatching : bool whether to render the tile with transformations removed ('local' coordinates) @@ -166,6 +173,8 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, qparams['filter'] = jbool(filter) if removeAllOption is not None: qparams['removeAllOption'] = jbool(removeAllOption) + if channel is not None: + qparams.update({'channels':channel}) logger.debug(request_url) r = session.get(request_url, params=qparams) @@ -180,7 +189,8 @@ def get_tile_image_data(stack, tileId, normalizeForMatching=True, @renderaccess -def get_section_image(stack, z, scale=1.0, filter=False, +def get_section_image(stack, z, scale=1.0, channel=None, + filter=False, maxTileSpecsToRender=None, img_format=None, host=None, port=None, owner=None, project=None, session=requests.session(), @@ -198,6 +208,8 @@ def get_section_image(stack, z, scale=1.0, filter=False, layer Z scale : float linear scale at which to render image (e.g. 0.5) + channel: str + channel name to render filter : bool whether or not to apply server side filtering maxTileSpecsToRender : int @@ -234,5 +246,8 @@ def get_section_image(stack, z, scale=1.0, filter=False, qparams = {'scale': scale, 'filter': jbool(filter)} if maxTileSpecsToRender is not None: qparams.update({'maxTileSpecsToRender': maxTileSpecsToRender}) + if channel is not None: + qparams.update({'channels':channel}) + r = session.get(request_url, params=qparams) return np.asarray(Image.open(io.BytesIO(r.content))) diff --git a/renderapi/image_pyramid.py b/renderapi/image_pyramid.py new file mode 100644 index 00000000..04ab4678 --- /dev/null +++ b/renderapi/image_pyramid.py @@ -0,0 +1,125 @@ +from collections import OrderedDict + +class MipMapLevel: + """MipMapLevel class to represent a level of an image pyramid. + Can be put in dictionary formatting using dict(mML) + + Attributes + ---------- + level : int + level of 2x downsampling represented by mipmaplevel + imageUrl : str or None + uri corresponding to image + maskUrl : str or None + uri corresponding to mask + + """ + def __init__(self, level, imageUrl=None, maskUrl=None): + self.level = level + self.imageUrl = imageUrl + self.maskUrl = maskUrl + + def to_dict(self): + """ + Returns + ------- + dict + json compatible dictionary representaton + """ + return dict(self.__iter__()) + + def _formatUrls(self): + d = {} + if self.imageUrl is not None: + d.update({'imageUrl': self.imageUrl}) + if self.maskUrl is not None: + d.update({'maskUrl': self.maskUrl}) + return d + + def __iter__(self): + return iter([(self.level, self._formatUrls())]) + + +class ImagePyramid: + '''Image Pyramid class representing a set of MipMapLevels which correspond + to mipmapped (continuously downsmapled by 2x) representations + of an image at level 0 + Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) + + Attributes + ---------- + mipMapLevels : :obj:`list` of :class:`MipMapLevel` + list of :class:`MipMapLevel` objects defining image pyramid + + ''' + def __init__(self, mipMapLevels=[]): + self.mipMapLevels = mipMapLevels + + def to_dict(self): + """return dictionary representation of this object""" + return dict(self.__iter__()) + + def to_ordered_dict(self, key=None): + """returns :class:`OrderedDict` represention of this object, + ordered by mipmapLevel + + Parameters + ---------- + key : func + function to sort ordered dict of + :class:`mipMapLevel` dicts (default is by level) + + Returns + ------- + OrderedDict + sorted dictionary of :class:`mipMapLevels` in ImagePyramid + + """ + return OrderedDict(sorted( + self.__iter__(), key=((lambda x: x[0]) if key + is None else key))) + + def append(self, mmL): + """appends a MipMapLevel to this ImagePyramid + + Parameters + ---------- + mml : :class:`MipMapLevel` + :class:`MipMapLevel` to append + """ + self.mipMapLevels.append(mmL) + + def update(self, mmL): + """updates the ImagePyramid with this MipMapLevel. + will overwrite existing mipMapLevels with same level + + Args: + mml (MipMapLevel): mipmap level to update in pyramid + """ + self.mipMapLevels = [ + l for l in self.mipMapLevels if l.level != mmL.level] + self.append(mmL) + + def get(self, to_get): + """gets a specific mipmap level in dictionary form + + Parameters + ---------- + to_get : int + level to get + + Returns + ------- + dict + representation of requested MipMapLevel + """ + return self.to_dict()[to_get] # TODO should this default + + @property + def levels(self): + """list of MipMapLevels in this ImagePyramid""" + return [int(i.level) for i in self.mipMapLevels] + + def __iter__(self): + return iter([ + l for sl in [list(mmL) for mmL in self.mipMapLevels] for l in sl]) \ No newline at end of file diff --git a/renderapi/layout.py b/renderapi/layout.py new file mode 100644 index 00000000..68039c82 --- /dev/null +++ b/renderapi/layout.py @@ -0,0 +1,107 @@ + +class Layout: + """Layout class to describe acquisition settings + + Attributes + ---------- + sectionId : str + sectionId this tile was taken from + scopeId : str + what microscope this came from + cameraId : str + camera this was taken with + imageRow : int + what row from a row,col layout this was taken + imageCol : int + column from a row,col layout this was taken + stageX : float + X stage coordinates for where this was taken + stageY : float + Y stage coordinates for where this taken + rotation : float + angle of camera when this was taken + pixelsize : float + effective size of pixels (in units of choice) + + """ + def __init__(self, sectionId=None, scopeId=None, cameraId=None, + imageRow=None, imageCol=None, stageX=None, stageY=None, + rotation=None, pixelsize=None, + force_pixelsize=True, **kwargs): + """Initialize Layout + + Parameters + ---------- + sectionId : str + sectionId this tile was taken from + scopeId : str + what microscope this came from + cameraId : str + camera this was taken with + imageRow : int + what row from a row,col layout this was taken + imageCol : int + column from a row,col layout this was taken + stageX : float + X stage coordinates for where this was taken + stageY : float + Y stage coordinates for where this taken + rotation : float + angle of camera when this was taken + pixelsize : float + effective size of pixels (in units of choice) + force_pixelsize : bool + whether to default pixelsize to 0.1 + + """ + self.sectionId = sectionId + self.scopeId = scopeId + self.cameraId = cameraId + self.imageRow = imageRow + self.imageCol = imageCol + self.stageX = stageX + self.stageY = stageY + self.rotation = rotation + if force_pixelsize: + pixelsize = 0.100 if pixelsize is None else pixelsize + self.pixelsize = pixelsize + + def to_dict(self): + """return a dictionary representation of this object + + Returns + ------- + dict + json compatible dictionary of this object + """ + d = {} + d['sectionId'] = self.sectionId + d['temca'] = self.scopeId + d['camera'] = self.cameraId + d['imageRow'] = self.imageRow + d['imageCol'] = self.imageCol + d['stageX'] = self.stageX + d['stageY'] = self.stageY + d['rotation'] = self.rotation + d['pixelsize'] = self.pixelsize + d = {k: v for k, v in d.items() if v is not None} + return d + + def from_dict(self, d): + """set this object equal to the fields found in dictionary + + Parameters + ---------- + d : dict + dictionary to use to update + """ + if d is not None: + self.sectionId = d.get('sectionId') + self.cameraId = d.get('camera') + self.scopeId = d.get('temca') + self.imageRow = d.get('imageRow') + self.imageCol = d.get('imageCol') + self.stageX = d.get('stageX') + self.stageY = d.get('stageY') + self.rotation = d.get('rotation') + self.pixelsize = d.get('pixelsize') \ No newline at end of file diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 004a7ab4..e2a71f38 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,125 +1,19 @@ #!/usr/bin/env python +import logging +import requests from .render import format_preamble, renderaccess from .utils import NullHandler from .stack import get_z_values_for_stack from .transform import TransformList from .errors import RenderError -from collections import OrderedDict -import logging -import requests +from .image_pyramid import MipMapLevel, ImagePyramid +from .layout import Layout +from .channel import Channel logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) -class Layout: - """Layout class to describe acquisition settings - - Attributes - ---------- - sectionId : str - sectionId this tile was taken from - scopeId : str - what microscope this came from - cameraId : str - camera this was taken with - imageRow : int - what row from a row,col layout this was taken - imageCol : int - column from a row,col layout this was taken - stageX : float - X stage coordinates for where this was taken - stageY : float - Y stage coordinates for where this taken - rotation : float - angle of camera when this was taken - pixelsize : float - effective size of pixels (in units of choice) - - """ - def __init__(self, sectionId=None, scopeId=None, cameraId=None, - imageRow=None, imageCol=None, stageX=None, stageY=None, - rotation=None, pixelsize=None, - force_pixelsize=True, **kwargs): - """Initialize Layout - - Parameters - ---------- - sectionId : str - sectionId this tile was taken from - scopeId : str - what microscope this came from - cameraId : str - camera this was taken with - imageRow : int - what row from a row,col layout this was taken - imageCol : int - column from a row,col layout this was taken - stageX : float - X stage coordinates for where this was taken - stageY : float - Y stage coordinates for where this taken - rotation : float - angle of camera when this was taken - pixelsize : float - effective size of pixels (in units of choice) - force_pixelsize : bool - whether to default pixelsize to 0.1 - - """ - self.sectionId = sectionId - self.scopeId = scopeId - self.cameraId = cameraId - self.imageRow = imageRow - self.imageCol = imageCol - self.stageX = stageX - self.stageY = stageY - self.rotation = rotation - if force_pixelsize: - pixelsize = 0.100 if pixelsize is None else pixelsize - self.pixelsize = pixelsize - - def to_dict(self): - """return a dictionary representation of this object - - Returns - ------- - dict - json compatible dictionary of this object - """ - d = {} - d['sectionId'] = self.sectionId - d['temca'] = self.scopeId - d['camera'] = self.cameraId - d['imageRow'] = self.imageRow - d['imageCol'] = self.imageCol - d['stageX'] = self.stageX - d['stageY'] = self.stageY - d['rotation'] = self.rotation - d['pixelsize'] = self.pixelsize - d = {k: v for k, v in d.items() if v is not None} - return d - - def from_dict(self, d): - """set this object equal to the fields found in dictionary - - Parameters - ---------- - d : dict - dictionary to use to update - """ - if d is not None: - self.sectionId = d.get('sectionId') - self.cameraId = d.get('camera') - self.scopeId = d.get('temca') - self.imageRow = d.get('imageRow') - self.imageCol = d.get('imageCol') - self.stageX = d.get('stageX') - self.stageY = d.get('stageY') - self.rotation = d.get('rotation') - self.pixelsize = d.get('pixelsize') - - class TileSpec: '''Fundamental class of render that store image tiles and their transformations @@ -177,11 +71,12 @@ class TileSpec: (DEPRECATED, use mipMapLevels, but will override) ''' + def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, minint=0, maxint=65535, layout=None, tforms=[], inputfilters=[], scale3Url=None, scale2Url=None, - scale1Url=None, json=None, mipMapLevels=[], **kwargs): + scale1Url=None, json=None, channels=None,mipMapLevels=[], **kwargs): if json is not None: self.from_dict(json) else: @@ -203,7 +98,7 @@ def __init__(self, tileId=None, z=None, width=None, height=None, self.scale3Url = scale3Url self.scale2Url = scale2Url self.scale1Url = scale1Url - + self.channels = channels if imageUrl is not None: self.ip.update(MipMapLevel( 0, imageUrl=imageUrl, maskUrl=maskUrl)) @@ -246,6 +141,8 @@ def to_dict(self): thedict['transforms']['type'] = 'list' # thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] thedict['transforms']['specList'] = [] + if self.channels is not None: + thedict['channels']=[ch.to_dict() for ch in self.channels] for t in self.tforms: strlist = {} # added by sharmi - if your speclist contains a speclist (can @@ -290,13 +187,19 @@ def from_dict(self, d): self.maxX = d.get('maxX', None) self.maxY = d.get('maxY', None) self.minY = d.get('minY', None) + mmld = d.get('mipmapLevels',{}) self.ip = ImagePyramid(mipMapLevels=[ - MipMapLevel( - int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) - for l, v in d['mipmapLevels'].items()]) + MipMapLevel( + int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) + for l, v in mmld.items()]) tfl = TransformList(json=d['transforms']) self.tforms = tfl.tforms + chd = d.get('channels',None) + if chd is None: + self.channels = None + else: + self.channels = [Channel(json=ch) for ch in chd] # TODO filters not implemented -- should skip ''' @@ -309,131 +212,6 @@ def from_dict(self, d): ''' -class MipMapLevel: - """MipMapLevel class to represent a level of an image pyramid. - Can be put in dictionary formatting using dict(mML) - - Attributes - ---------- - level : int - level of 2x downsampling represented by mipmaplevel - imageUrl : str or None - uri corresponding to image - maskUrl : str or None - uri corresponding to mask - - """ - def __init__(self, level, imageUrl=None, maskUrl=None): - self.level = level - self.imageUrl = imageUrl - self.maskUrl = maskUrl - - def to_dict(self): - """ - Returns - ------- - dict - json compatible dictionary representaton - """ - return dict(self.__iter__()) - - def _formatUrls(self): - d = {} - if self.imageUrl is not None: - d.update({'imageUrl': self.imageUrl}) - if self.maskUrl is not None: - d.update({'maskUrl': self.maskUrl}) - return d - - def __iter__(self): - return iter([(self.level, self._formatUrls())]) - - -class ImagePyramid: - '''Image Pyramid class representing a set of MipMapLevels which correspond - to mipmapped (continuously downsmapled by 2x) representations - of an image at level 0 - Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) - - Attributes - ---------- - mipMapLevels : :obj:`list` of :class:`MipMapLevel` - list of :class:`MipMapLevel` objects defining image pyramid - - ''' - def __init__(self, mipMapLevels=[]): - self.mipMapLevels = mipMapLevels - - def to_dict(self): - """return dictionary representation of this object""" - return dict(self.__iter__()) - - def to_ordered_dict(self, key=None): - """returns :class:`OrderedDict` represention of this object, - ordered by mipmapLevel - - Parameters - ---------- - key : func - function to sort ordered dict of - :class:`mipMapLevel` dicts (default is by level) - - Returns - ------- - OrderedDict - sorted dictionary of :class:`mipMapLevels` in ImagePyramid - - """ - return OrderedDict(sorted( - self.__iter__(), key=((lambda x: x[0]) if key - is None else key))) - - def append(self, mmL): - """appends a MipMapLevel to this ImagePyramid - - Parameters - ---------- - mml : :class:`MipMapLevel` - :class:`MipMapLevel` to append - """ - self.mipMapLevels.append(mmL) - - def update(self, mmL): - """updates the ImagePyramid with this MipMapLevel. - will overwrite existing mipMapLevels with same level - - Args: - mml (MipMapLevel): mipmap level to update in pyramid - """ - self.mipMapLevels = [ - l for l in self.mipMapLevels if l.level != mmL.level] - self.append(mmL) - - def get(self, to_get): - """gets a specific mipmap level in dictionary form - - Parameters - ---------- - to_get : int - level to get - - Returns - ------- - dict - representation of requested MipMapLevel - """ - return self.to_dict()[to_get] # TODO should this default - - @property - def levels(self): - """list of MipMapLevels in this ImagePyramid""" - return [int(i.level) for i in self.mipMapLevels] - - def __iter__(self): - return iter([ - l for sl in [list(mmL) for mmL in self.mipMapLevels] for l in sl]) - - @renderaccess def get_tile_spec_renderparameters(stack, tile, host=None, port=None, owner=None, project=None, @@ -638,7 +416,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( - z, x, y, width, height, scale) + z, x, y, width, height, scale) logger.debug(request_url) r = session.get(request_url) try: diff --git a/test/test_channel.py b/test/test_channel.py new file mode 100644 index 00000000..3c536e80 --- /dev/null +++ b/test/test_channel.py @@ -0,0 +1,29 @@ +import renderapi +import pytest +import json + +d = { + "name":"DAPI", + "maxIntensity":255, + "minIntensity":0, + "mipmapLevels":{ + "0":{ + "imageUrl":"file:///not/a/path", + "maskUrl":"file:///not/a/mask" + } + } +} + +def test_channel(): + mml = renderapi.image_pyramid.MipMapLevel(0, + imageUrl='file:///not/a/path', + maskUrl='file:///not/a/mask') + ip = renderapi.image_pyramid.ImagePyramid(mipMapLevels=[mml]) + channel = renderapi.channel.Channel(name='DAPI', + maxIntensity=255, + minIntensity=0, + ip=ip) + + a=json.loads(renderapi.utils.renderdumps(channel)) + b=json.loads(renderapi.utils.renderdumps(d)) + assert(a==b) diff --git a/test_requirements.txt b/test_requirements.txt index 18650732..2f722abe 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -7,4 +7,5 @@ pytest-pep8>=1.0.6 pytest-xdist>=1.14 flake8>=3.0.4 pylint>=1.5.4 -scipy \ No newline at end of file +scipy +jinja2 diff --git a/tox.ini b/tox.ini index 33ea9451..143a2850 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ envlist = py27, py36 [testenv] -commands = pytest test +commands = pytest deps = -rrequirements.txt - -rtest_requirements.txt \ No newline at end of file + -rtest_requirements.txt +passenv = RENDER_HOST RENDER_PORT RENDER_EXAMPLE_DATA RENDER_CLIENT_SCRIPTS RENDER_PYTHON_TEST_POOL_SIZE RENDER_JAVA_HOME From 6b6429e67384ca6d9cb8cea54503b75a8d9f1eb9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 26 Oct 2017 10:36:33 -0400 Subject: [PATCH 534/766] added transform labels finished adding labels as options to transforms added a label to simple transform pep8 fixed handling cases with no metadata or labels added labels to polynomial transform Changed metadata to metaData to get persistence of transform labels working. Although I now realize metadata is the proper spelling, the camel case version has been propagated too widely across the render code base to quickly correct it. Hopefully this doesn't offend anyone's sensibilities too much (if it does, let me know). first attempt at fixing test by making labels an array --- integration_tests/test_stack_integrated.py | 2 +- renderapi/transform.py | 47 +++++++++++++++++----- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 9b9a8b12..54bf5958 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -34,7 +34,7 @@ def render(): @pytest.fixture def simpletilespec(): mml = renderapi.tilespec.MipMapLevel(0, '/not/a/path.jpg') - tform = renderapi.transform.AffineModel() + tform = renderapi.transform.AffineModel(labels=['simple']) layout = renderapi.tilespec.Layout(sectionId="section0", scopeId="testscope", cameraId="testcamera", diff --git a/renderapi/transform.py b/renderapi/transform.py index 0f79dd29..28305b13 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -326,7 +326,7 @@ class Transform(object): """ def __init__(self, className=None, dataString=None, - transformId=None, json=None): + transformId=None, labels=None, json=None): """Initialize Transform Parameters @@ -338,6 +338,8 @@ def __init__(self, className=None, dataString=None, by mpicbg java class library transformId : str, optional unique Id for this transform (optional) + labels : list of str + list of labels to give this transform json : dict json compatible representation of this transform (supersedes className, dataString, and transformId if not None) @@ -348,6 +350,7 @@ def __init__(self, className=None, dataString=None, self.className = className self.dataString = dataString self.transformId = transformId + self.labels = labels def to_dict(self): """serialization routine @@ -363,6 +366,8 @@ def to_dict(self): d['dataString'] = self.dataString if self.transformId is not None: d['id'] = self.transformId + if self.labels is not None: + d['metaData'] = {'labels': self.labels} return d def from_dict(self, d): @@ -376,7 +381,12 @@ def from_dict(self, d): self.className = d['className'] self.transformId = d.get('id', None) self._process_dataString(d['dataString']) - + md = d.get('metaData', None) + if md is not None: + self.labels = md.get('labels', None) + else: + self.labels = None + def _process_dataString(self, datastring): """method meant to set state of transform from datastring generic implementation only saves datastring at self.dataString. @@ -427,6 +437,8 @@ class AffineModel(Transform): y'+=B1 transformId : str, optional unique transformId for this transform + labels : list of str + list of labels to give this transform M : numpy.array 3x3 numpy array representing 2d Affine with homogeneous coordinates populates with values from M00, M01, M10, M11, B0, B1 with load_M() @@ -436,7 +448,7 @@ class AffineModel(Transform): className = 'mpicbg.trakem2.transform.AffineModel2D' def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, - transformId=None, json=None): + transformId=None, labels=None, json=None): """Initialize AffineModel, defaulting to identity Parameters @@ -455,6 +467,8 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, y'+=B1 transformId : str unique transformId for this transform (optional) + labels : list of str + list of labels to give this transform json : dict json compatible representation of this transform (will supersede all other parameters if not None) @@ -470,6 +484,7 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, self.B0 = B0 self.B1 = B1 self.className = 'mpicbg.trakem2.transform.AffineModel2D' + self.labels = labels self.load_M() self.transformId = transformId @@ -744,6 +759,8 @@ class TranslationModel(AffineModel): y'+=B1 transformId : str, optional unique transformId for this transform + labels : list of str + list of labels to give this transform M : numpy.array 3x3 numpy array representing 2d Affine with homogeneous coordinates populates with values from M00, M01, M10, M11, B0, B1 with load_M() @@ -833,6 +850,8 @@ class RigidModel(AffineModel): y'+=B1 transformId : str, optional unique transformId for this transform + labels : list of str + list of labels to give this transform M : numpy.array 3x3 numpy array representing 2d Affine with homogeneous coordinates populates with values from M00, M01, M10, M11, B0, B1 with load_M() @@ -960,6 +979,8 @@ class SimilarityModel(RigidModel): y'+=B1 transformId : str, optional unique transformId for this transform + labels : list of str + list of labels to give this transform M : numpy.array 3x3 numpy array representing 2d Affine with homogeneous coordinates populates with values from M00, M01, M10, M11, B0, B1 with load_M() @@ -1018,7 +1039,7 @@ class Polynomial2DTransform(Transform): def __init__(self, dataString=None, src=None, dst=None, order=2, force_polynomial=True, params=None, identity=False, - json=None, **kwargs): + labels=None,json=None, **kwargs): """Initialize Polynomial2DTransform This provides 5 different ways to initialize the transform which are mutually exclusive and applied in the order specified here. @@ -1064,6 +1085,7 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, raise NotImplementedError('Falling back to Affine model is ' 'not supported {}') self.transformId = None + self.labels = labels @property def is_affine(self): @@ -1240,7 +1262,7 @@ def _format_raveled_params(raveled_params): a (2,K/2) matrix of parameters, with first row for x and 2nd row for y """ - halfway = int (len(raveled_params) / 2) + halfway = int(len(raveled_params) / 2) return np.array( [[float(d) for d in raveled_params[:halfway]], [float(d) for d in raveled_params[halfway:]]]) @@ -1377,6 +1399,8 @@ class NonLinearCoordinateTransform(Transform): ---------- dataString: str or None data string of transformation + labels : list of str + list of labels to give this transform json: dict or None json compatible dictionary representation of the transformation @@ -1390,12 +1414,15 @@ class NonLinearCoordinateTransform(Transform): className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' - def __init__(self, dataString=None, json=None, transformId=None): + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): if json is not None: self.from_dict(json) else: if dataString is not None: self._process_dataString(dataString) + if labels is not None: + self.labels = labels self.transformId = transformId self.className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' @@ -1407,14 +1434,14 @@ def _process_dataString(self, dataString): self.length = int(fields[1]) # cutoff whitespace if there - fields = fields[0:2+4*self.length+2] + fields = fields[0:2 + 4 * self.length + 2] # last 2 fields are width and height self.width = int(fields[-2]) self.height = int(fields[-1]) data = np.array(fields[2:-2], dtype='float32') try: - self.beta = data[0:2*self.length].reshape(self.length, 2) + self.beta = data[0:2 * self.length].reshape(self.length, 2) except ValueError as e: raise RenderError( 'Incorrect number of coefficients in ' @@ -1423,8 +1450,8 @@ def _process_dataString(self, dataString): raise RenderError("not correct number of coefficents") # normMean and normVar follow - self.normMean = data[self.length*2:self.length*3] - self.normVar = data[self.length*3:self.length*4] + self.normMean = data[self.length * 2:self.length * 3] + self.normVar = data[self.length * 3:self.length * 4] if not (self.normMean.shape[0] == self.length): raise RenderError( "incorrect number of normMean coefficents " From bf770ced4974b80bc21cd2631b20179970db8123 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 1 Nov 2017 13:05:39 -0700 Subject: [PATCH 535/766] removed shell scripts which don't work and were allen specific --- docker_base_setup.sh | 5 ----- docker_setup.sh | 5 ----- 2 files changed, 10 deletions(-) delete mode 100755 docker_base_setup.sh delete mode 100755 docker_setup.sh diff --git a/docker_base_setup.sh b/docker_base_setup.sh deleted file mode 100755 index 19599ecf..00000000 --- a/docker_base_setup.sh +++ /dev/null @@ -1,5 +0,0 @@ -docker pull atbigdawg:5000/fcollman/render:latest -docker tag atbigdawg:5000/fcollman/render:latest fcollman/render:latest -docker build -t fcollman/render-python-base:latest -f Dockerfile.base . -docker tag fcollman/render-python-base:latest atbigdawg:5000/fcollman/render-python-base:latest -docker push atbigdawg:5000/fcollman/render-python-base:latest diff --git a/docker_setup.sh b/docker_setup.sh deleted file mode 100755 index 2796200a..00000000 --- a/docker_setup.sh +++ /dev/null @@ -1,5 +0,0 @@ -docker pull atbigdawg:5000/fcollman/render-python-base:latest -docker tag atbigdawg:5000/fcollman/render-python-base:latest fcollman/render-python-base:latest -docker build -t fcollman/render-python:latest . -docker tag fcollman/render-python:latest atbigdawg:5000/fcollman/render-python:latest -docker push atbigdawg:5000/fcollman/render-python:latest From 132d22aa4834d79b754ccbd081536b0b80fa6c2a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 1 Nov 2017 20:32:14 -0700 Subject: [PATCH 536/766] added badgets --- README.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 41601a9e..4b881551 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,9 @@ .. image:: https://readthedocs.org/projects/render-python/badge/ :target: http://render-python.readthedocs.io/en/latest/ - :alt: Documentation Status + :alt: Documentation Status +.. image:: https://travis-ci.org/fcollman/render-python.svg?branch=master + :target: https://travis-ci.org/fcollman/render-python + :alt: Build Status render-python ############# @@ -27,4 +30,4 @@ Documentation ############# http://render-python.readthedocs.io/en/latest/ -.. _render : \ No newline at end of file +.. _render : From 3690dff47160ec4600633c3d9b93bbef71e463a4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 3 Nov 2017 14:09:27 -0700 Subject: [PATCH 537/766] improved documentation of channels interface with examples and weighted average --- renderapi/image.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index c5e07add..29d53af0 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -53,7 +53,8 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, scale : float scale to render image at (default 1.0) channel : str - channel name to render + channel name to render, (e.g. 'DAPI') or a weighted average of channels of the format + e.g 'DAPI___.8___GFP___.2' binaryMask : bool whether to treat maskimage as binary maxTileSpecsToRender : int @@ -124,7 +125,8 @@ def get_tile_image_data(stack, tileId, channel=None,normalizeForMatching=True, tileId : str tileId of tile to render channel : str - channel name to render + channel name to render, (e.g. 'DAPI') or a weighted average of channels of the format + e.g 'DAPI___.8___GFP___.2' normalizeForMatching : bool whether to render the tile with transformations removed ('local' coordinates) @@ -209,7 +211,8 @@ def get_section_image(stack, z, scale=1.0, channel=None, scale : float linear scale at which to render image (e.g. 0.5) channel: str - channel name to render + channel name to render, (e.g. 'DAPI') or a weighted average of channels of the format + e.g 'DAPI___.8___GFP___.2' filter : bool whether or not to apply server side filtering maxTileSpecsToRender : int From cb5fb9081392adfbb58371480534f6a9d2e4a70c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 3 Nov 2017 14:16:29 -0700 Subject: [PATCH 538/766] changed exception raising to only be for check_call and check_output, changed failed test parameters in kind --- integration_tests/test_client_integrated.py | 2 +- renderapi/client.py | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 4f915a87..ae05220b 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -78,7 +78,7 @@ def test_import_jsonfiles_validate_client( validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) -@pytest.mark.parametrize('call_mode',('call','check_call','check_output')) +@pytest.mark.parametrize('call_mode',('check_call','check_output')) def test_failed_jsonfiles_validate_client( render, render_example_tilespec_and_transforms,call_mode): stack = 'test_failed_import_jsonfiles_validate_client' diff --git a/renderapi/client.py b/renderapi/client.py index 46b9c3e7..74941a0c 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -422,18 +422,14 @@ def call_run_ws_client(className, add_args=[], renderclient=None, 'Unknown subprocess mode {} specified -- ' 'using default subprocess.call'.format(subprocess_mode)) args = map(str, [client_script, memGB, className] + add_args) - sub_mode = subprocess_modes.get(subprocess_mode, subprocess.call) + sub_mode = subprocess_modes.get(subprocess_mode, subprocess.check_call) try: ret_val = sub_mode(args) except subprocess.CalledProcessError as e: raise ClientScriptError('client_script call {} failed'.format(args)) - # if you are using call, then returning 0 means the subprocess failed - # and we should raise an exception - if (ret_val != 0) and (sub_mode == subprocess.call): - raise ClientScriptError('client_script call {} failed'.format(args)) - else: - return ret_val + return ret_val + def get_param(var, flag): From cc07ad005bd55ed481ebe981980290cb0f55902a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 3 Nov 2017 17:59:59 -0700 Subject: [PATCH 539/766] bumping version before master merge --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0d6914f8..aa98afff 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.4.0', + version='1.5.0', description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', From cace929305ee689a317a96f679f141cc0a6e45d5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 3 Nov 2017 14:06:38 -0700 Subject: [PATCH 540/766] This adds more options available in the section client to the python interface section client wip added more arguments to section client added padFileNameWithZeros option added pad with zeros --- renderapi/client.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 74941a0c..f2ba5817 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -684,8 +684,9 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, @renderaccess def renderSectionClient(stack, rootDirectory, zs, scale=None, maxIntensity=None, minIntensity=None, bounds=None, - format=None, - doFilter=None, fillWithNoise=None, + format=None, channel=None, customOutputFolder=None, + customSubFolder=None,padFileNamesWithZeros=None, + doFilter=None, fillWithNoise=None, imageType=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): @@ -710,6 +711,16 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, dictionary with keys of minX maxX minY maxY format : str output image format in 'PNG', 'TIFF', 'JPEG' + channel : str + channel to render out (use on multichannel stack) + customOutputFolder : str + folder to save all images in (overrides default of sections_at_%scale) + customSubFolder : str + folder to save all images in under outputFolder (overrides default of none) + padFileNamesWithZeros: bool + whether to pad file names with zeros to make sortable + imageType: int + 8,16,24 to specify what kind of image type to save doFilter : str string representing java boolean for whether to render image with default filter (varies with render version) @@ -742,6 +753,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, get_param(minIntensity, '--minIntensity') + get_param(maxIntensity, '--maxIntensity') + get_param(fillWithNoise, '--fillWithNoise') + + get_param(customOutputFolder, '--customOutputFolder')+ + get_param(imageType,'--imageType')+ + get_param(channel,'--channels')+ + get_param(customSubFolder,'--customSubFolder')+ + get_param(padFileNamesWithZeros,'--padFileNamesWithZeros')+ bound_param + zs) call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, From 9e4c93922f369dd137020fb166fef77b18ca37d9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 15 Nov 2017 14:56:16 -0500 Subject: [PATCH 541/766] typo --- renderapi/channel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/channel.py b/renderapi/channel.py index 8509c9e1..267c19d8 100644 --- a/renderapi/channel.py +++ b/renderapi/channel.py @@ -11,7 +11,7 @@ def __init__(self,name=None,maxIntensity=None,minIntensity=None,ip=None,json=Non name of channel maxIntensity: int maximum intensity to display (optional) - minIntesnity: int + minIntensity: int minimum default intensity to display (optional) ip: ImagePyramid set of mipmaplevel images for this channel @@ -53,4 +53,4 @@ def from_dict(self,d): MipMapLevel( int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) for l, v in d['mipmapLevels'].items()]) - \ No newline at end of file + From 220eb30f7dae44e22c245cd1c2d518b20784bec5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 26 Nov 2017 00:28:26 -0800 Subject: [PATCH 542/766] render: add friendly classmethods to support RenderClient options --- renderapi/render.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index ad08f8ef..7893781f 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -33,7 +33,7 @@ class Render(object): """ def __init__(self, host=None, port=None, owner=None, project=None, - client_scripts=None): + client_scripts=None, **kwargs): self.DEFAULT_HOST = host self.DEFAULT_PORT = port self.DEFAULT_PROJECT = project @@ -151,6 +151,8 @@ class RenderClient(Render): java clients (default '1G' for 1 GB) """ + client_script_wrapper = 'run_ws_client.sh' + def __init__(self, client_script=None, memGB=None, validate_client=True, *args, **kwargs): """Initialize RenderClient object extending Render to @@ -175,11 +177,20 @@ def __init__(self, client_script=None, memGB=None, validate_client=True, super(RenderClient, self).__init__(**kwargs) if validate_client: if client_script is None: - raise ClientScriptError('No RenderClient script specified!') - elif not os.path.isfile(client_script): + if self.DEFAULT_CLIENT_SCRIPTS is None: + raise ClientScriptError( + 'No RenderClient script specified!') + else: + logger.debug("Attempting to derive client script " + "from client_scripts variable {}".format( + self.DEFAULT_CLIENT_SCRIPTS)) + client_script = self.clientscript_from_clientscripts( + self.DEFAULT_CLIENT_SCRIPTS) + + if not os.path.isfile(client_script): raise ClientScriptError('Client script {} not found!'.format( client_script)) - if 'run_ws_client.sh' not in os.path.basename(client_script): + if self.client_script_wrapper not in os.path.basename(client_script): logger.warning( 'Unrecognized client script {}!'.format(client_script)) self.client_script = client_script @@ -190,6 +201,10 @@ def __init__(self, client_script=None, memGB=None, validate_client=True, memGB = '1G' self.memGB = memGB + @classmethod + def clientscript_from_clientscripts(cls, client_scripts): + return os.path.join(client_scripts, cls.client_script_wrapper) + def make_kwargs(self, *args, **kwargs): """method to fill in default properties of RenderClient object @@ -329,7 +344,8 @@ def connect(host=None, port=None, owner=None, project=None, if client_script is None: if 'RENDER_CLIENT_SCRIPT' not in os.environ: # client_script = str(raw_input("Enter Render Client Script: ")) - client_script = os.path.join(client_scripts, 'run_ws_client.sh') + client_script = RenderClient.clientscript_from_clientscripts( + client_scripts) else: client_script = str(os.environ['RENDER_CLIENT_SCRIPT']) From ac8b30ec4f552205046d7180695d7f55f5f4417e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sun, 26 Nov 2017 00:29:39 -0800 Subject: [PATCH 543/766] client: add renderclientaccess decorator to address #52 and #16 --- renderapi/client.py | 88 +++++++++++++++++++++++++++------ test/test_client.py | 115 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 180 insertions(+), 23 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 338162be..c0a6e710 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,9 +8,10 @@ import logging import subprocess import tempfile +from decorator import decorator from .errors import ClientScriptError -from .utils import NullHandler, renderdump_temp -from .render import RenderClient, renderaccess +from .utils import NullHandler, renderdump_temp, fitargspec +from .render import RenderClient, renderaccess, Render from .stack import set_stack_state, make_stack_params from pathos.multiprocessing import ProcessingPool as Pool @@ -19,6 +20,61 @@ logger.addHandler(NullHandler()) +@decorator +def renderclientaccess(f, *args, **kwargs): + """Decorator allowing functions asking for host, port, owner, project, + client_script to default to a connection defined by :class:`RenderClient` + object using its :func:`RenderClient.make_kwargs` method. + Will also attempt to derive a :class:`RenderClient` from an input + :class:`Render` object and fail if client scripts cannot be reached. + + Parameters + ---------- + f : func + function to decorate + Returns + ------- + obj + output of decorated function + """ + args, kwargs = fitargspec(f, args, kwargs) + render = kwargs.get('render') + if render is not None: + if not isinstance(render, RenderClient): + if isinstance(render, Render): + render = RenderClient(**render.make_kwargs(**kwargs)) + else: + raise ValueError( + 'invalid RenderClient object type {} specified!'.format( + type(render))) + return f(*args, **render.make_kwargs(**kwargs)) + else: + try: + client_script = kwargs.get('client_script') + cs_valid = os.path.isfile(client_script) + except TypeError as e: + try: + if os.path.isdir(kwargs.get('client_scripts')): + client_script = os.path.join(client_scripts, + 'run_ws_client.sh') + cs_valid = os.path.isfile(client_script) + else: + raise ClientScriptError( + 'invalid client_scripts directory {}'.format( + kwargs.get('client_scripts'))) + except TypeError as e: + raise ClientScriptError( + 'No client script information specified: ' + 'client_scripts={} client_script={}'.format( + kwargs.get('client_scripts'), + kwargs.get('client_script'))) + if not cs_valid: + # TODO should also check for executability + raise ClientScriptError( + 'invalid client script: {} not a file'.format(client_script)) + return f(*args, **kwargs) + + class WithPool(Pool): """pathos ProcessingPool with functioning __exit__ call @@ -42,7 +98,7 @@ def __exit__(self, *args, **kwargs): super(WithPool, self)._clear() -@renderaccess +@renderclientaccess def import_single_json_file(stack, jsonfile, transformFile=None, client_scripts=None, host=None, port=None, owner=None, project=None, render=None, **kwargs): @@ -77,7 +133,7 @@ def import_single_json_file(stack, jsonfile, transformFile=None, logger.debug(proc.stdout.read()) -@renderaccess +@renderclientaccess def import_jsonfiles_and_transforms_parallel_by_z( stack, jsonfiles, transformfiles, poolsize=20, client_scripts=None, host=None, port=None, owner=None, @@ -117,7 +173,7 @@ def import_jsonfiles_and_transforms_parallel_by_z( set_stack_state(stack, 'COMPLETE', host, port, owner, project) -@renderaccess +@renderclientaccess def import_jsonfiles_parallel( stack, jsonfiles, poolsize=20, transformFile=None, client_scripts=None, host=None, port=None, owner=None, @@ -198,7 +254,7 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, set_stack_state(stack, 'COMPLETE', host, port, owner, project) -@renderaccess +@renderclientaccess def import_jsonfiles_validate_client(stack, jsonfiles, transformFile=None, client_scripts=None, host=None, port=None, owner=None, @@ -249,7 +305,7 @@ def import_jsonfiles_validate_client(stack, jsonfiles, set_stack_state(stack, 'COMPLETE', host, port, owner, project) -@renderaccess +@renderclientaccess def import_tilespecs(stack, tilespecs, sharedTransforms=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, @@ -285,7 +341,7 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, os.remove(trjson) -@renderaccess +@renderclientaccess def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, subprocess_mode=None, poolsize=20, close_stack=True, host=None, port=None, @@ -329,7 +385,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, # TODO handle fromJson and toJson persistence in these calls -@renderaccess +@renderclientaccess def local_to_world_array(stack, points, tileId, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, @@ -355,7 +411,7 @@ def local_to_world_array(stack, points, tileId, subprocess_mode=None, raise NotImplementedError('Whoops') -@renderaccess +@renderclientaccess def world_to_local_array(stack, points, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, @@ -443,7 +499,7 @@ def get_param(var, flag): return ([flag, var] if var is not None else []) -@renderaccess +@renderclientaccess def importJsonClient(stack, tileFiles=None, transformFile=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, @@ -473,7 +529,7 @@ def importJsonClient(stack, tileFiles=None, transformFile=None, client_script=client_script, memGB=memGB) -@renderaccess +@renderclientaccess def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, baseowner=None, baseproject=None, basestack=None, xyNeighborFactor=None, zNeighborDistance=None, @@ -584,7 +640,7 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, return jsondata -@renderaccess +@renderclientaccess def importTransformChangesClient(stack, targetStack, transformFile, targetOwner=None, targetProject=None, changeMode=None, close_stack=True, @@ -642,7 +698,7 @@ def importTransformChangesClient(stack, targetStack, transformFile, set_stack_state(stack, 'COMPLETE', host, port, owner, project) -@renderaccess +@renderclientaccess def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, numberOfThreads=None, subprocess_mode=None, host=None, port=None, owner=None, @@ -688,7 +744,7 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, return jsondata -@renderaccess +@renderclientaccess def renderSectionClient(stack, rootDirectory, zs, scale=None, maxIntensity=None, minIntensity=None, format=None, doFilter=None, fillWithNoise=None, @@ -734,7 +790,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, subprocess_mode=subprocess_mode, add_args=argvs) -@renderaccess +@renderclientaccess def transformSectionClient(stack, transformId, transformClass, transformData, zValues, targetProject=None, targetStack=None, replaceLast=None, subprocess_mode=None, diff --git a/test/test_client.py b/test/test_client.py index 5562657e..a50dcb69 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -1,4 +1,5 @@ import os +import pytest import renderapi import rendersettings @@ -9,6 +10,8 @@ 'project': 'renderproject', 'client_scripts': '/path/to/client_scripts' } + + def test_render_client(): r = renderapi.render.connect(**args) @@ -51,14 +54,112 @@ def test_environment_variables_client(): renvkwargs=rendersettings.DEFAULT_RENDER_CLIENT_ENVIRONMENT_VARIABLES, validate_client=False) + @renderapi.render.renderaccess -def my_decorated(myparameter, owner=None, host=None, port=None, - project=None,client_scripts=None, **kwargs): - return (owner,host,port,project,client_scripts) +def renderaccess_decorated(myparameter, owner=None, host=None, port=None, + project=None, client_scripts=None, **kwargs): + return (owner, host, port, project, client_scripts) + -def test_decorator(): +@renderapi.client.renderclientaccess +def renderclientaccess_decorated(myparameter, owner=None, host=None, + port=None, project=None, + client_scripts=None, client_script=None, + **kwargs): + return (owner, host, port, project, client_scripts, client_script) + + +def test_decorator(my_decorated=renderaccess_decorated): r = renderapi.render.connect(**args) - (owner,host,port,project,client_scripts)=my_decorated(5,render=r) + (owner, host, port, project, client_scripts) = my_decorated(5, render=r) assert(owner == args['owner']) - (owner,host,port,project,client_scripts)=my_decorated(5,owner='newowner',render=r) - assert(owner == 'newowner') + (owner, host, port, project, client_scripts) = my_decorated( + 5, owner='newowner', render=r) + assert(owner == 'newowner') + + +def test_renderaccess_decorator(tmpdir): + def checkexpected(expectation, values): + return all([i == j for i, j in zip(expectation, values)]) + + newargs = dict(args, **{'client_scripts': str(tmpdir)}) + + assert not os.path.isfile( + renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + expected = (newargs['owner'], newargs['host'], newargs['port'], + newargs['project'], newargs['client_scripts'], + renderapi.render.RenderClient.clientscript_from_clientscripts( + newargs['client_scripts'])) + + with open(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir)), 'w') as f: + # test that renderclientaccess decorated funtion works with Render + # objects missing client_script + assert checkexpected(expected, renderclientaccess_decorated( + 5, render=renderapi.render.Render(**newargs))) + # test that RenderClient objects continue to work + assert checkexpected(expected, renderclientaccess_decorated( + 5, render=renderapi.render.RenderClient(**newargs))) + # test with renderapi.connect set RenderObjects + assert checkexpected(expected, renderclientaccess_decorated( + 5, render=renderapi.connect(force_http=False, **newargs))) + + os.remove(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + +def test_renderclientaccess_decorator_fail(tmpdir): + # test that common methods of defining renderclient options fail quickly + newargs = dict(args, **{'client_scripts': str(tmpdir)}) + + assert not os.path.isfile( + renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + with pytest.raises(renderapi.errors.ClientScriptError): + _ = renderclientaccess_decorated( + 5, render=renderapi.render.Render(**newargs)) + + with pytest.raises(renderapi.errors.ClientScriptError): + _ = renderclientaccess_decorated( + 5, render=renderapi.render.RenderClient(**newargs)) + + with pytest.raises(renderapi.errors.ClientScriptError): + _ = renderclientaccess_decorated( + 5, render=renderapi.connect(force_http=False, **newargs)) + + +def test_renderclientaccess_override(tmpdir): + def checkexpected(expectation, values): + return all([i == j for i, j in zip(expectation, values)]) + + newargs = dict(args, **{'client_scripts': str(tmpdir)}) + + assert not os.path.isfile( + renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) + + expected = ('newowner', newargs['host'], newargs['port'], + newargs['project'], newargs['client_scripts'], + renderapi.render.RenderClient.clientscript_from_clientscripts( + newargs['client_scripts'])) + + with open(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir)), 'w') as f: + # test that renderclientaccess decorated funtion works with Render + # objects missing client_script + assert checkexpected(expected, renderclientaccess_decorated( + 5, owner='newowner', render=renderapi.render.Render(**newargs))) + # test that RenderClient objects continue to work + assert checkexpected(expected, renderclientaccess_decorated( + 5, owner='newowner', + render=renderapi.render.RenderClient(**newargs))) + # test with renderapi.connect set RenderObjects + assert checkexpected(expected, renderclientaccess_decorated( + 5, owner='newowner', + render=renderapi.connect(force_http=False, **newargs))) + + os.remove(renderapi.render.RenderClient.clientscript_from_clientscripts( + str(tmpdir))) From 9a3d11f4b1d0789867392214433e8eeb70799631 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 1 Dec 2017 15:38:39 -0800 Subject: [PATCH 544/766] changed to scm_version trivial change reverting whooops --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aa98afff..83833ed9 100644 --- a/setup.py +++ b/setup.py @@ -29,13 +29,14 @@ def run_tests(self): required = f.read().splitlines() setup(name='render-python', - version='1.5.0', + use_scm_version=True, description=' a python API to interact via python with render ' 'databases see https://github.com/saalfeldlab/render', author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', packages=['renderapi'], + setup_requires=['setuptools_scm'], install_requires=required, tests_require=test_required, cmdclass={'test': PyTest},) From 913eb2a7c65271f217cc25c3d267d1cedc4b280a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 1 Dec 2017 16:19:04 -0800 Subject: [PATCH 545/766] removing .git from .dockerignore --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index f6154099..978ac6f2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ -.git docker_setup.sh Dockerfile.base Dockerfile From ffe8a7774be9e3aa416ecab778e7345128cd8a3d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 2 Dec 2017 09:32:49 -0800 Subject: [PATCH 546/766] changed dockerfile to install --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 78249924..03c5f0cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,9 @@ FROM fcollman/render-python-base:latest MAINTAINER Forrest Collman (forrest.collman@gmail.com) -RUN mkdir -p /shared/render-python -COPY . /shared/render-python -RUN pip install -e /shared/render-python WORKDIR /shared/render-python +COPY . /shared/render-python +RUN python setup.py install ENTRYPOINT [ "/usr/bin/tini", "--" ] CMD [ "/bin/bash" ] From b2ca1afddf205e332e0dc78e012acf4e503a8cf0 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 2 Dec 2017 10:23:00 -0800 Subject: [PATCH 547/766] remove Dockerfile from .dockerignore --- .dockerignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 978ac6f2..a04aefb7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,2 @@ docker_setup.sh -Dockerfile.base -Dockerfile docker_base_setup.sh From bc2d48e6bfaa7208d8a3926d46ee2fde3528c82c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Sat, 2 Dec 2017 10:24:23 -0800 Subject: [PATCH 548/766] changed to pip install --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 03c5f0cc..5751a400 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Forrest Collman (forrest.collman@gmail.com) WORKDIR /shared/render-python COPY . /shared/render-python -RUN python setup.py install +RUN pip install /shared/render-python ENTRYPOINT [ "/usr/bin/tini", "--" ] CMD [ "/bin/bash" ] From a8b9bbfe965ab65a70fc09f539de68fdb24ffba9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 15 Jan 2018 16:47:57 -0800 Subject: [PATCH 549/766] added delete collection and test --- .../test_pointmatch_integrated.py | 10 +++++ renderapi/pointmatch.py | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 4f8cd910..87c6e1cf 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -190,3 +190,13 @@ def test_delete_point_matches_between_groups(render): collection, '0', '1', render=render) groups = renderapi.pointmatch.get_match_groupIds(collection, render=render) assert len(groups) == 2 + +def test_delete_collection(render): + collection = 'test_delete_collection' + owner = 'test' + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) + + renderapi.pointmatch.delete_collection(collection,render=render) + collections = renderapi.pointmatch.get_matchcollections(render=render) + assert(collection not in collections) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 0f7cfd13..679744f7 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -649,3 +649,47 @@ def import_matches(matchCollection, data, owner=None, host=None, port=None, r = session.put(request_url, data=data, headers={ "content-type": "application/json", "Accept": "application/json"}) return r + + +@renderaccess +def delete_collection(matchCollection, owner=None, host=None, port=None, + session=requests.session(), render=None, **kwargs): + """delete match collection from render database + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + matchCollection : str + matchCollection name to delete + owner : unicode + matchCollection owner (fallback to render.DEFAULT_OWNER) + (note match owner != stack owner always) + render : Render + Render connection object + session : requests.session.Session + requests session + + Returns + ------- + requests.response.Reponse + server response + + Raises + ------ + RenderError + if cannot get a proper reponse from server + + """ + request_url = format_baseurl(host, port) + \ + "/owner/%s/matchCollection/%s" % (owner, matchCollection) + logger.debug(request_url) + try: + r = session.delete(request_url) + return r + except Exception as e: + logger.error(e) + logger.error(request_url) + logger.error(r.text) + raise RenderError(r.text) + From 89344072390a5c36a3f21553c3e1aa68d221409b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 16 Jan 2018 09:20:03 -0800 Subject: [PATCH 550/766] adding codecov --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index edf871ee..133f8d48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,7 @@ addons: - oracle-java8-set-default - maven install: + - pip install codecov - pip install -r requirements.txt - pip install -r test_requirements.txt before_install: @@ -44,4 +45,5 @@ cache: - $HOME/.m2 secure: "x3rK0ICjrT1yAguEgq7/UyNDDCbbr46j7DsHLzEEgoM4l5crRECfdclgws4s4Z7RH6TRrW3+CKvhJuq/71QRIWBUN/ZUK9REEvgC13mOFFN7PQlhQGHTQPSiJ1oUTtXG5KenNBOOgz9q4vJNFnahn+L+GuN2TiAYRJGbyE8G5A5uFLIhXHVb+Xo295XkarFtX8EFKQmJJzdyFgqU+NsKW2gbq0hIASmQi3swJZhmbzQXhaj0gCuLjQfnDR+3qHzE+v5mQfObk4v6FRf3mdnZRVqV4S67yKONDK5LCkuI69C/1EKPiAfEEg5RKcRIzQlDjApgYVFf+jFjrVy6RLU9xOp2dstSFmynL96N+K4HoRAw53F08WokBih5hbsEwGAb/Fat1fLVy1hqmboF4d5Fy42TXrPmHgkqwlMXABPVqKwoWQTo0sANQdOVNWaF6NyMDTkUwtAzD9IG+Qwu/9v1zleT0VQ92Uk0s8wTDlkVny+8XGv4Pi2sNBcgG+huECNAOQKCeFbMn0LfMoGaKDXNCW4OHp59wXXXJQHAaN8xujwGpwQxJKb3iwf6uY9wnBpfKtaaCqOcP3Y/WhuiuKf4477bZPoQXD+2DFAPlC5nVjn+LzzN1HigdPCWn8F+SianZsL/iuxgEqmiW6OFGGzpqD4zHFNACky3fTS7yBLnd+Q=" after_success: - - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=cdown/srt .travis/merge_script.sh" \ No newline at end of file + - codecov -t 3f12d985-af62-455d-a11d-9669c039640d + - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=cdown/srt .travis/merge_script.sh" From d688ed06f44b1d2cadd483943698265e76fe007c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 16 Jan 2018 09:20:53 -0800 Subject: [PATCH 551/766] adding codecov --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4b881551..83ed3ece 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,9 @@ .. image:: https://travis-ci.org/fcollman/render-python.svg?branch=master :target: https://travis-ci.org/fcollman/render-python :alt: Build Status - +.. image:: https://codecov.io/gh/fcollman/render-python/branch/master/graph/badge.svg + :target: https://codecov.io/gh/fcollman/render-python + render-python ############# From 16533f7efa6102d741cb5bc11c475710177f9267 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 16 Jan 2018 17:52:01 -0800 Subject: [PATCH 552/766] fixing travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 133f8d48..f3d8b20f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,4 +46,4 @@ cache: secure: "x3rK0ICjrT1yAguEgq7/UyNDDCbbr46j7DsHLzEEgoM4l5crRECfdclgws4s4Z7RH6TRrW3+CKvhJuq/71QRIWBUN/ZUK9REEvgC13mOFFN7PQlhQGHTQPSiJ1oUTtXG5KenNBOOgz9q4vJNFnahn+L+GuN2TiAYRJGbyE8G5A5uFLIhXHVb+Xo295XkarFtX8EFKQmJJzdyFgqU+NsKW2gbq0hIASmQi3swJZhmbzQXhaj0gCuLjQfnDR+3qHzE+v5mQfObk4v6FRf3mdnZRVqV4S67yKONDK5LCkuI69C/1EKPiAfEEg5RKcRIzQlDjApgYVFf+jFjrVy6RLU9xOp2dstSFmynL96N+K4HoRAw53F08WokBih5hbsEwGAb/Fat1fLVy1hqmboF4d5Fy42TXrPmHgkqwlMXABPVqKwoWQTo0sANQdOVNWaF6NyMDTkUwtAzD9IG+Qwu/9v1zleT0VQ92Uk0s8wTDlkVny+8XGv4Pi2sNBcgG+huECNAOQKCeFbMn0LfMoGaKDXNCW4OHp59wXXXJQHAaN8xujwGpwQxJKb3iwf6uY9wnBpfKtaaCqOcP3Y/WhuiuKf4477bZPoQXD+2DFAPlC5nVjn+LzzN1HigdPCWn8F+SianZsL/iuxgEqmiW6OFGGzpqD4zHFNACky3fTS7yBLnd+Q=" after_success: - codecov -t 3f12d985-af62-455d-a11d-9669c039640d - - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=cdown/srt .travis/merge_script.sh" + - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=fcollman/render-python .travis/merge_script.sh" From 2b13ccee21aff94c83092d32dfdae47871e2bb3b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 17 Jan 2018 08:03:55 -0800 Subject: [PATCH 553/766] trying to fix automerge --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f3d8b20f..1894bcc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,7 +43,7 @@ cache: pip: true directories: - $HOME/.m2 -secure: "x3rK0ICjrT1yAguEgq7/UyNDDCbbr46j7DsHLzEEgoM4l5crRECfdclgws4s4Z7RH6TRrW3+CKvhJuq/71QRIWBUN/ZUK9REEvgC13mOFFN7PQlhQGHTQPSiJ1oUTtXG5KenNBOOgz9q4vJNFnahn+L+GuN2TiAYRJGbyE8G5A5uFLIhXHVb+Xo295XkarFtX8EFKQmJJzdyFgqU+NsKW2gbq0hIASmQi3swJZhmbzQXhaj0gCuLjQfnDR+3qHzE+v5mQfObk4v6FRf3mdnZRVqV4S67yKONDK5LCkuI69C/1EKPiAfEEg5RKcRIzQlDjApgYVFf+jFjrVy6RLU9xOp2dstSFmynL96N+K4HoRAw53F08WokBih5hbsEwGAb/Fat1fLVy1hqmboF4d5Fy42TXrPmHgkqwlMXABPVqKwoWQTo0sANQdOVNWaF6NyMDTkUwtAzD9IG+Qwu/9v1zleT0VQ92Uk0s8wTDlkVny+8XGv4Pi2sNBcgG+huECNAOQKCeFbMn0LfMoGaKDXNCW4OHp59wXXXJQHAaN8xujwGpwQxJKb3iwf6uY9wnBpfKtaaCqOcP3Y/WhuiuKf4477bZPoQXD+2DFAPlC5nVjn+LzzN1HigdPCWn8F+SianZsL/iuxgEqmiW6OFGGzpqD4zHFNACky3fTS7yBLnd+Q=" +secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" after_success: - codecov -t 3f12d985-af62-455d-a11d-9669c039640d - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=fcollman/render-python .travis/merge_script.sh" From 43388b7892948e27c4ceac74d5dbf49bbfa7bb8c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 17 Jan 2018 14:14:09 -0800 Subject: [PATCH 554/766] trying to fix travis automerge again --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1894bcc0..b3f53284 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,6 +36,8 @@ before_install: - docker-compose up -d env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts + global: + - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below @@ -43,7 +45,6 @@ cache: pip: true directories: - $HOME/.m2 -secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" after_success: - codecov -t 3f12d985-af62-455d-a11d-9669c039640d - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=fcollman/render-python .travis/merge_script.sh" From fd778c3f7f2e6581f4862483fb825ddb5b1c436c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 17 Jan 2018 14:14:58 -0800 Subject: [PATCH 555/766] fix again --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b3f53284..4be23bb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,7 @@ before_install: - docker-compose up -d env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts - global: - - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" + secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From 2a7d981c57ac36b0404bc4bfa102a13fa96b51c3 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 17 Jan 2018 14:15:27 -0800 Subject: [PATCH 556/766] again.. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4be23bb8..f35306f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,7 +36,7 @@ before_install: - docker-compose up -d env: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" + - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From 11f4840cf60a39bc4186e26cd44fdfb96a0e7aaa Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 17 Jan 2018 14:36:05 -0800 Subject: [PATCH 557/766] trying again.. --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f35306f0..53b2ebcf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,8 +35,9 @@ before_install: - export RENDER_EXAMPLE_DATA=/tmp - docker-compose up -d env: - - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts - - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" + global: + - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts + - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below From 9d719aecbf86f4b93b9ef52e0ff7ea5e0cf22bbb Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 17 Jan 2018 15:49:38 -0800 Subject: [PATCH 558/766] removing push to develop --- .travis/merge_script.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis/merge_script.sh b/.travis/merge_script.sh index c172cbce..16da047f 100755 --- a/.travis/merge_script.sh +++ b/.travis/merge_script.sh @@ -31,4 +31,3 @@ push_uri="https://$GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO" # Redirect to /dev/null to avoid secret leakage git push "$push_uri" "$BRANCH_TO_MERGE_INTO" >/dev/null 2>&1 -git push "$push_uri" :"$TRAVIS_BRANCH" >/dev/null 2>&1 From f1d6c4a3aba82c263d9f34990edd71c15c55b4e2 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 2 Feb 2018 01:41:03 -0800 Subject: [PATCH 559/766] stack: add option to get full stack metadata --- renderapi/stack.py | 49 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 17afc51e..94164327 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -124,9 +124,10 @@ def set_stack_metadata(stack, sv, host=None, port=None, owner=None, @renderaccess -def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): - """get the stack metadata for a stack +def get_full_stack_metadata(stack, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + """get stack metadata for stack :func:`renderapi.render.renderaccess` decorated function @@ -141,28 +142,60 @@ def get_stack_metadata(stack, host=None, port=None, owner=None, project=None, Returns ------- - StackVersion + dict metadata of the stack Raises ------ RenderError - """ request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) r = session.get(request_url) + try: - sv = StackVersion() - sv.from_dict(r.json()['currentVersion']) - return sv + return r.json() except Exception as e: logger.error(e) logger.error(r.text) raise RenderError(r.text) +def get_stack_metadata(*args, **kwargs): + """get the stack version metadata for a stack + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to get the metadata for + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + StackVersion + metadata of the stack + + Raises + ------ + RenderError + + """ + j = get_full_stack_metadata(*args, **kwargs) + try: + sv = StackVersion() + sv.from_dict(j['currentVersion']) + return sv + except Exception as e: + logger.error(e) + raise RenderError(e) + + @renderaccess def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, From 87bf3bbd80335fc4fccc0ffd4d9ab3f0801dd978 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 2 Feb 2018 18:49:11 -0800 Subject: [PATCH 560/766] tests: change hardcoded render ports --- integration_tests/test_coordinate_integrated.py | 2 +- integration_tests/test_pointmatch_integrated.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 16745686..5600c724 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -26,7 +26,7 @@ def render(): render_test_parameters = { 'host': render_host, - 'port': 8080, + 'port': render_port, 'owner': 'test_coordinate', 'project': 'test_coordinate_project', 'client_scripts': client_script_location diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 87c6e1cf..62cb6028 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -88,7 +88,7 @@ def render(): render_test_parameters = { 'host': render_host, - 'port': 8080, + 'port': render_port, 'owner': 'test', 'project': 'test_pointmatch_project', 'client_scripts': client_script_location From e259bd8c36c3baf9e173b223826c03adaba56a37 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Sat, 3 Feb 2018 10:37:16 -0800 Subject: [PATCH 561/766] tests: allow RENDERAPP_EXAMPLE_DATA to override RENDER_EXAMPLE_DATA as final step to allow non-deployment testing --- integration_tests/test_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py index 9ed3ae47..ee7b2aaf 100644 --- a/integration_tests/test_data.py +++ b/integration_tests/test_data.py @@ -8,6 +8,7 @@ def render_json_template(env, template_file, **kwargs): return d example_dir = os.environ.get('RENDER_EXAMPLE_DATA','/var/www/render/examples/') +renderapp_example_dir = os.environ.get('RENDERAPP_EXAMPLE_DATA', example_dir) test_files_dir = os.path.join(os.path.dirname(__file__), 'test_files') example_env = Environment(loader=FileSystemLoader(test_files_dir)) @@ -31,9 +32,8 @@ def render_json_template(env, template_file, **kwargs): tform_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_transforms.json') test_pool_size = os.environ.get('RENDER_PYTHON_TEST_POOL_SIZE',3) -multi_channel_dir = os.path.join(example_dir,'multichannel-test') +multi_channel_dir = os.path.join(renderapp_example_dir, 'multichannel-test') test_2_channels_d = render_json_template(example_env, 'test_2_channels.json', multi_channel_example_dir=multi_channel_dir) - From 7c3b2c0a2cfc7c88e84f9ebaf8be0b55d39d2240 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 15 Feb 2018 09:40:26 -0800 Subject: [PATCH 562/766] basic smoke test implementation of pointmatchclient --- integration_tests/test_client_integrated.py | 9 + renderapi/client.py | 193 +++++++++++++++++++- requirements.txt | 1 + 3 files changed, 202 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index ae05220b..540620ff 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -242,3 +242,12 @@ def test_transformSectionClient(render, teststack, assert all([ts.tforms[-1].to_dict() == tform.to_dict() for ts in output_ts]) renderapi.stack.delete_stack(deststack, render=render) + +def test_point_match_client(teststack, render): + collection = 'test_collection' + zvalues = np.array(renderapi.stack.get_z_values_for_stack( + teststack, render=render)) + tilepairjson = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), render=render) + tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs']] + renderapi.client.pointMatchClient(teststack,collection,tile_pairs,render=render) diff --git a/renderapi/client.py b/renderapi/client.py index 3ba8aa56..b433e45b 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -11,9 +11,10 @@ from decorator import decorator from .errors import ClientScriptError from .utils import NullHandler, renderdump_temp, fitargspec -from .render import RenderClient, renderaccess, Render +from .render import RenderClient, renderaccess, Render, format_preamble, format_baseurl from .stack import set_stack_state, make_stack_params from pathos.multiprocessing import ProcessingPool as Pool +import attr # setup logger logger = logging.getLogger(__name__) @@ -859,3 +860,193 @@ def transformSectionClient(stack, transformId, transformClass, transformData, call_run_ws_client('org.janelia.render.client.TransformSectionClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) + +@renderclientaccess +def get_canvas_url_template(stack, filter=False, + renderWithoutMask=False, normalizeForMatching=True, + excludeTransformsAfterLast=None, + excludeFirstTransformAndAllAfter=None,excludeAllTransforms=False, + host=None, port=None, owner=None, project=None, client_script=None, + render=None, **kwargs): + """function for making a render-parameters url template for point matching + + Parameters + ---------- + stack: str + render stack name + filter: bool + whether to apply default filtering to tile (default=False) + renderWithoutMask: bool + whether to exclude the mask when rendering tile (default=False) + normalizeForMatching: bool + whether to apply traditional 'normalizeForMatching' transform manipulation to image + this removes the last transform from the transformList, then if there are more than 3 transforms + continues to remove transforms until there are exactly 3. Then assumes the image will be near 0,0 + with a width/height that is about equal to the raw image width/height. This is true for Janelia's + conventions for transformation alignment, but use at your own risk. (default=True) + excludeTransformsAfterLast: str or None + alternative to normalizeForMatching, which uses transformLabels. Will remove all transformations + after the last transformation with this transform label. i.e. if all lens corrections have a 'lens' + label. Then this will remove all non-lens transformations from the list. + This is more general than normalizeForMatching=true, but requires you have transform labels applied. + default = None + excludeFirstTransformAndAllAfter: str + alternative to normalizeForMatching which finds the first transform in the list with a given label + and then removes that transform and all transforms that follow it. i.e. if you had a compound list + of transformations, and you had labelled the first non-local transform 'montage' then setting + excludeFirstTransformAndAllAfter='montage' would remove that montage transform and any other + transforms that you had applied after it. default= None. + excludeAllTransforms: bool + alternative to normalizeForMatching which simply removes all transforms from the list. + default=False + """ + request_url = format_preamble(host, port, owner, project, stack) + tile_base_url = request_url+"/tile" + url_suffix = "render-parameters" + if filter: + url_suffix += '?filter=true' + else: + url_suffix += '?filter=false' + + if normalizeForMatching: + url_suffix += '&normalizeForMatching=true' + if normalizeForMatching: + url_suffix += '&normalizeForMatching=false' + + if renderWithoutMask: + url_suffix += '&renderWithoutMask=true' + else: + url_suffix += '&renderWithoutMask=false' + + if excludeTransformsAfterLast is not None: + url_suffix += '&excludeTransformsAfterLast={}'.format(excludeTransformsAfterLast) + if excludeFirstTransformAndAllAfter is not None: + url_suffix += '&excludeFirstTransformAndAllAfter={}'.format(excludeFirstTransformAndAllAfter) + if excludeAllTransforms: + url_suffix += '&excludeAllTransforms=true' + + canvas_url_template = "%s/{}/%s"%(tile_base_url, + url_suffix) + return canvas_url_template + +class SiftOptions(object): + SIFTfdSize = attr.ib(default=89) + SIFTmaxScale = attr.ib(default=.85) + SIFTminScale = attr.ib(default=.5) + SIFTsteps = attr.ib(default=3) + matchIterations = attr.ib(default=1000) + matchMaxEpsilon = attr.ib(default=20.0) + matchMaxNumInliers = attr.ib(default=500) + matchMaxTrust = attr.ib(default=3.0) + matchMinInlierRatio = attr.ib(default=0.0) + matchMinNumInliers = attr.ib(default=8) + matchModelType = attr.ib(default='AFFINE') + matchRod = attr.ib(default=0.92) + renderScale = attr.ib(default=.35) + + @matchModelType.validator + def checkModel(self,attribute,value): + assert value in ['AFFINE','RIGID','TRANSLATION','SIMILARITY'] + + @renderScale.validator + def checkScale(self,attribute,value): + assert (value>0.0) + assert (value<=1.0) + + def to_java_args(self): + args = [] + for key,value in self.__dict__.items(): + args.append("--{}".format(key)) + args.append("{}".value) + return args + +@renderclientaccess +def pointMatchClient(stack, collection, tile_pairs, + sift_options=SiftOptions(), + pointMatchRender=None, + debugDirectory=None, + filter=False, + renderWithoutMask=False, normalizeForMatching=True, + excludeTransformsAfterLast=None, excludeAllTransforms=None, + excludeFirstTransformAndAllAfter=None, + subprocess_mode=None, + host=None, port=None, + owner=None, project=None, client_script=None, + memGB=None, render=None, **kwargs): + """run SiftPointMatchClient.java + + Parameters + ---------- + stack : str + stack containing the tiles + collection : str + point match collection to save results into + tile_pairs : iterable + list of iterables of length 2 containing tileIds to calculate point matches between + sift_options: SiftOptions + options for running point matching + pointMatchRender : renderapi.render.renderaccess + renderaccess object specifying the render server to store point matches in + defaults to values specified by render and its keyword argument overrides + debugDirectory : str + directory to store debug results (optional) + filter: bool + whether to apply default filtering to tile (default=False) + renderWithoutMask: bool + whether to exclude the mask when rendering tile (default=False) + normalizeForMatching: bool + whether to apply traditional 'normalizeForMatching' transform manipulation to image + this removes the last transform from the transformList, then if there are more than 3 transforms + continues to remove transforms until there are exactly 3. Then assumes the image will be near 0,0 + with a width/height that is about equal to the raw image width/height. This is true for Janelia's + conventions for transformation alignment, but use at your own risk. (default=True) + excludeTransformsAfterLast: str or None + alternative to normalizeForMatching, which uses transformLabels. Will remove all transformations + after the last transformation with this transform label. i.e. if all lens corrections have a 'lens' + label. Then this will remove all non-lens transformations from the list. + This is more general than normalizeForMatching=true, but requires you have transform labels applied. + default = None + excludeFirstTransformAndAllAfter: str + alternative to normalizeForMatching which finds the first transform in the list with a given label + and then removes that transform and all transforms that follow it. i.e. if you had a compound list + of transformations, and you had labelled the first non-local transform 'montage' then setting + excludeFirstTransformAndAllAfter='montage' would remove that montage transform and any other + transforms that you had applied after it. default= None. + excludeAllTransforms: bool + alternative to normalizeForMatching which simply removes all transforms from the list. + default=False + + """ + if pointMatchRender is None: + pointMatchRender = Render(host, port, owner, project, client_script) + + baseDataUrl = format_baseurl(pointMatchRender.DEFAULT_KWARGS['host'], + pointMatchRender.DEFAULT_KWARGS['port']) + argvs = [] + argvs += ['--baseDataUrl', baseDataUrl] + argvs += ['--owner', pointMatchRender.DEFAULT_KWARGS['owner']] + argvs += ['--collection', collection] + #argvs += ['--matchStorageFile', os.path.join(outdir, 'matches.json')] + if debugDirectory is not None: + argvs += ['--debugDirectory', debugDirectory] + argvs += sift_options.to_java_args() + + canvas_url_template = get_canvas_url_template(stack, + filter, + renderWithoutMask, + normalizeForMatching, + excludeTransformsAfterLast, + excludeFirstTransformAndAllAfter, + excludeAllTransforms, + host=host, + port=port, + owner=owner, + project=project, + client_script=client_script) + + for tile1,tile2 in tile_pairs: + argvs += [canvas_url_template.format(tile1),canvas_url_template.format(tile2)] + + call_run_ws_client('org.janelia.render.client.PointMatchClient', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs) diff --git a/requirements.txt b/requirements.txt index f56d3209..cf41d8de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ dill>=0.2.6 pathos sphinxcontrib-napoleon decorator +attrs \ No newline at end of file From d748f2de1778404a3f8b8c5076357e4c12025ca9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 11:40:14 -0800 Subject: [PATCH 563/766] removing attrs --- integration_tests/test_client_integrated.py | 2 +- renderapi/client.py | 45 ++++++++++----------- requirements.txt | 1 - 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 540620ff..95229608 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -249,5 +249,5 @@ def test_point_match_client(teststack, render): teststack, render=render)) tilepairjson = renderapi.client.tilePairClient( teststack, np.min(zvalues), np.max(zvalues), render=render) - tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs']] + tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs'][0:1]] renderapi.client.pointMatchClient(teststack,collection,tile_pairs,render=render) diff --git a/renderapi/client.py b/renderapi/client.py index b433e45b..0bc68256 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -930,34 +930,33 @@ def get_canvas_url_template(stack, filter=False, return canvas_url_template class SiftOptions(object): - SIFTfdSize = attr.ib(default=89) - SIFTmaxScale = attr.ib(default=.85) - SIFTminScale = attr.ib(default=.5) - SIFTsteps = attr.ib(default=3) - matchIterations = attr.ib(default=1000) - matchMaxEpsilon = attr.ib(default=20.0) - matchMaxNumInliers = attr.ib(default=500) - matchMaxTrust = attr.ib(default=3.0) - matchMinInlierRatio = attr.ib(default=0.0) - matchMinNumInliers = attr.ib(default=8) - matchModelType = attr.ib(default='AFFINE') - matchRod = attr.ib(default=0.92) - renderScale = attr.ib(default=.35) - @matchModelType.validator - def checkModel(self,attribute,value): - assert value in ['AFFINE','RIGID','TRANSLATION','SIMILARITY'] - - @renderScale.validator - def checkScale(self,attribute,value): - assert (value>0.0) - assert (value<=1.0) + def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, + SIFTminScale=None, SIFTsteps=None, matchIterations=None, + matchMaxEpsilon=None, matchMaxNumInliers=None, + matchMaxTrust=None, matchMinInlierRatio=None, matchMinNumInliers=None, + matchModelType=None, matchRod=None, renderScale=None, **kwargs): + self.SIFTfdSize=SIFTfdSize + self.SIFTmaxScale=SIFTmaxScale + self.SIFTminScale=SIFTminScale + self.SIFTsteps=SIFTsteps + self.matchIterations=matchIterations + self.matchMaxEpsilon=matchMaxEpsilon + self.matchMaxNumInliers=matchMaxNumInliers + self.matchMaxTrust=matchMaxTrust + self.matchMinInlierRatio=matchMinInlierRatio + self.matchMinNumInliers=matchMinNumInliers + self.matchMinNumInliers=matchMinNumInliers + self.matchModelType=matchModelType + self.matchRod=matchRod + self.renderScale=renderScale def to_java_args(self): args = [] for key,value in self.__dict__.items(): - args.append("--{}".format(key)) - args.append("{}".value) + if value is not None: + args.append("--{}".format(key)) + args.append("{}".value) return args @renderclientaccess diff --git a/requirements.txt b/requirements.txt index cf41d8de..f56d3209 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,3 @@ dill>=0.2.6 pathos sphinxcontrib-napoleon decorator -attrs \ No newline at end of file From 0c7dd797e5aa7092acb81b3a159686cfcd12150f Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 13:26:45 -0800 Subject: [PATCH 564/766] fixing tests --- integration_tests/test_client_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 95229608..a2f3f221 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -244,7 +244,7 @@ def test_transformSectionClient(render, teststack, renderapi.stack.delete_stack(deststack, render=render) def test_point_match_client(teststack, render): - collection = 'test_collection' + collection = 'test_client_collection' zvalues = np.array(renderapi.stack.get_z_values_for_stack( teststack, render=render)) tilepairjson = renderapi.client.tilePairClient( From 5b87085a5996a5fd224d2869bec03cd0279d8d49 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 13:51:26 -0800 Subject: [PATCH 565/766] adding assert --- integration_tests/test_client_integrated.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index a2f3f221..9867cb47 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -251,3 +251,6 @@ def test_point_match_client(teststack, render): teststack, np.min(zvalues), np.max(zvalues), render=render) tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs'][0:1]] renderapi.client.pointMatchClient(teststack,collection,tile_pairs,render=render) + pms = rendearpi.pointmatch.get_matches_involving_tile(collection,tp['p']['groupId'],tp['p']['id'],render=render) + assert(len(pms)>0) + \ No newline at end of file From 448df6f9c0ca3d077eedca20229744a2a9e7e187 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 14:01:36 -0800 Subject: [PATCH 566/766] fix duplicate normalizeForMatching --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 0bc68256..8fcb3f46 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -910,7 +910,7 @@ def get_canvas_url_template(stack, filter=False, if normalizeForMatching: url_suffix += '&normalizeForMatching=true' - if normalizeForMatching: + else: url_suffix += '&normalizeForMatching=false' if renderWithoutMask: From d2e79a8cc78aa0457254b40fbec1dd7f68b2c611 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 14:01:42 -0800 Subject: [PATCH 567/766] fix typo in test --- integration_tests/test_client_integrated.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 9867cb47..53e6c01b 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -251,6 +251,5 @@ def test_point_match_client(teststack, render): teststack, np.min(zvalues), np.max(zvalues), render=render) tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs'][0:1]] renderapi.client.pointMatchClient(teststack,collection,tile_pairs,render=render) - pms = rendearpi.pointmatch.get_matches_involving_tile(collection,tp['p']['groupId'],tp['p']['id'],render=render) + pms = renderapi.pointmatch.get_matches_involving_tile(collection,tp['p']['groupId'],tp['p']['id'],render=render) assert(len(pms)>0) - \ No newline at end of file From f7570429daa9b317c23f598afb9fb9245e81f009 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 14:22:03 -0800 Subject: [PATCH 568/766] bug fix in SiftOptions --- renderapi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 8fcb3f46..64845322 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -956,7 +956,7 @@ def to_java_args(self): for key,value in self.__dict__.items(): if value is not None: args.append("--{}".format(key)) - args.append("{}".value) + args.append("{}".format(value)) return args @renderclientaccess From 0a72520b63b47340015021667ceb0a5a80a5e6d1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 14:22:09 -0800 Subject: [PATCH 569/766] fixed test --- integration_tests/test_client_integrated.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 53e6c01b..25082366 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -249,7 +249,14 @@ def test_point_match_client(teststack, render): teststack, render=render)) tilepairjson = renderapi.client.tilePairClient( teststack, np.min(zvalues), np.max(zvalues), render=render) + tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs'][0:1]] - renderapi.client.pointMatchClient(teststack,collection,tile_pairs,render=render) + sift_options = renderapi.client.SiftOptions(renderScale=.25) + renderapi.client.pointMatchClient(teststack, + collection, + tile_pairs, + sift_options=sift_options, + render=render) + tp = tilepairjson['neighborPairs'][0] pms = renderapi.pointmatch.get_matches_involving_tile(collection,tp['p']['groupId'],tp['p']['id'],render=render) assert(len(pms)>0) From a371b6e620cc5132d1bd28cdc488745156b0e4a9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 16 Feb 2018 15:19:20 -0800 Subject: [PATCH 570/766] added debugDirectory to test --- integration_tests/test_client_integrated.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 25082366..6a8acc9d 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -243,7 +243,7 @@ def test_transformSectionClient(render, teststack, for ts in output_ts]) renderapi.stack.delete_stack(deststack, render=render) -def test_point_match_client(teststack, render): +def test_point_match_client(teststack, render,tmpdir): collection = 'test_client_collection' zvalues = np.array(renderapi.stack.get_z_values_for_stack( teststack, render=render)) @@ -255,6 +255,7 @@ def test_point_match_client(teststack, render): renderapi.client.pointMatchClient(teststack, collection, tile_pairs, + debugDirectory=tmpdir, sift_options=sift_options, render=render) tp = tilepairjson['neighborPairs'][0] From 79d052d6015c9f2a9d9a133b5a7422c3b2c5e8b3 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Thu, 22 Feb 2018 16:50:32 -0800 Subject: [PATCH 571/766] pointmatchclient: different sift parameter handling (java based) --- integration_tests/test_client_integrated.py | 4 +- renderapi/client.py | 139 ++++++++++++++------ 2 files changed, 100 insertions(+), 43 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 6a8acc9d..cb036b91 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -249,9 +249,9 @@ def test_point_match_client(teststack, render,tmpdir): teststack, render=render)) tilepairjson = renderapi.client.tilePairClient( teststack, np.min(zvalues), np.max(zvalues), render=render) - + tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs'][0:1]] - sift_options = renderapi.client.SiftOptions(renderScale=.25) + sift_options = renderapi.client.SiftPointMatchOptions(renderScale=.25) renderapi.client.pointMatchClient(teststack, collection, tile_pairs, diff --git a/renderapi/client.py b/renderapi/client.py index 64845322..07f0f400 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -862,14 +862,14 @@ def transformSectionClient(stack, transformId, transformClass, transformData, subprocess_mode=subprocess_mode, add_args=argvs) @renderclientaccess -def get_canvas_url_template(stack, filter=False, +def get_canvas_url_template(stack, filter=False, renderWithoutMask=False, normalizeForMatching=True, - excludeTransformsAfterLast=None, + excludeTransformsAfterLast=None, excludeFirstTransformAndAllAfter=None,excludeAllTransforms=False, host=None, port=None, owner=None, project=None, client_script=None, render=None, **kwargs): """function for making a render-parameters url template for point matching - + Parameters ---------- stack: str @@ -921,53 +921,107 @@ def get_canvas_url_template(stack, filter=False, if excludeTransformsAfterLast is not None: url_suffix += '&excludeTransformsAfterLast={}'.format(excludeTransformsAfterLast) if excludeFirstTransformAndAllAfter is not None: - url_suffix += '&excludeFirstTransformAndAllAfter={}'.format(excludeFirstTransformAndAllAfter) + url_suffix += '&excludeFirstTransformAndAllAfter={}'.format(excludeFirstTransformAndAllAfter) if excludeAllTransforms: url_suffix += '&excludeAllTransforms=true' - + canvas_url_template = "%s/{}/%s"%(tile_base_url, url_suffix) return canvas_url_template -class SiftOptions(object): - - def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, - SIFTminScale=None, SIFTsteps=None, matchIterations=None, - matchMaxEpsilon=None, matchMaxNumInliers=None, - matchMaxTrust=None, matchMinInlierRatio=None, matchMinNumInliers=None, - matchModelType=None, matchRod=None, renderScale=None, **kwargs): - self.SIFTfdSize=SIFTfdSize - self.SIFTmaxScale=SIFTmaxScale - self.SIFTminScale=SIFTminScale - self.SIFTsteps=SIFTsteps - self.matchIterations=matchIterations - self.matchMaxEpsilon=matchMaxEpsilon - self.matchMaxNumInliers=matchMaxNumInliers - self.matchMaxTrust=matchMaxTrust - self.matchMinInlierRatio=matchMinInlierRatio - self.matchMinNumInliers=matchMinNumInliers - self.matchMinNumInliers=matchMinNumInliers - self.matchModelType=matchModelType - self.matchRod=matchRod - self.renderScale=renderScale + +class ArgumentParameters(object): + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def sanitize_cmd(cmd): + def jbool_str(c): + return str(c) if type(c) is not bool else "true" if c else "false" + if any([i is None for i in cmd]): + # FIXME exception class + raise Exception( + 'missing argument in command "{}"'.format(map(str, cmd))) + return map(jbool_str, cmd) + + @staticmethod + def get_cmd_opt(v, flag=None): + return [] if v is None else [v] if flag is None else [flag, v] + + @staticmethod + def get_flag_cmd(v, flag=None): + # for arity 0 + raise NotImplementedError( + "flag commands are not supported by ArgumentParameters") + return [flag] if v else [] def to_java_args(self): args = [] - for key,value in self.__dict__.items(): - if value is not None: - args.append("--{}".format(key)) - args.append("{}".format(value)) - return args + for key, value in self.__dict__.items(): + if (value is not None) and not (key == 'kwargs'): + args += self.get_cmd_opt(value, "--{}".format(key)) + return self.sanitize_cmd(args) + + +class FeatureRenderParameters(ArgumentParameters): + def __init__(self, renderScale=None, renderWithFilter=None, + renderWithoutMask=None, renderFullScaleWidth=None, + renderFullScaleHeight=None, fillWithNoise=None, **kwargs): + self.renderScale = renderScale + self.renderWithFilter = renderWithFilter + self.renderWithoutMask = renderWithoutMask + self.renderFullScaleWidth = renderFullScaleWidth + self.renderFullScaleHeight = renderFullScaleHeight + self.fillWithNoise = fillWithNoise + + +class FeatureExtractionParameters(ArgumentParameters): + def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, + SIFTminScale=None, SIFTsteps=None, **kwargs): + super(FeatureExtractionParameters, self).__init__(**kwargs) + self.SIFTfdSize = SIFTfdSize + self.SIFTmaxScale = SIFTmaxScale + self.SIFTminScale = SIFTminScale + self.SIFTsteps = SIFTsteps + + +class MatchDerivationParameters(ArgumentParameters): + def __init__(self, matchIterations=None, + matchMaxEpsilon=None, matchMaxNumInliers=None, + matchMaxTrust=None, matchMinInlierRatio=None, + matchMinNumInliers=None, + matchModelType=None, matchRod=None, **kwargs): + super(MatchDerivationParameters, self).__init__(**kwargs) + self.matchIterations = matchIterations + self.matchMaxEpsilon = matchMaxEpsilon + self.matchMaxNumInliers = matchMaxNumInliers + self.matchMaxTrust = matchMaxTrust + self.matchMinInlierRatio = matchMinInlierRatio + self.matchMinNumInliers = matchMinNumInliers + self.matchMinNumInliers = matchMinNumInliers + self.matchModelType = matchModelType + self.matchRod = matchRod + + + +class SiftPointMatchOptions(MatchDerivationParameters, + FeatureExtractionParameters): + def __init__(self, renderScale=None, fillWithNoise=None, **kwargs): + # TODO add missing parameters + super(SiftPointMatchOptions, self).__init__(**kwargs) + self.renderScale = renderScale + self.fillWithNoise = fillWithNoise + @renderclientaccess def pointMatchClient(stack, collection, tile_pairs, - sift_options=SiftOptions(), + sift_options=None, pointMatchRender=None, - debugDirectory=None, - filter=False, + debugDirectory=None, + filter=False, renderWithoutMask=False, normalizeForMatching=True, excludeTransformsAfterLast=None, excludeAllTransforms=None, - excludeFirstTransformAndAllAfter=None, + excludeFirstTransformAndAllAfter=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, @@ -1016,11 +1070,14 @@ def pointMatchClient(stack, collection, tile_pairs, default=False """ + sift_options = (SiftPointMatchOptions(**kwargs) if sift_options is None + else sift_options) + if pointMatchRender is None: pointMatchRender = Render(host, port, owner, project, client_script) - - baseDataUrl = format_baseurl(pointMatchRender.DEFAULT_KWARGS['host'], - pointMatchRender.DEFAULT_KWARGS['port']) + + baseDataUrl = format_baseurl(pointMatchRender.DEFAULT_KWARGS['host'], + pointMatchRender.DEFAULT_KWARGS['port']) argvs = [] argvs += ['--baseDataUrl', baseDataUrl] argvs += ['--owner', pointMatchRender.DEFAULT_KWARGS['owner']] @@ -1031,8 +1088,8 @@ def pointMatchClient(stack, collection, tile_pairs, argvs += sift_options.to_java_args() canvas_url_template = get_canvas_url_template(stack, - filter, - renderWithoutMask, + filter, + renderWithoutMask, normalizeForMatching, excludeTransformsAfterLast, excludeFirstTransformAndAllAfter, @@ -1045,7 +1102,7 @@ def pointMatchClient(stack, collection, tile_pairs, for tile1,tile2 in tile_pairs: argvs += [canvas_url_template.format(tile1),canvas_url_template.format(tile2)] - + call_run_ws_client('org.janelia.render.client.PointMatchClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) From 8f2fd76c6d9b41f116fe639b9ef6dcc2645d0c62 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 23 Feb 2018 10:30:40 -0800 Subject: [PATCH 572/766] client: change exception in argumentparameters, some formatting, remove NotImplementedError. --- renderapi/client.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 07f0f400..5391f341 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -488,7 +488,6 @@ def call_run_ws_client(className, add_args=[], renderclient=None, return ret_val - def get_param(var, flag): return ([flag, var] if var is not None else []) @@ -861,13 +860,14 @@ def transformSectionClient(stack, transformId, transformClass, transformData, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs) + @renderclientaccess -def get_canvas_url_template(stack, filter=False, - renderWithoutMask=False, normalizeForMatching=True, - excludeTransformsAfterLast=None, - excludeFirstTransformAndAllAfter=None,excludeAllTransforms=False, - host=None, port=None, owner=None, project=None, client_script=None, - render=None, **kwargs): +def get_canvas_url_template( + stack, filter=False, renderWithoutMask=False, + normalizeForMatching=True, excludeTransformsAfterLast=None, + excludeFirstTransformAndAllAfter=None, excludeAllTransforms=False, + host=None, port=None, owner=None, project=None, client_script=None, + render=None, **kwargs): """function for making a render-parameters url template for point matching Parameters @@ -919,13 +919,15 @@ def get_canvas_url_template(stack, filter=False, url_suffix += '&renderWithoutMask=false' if excludeTransformsAfterLast is not None: - url_suffix += '&excludeTransformsAfterLast={}'.format(excludeTransformsAfterLast) + url_suffix += '&excludeTransformsAfterLast={}'.format( + excludeTransformsAfterLast) if excludeFirstTransformAndAllAfter is not None: - url_suffix += '&excludeFirstTransformAndAllAfter={}'.format(excludeFirstTransformAndAllAfter) + url_suffix += '&excludeFirstTransformAndAllAfter={}'.format( + excludeFirstTransformAndAllAfter) if excludeAllTransforms: url_suffix += '&excludeAllTransforms=true' - canvas_url_template = "%s/{}/%s"%(tile_base_url, + canvas_url_template = "%s/{}/%s" % (tile_base_url, url_suffix) return canvas_url_template @@ -939,8 +941,7 @@ def sanitize_cmd(cmd): def jbool_str(c): return str(c) if type(c) is not bool else "true" if c else "false" if any([i is None for i in cmd]): - # FIXME exception class - raise Exception( + raise ClientScriptError( 'missing argument in command "{}"'.format(map(str, cmd))) return map(jbool_str, cmd) @@ -951,8 +952,6 @@ def get_cmd_opt(v, flag=None): @staticmethod def get_flag_cmd(v, flag=None): # for arity 0 - raise NotImplementedError( - "flag commands are not supported by ArgumentParameters") return [flag] if v else [] def to_java_args(self): @@ -967,6 +966,7 @@ class FeatureRenderParameters(ArgumentParameters): def __init__(self, renderScale=None, renderWithFilter=None, renderWithoutMask=None, renderFullScaleWidth=None, renderFullScaleHeight=None, fillWithNoise=None, **kwargs): + super(FeatureRenderParameters, self).__init__(**kwargs) self.renderScale = renderScale self.renderWithFilter = renderWithFilter self.renderWithoutMask = renderWithoutMask @@ -1003,7 +1003,6 @@ def __init__(self, matchIterations=None, self.matchRod = matchRod - class SiftPointMatchOptions(MatchDerivationParameters, FeatureExtractionParameters): def __init__(self, renderScale=None, fillWithNoise=None, **kwargs): @@ -1020,7 +1019,8 @@ def pointMatchClient(stack, collection, tile_pairs, debugDirectory=None, filter=False, renderWithoutMask=False, normalizeForMatching=True, - excludeTransformsAfterLast=None, excludeAllTransforms=None, + excludeTransformsAfterLast=None, + excludeAllTransforms=None, excludeFirstTransformAndAllAfter=None, subprocess_mode=None, host=None, port=None, @@ -1082,12 +1082,12 @@ def pointMatchClient(stack, collection, tile_pairs, argvs += ['--baseDataUrl', baseDataUrl] argvs += ['--owner', pointMatchRender.DEFAULT_KWARGS['owner']] argvs += ['--collection', collection] - #argvs += ['--matchStorageFile', os.path.join(outdir, 'matches.json')] if debugDirectory is not None: argvs += ['--debugDirectory', debugDirectory] argvs += sift_options.to_java_args() - canvas_url_template = get_canvas_url_template(stack, + canvas_url_template = get_canvas_url_template( + stack, filter, renderWithoutMask, normalizeForMatching, @@ -1100,8 +1100,9 @@ def pointMatchClient(stack, collection, tile_pairs, project=project, client_script=client_script) - for tile1,tile2 in tile_pairs: - argvs += [canvas_url_template.format(tile1),canvas_url_template.format(tile2)] + for tile1, tile2 in tile_pairs: + argvs += [canvas_url_template.format(tile1), + canvas_url_template.format(tile2)] call_run_ws_client('org.janelia.render.client.PointMatchClient', memGB=memGB, client_script=client_script, From 59011d8f8bf0c2c062bb715982cb382277179d53 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 23 Feb 2018 14:47:38 -0800 Subject: [PATCH 573/766] client: subprocess calls now accept Popen kwargs from top level calls --- renderapi/client.py | 67 ++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 5391f341..05616cf9 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -14,7 +14,6 @@ from .render import RenderClient, renderaccess, Render, format_preamble, format_baseurl from .stack import set_stack_state, make_stack_params from pathos.multiprocessing import ProcessingPool as Pool -import attr # setup logger logger = logging.getLogger(__name__) @@ -127,7 +126,8 @@ def import_single_json_file(stack, jsonfile, transformFile=None, host, port, owner, project, stack) call_run_ws_client('org.janelia.render.client.ImportJsonClient', stack_params + transform_params + [jsonfile], - client_script=client_script, memGB=memGB, subprocess_mode=subprocess_mode) + client_script=client_script, memGB=memGB, + subprocess_mode=subprocess_mode, **kwargs) @renderclientaccess @@ -202,7 +202,7 @@ def import_jsonfiles_parallel( transformFile=transformFile, client_scripts=client_scripts, host=host, port=port, owner=owner, - project=project) + project=project, **kwargs) with WithPool(poolsize) as pool: pool.map(partial_import, jsonfiles) @@ -240,7 +240,8 @@ def import_jsonfiles(stack, jsonfiles, transformFile=None, subprocess_mode=None, host, port, owner, project, stack) call_run_ws_client('org.janelia.render.client.ImportJsonClient', stack_params + transform_params + jsonfiles, - client_script=client_script, memGB=memGB, subprocess_mode=subprocess_mode) + client_script=client_script, memGB=memGB, + subprocess_mode=subprocess_mode, **kwargs) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -287,7 +288,8 @@ def import_jsonfiles_validate_client(stack, jsonfiles, validator_params + transform_params + jsonfiles, client_script=client_script, - memGB=memGB, subprocess_mode=subprocess_mode) + memGB=memGB, subprocess_mode=subprocess_mode, + **kwargs) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -322,7 +324,7 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, trjson if sharedTransforms is not None else None), subprocess_mode=subprocess_mode, host=host, port=port, owner=owner, project=project, - client_script=client_script, memGB=memGB) + client_script=client_script, memGB=memGB, **kwargs) os.remove(tsjson) if sharedTransforms is not None: @@ -426,8 +428,27 @@ def world_to_local_array(stack, points, subprocess_mode=None, raise NotImplementedError('Whoops.') +def run_subprocess_mode(args, subprocess_mode=None, **kwargs): + subprocess_options = ['bufsize', 'executable', 'stdin', 'stdout', + 'stderr', 'preexec_fn', 'close_fds', 'shell', + 'cwd', 'env', 'universal_newlines', 'startupinfo', + 'creationflags'] + subprocess_kwargs = {k: v for k, v in kwargs.items() + if k in subprocess_options} + subprocess_modes = {'call': subprocess.call, + 'check_call': subprocess.check_call, + 'check_output': subprocess.check_output, + None: subprocess.check_call} + if subprocess_mode not in subprocess_modes: + logger.warning( + 'Unknown subprocess mode {} specified -- ' + 'using default subprocess.check_call'.format(subprocess_mode)) + sub_mode = subprocess_modes.get(subprocess_mode, subprocess.check_call) + return sub_mode(args, **subprocess_kwargs) + + def call_run_ws_client(className, add_args=[], renderclient=None, - memGB=None, client_script=None, subprocess_mode=None, + memGB=None, client_script=None, **kwargs): """simple call for run_ws_client.sh -- all arguments set in add_args @@ -465,23 +486,15 @@ def call_run_ws_client(className, add_args=[], renderclient=None, subprocess_mode=subprocess_mode, **renderclient.make_kwargs( memGB=memGB, - client_script=client_script)) + client_script=client_script, + **kwargs)) if memGB is None: logger.warning('call_run_ws_client requires memory specification -- ' 'defaulting to 1G') memGB = '1G' - - subprocess_modes = {'call': subprocess.call, - 'check_call': subprocess.check_call, - 'check_output': subprocess.check_output} - if subprocess_mode not in subprocess_modes: - logger.warning( - 'Unknown subprocess mode {} specified -- ' - 'using default subprocess.call'.format(subprocess_mode)) args = map(str, [client_script, memGB, className] + add_args) - sub_mode = subprocess_modes.get(subprocess_mode, subprocess.check_call) try: - ret_val = sub_mode(args) + ret_val = run_subprocess_mode(args, **kwargs) except subprocess.CalledProcessError as e: raise ClientScriptError('client_script call {} failed'.format(args)) @@ -519,7 +532,7 @@ def importJsonClient(stack, tileFiles=None, transformFile=None, else [tileFiles])) call_run_ws_client('org.janelia.render.client.ImportJsonClient', add_args=argvs, subprocess_mode=subprocess_mode, - client_script=client_script, memGB=memGB) + client_script=client_script, memGB=memGB, **kwargs) @renderclientaccess @@ -623,7 +636,7 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, call_run_ws_client('org.janelia.render.client.TilePairClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, - add_args=argvs) + add_args=argvs, **kwargs) with open(outjson, 'r') as f: jsondata = json.load(f) @@ -686,7 +699,7 @@ def importTransformChangesClient(stack, targetStack, transformFile, call_run_ws_client( 'org.janelia.render.client.ImportTransformChangesClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, - add_args=argvs) + add_args=argvs, **kwargs) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -729,7 +742,8 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, get_param(numberOfThreads, '--numberOfThreads')) call_run_ws_client('org.janelia.render.client.CoordinateClient', memGB=memGB, client_script=client_script, - subprocess_mode=subprocess_mode, add_args=argvs) + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) with open(toJson, 'r') as f: jsondata = json.load(f) @@ -817,7 +831,8 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, bound_param + zs) call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, - subprocess_mode=subprocess_mode, add_args=argvs) + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) @renderclientaccess @@ -858,7 +873,8 @@ def transformSectionClient(stack, transformId, transformClass, transformData, '--transformData', transformData] + zValues) call_run_ws_client('org.janelia.render.client.TransformSectionClient', memGB=memGB, client_script=client_script, - subprocess_mode=subprocess_mode, add_args=argvs) + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) @renderclientaccess @@ -1106,4 +1122,5 @@ def pointMatchClient(stack, collection, tile_pairs, call_run_ws_client('org.janelia.render.client.PointMatchClient', memGB=memGB, client_script=client_script, - subprocess_mode=subprocess_mode, add_args=argvs) + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) From 4fc423e30c634606bb250e52bd46a32d3123cbab Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 6 Mar 2018 17:00:33 -0800 Subject: [PATCH 574/766] client: remove unused parameters --- renderapi/client.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 5391f341..55570e24 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -962,19 +962,6 @@ def to_java_args(self): return self.sanitize_cmd(args) -class FeatureRenderParameters(ArgumentParameters): - def __init__(self, renderScale=None, renderWithFilter=None, - renderWithoutMask=None, renderFullScaleWidth=None, - renderFullScaleHeight=None, fillWithNoise=None, **kwargs): - super(FeatureRenderParameters, self).__init__(**kwargs) - self.renderScale = renderScale - self.renderWithFilter = renderWithFilter - self.renderWithoutMask = renderWithoutMask - self.renderFullScaleWidth = renderFullScaleWidth - self.renderFullScaleHeight = renderFullScaleHeight - self.fillWithNoise = fillWithNoise - - class FeatureExtractionParameters(ArgumentParameters): def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, SIFTminScale=None, SIFTsteps=None, **kwargs): From 3774b4c73020a3b5cefcadbad3de10ec50429df3 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 6 Mar 2018 17:45:11 -0800 Subject: [PATCH 575/766] client: fix potential issue with renderclientaccess --- renderapi/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 7540dced..8cde85e3 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -54,14 +54,15 @@ def renderclientaccess(f, *args, **kwargs): cs_valid = os.path.isfile(client_script) except TypeError as e: try: - if os.path.isdir(kwargs.get('client_scripts')): + client_scripts = kwargs.get('client_scripts') + if os.path.isdir(client_scripts): client_script = os.path.join(client_scripts, 'run_ws_client.sh') cs_valid = os.path.isfile(client_script) else: raise ClientScriptError( 'invalid client_scripts directory {}'.format( - kwargs.get('client_scripts'))) + client_scripts)) except TypeError as e: raise ClientScriptError( 'No client script information specified: ' From 34b896d63cd46d6c7073356eafe523bce5176d5e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 7 Mar 2018 21:37:07 -0800 Subject: [PATCH 576/766] test: add failing test for error in call_run_ws_client --- integration_tests/test_client_integrated.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index cb036b91..9ede85a3 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -261,3 +261,14 @@ def test_point_match_client(teststack, render,tmpdir): tp = tilepairjson['neighborPairs'][0] pms = renderapi.pointmatch.get_matches_involving_tile(collection,tp['p']['groupId'],tp['p']['id'],render=render) assert(len(pms)>0) + + +def test_call_run_ws_client_renderclient(render, teststack): + # class for this test should be something relatively lightweight.... + test_class = 'org.janelia.render.client.ValidateTilesClient' + zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) + args = renderapi.stack.make_stack_params( + render.DEFAULT_HOST, render.DEFAULT_PORT, render.DEFAULT_OWNER, + render.DEFAULT_PROJECT, teststack) + [zvalues[0]] + assert not renderapi.client.call_run_ws_client( + test_class, add_args=args, subprocess_mode='call', renderclient=render) From 6091286d68a3a8bbc10886ef222c41f2d69bbed9 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 7 Mar 2018 21:37:51 -0800 Subject: [PATCH 577/766] client: fix call_run_ws_client running with renderclient --- renderapi/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/renderapi/client.py b/renderapi/client.py index 8cde85e3..87c16495 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -484,7 +484,6 @@ def call_run_ws_client(className, add_args=[], renderclient=None, if renderclient is not None: if isinstance(renderclient, RenderClient): return call_run_ws_client(className, add_args=add_args, - subprocess_mode=subprocess_mode, **renderclient.make_kwargs( memGB=memGB, client_script=client_script, From 820a0476c62422d13ca54f73e51cb2233e9c357b Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 30 Mar 2018 16:32:13 -0700 Subject: [PATCH 578/766] adding 16 bit tiff support --- renderapi/image.py | 1 + 1 file changed, 1 insertion(+) diff --git a/renderapi/image.py b/renderapi/image.py index 29d53af0..eb377c11 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -21,6 +21,7 @@ 'tif': 'tiff-image', '.tif': 'tiff-image', 'tiff': 'tiff-image', + 'tiff16':'tiff16-image' None: 'png-image'} # Default to png From d031fd23999dfc35b83659e4685a0dd4c95b1d7d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 30 Mar 2018 16:34:57 -0700 Subject: [PATCH 579/766] typo --- renderapi/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/image.py b/renderapi/image.py index eb377c11..1895d8d4 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -21,7 +21,7 @@ 'tif': 'tiff-image', '.tif': 'tiff-image', 'tiff': 'tiff-image', - 'tiff16':'tiff16-image' + 'tiff16':'tiff16-image', None: 'png-image'} # Default to png From c2b6ffa2be6b4d6f4fb17cce56ce6f32fcb35d61 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 30 Mar 2018 16:52:47 -0700 Subject: [PATCH 580/766] fixing tests --- integration_tests/test_stack_integrated.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index ae7904d6..4eb35272 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -309,7 +309,8 @@ def test_bb_image(render, teststack, **kwargs): dr = data.ravel() assert data.shape[0] == (np.floor(height*.25)) assert data.shape[1] == (np.floor(width*.25)) - assert data.shape[2] >= 3 + if len(data.shape)>2: + assert data.shape[2] >= 3 def test_bb_image_options(render, teststack): From b5bfe30377192f94e07e726dca2918eed0cdb150 Mon Sep 17 00:00:00 2001 From: Sharmi Date: Wed, 4 Apr 2018 17:02:24 -0700 Subject: [PATCH 581/766] Removed removeAllOption and added excludeAllTransforms --- renderapi/image.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index 1895d8d4..c8ca0bdf 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -111,7 +111,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, @renderaccess def get_tile_image_data(stack, tileId, channel=None,normalizeForMatching=True, - removeAllOption=False, scale=None, + excludeAllTransforms=False, scale=None, filter=None, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): @@ -174,8 +174,8 @@ def get_tile_image_data(stack, tileId, channel=None,normalizeForMatching=True, qparams['scale'] = scale if filter is not None: qparams['filter'] = jbool(filter) - if removeAllOption is not None: - qparams['removeAllOption'] = jbool(removeAllOption) + if excludeAllTransforms is not None: + qparams['excludeAllTransforms'] = jbool(excludeAllTransforms) if channel is not None: qparams.update({'channels':channel}) logger.debug(request_url) From d6bc7c5f0b99b93ce58319351fda48b087f10c2c Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 6 Apr 2018 13:12:20 -0700 Subject: [PATCH 582/766] Pathos cut (#79) * trying to cut pathos from library * stripping pathos * fixing tests * added enter * added a simple code coverage test of parallel calls * missing import * removing test of code coverage * removing test --- integration_tests/test_client_integrated.py | 8 ++++---- renderapi/client.py | 15 ++++++++++----- requirements.txt | 2 -- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 9ede85a3..e60b5697 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -6,10 +6,8 @@ import sys import json import numpy as np -import dill from test_data import (render_host, render_port, client_script_location, tilespec_file, tform_file, test_pool_size) -from pathos.multiprocessing import ProcessingPool as Pool import PIL root = logging.getLogger() @@ -101,14 +99,15 @@ def test_import_jsonfiles_parallel( validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) - +def square(x): + return x**2 def test_import_jsonfiles_parallel_multiple( render, render_example_tilespec_and_transforms, poolsize=test_pool_size): stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] mylist = range(10) for stack in stacks: with renderapi.client.WithPool(poolsize) as pool: - results = pool.map(lambda x: x**2, mylist) + results = pool.map(square, mylist) test_import_jsonfiles_parallel( render, render_example_tilespec_and_transforms, stack, poolsize=poolsize) @@ -272,3 +271,4 @@ def test_call_run_ws_client_renderclient(render, teststack): render.DEFAULT_PROJECT, teststack) + [zvalues[0]] assert not renderapi.client.call_run_ws_client( test_class, add_args=args, subprocess_mode='call', renderclient=render) + diff --git a/renderapi/client.py b/renderapi/client.py index 87c16495..710409f1 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -13,7 +13,7 @@ from .utils import NullHandler, renderdump_temp, fitargspec from .render import RenderClient, renderaccess, Render, format_preamble, format_baseurl from .stack import set_stack_state, make_stack_params -from pathos.multiprocessing import ProcessingPool as Pool +from multiprocessing.pool import Pool # setup logger logger = logging.getLogger(__name__) @@ -77,15 +77,15 @@ def renderclientaccess(f, *args, **kwargs): class WithPool(Pool): - """pathos ProcessingPool with functioning __exit__ call + """Multiprocessing.pool.Pool with functioning __exit__ call Parameters ---------- *args variable length argument list matching input - to pathos.multiprocessing.Pool + to multiprocessing.pool.Pool **kwargs - keyword argument input matching pathos.multiprocessing.Pool + keyword argument input matching multiprocessing.pool.Pool Examples -------- @@ -96,8 +96,12 @@ class WithPool(Pool): def __init__(self, *args, **kwargs): super(WithPool, self).__init__(*args, **kwargs) + def __enter__(self): + return self + def __exit__(self, *args, **kwargs): - super(WithPool, self)._clear() + self.close() + self.join() @renderclientaccess @@ -1111,3 +1115,4 @@ def pointMatchClient(stack, collection, tile_pairs, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f56d3209..27c007c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ requests numpy pillow -dill>=0.2.6 -pathos sphinxcontrib-napoleon decorator From d5e271d5d0719544936e8db626ba087a1077cf3c Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 17 Apr 2018 14:06:48 -0700 Subject: [PATCH 583/766] test: add test for #81 --- test/test_transform.py | 43 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/test/test_transform.py b/test/test_transform.py index b266de54..e20411b1 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -4,6 +4,8 @@ import scipy.linalg import rendersettings import importlib +import pytest + def cross_py23_reload(module): try: @@ -11,6 +13,7 @@ def cross_py23_reload(module): except NameError as e: importlib.reload(module) + def test_affine_rot_90(): am = renderapi.transform.AffineModel() # setup a 90 degree clockwise rotation @@ -415,4 +418,42 @@ def test_non_linear_transform(): xyp=lens_tform.tform(xy) dv = xyp-xy mean_disp= np.mean(np.sqrt(np.sum(dv**2,axis=1))) - assert((mean_disp-0.7570507)<.01) \ No newline at end of file + assert((mean_disp-0.7570507)<.01) + + +@pytest.mark.parametrize("transform_class,transform_json", [ + (renderapi.transform.Transform, { + 'className': 'some_class', + 'dataString': '1234_some_data', + 'id': 'myTransform', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}), + (renderapi.transform.AffineModel, { + 'className': 'mpicbg.trakem2.transform.AffineModel2D', + 'dataString': ( + "-0.6433267575 0.7655917209 -0.7655917209 " + "-0.6433267575 115071.0014383696 23073.7167961992"), + 'id': 'myAffineModel', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}), + (renderapi.transform.Polynomial2DTransform, { + 'className': 'mpicbg.trakem2.transform.PolynomialTransform2D', + 'dataString': ( + '67572.7356991 0.972637082773 -0.0266434803369 ' + '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' + '5446.85340052 0.0224047626583 0.961202608454 ' + '-3.36753624487E-07 -8.97219078255E-07 -5.49854010072E-06'), + 'id': 'myPolynomialModel', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}), + (renderapi.transform.NonLinearCoordinateTransform, + dict(rendersettings.NONLINEAR_TRANSFORM_KWARGS, **{ + 'id': 'myNLCTransform', + 'metaData': {'labels': ['my_first_label', 'my_second_label']}}))]) +def test_load_json_transforms(transform_class, transform_json): + tform = transform_class(json=transform_json) + tform_d = tform.to_dict() + + # avoid comparing datastrings as strings because of rounding + assert (tform.transformId == tform_d['id'] == transform_json['id']) + assert (tform.labels == tform_d['metaData']['labels'] == + transform_json['metaData']['labels']) + assert (tform.className == tform_d['className'] == + transform_json['className']) From a238e960ca3170e88b65948c8302b2206161d0ca Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 17 Apr 2018 14:40:44 -0700 Subject: [PATCH 584/766] transform: quick resolution to #81 --- renderapi/transform.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 28305b13..d10d4994 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -386,7 +386,7 @@ def from_dict(self, d): self.labels = md.get('labels', None) else: self.labels = None - + def _process_dataString(self, datastring): """method meant to set state of transform from datastring generic implementation only saves datastring at self.dataString. @@ -1039,7 +1039,7 @@ class Polynomial2DTransform(Transform): def __init__(self, dataString=None, src=None, dst=None, order=2, force_polynomial=True, params=None, identity=False, - labels=None,json=None, **kwargs): + labels=None, transformId=None, json=None, **kwargs): """Initialize Polynomial2DTransform This provides 5 different ways to initialize the transform which are mutually exclusive and applied in the order specified here. @@ -1084,7 +1084,7 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, if not force_polynomial and self.is_affine: raise NotImplementedError('Falling back to Affine model is ' 'not supported {}') - self.transformId = None + self.transformId = transformId self.labels = labels @property @@ -1423,8 +1423,8 @@ def __init__(self, dataString=None, json=None, transformId=None, self._process_dataString(dataString) if labels is not None: self.labels = labels - self.transformId = transformId - self.className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' + self.transformId = transformId + self.className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' def _process_dataString(self, dataString): From d447bcebecfb29a168e58e73968770bbfb8e4157 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 17 Apr 2018 17:20:57 -0700 Subject: [PATCH 585/766] processingpools: give the people a choice --- renderapi/client.py | 14 +++++++------- renderapi/external/processpools/pool_pathos.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 renderapi/external/processpools/pool_pathos.py diff --git a/renderapi/client.py b/renderapi/client.py index 710409f1..b9a244d0 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -98,7 +98,7 @@ def __init__(self, *args, **kwargs): def __enter__(self): return self - + def __exit__(self, *args, **kwargs): self.close() self.join() @@ -137,7 +137,7 @@ def import_single_json_file(stack, jsonfile, transformFile=None, @renderclientaccess def import_jsonfiles_and_transforms_parallel_by_z( - stack, jsonfiles, transformfiles, poolsize=20, + stack, jsonfiles, transformfiles, poolsize=20, mpPool=WithPool, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): """imports json files and transform files in parallel @@ -168,7 +168,7 @@ def import_jsonfiles_and_transforms_parallel_by_z( partial_import = partial(import_single_json_file, stack, render=render, client_scripts=client_scripts, host=host, port=port, owner=owner, project=project) - with WithPool(poolsize) as pool: + with mpPool(poolsize) as pool: pool.map(partial_import, jsonfiles, transformfiles) if close_stack: @@ -177,7 +177,7 @@ def import_jsonfiles_and_transforms_parallel_by_z( @renderclientaccess def import_jsonfiles_parallel( - stack, jsonfiles, poolsize=20, transformFile=None, + stack, jsonfiles, poolsize=20, transformFile=None, mpPool=WithPool, client_scripts=None, host=None, port=None, owner=None, project=None, close_stack=True, render=None, **kwargs): """import jsons using client script in parallel @@ -208,7 +208,7 @@ def import_jsonfiles_parallel( client_scripts=client_scripts, host=host, port=port, owner=owner, project=project, **kwargs) - with WithPool(poolsize) as pool: + with mpPool(poolsize) as pool: pool.map(partial_import, jsonfiles) if close_stack: @@ -339,6 +339,7 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, @renderclientaccess def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, subprocess_mode=None, poolsize=20, + mpPool=WithPool, close_stack=True, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, @@ -373,7 +374,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, # TODO this is a weird way to do splits.... is that okay? tilespec_groups = [tilespecs[i::poolsize] for i in range(poolsize)] - with WithPool(poolsize) as pool: + with mpPool(poolsize) as pool: pool.map(partial_import, tilespec_groups) if close_stack: set_stack_state(stack, 'COMPLETE', host, port, owner, project) @@ -1115,4 +1116,3 @@ def pointMatchClient(stack, collection, tile_pairs, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) - \ No newline at end of file diff --git a/renderapi/external/processpools/pool_pathos.py b/renderapi/external/processpools/pool_pathos.py new file mode 100644 index 00000000..f5344c42 --- /dev/null +++ b/renderapi/external/processpools/pool_pathos.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +""" +external processing pool support for Pathos (legacy mode) +""" +from pathos.multiprocessing import ProcessingPool as Pool + + +class PathosWithPool(Pool): + def __init__(self, *args, **kwargs): + super(PathosWithPool, self).__init__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + super(PathosWithPool, self)._clear() + + +WithPool = PathosWithPool + +__all__ = ['PathosWithPool', 'WithPool'] From 96b2f96cb3d7c1e6620d600e73360ac21e7574f4 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 17 Apr 2018 17:28:02 -0700 Subject: [PATCH 586/766] processpools: add inits for good measure --- renderapi/external/__init__.py | 0 renderapi/external/processpools/__init__.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 renderapi/external/__init__.py create mode 100644 renderapi/external/processpools/__init__.py diff --git a/renderapi/external/__init__.py b/renderapi/external/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/renderapi/external/processpools/__init__.py b/renderapi/external/processpools/__init__.py new file mode 100644 index 00000000..e69de29b From ed63e23f13ec765d4ff4d3a5f4d23deb3ac7c586 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 17 Apr 2018 18:09:06 -0700 Subject: [PATCH 587/766] serverside bounding box (#86) * fixing renderdumps in put * adding putting resolved tilespecs and test * fix test name * removing unnecessary to_dict * adding interface for just passing tilespecs and transforms * added use_rest options and tests to client --- integration_tests/test_client_integrated.py | 16 ++++++-- integration_tests/test_stack_integrated.py | 27 ++++++++++++- renderapi/client.py | 15 +++++++ renderapi/resolvedtiles.py | 43 ++++++++++++++++++++- renderapi/utils.py | 2 +- 5 files changed, 95 insertions(+), 8 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index e60b5697..bb5dda0b 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -86,21 +86,28 @@ def test_failed_jsonfiles_validate_client( stack, ['not_a_file'], render=render, subprocess_mode=call_mode) +@pytest.mark.parametrize('use_rest,stack', + [(True,'test_import_jsonfiles_parallel'), + (False,'test_import_jsonfiles_parallel_rest')]) def test_import_jsonfiles_parallel( - render, render_example_tilespec_and_transforms, - stack='test_import_jsonfiles_parallel', poolsize=test_pool_size): + render, render_example_tilespec_and_transforms, + stack, use_rest, + poolsize=test_pool_size): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files( render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles_parallel( stack, tfiles, transformFile=transformFile, - render=render, poolsize=poolsize) + render=render, poolsize=poolsize, use_rest=use_rest) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) def square(x): return x**2 +#this test was added in order to validate that multiple WithPools would work +#pathos was breaking when we did this before. Should now be not relevant, +#but who ever deletes a test if you don't have to. def test_import_jsonfiles_parallel_multiple( render, render_example_tilespec_and_transforms, poolsize=test_pool_size): stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] @@ -109,7 +116,8 @@ def test_import_jsonfiles_parallel_multiple( with renderapi.client.WithPool(poolsize) as pool: results = pool.map(square, mylist) test_import_jsonfiles_parallel( - render, render_example_tilespec_and_transforms, stack, poolsize=poolsize) + render, render_example_tilespec_and_transforms, stack, + use_rest=False,poolsize=poolsize) def test_import_tilespecs_parallel(render, diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 4eb35272..d8f80beb 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -433,5 +433,28 @@ def test_get_resolvedtiles_from_z(render, teststack, matching_ts = next(ts for ts in resolved_tiles.tilespecs if ts.tileId == tsz[0].tileId) assert (len(matching_ts.tforms)==len(tsz[0].tforms)) - - +def test_put_resolved_tiles_scratch(render,render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + out_stack = 'resolved_test_stack' + renderapi.stack.create_stack(out_stack,render=render) + resolved_tilespecs = renderapi.resolvedtiles.ResolvedTiles(tilespecs,tforms) + r=renderapi.resolvedtiles.put_tilespecs(out_stack,resolved_tilespecs, + render=render) + tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack(out_stack, + render=render) + assert(len(tilespecs_out)==len(resolved_tilespecs.tilespecs)) + +def test_put_tilespecs_and_tforms(render,render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + out_stack = 'resolved_test_stack2' + renderapi.stack.create_stack(out_stack,render=render) + r=renderapi.resolvedtiles.put_tilespecs(out_stack,tilespecs=tilespecs, + shared_transforms=tforms,render=render) + tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack(out_stack, + render=render) + assert(len(tilespecs_out)==len(tilespecs)) + +def test_put_tilespecs_fail(render): + with pytest.raises(renderapi.errors.RenderError): + r=renderapi.resolvedtiles.put_tilespecs('fail_stack',render=render) + \ No newline at end of file diff --git a/renderapi/client.py b/renderapi/client.py index b9a244d0..7c3d0d04 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -13,6 +13,7 @@ from .utils import NullHandler, renderdump_temp, fitargspec from .render import RenderClient, renderaccess, Render, format_preamble, format_baseurl from .stack import set_stack_state, make_stack_params +from .resolvedtiles import put_tilespecs from multiprocessing.pool import Pool # setup logger @@ -302,6 +303,7 @@ def import_jsonfiles_validate_client(stack, jsonfiles, @renderclientaccess def import_tilespecs(stack, tilespecs, sharedTransforms=None, + use_rest=False,deriveData=True, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): @@ -316,10 +318,22 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, list of tilespecs to import sharedTransforms : :obj:`list` of :class:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional list of shared referenced transforms to be ingested + use_rest: bool + whether to import the tilespecs using the post method directly with deriveData=True + deriveData: bool + if doing use_rest, will determine whether to have the server calculate bounds (default=True) render : renderapi.render.Render render connect object """ + if use_rest: + put_tilespecs(stack, + deriveData=deriveData, + tilespecs=tilespecs, + shared_transforms=sharedTransforms, + host=host,port=port,owner=owner,project=project,**kwargs) + + tsjson = renderdump_temp(tilespecs) if sharedTransforms is not None: @@ -338,6 +352,7 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, @renderclientaccess def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, + subprocess_mode=None, poolsize=20, mpPool=WithPool, close_stack=True, host=None, port=None, diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py index d76f8e37..ea930d7a 100644 --- a/renderapi/resolvedtiles.py +++ b/renderapi/resolvedtiles.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from .tilespec import TileSpec from .transform import load_transform_json -from .utils import NullHandler +from .utils import NullHandler, put_json, jbool from .render import format_preamble, renderaccess from .errors import RenderError import logging @@ -52,6 +52,47 @@ def from_dict(self, d): A list of tilespecs stored in this ResolvedTiles with the transformations dereferenced """ +@renderaccess +def put_tilespecs(stack,resolved_tiles=None,deriveData=True, + tilespecs=None,shared_transforms=None, + host=None,port=None,owner=None,project=None, + session=requests.session(),render=None,**kwargs): + """upload resolved tiles to the server + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + render stack + resolved_tiles: renderapi.resolvedtiles.ResolvedTiles + resolved tiles to upload + deriveData: bool + whether or not to calculate bounding boxes serverside + tilespecs: list[renderapi.tilespec.Tilespec] + list of tilespecs to upload + sharedTransforms: list[renderapi.transform.Transform] + list of shared transforms to upload + render: renderapi.render.Render + render connect object + + Returns + ------- + requests.response.Reponse + server response + """ + request_url = format_preamble( + host, port, owner, project, stack) + '/resolvedTiles' + qparams = {} if deriveData is None else {'deriveData': jbool(deriveData)} + logger.debug(request_url) + if resolved_tiles is None: + if (tilespecs is None): + raise RenderError("need to pass resolved_tiles or tilespecs") + resolved_tiles = ResolvedTiles(tilespecs=tilespecs, + transformList=shared_transforms) + r=put_json(session,request_url,resolved_tiles,qparams) + logger.debug(r) + return r @renderaccess def get_resolved_tiles_from_z(stack, z, host=None, port=None, diff --git a/renderapi/utils.py b/renderapi/utils.py index 8eebdbad..2956bbf8 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -130,7 +130,7 @@ def put_json(session, request_url, d, params=None): headers = {"content-type": "application/json"} if d is not None: - payload = json.dumps(d) + payload = renderdumps(d) else: payload = None headers['Accept'] = "application/json" From 5ea1f9ef6e06d48ec139c12035a0b309641fbe42 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 17 Apr 2018 18:35:50 -0700 Subject: [PATCH 588/766] update documenation --- docs/api/renderapi.external.processpools.rst | 22 ++++++++++++++++++++ docs/api/renderapi.external.rst | 17 +++++++++++++++ docs/api/renderapi.rst | 8 +++++++ 3 files changed, 47 insertions(+) create mode 100644 docs/api/renderapi.external.processpools.rst create mode 100644 docs/api/renderapi.external.rst diff --git a/docs/api/renderapi.external.processpools.rst b/docs/api/renderapi.external.processpools.rst new file mode 100644 index 00000000..ff40f8c8 --- /dev/null +++ b/docs/api/renderapi.external.processpools.rst @@ -0,0 +1,22 @@ +renderapi\.external\.processpools package +========================================= + +Submodules +---------- + +renderapi\.external\.processpools\.pool\_pathos module +------------------------------------------------------ + +.. automodule:: renderapi.external.processpools.pool_pathos + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: renderapi.external.processpools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/renderapi.external.rst b/docs/api/renderapi.external.rst new file mode 100644 index 00000000..6756ed9d --- /dev/null +++ b/docs/api/renderapi.external.rst @@ -0,0 +1,17 @@ +renderapi\.external package +=========================== + +Subpackages +----------- + +.. toctree:: + + renderapi.external.processpools + +Module contents +--------------- + +.. automodule:: renderapi.external + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/renderapi.rst b/docs/api/renderapi.rst index 15e1cd8f..e99dcab8 100644 --- a/docs/api/renderapi.rst +++ b/docs/api/renderapi.rst @@ -68,6 +68,14 @@ renderapi\.tilespec module :undoc-members: :show-inheritance: +renderapi\.resolvedtiles module +-------------------------- + +.. automodule:: renderapi.resolvedtiles + :members: + :undoc-members: + :show-inheritance: + renderapi\.transform module --------------------------- From ad7d18e9f7f364a84c2abd7066e88f6e03e9e3e5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 17 Apr 2018 18:36:53 -0700 Subject: [PATCH 589/766] update documenation (#88) --- docs/api/renderapi.external.processpools.rst | 22 ++++++++++++++++++++ docs/api/renderapi.external.rst | 17 +++++++++++++++ docs/api/renderapi.rst | 8 +++++++ 3 files changed, 47 insertions(+) create mode 100644 docs/api/renderapi.external.processpools.rst create mode 100644 docs/api/renderapi.external.rst diff --git a/docs/api/renderapi.external.processpools.rst b/docs/api/renderapi.external.processpools.rst new file mode 100644 index 00000000..ff40f8c8 --- /dev/null +++ b/docs/api/renderapi.external.processpools.rst @@ -0,0 +1,22 @@ +renderapi\.external\.processpools package +========================================= + +Submodules +---------- + +renderapi\.external\.processpools\.pool\_pathos module +------------------------------------------------------ + +.. automodule:: renderapi.external.processpools.pool_pathos + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: renderapi.external.processpools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/renderapi.external.rst b/docs/api/renderapi.external.rst new file mode 100644 index 00000000..6756ed9d --- /dev/null +++ b/docs/api/renderapi.external.rst @@ -0,0 +1,17 @@ +renderapi\.external package +=========================== + +Subpackages +----------- + +.. toctree:: + + renderapi.external.processpools + +Module contents +--------------- + +.. automodule:: renderapi.external + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/renderapi.rst b/docs/api/renderapi.rst index 15e1cd8f..e99dcab8 100644 --- a/docs/api/renderapi.rst +++ b/docs/api/renderapi.rst @@ -68,6 +68,14 @@ renderapi\.tilespec module :undoc-members: :show-inheritance: +renderapi\.resolvedtiles module +-------------------------- + +.. automodule:: renderapi.resolvedtiles + :members: + :undoc-members: + :show-inheritance: + renderapi\.transform module --------------------------- From 7d1630fef93d6b84f5c1d9e96d5a7961964bb14d Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Wed, 18 Apr 2018 16:35:29 -0700 Subject: [PATCH 590/766] adding find_packages to setup.py (#89) --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 83833ed9..5a8e826c 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -from setuptools import setup +from setuptools import setup, find_packages import sys from setuptools.command.test import test as TestCommand @@ -35,7 +35,7 @@ def run_tests(self): author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', author_email='forrest.collman@gmail.com', url='https://github.com/fcollman/render-python', - packages=['renderapi'], + packages=find_packages(), setup_requires=['setuptools_scm'], install_requires=required, tests_require=test_required, From 63bfd8f075a6a48aba918c5bc327466e819a27d8 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Apr 2018 12:21:41 -0700 Subject: [PATCH 591/766] wrap all gets,puts,posts,deletes in common wrapper (#91) * fixed test lint problem * wip * fixing mistakes in call * added delete and post * added stream * pep8 and kwargs fix * added six to requirements * putting back raw_input for code cov --- integration_tests/test_stack_integrated.py | 2 +- renderapi/coordinate.py | 17 +-- renderapi/pointmatch.py | 141 ++++++--------------- renderapi/render.py | 21 +-- renderapi/resolvedtiles.py | 10 +- renderapi/stack.py | 60 ++------- renderapi/tilespec.py | 38 +----- renderapi/utils.py | 76 +++++++++-- requirements.txt | 1 + 9 files changed, 139 insertions(+), 227 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index d8f80beb..97874d09 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -360,7 +360,7 @@ def test_section_image(render, teststack, **kwargs): def test_section_image_options(render, teststack): - img = test_section_image(render, teststack, filter=True, + test_section_image(render, teststack, filter=True, maxTileSpecsToRender=50) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 5d1b6178..a9f35230 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -3,7 +3,7 @@ coordinate mapping functions for render api ''' from .render import format_preamble, renderaccess -from .utils import NullHandler, renderdumps, renderdump +from .utils import NullHandler, renderdumps, renderdump, get_json from .client import coordinateClient from .errors import RenderError import requests @@ -58,12 +58,7 @@ def world_to_local_coordinates(stack, z, x, y, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) + return get_json(session,request_url) @renderaccess @@ -108,12 +103,8 @@ def local_to_world_coordinates(stack, tileId, x, y, request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) + return get_json(session,request_url) + @renderaccess diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 679744f7..fc926362 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -6,7 +6,7 @@ import logging from .render import format_baseurl, renderaccess from .errors import RenderError -from .utils import NullHandler +from .utils import NullHandler, get_json, put_json, rest_delete import json logger = logging.getLogger(__name__) @@ -41,13 +41,8 @@ def get_matchcollection_owners(host=None, port=None, """ request_url = format_baseurl(host, port) + \ "/matchCollectionOwners" - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess @@ -79,13 +74,8 @@ def get_matchcollections(owner=None, host=None, port=None, """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollections" % owner - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess @@ -120,17 +110,13 @@ def get_match_groupIds(matchCollection, owner=None, host=None, """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, + stream=True, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): @@ -147,6 +133,8 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, groupId to query mergeCollections : :obj:`list` of :obj:`str` other matchCollections to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) owner : unicode matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) @@ -170,17 +158,12 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, owner, matchCollection, groupId) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url,stream=stream) @renderaccess def get_matches_within_group(matchCollection, groupId, mergeCollections=None, + stream=True, owner=None, host=None, port=None, session=requests.session(), render=None, **kwargs): @@ -198,6 +181,8 @@ def get_matches_within_group(matchCollection, groupId, mergeCollections=None, mergeCollections : :obj:`list` of :obj:`str` or None other matchCollections to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) owner : unicode matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) @@ -221,18 +206,12 @@ def get_matches_within_group(matchCollection, groupId, mergeCollections=None, owner, matchCollection, groupId) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url,stream=stream) @renderaccess def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, - mergeCollections=None, + mergeCollections=None, stream=True, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): @@ -253,6 +232,8 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, mergeCollections : :obj:`list` of :obj:`str` or None other matchCollections to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) owner : unicode matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) @@ -277,13 +258,8 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, owner, matchCollection, pgroup, qgroup) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url,stream=stream) + def add_merge_collections(request_url, mcs): @@ -360,17 +336,13 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, owner, matchCollection, pgroup, pid, qgroup, qid)) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, + stream=True, render=None, owner=None, host=None, port=None, session=requests.session(), **kwargs): @@ -387,6 +359,8 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, source group to query mergeCollections : :obj:`list` of :obj:`str` or None other matchCollections to aggregate into answer + stream : bool + whether to invoke streaming (default=True) owner : unicode matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) @@ -410,13 +384,8 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, owner, matchCollection, pgroup) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url,stream=stream) + @renderaccess @@ -454,13 +423,8 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess @@ -499,18 +463,13 @@ def get_match_groupIds_to_only(matchCollection, mergeCollections=None, "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess def get_matches_involving_tile(matchCollection, groupId, id, - mergeCollections=None, + mergeCollections=None, stream=True, owner=None, host=None, port=None, session=requests.session(), **kwargs): """get all the matches involving a specific tile @@ -529,6 +488,8 @@ def get_matches_involving_tile(matchCollection, groupId, id, id to query mergeCollections : :obj:`list` of :obj:`str`, optional other matchCollections to aggregate into answer + stream: bool + whether to invoke streaming on get (default True) owner : unicode matchCollection owner (fallback to render.DEFAULT_OWNER) (note match owner != stack owner always) @@ -552,13 +513,8 @@ def get_matches_involving_tile(matchCollection, groupId, id, owner, matchCollection, groupId, id) request_url = add_merge_collections(request_url, mergeCollections) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url,stream=stream) + @renderaccess @@ -604,14 +560,8 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/matchesWith/{}".format( owner, matchCollection, pGroupId, qGroupId) - try: - r = session.delete(request_url) - return r - except Exception as e: - logger.error(e) - logger.error(request_url) - logger.error(r.text) - raise RenderError(r.text) + r = rest_delete(session,request_url) + @renderaccess @@ -644,11 +594,7 @@ def import_matches(matchCollection, data, owner=None, host=None, port=None, request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/matches" % (owner, matchCollection) logger.debug(request_url) - if not isinstance(data, str): - data = json.dumps(data) - r = session.put(request_url, data=data, headers={ - "content-type": "application/json", "Accept": "application/json"}) - return r + return put_json(session,request_url,data) @renderaccess @@ -684,12 +630,5 @@ def delete_collection(matchCollection, owner=None, host=None, port=None, request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s" % (owner, matchCollection) logger.debug(request_url) - try: - r = session.delete(request_url) - return r - except Exception as e: - logger.error(e) - logger.error(request_url) - logger.error(r.text) - raise RenderError(r.text) + r = rest_delete(session,request_url) diff --git a/renderapi/render.py b/renderapi/render.py index 7893781f..f3f661f6 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -2,9 +2,10 @@ import logging import os import requests -from .utils import defaultifNone, NullHandler, fitargspec +from .utils import defaultifNone, NullHandler, fitargspec, get_json from .errors import ClientScriptError, RenderError from decorator import decorator +from six.moves import input as raw_input logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -482,13 +483,8 @@ def get_owners(host=None, port=None, session=requests.session(), """ request_url = "%s/owners/" % format_baseurl(host, port) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess @@ -517,13 +513,8 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) logger.debug(request_url) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py index ea930d7a..958c0814 100644 --- a/renderapi/resolvedtiles.py +++ b/renderapi/resolvedtiles.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from .tilespec import TileSpec from .transform import load_transform_json -from .utils import NullHandler, put_json, jbool +from .utils import NullHandler, put_json, jbool, get_json from .render import format_preamble, renderaccess from .errors import RenderError import logging @@ -123,11 +123,5 @@ def get_resolved_tiles_from_z(stack, z, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/resolvedTiles' % (z) logger.debug(request_url) - r = session.get(request_url) - try: - d = r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + d= get_json(session,request_url) return ResolvedTiles(json=d) diff --git a/renderapi/stack.py b/renderapi/stack.py index 94164327..947cb426 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,9 +3,10 @@ from time import strftime import requests from .errors import RenderError -from .utils import jbool, NullHandler, post_json, put_json +from .utils import jbool, NullHandler, post_json, put_json,rest_delete from .render import (format_baseurl, format_preamble, renderaccess) +from .utils import get_json import json logger = logging.getLogger(__name__) @@ -152,14 +153,8 @@ def get_full_stack_metadata(stack, host=None, port=None, owner=None, request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) - r = session.get(request_url) + return get_json(session,request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) def get_stack_metadata(*args, **kwargs): @@ -320,7 +315,7 @@ def delete_stack(stack, host=None, port=None, owner=None, """ request_url = format_preamble(host, port, owner, project, stack) - r = session.delete(request_url) + r = rest_delete(session,request_url) logger.debug(r.text) return r @@ -351,7 +346,7 @@ def delete_section(stack, z, host=None, port=None, owner=None, """ request_url = '{}/z/{}'.format( format_preamble(host, port, owner, project, stack), z) - r = session.delete(request_url) + r = rest_delete(session,request_url) logger.debug(r.text) return r @@ -384,7 +379,7 @@ def delete_tile(stack, tileId, host=None, port=None, owner=None, """ request_url = '{}/tile/{}'.format( format_preamble(host, port, owner, project, stack), tileId) - r = session.delete(request_url) + r = rest_delete(session,request_url) logger.debug(r.text) return r @@ -536,13 +531,8 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" logger.debug(request_url) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + def get_z_value_for_section(stack, sectionId, **kwargs): @@ -596,13 +586,7 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/bounds' % (z) - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) @renderaccess @@ -633,13 +617,8 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, """ request_url = format_preamble( host, port, owner, project, stack) + '/bounds' - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess @@ -717,13 +696,7 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, """ request_url = format_preamble( host, port, owner, project, stack) + '/sectionData' - r = session.get(request_url) - try: - return r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) @renderaccess @@ -756,13 +729,8 @@ def get_section_z_value(stack, sectionId, host=None, port=None, """ request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % sectionId - r = session.get(request_url) - try: - return float(r.json()) - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index e2a71f38..fb3dd8c2 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -2,7 +2,7 @@ import logging import requests from .render import format_preamble, renderaccess -from .utils import NullHandler +from .utils import NullHandler, get_json from .stack import get_z_values_for_stack from .transform import TransformList from .errors import RenderError @@ -244,14 +244,8 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/render-parameters" % (tile) - r = session.get(request_url) - try: - tilespec_json = r.json() - return tilespec_json - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + return get_json(session,request_url) + @renderaccess @@ -318,14 +312,8 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/" % (tile) - r = session.get(request_url) - try: - tilespec_json = r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) - return TileSpec(json=tilespec_json) + return TileSpec(json=get_json(session,request_url)) + @renderaccess @@ -418,13 +406,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( z, x, y, width, height, scale) logger.debug(request_url) - r = session.get(request_url) - try: - tilespecs_json = r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + tilespecs_json = get_json(session,request_url) return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json['tileSpecs']] @@ -456,13 +438,7 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) logger.debug(request_url) - r = session.get(request_url) - try: - tilespecs_json = r.json() - except Exception as e: - logger.error(e) - logger.error(r.text) - raise RenderError(r.text) + tilespecs_json = get_json(session,request_url) if len(tilespecs_json) == 0: return None diff --git a/renderapi/utils.py b/renderapi/utils.py index 2956bbf8..82a6ea8e 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -9,6 +9,7 @@ import json from .errors import RenderError import numpy +import requests class NullHandler(logging.Handler): """handler to avoid logging errors for, e.g., missing logger setup""" @@ -93,14 +94,30 @@ def post_json(session, request_url, d, params=None): headers['Accept'] = "application/json" r = session.post(request_url, data=payload, params=params, headers=headers) - try: - return r - except Exception as e: - logger.error(e) - logger.error(r.text) + if r.status_code not in [200,201,204]: raise RenderError( - 'cannot post {} to {} with params {}'.format( - d, request_url, params)) + 'cannot post {} to {} with params {} returned status_code {}'.format( + d, request_url, params,r.status_code)) + return r + +def rest_delete(session,request_url,params=None): + """DELETE requests with RenderError handling + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + Returns + ------- + requests.response + server response + """ + r = session.delete(request_url) + if r.status_code not in [200,202,204]: + raise RenderError("delete of {} returned {}".format(r.url,r.status_code)) + return r def put_json(session, request_url, d, params=None): @@ -136,15 +153,50 @@ def put_json(session, request_url, d, params=None): headers['Accept'] = "application/json" r = session.put(request_url, data=payload, params=params, headers=headers) + if r.status_code not in [200,201,204]: + raise RenderError( + 'put {} to {} returned status code {}'.format( + d, r.url,r.status_code)) + return r + + +def get_json(session,request_url,params=None,stream=False,**kwargs): + """get_json wrapper for requests to handle errors + + Parameters + ---------- + session : requests.session.Session + requests session + request_url : str + url + params : dict + requests parameters + stream: bool + requests whether to stream + kwargs: dict + kwargs to shout into the dark + Returns + ------- + dict + json response from server + + Raises + ------ + RenderError + if cannot get json successfully + """ + + r = session.get(request_url,params=params,stream=stream) + if r.status_code != 200: + message = "request to {} returned error code {} with message {}" + raise RenderError(message.format(r.url,r.status_code,r.text)) try: - return r + return r.json() except Exception as e: logger.error(e) logger.error(r.text) - raise RenderError( - 'cannot put {} to {} with params {}'.format( - d, request_url, params)) - + raise RenderError(r.text) + def renderdumps(obj, *args, **kwargs): """json.dumps using the RenderEncode diff --git a/requirements.txt b/requirements.txt index 27c007c4..1ec45f7d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ numpy pillow sphinxcontrib-napoleon decorator +six \ No newline at end of file From e3c875219f47b11312eec6a14dbf21a475f7209a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Apr 2018 13:15:35 -0700 Subject: [PATCH 592/766] fixed use rest interface (#93) --- renderapi/client.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 7c3d0d04..fd39df2d 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -332,27 +332,25 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, tilespecs=tilespecs, shared_transforms=sharedTransforms, host=host,port=port,owner=owner,project=project,**kwargs) + else: + tsjson = renderdump_temp(tilespecs) + if sharedTransforms is not None: + trjson = renderdump_temp(sharedTransforms) - tsjson = renderdump_temp(tilespecs) - - if sharedTransforms is not None: - trjson = renderdump_temp(sharedTransforms) - - importJsonClient(stack, tileFiles=[tsjson], transformFile=( - trjson if sharedTransforms is not None else None), - subprocess_mode=subprocess_mode, host=host, port=port, - owner=owner, project=project, - client_script=client_script, memGB=memGB, **kwargs) + importJsonClient(stack, tileFiles=[tsjson], transformFile=( + trjson if sharedTransforms is not None else None), + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs) - os.remove(tsjson) - if sharedTransforms is not None: - os.remove(trjson) + os.remove(tsjson) + if sharedTransforms is not None: + os.remove(trjson) @renderclientaccess -def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, - +def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, subprocess_mode=None, poolsize=20, mpPool=WithPool, close_stack=True, host=None, port=None, @@ -379,6 +377,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, mark render stack as COMPLETE after successful import render : :class:renderapi.render.Render render connect object + kwargs: dict .. all other kwargs to pass on to renderapi.client.import_tilespecs """ set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial( From 7321f11f3dbbb170d75fd5814d2d2a1c00711af1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Apr 2018 14:56:35 -0700 Subject: [PATCH 593/766] Estpoints update (#94) * updating estimate_dstpts to allow reference transforms and transformLists * fixed error handling and tests * duplicate import --- renderapi/transform.py | 18 +++++++++++++++--- test/test_transform.py | 28 +++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index d10d4994..84c90935 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -103,7 +103,7 @@ def from_dict(self, d): self.tforms.append(load_transform_json(td)) return self.tforms - + def load_transform_json(d, default_type='leaf'): """function to get the proper deserialization function @@ -1365,7 +1365,7 @@ def fromAffine(aff): [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) -def estimate_dstpts(transformlist, src=None): +def estimate_dstpts(transformlist, src=None, reference_tforms=None): """estimate destination points for list of transforms. Recurses through lists. @@ -1384,7 +1384,19 @@ def estimate_dstpts(transformlist, src=None): dstpts = src for tform in transformlist: if isinstance(tform, list): - dstpts = estimate_dstpts(tform, dstpts) + dstpts = estimate_dstpts(tform, dstpts,reference_tforms) + elif isinstance(tform,TransformList): + dstpts = estimate_dstpts(tform.tforms,dstpts,reference_tforms) + elif isinstance(tform,ReferenceTransform): + try: + tform_deref= next(tf for tf in reference_tforms if tf.transformId==tform.refId) + except TypeError: + raise RenderError("you supplied a set of tranforms that includes a reference transform,\ + but didn't supply a set of reference transforms to enable dereferencing") + except StopIteration: + raise RenderError("the list of transforms you provided references transorm {} but that transform\ + could not be found in the list of reference transforms".format(tform.refId)) + dstpts=estimate_dstpts([tform_deref],dstpts,reference_tforms) else: dstpts = tform.tform(dstpts) return dstpts diff --git a/test/test_transform.py b/test/test_transform.py index e20411b1..179a9535 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -6,7 +6,6 @@ import importlib import pytest - def cross_py23_reload(module): try: reload(module) @@ -457,3 +456,30 @@ def test_load_json_transforms(transform_class, transform_json): transform_json['metaData']['labels']) assert (tform.className == tform_d['className'] == transform_json['className']) + +@pytest.fixture(scope='module') +def referenced_tilespecs_and_transforms(): + with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS,'r') as fp: + ds = json.load(fp) + tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ds] + + with open(rendersettings.REFERENCE_TRANSFORM_SPECS,'r') as fp: + ds = json.load(fp) + transforms = [renderapi.transform.load_transform_json(tf) for tf in ds] + return tilespecs,transforms + +def test_estimate_dstpoints_reference(referenced_tilespecs_and_transforms): + tilespecs,transforms = referenced_tilespecs_and_transforms + + ticks = np.arange(0,2048,64,np.float) + xx,yy = np.meshgrid(ticks,ticks) + x = np.ravel(xx).T + y = np.ravel(yy).T + xy = np.vstack((x,y)).T + + xyt=renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy,transforms) + assert(xy.shape==xyt.shape) + with pytest.raises(renderapi.errors.RenderError): + renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy) + with pytest.raises(renderapi.errors.RenderError): + renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy,[transforms[2]]) From 19981b00211f99f377b096af5228a1b82b92f013 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Apr 2018 17:25:20 -0700 Subject: [PATCH 594/766] better error messages on rest calls --- renderapi/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 82a6ea8e..07ac9372 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -96,8 +96,8 @@ def post_json(session, request_url, d, params=None): headers=headers) if r.status_code not in [200,201,204]: raise RenderError( - 'cannot post {} to {} with params {} returned status_code {}'.format( - d, request_url, params,r.status_code)) + 'cannot post {} to {} with params {} returned status_code {} with message {}'\ + .format(d, request_url, params,r.status_code,r.text)) return r def rest_delete(session,request_url,params=None): @@ -116,7 +116,8 @@ def rest_delete(session,request_url,params=None): """ r = session.delete(request_url) if r.status_code not in [200,202,204]: - raise RenderError("delete of {} returned {}".format(r.url,r.status_code)) + raise RenderError("delete of {} returned {} with message {}"\ + .format(r.url, r.status_code,r.text)) return r @@ -155,8 +156,8 @@ def put_json(session, request_url, d, params=None): headers=headers) if r.status_code not in [200,201,204]: raise RenderError( - 'put {} to {} returned status code {}'.format( - d, r.url,r.status_code)) + 'put {} to {} returned status code {} with message {}'.format( + d, r.url,r.status_code,r.text)) return r From 5b04457d69ba01884cce86ef623672efb65f58d1 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Apr 2018 17:31:21 -0700 Subject: [PATCH 595/766] added more tests to failed rest operations --- integration_tests/test_stack_integrated.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 97874d09..e4e09fe4 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -457,4 +457,21 @@ def test_put_tilespecs_and_tforms(render,render_example_tilespec_and_transforms) def test_put_tilespecs_fail(render): with pytest.raises(renderapi.errors.RenderError): r=renderapi.resolvedtiles.put_tilespecs('fail_stack',render=render) + +def test_bad_delete(render): + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.delete_stack('not_a_stack',render=render) + +def test_bad_get(render): + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.get_stack_metadata('not_a_stack',render=render) + +def test_bad_post(render): + sv = renderapi.stack.StackVersion() + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.set_stack_metadata('not_a_stack',sv,render=render) + +def test_bad_put(render): + with pytest.raises(renderapi.errors.RenderError): + renderapi.stack.clone_stack('not_a_stack','not_getting_here',render=render) \ No newline at end of file From f1ededd630aa1a20a3295ea3253a59ab69d76b48 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 19 Apr 2018 18:16:58 -0700 Subject: [PATCH 596/766] fix tests --- integration_tests/test_stack_integrated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index e4e09fe4..c60b99c6 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -460,7 +460,7 @@ def test_put_tilespecs_fail(render): def test_bad_delete(render): with pytest.raises(renderapi.errors.RenderError): - renderapi.stack.delete_stack('not_a_stack',render=render) + renderapi.stack.delete_section('not_a_stack',0.0,render=render) def test_bad_get(render): with pytest.raises(renderapi.errors.RenderError): @@ -469,7 +469,7 @@ def test_bad_get(render): def test_bad_post(render): sv = renderapi.stack.StackVersion() with pytest.raises(renderapi.errors.RenderError): - renderapi.stack.set_stack_metadata('not_a_stack',sv,render=render) + renderapi.stack.set_stack_state('not_a_stack','LOADING',render=render) def test_bad_put(render): with pytest.raises(renderapi.errors.RenderError): From 77ee59c1b13f60f26fde49f8b4de77b105631232 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 19 Apr 2018 18:18:08 -0700 Subject: [PATCH 597/766] import_tilespecs_parallel: only start processes for tsgroups with members (#96) --- renderapi/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index fd39df2d..71e5d6d6 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -350,7 +350,7 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, @renderclientaccess -def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, +def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, subprocess_mode=None, poolsize=20, mpPool=WithPool, close_stack=True, host=None, port=None, @@ -387,7 +387,8 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, memGB=memGB, **kwargs) # TODO this is a weird way to do splits.... is that okay? - tilespec_groups = [tilespecs[i::poolsize] for i in range(poolsize)] + tilespec_groups = [g for g in + (tilespecs[i::poolsize] for i in range(poolsize)) if g] with mpPool(poolsize) as pool: pool.map(partial_import, tilespec_groups) if close_stack: From 284ae8c730ea30989336c4403e863d5f8e96e848 Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Thu, 19 Apr 2018 18:19:43 -0700 Subject: [PATCH 598/766] added bbox_transformed() to tilespec class (#84) * added bbox_transformed() to tilespec class, to generate shapely polygon with transforms * removed shapely from bbox_transformed * added a unit test for bbox_transformed and error handling for non-leaf transforms * fixed assert statement in test_bbox... * simplified unit test * incorporating Russel's suggestion for cleaner code and proper error handling * updating estimate_dstpts to allow reference transforms and transformLists * fixed error handling and tests * duplicate import * updating bbox_transformed to use estimate_dstpts --- integration_tests/test_client_integrated.py | 14 +++++++ renderapi/tilespec.py | 41 ++++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index bb5dda0b..ab4cbeeb 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -103,6 +103,20 @@ def test_import_jsonfiles_parallel( validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) +def test_bbox_transformed(render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + ts = tilespecs[0] + xy = ts.bbox_transformed(ndiv_inner=0,tf_limit=0) + assert xy.shape == (5,2) + assert np.abs((xy[2,:]-np.array([ts.width,ts.height])).sum()) < 1e-10 + xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=0) + assert xy.shape == (9,2) + #xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=4) + #assert xy.shape == (9,2) + #xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=None) + #assert xy.shape == (9,2) + + def square(x): return x**2 #this test was added in order to validate that multiple WithPools would work diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index fb3dd8c2..fcf24cce 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,10 +1,11 @@ #!/usr/bin/env python import logging import requests +import numpy as np from .render import format_preamble, renderaccess from .utils import NullHandler, get_json from .stack import get_z_values_for_stack -from .transform import TransformList +from .transform import TransformList,estimate_dstpts from .errors import RenderError from .image_pyramid import MipMapLevel, ImagePyramid from .layout import Layout @@ -118,6 +119,44 @@ def bbox(self): 'undefined bounding box for tile {}'.format(self.tileId)) return box + def bbox_transformed(self,ndiv_inner=0,tf_limit=None, reference_tforms=None): + """method to return Nx2 transformed coordinates of bounding box + Paramters + --------- + ndiv_inner : starting with just corner points, add intermediate + points to the boundary, recursively, ndiv_inner times + tf_limit : + 0 returns the raw bounding box + 1 returns the bounding box with the first transform applied + ... + None all transforms are applied + + Returns + ------- + Nx2 array ready for input to shapely.Polygon() + """ + #start with closed Nx2 array of corners + xy = np.zeros((5,2)).astype('float') + xy[0,:] = [0,0] + xy[1,:] = [0,self.height] + xy[2,:] = [self.width,self.height] + xy[3,:] = [self.width,0] + xy[4,:] = [0,0] + + #recursively add points to the boundary + while ndiv_inner>0: + sz = 2*xy.shape[0]-1 + newxy = np.zeros((sz,2)).astype('float') + newxy[0::2,:] = xy[:,:] + newxy[1:sz:2,:] = 0.5*(newxy[0:(sz-2):2,:] + newxy[2:sz:2,:]) + xy = newxy + ndiv_inner-=1 + + xy = estimate_dstpts(self.tforms[0:tf_limit], \ + src=xy, reference_tforms=reference_tforms) + + return xy + def to_dict(self): """method to produce a json tilespec for this tile returns a json compatible dictionary From 89a36ceb485e603f5dd8b5d7b885d62ec7310dcd Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 00:49:09 -0700 Subject: [PATCH 599/766] flake8: fix init --- renderapi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 25146c65..75a42151 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -15,4 +15,4 @@ __all__ = ['render', 'client', 'tilespec', 'errors', 'stack', 'image', 'pointmatch', 'coordinate', - 'connect', 'transform', 'resolvedtiles','Render'] + 'connect', 'transform', 'resolvedtiles', 'Render'] From 8af957ecbbffd0020c73710ccd37a09fa960f9ef Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 07:10:14 -0700 Subject: [PATCH 600/766] wasn't actually hitting a bad post --- integration_tests/test_stack_integrated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index c60b99c6..a1a164fe 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -469,7 +469,7 @@ def test_bad_get(render): def test_bad_post(render): sv = renderapi.stack.StackVersion() with pytest.raises(renderapi.errors.RenderError): - renderapi.stack.set_stack_state('not_a_stack','LOADING',render=render) + renderapi.stack.create_stack('not_a___stack',render=render) def test_bad_put(render): with pytest.raises(renderapi.errors.RenderError): From 929de87a6be2da0d5e80b86853b5e43c553d36bd Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 07:18:45 -0700 Subject: [PATCH 601/766] added simple tests for non renderdumps passthrough to simplier serialize --- test/test_utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index ea3733fc..b7963724 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,8 +1,19 @@ import renderapi - +import pytest def test_jbool(): assert(renderapi.utils.jbool(True) == 'true') assert(renderapi.utils.jbool(False) == 'false') assert(renderapi.utils.jbool(0) == 'false') assert(renderapi.utils.jbool(1) == 'true') + +def test_renderdumps_simple(): + s=renderapi.utils.renderdumps({'a':1}) + assert(s=='{"a": 1}') + + s=renderapi.utils.renderdumps(5) + assert(s=='5') + +def test_renderdumps_fails(): + with pytest.raises(AttributeError): + renderapi.utils.renderdumps(pytest) \ No newline at end of file From 73e232f1518e14ae550453849d3bbfc7de5f4227 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:22:05 -0700 Subject: [PATCH 602/766] client: fix flake8 for client --- renderapi/client.py | 59 ++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/renderapi/client.py b/renderapi/client.py index 71e5d6d6..e1dbba59 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,13 +8,14 @@ import logging import subprocess import tempfile +from multiprocessing.pool import Pool from decorator import decorator from .errors import ClientScriptError from .utils import NullHandler, renderdump_temp, fitargspec -from .render import RenderClient, renderaccess, Render, format_preamble, format_baseurl +from .render import (RenderClient, renderaccess, Render, + format_preamble, format_baseurl) from .stack import set_stack_state, make_stack_params from .resolvedtiles import put_tilespecs -from multiprocessing.pool import Pool # setup logger logger = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def renderclientaccess(f, *args, **kwargs): try: client_script = kwargs.get('client_script') cs_valid = os.path.isfile(client_script) - except TypeError as e: + except TypeError: try: client_scripts = kwargs.get('client_scripts') if os.path.isdir(client_scripts): @@ -64,7 +65,7 @@ def renderclientaccess(f, *args, **kwargs): raise ClientScriptError( 'invalid client_scripts directory {}'.format( client_scripts)) - except TypeError as e: + except TypeError: raise ClientScriptError( 'No client script information specified: ' 'client_scripts={} client_script={}'.format( @@ -107,8 +108,8 @@ def __exit__(self, *args, **kwargs): @renderclientaccess def import_single_json_file(stack, jsonfile, transformFile=None, - subprocess_mode=None, - client_script=None, memGB=None, host=None, port=None, + subprocess_mode=None, client_script=None, + memGB=None, host=None, port=None, owner=None, project=None, render=None, **kwargs): """calls client script to import given jsonfile @@ -217,10 +218,10 @@ def import_jsonfiles_parallel( @renderaccess -def import_jsonfiles(stack, jsonfiles, transformFile=None, subprocess_mode=None, - client_script=None, memGB=None, host=None, port=None, - owner=None, project=None, close_stack=True, - render=None, **kwargs): +def import_jsonfiles(stack, jsonfiles, transformFile=None, + subprocess_mode=None, client_script=None, memGB=None, + host=None, port=None, owner=None, project=None, + close_stack=True, render=None, **kwargs): """import jsons using client script serially Parameters @@ -285,7 +286,6 @@ def import_jsonfiles_validate_client(stack, jsonfiles, else: raise NotImplementedError('No custom validation handling!') - my_env = os.environ.copy() stack_params = make_stack_params(host, port, owner, project, stack) set_stack_state(stack, 'LOADING', host, port, owner, project) @@ -303,7 +303,7 @@ def import_jsonfiles_validate_client(stack, jsonfiles, @renderclientaccess def import_tilespecs(stack, tilespecs, sharedTransforms=None, - use_rest=False,deriveData=True, + use_rest=False, deriveData=True, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): @@ -325,13 +325,14 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, render : renderapi.render.Render render connect object - """ + """ # noqa: E501 if use_rest: put_tilespecs(stack, deriveData=deriveData, tilespecs=tilespecs, shared_transforms=sharedTransforms, - host=host,port=port,owner=owner,project=project,**kwargs) + host=host, port=port, owner=owner, + project=project, **kwargs) else: tsjson = renderdump_temp(tilespecs) @@ -378,7 +379,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, render : :class:renderapi.render.Render render connect object kwargs: dict .. all other kwargs to pass on to renderapi.client.import_tilespecs - """ + """ # noqa: E501 set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial( import_tilespecs, stack, sharedTransforms=sharedTransforms, @@ -515,7 +516,7 @@ def call_run_ws_client(className, add_args=[], renderclient=None, args = map(str, [client_script, memGB, className] + add_args) try: ret_val = run_subprocess_mode(args, **kwargs) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError: raise ClientScriptError('client_script call {} failed'.format(args)) return ret_val @@ -755,7 +756,7 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, ------- :obj:`list` of :obj:`dict` for local to world or :obj:`list` of :obj:`list` of :obj:`dict` for world to local list representing mapped coordinates - """ + """ # noqa: E501 argvs = (make_stack_params(host, port, owner, project, stack) + ['--z', z, '--fromJson', fromJson, '--toJson', toJson] + (['--localToWorld'] if localToWorld else []) + @@ -775,7 +776,7 @@ def coordinateClient(stack, z, fromJson=None, toJson=None, localToWorld=None, def renderSectionClient(stack, rootDirectory, zs, scale=None, maxIntensity=None, minIntensity=None, bounds=None, format=None, channel=None, customOutputFolder=None, - customSubFolder=None,padFileNamesWithZeros=None, + customSubFolder=None, padFileNamesWithZeros=None, doFilter=None, fillWithNoise=None, imageType=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, @@ -818,7 +819,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, string representing java boolean for whether to replace saturated image values with uniform noise - """ + """ # noqa: E501 if bounds is not None: try: if bounds['maxX'] < bounds['minX']: @@ -828,11 +829,13 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, raise ClientScriptError('maxY:{} is less than minY:{}'.format( bounds['maxY'], bounds['minY'])) bound_list = ','.join(map(lambda x: str(int(x)), - [bounds['minX'], bounds['maxX'], bounds['minY'], bounds['maxY']])) + [bounds['minX'], bounds['maxX'], + bounds['minY'], bounds['maxY']])) bound_param = ['--bounds', bound_list] except KeyError as e: raise ClientScriptError( - 'bounds does not contain correct keys {}'.format(bounds)) + 'bounds does not contain correct keys {}. Missing {}'.format( + bounds, e)) else: bound_param = [] @@ -843,11 +846,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, get_param(minIntensity, '--minIntensity') + get_param(maxIntensity, '--maxIntensity') + get_param(fillWithNoise, '--fillWithNoise') + - get_param(customOutputFolder, '--customOutputFolder')+ - get_param(imageType,'--imageType')+ - get_param(channel,'--channels')+ - get_param(customSubFolder,'--customSubFolder')+ - get_param(padFileNamesWithZeros,'--padFileNamesWithZeros')+ + get_param(customOutputFolder, '--customOutputFolder') + + get_param(imageType, '--imageType') + + get_param(channel, '--channels') + + get_param(customSubFolder, '--customSubFolder') + + get_param(padFileNamesWithZeros, '--padFileNamesWithZeros') + bound_param + zs) call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, @@ -935,7 +938,7 @@ def get_canvas_url_template( excludeAllTransforms: bool alternative to normalizeForMatching which simply removes all transforms from the list. default=False - """ + """ # noqa: E501 request_url = format_preamble(host, port, owner, project, stack) tile_base_url = request_url+"/tile" url_suffix = "render-parameters" @@ -1092,7 +1095,7 @@ def pointMatchClient(stack, collection, tile_pairs, alternative to normalizeForMatching which simply removes all transforms from the list. default=False - """ + """ # noqa: E501 sift_options = (SiftPointMatchOptions(**kwargs) if sift_options is None else sift_options) From b95b390b138792d89cd3fb77817c4faf5a824e5c Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:22:55 -0700 Subject: [PATCH 603/766] channel: channel conform to flake8 --- renderapi/channel.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/renderapi/channel.py b/renderapi/channel.py index 267c19d8..48e10b89 100644 --- a/renderapi/channel.py +++ b/renderapi/channel.py @@ -1,9 +1,11 @@ from .image_pyramid import ImagePyramid, MipMapLevel + class Channel: '''class for storing channels of different mipmapsources''' - def __init__(self,name=None,maxIntensity=None,minIntensity=None,ip=None,json=None): + def __init__(self, name=None, maxIntensity=None, minIntensity=None, + ip=None, json=None): ''' Parameters ========== @@ -22,35 +24,34 @@ def __init__(self,name=None,maxIntensity=None,minIntensity=None,ip=None,json=Non if json is not None: self.from_dict(json) else: - self.name=name - self.maxIntensity=maxIntensity - self.minIntensity=minIntensity + self.name = name + self.maxIntensity = maxIntensity + self.minIntensity = minIntensity self.ip = ip def to_dict(self): - ''' method for serializing this class to a json compatible dictionary''' + '''method for serializing this class to a json compatible dictionary''' d = {} - d['name']=self.name + d['name'] = self.name if self.minIntensity is not None: - d['minIntensity']=self.minIntensity + d['minIntensity'] = self.minIntensity if self.maxIntensity is not None: - d['maxIntensity']=self.maxIntensity + d['maxIntensity'] = self.maxIntensity d['mipmapLevels'] = self.ip.to_ordered_dict() return d - - def from_dict(self,d): - ''' method for deserializing this class from a json compatible dictionary + + def from_dict(self, d): + '''method for deserializing this class from a json compatible dictionary Parameters ========== d: dict json compatible dictionary representation of this channel ''' - self.name=d['name'] - self.minIntensity=d['minIntensity'] - self.maxIntensity=d['maxIntensity'] + self.name = d['name'] + self.minIntensity = d['minIntensity'] + self.maxIntensity = d['maxIntensity'] self.ip = ImagePyramid(mipMapLevels=[ MipMapLevel( int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) for l, v in d['mipmapLevels'].items()]) - From 72a6f133ad57e9d3bafe5a80b09fe6e0a2d9c864 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:23:51 -0700 Subject: [PATCH 604/766] flake8: coordinate flake8 --- renderapi/coordinate.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index a9f35230..e022572c 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -58,7 +58,7 @@ def world_to_local_coordinates(stack, z, x, y, host=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/z/%d/world-to-local-coordinates/%f,%f" % (z, x, y) - return get_json(session,request_url) + return get_json(session, request_url) @renderaccess @@ -103,8 +103,7 @@ def local_to_world_coordinates(stack, tileId, x, y, request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/local-to-world-coordinates/%f,%f" % (tileId, x, y) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -252,7 +251,7 @@ def package_point_match_data_into_json(dataarray, tileId, dict dictionary representation of those points and tileId following - + :: { @@ -513,7 +512,7 @@ def map_coordinates_clientside(stack, jsondata, z, host, port, owner, json json data as would be returned by client calls of local>world or world>local - """ + """ # noqa: E501 # write point match json to temp file on disk with tempfile.NamedTemporaryFile( prefix='render_coordinates_in_', suffix='.json', From 34da65a7cc02415a49ceabbc8744b37324c8aa8e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:24:19 -0700 Subject: [PATCH 605/766] image_pyramid: flake8 image pyramid --- renderapi/image_pyramid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/image_pyramid.py b/renderapi/image_pyramid.py index 04ab4678..65ff1ba5 100644 --- a/renderapi/image_pyramid.py +++ b/renderapi/image_pyramid.py @@ -1,5 +1,6 @@ from collections import OrderedDict + class MipMapLevel: """MipMapLevel class to represent a level of an image pyramid. Can be put in dictionary formatting using dict(mML) @@ -122,4 +123,4 @@ def levels(self): def __iter__(self): return iter([ - l for sl in [list(mmL) for mmL in self.mipMapLevels] for l in sl]) \ No newline at end of file + l for sl in [list(mmL) for mmL in self.mipMapLevels] for l in sl]) From c9b1855caf41729c2e88d2780b1f8b8e189a046e Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:25:00 -0700 Subject: [PATCH 606/766] image: flake8 image --- renderapi/image.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/renderapi/image.py b/renderapi/image.py index c8ca0bdf..a5f812c4 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -21,7 +21,7 @@ 'tif': 'tiff-image', '.tif': 'tiff-image', 'tiff': 'tiff-image', - 'tiff16':'tiff16-image', + 'tiff16': 'tiff16-image', None: 'png-image'} # Default to png @@ -75,7 +75,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, Raises ------ RenderError - """ + """ # noqa: E501 try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: # pragma: no cover @@ -97,7 +97,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, if maxTileSpecsToRender is not None: qparams['maxTileSpecsToRender'] = maxTileSpecsToRender if channel is not None: - qparams.update({'channels':channel}) + qparams.update({'channels': channel}) r = session.get(request_url, params=qparams) try: @@ -110,7 +110,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, @renderaccess -def get_tile_image_data(stack, tileId, channel=None,normalizeForMatching=True, +def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, excludeAllTransforms=False, scale=None, filter=None, host=None, port=None, owner=None, project=None, img_format=None, @@ -157,7 +157,7 @@ def get_tile_image_data(stack, tileId, channel=None,normalizeForMatching=True, ------ RenderError - """ + """ # noqa: E501 try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: # pragma: no cover @@ -177,7 +177,7 @@ def get_tile_image_data(stack, tileId, channel=None,normalizeForMatching=True, if excludeAllTransforms is not None: qparams['excludeAllTransforms'] = jbool(excludeAllTransforms) if channel is not None: - qparams.update({'channels':channel}) + qparams.update({'channels': channel}) logger.debug(request_url) r = session.get(request_url, params=qparams) @@ -234,12 +234,12 @@ def get_section_image(stack, z, scale=1.0, channel=None, Examples -------- :: - + >>> import renderapi >>> render = renderapi.render.connect('server',8080,'me','myproject') >>> img = render.run(renderapi.stack.get_section_image,'mystack',3.0) - """ + """ # noqa: E501 try: image_ext = IMAGE_FORMATS[img_format] except KeyError as e: # pragma: no cover @@ -251,7 +251,7 @@ def get_section_image(stack, z, scale=1.0, channel=None, if maxTileSpecsToRender is not None: qparams.update({'maxTileSpecsToRender': maxTileSpecsToRender}) if channel is not None: - qparams.update({'channels':channel}) + qparams.update({'channels': channel}) r = session.get(request_url, params=qparams) return np.asarray(Image.open(io.BytesIO(r.content))) From 1533b56884fda2e00b89596844b81a7b700e05d2 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:25:45 -0700 Subject: [PATCH 607/766] tilespec: PEP8 tilespec --- renderapi/layout.py | 3 +-- renderapi/tilespec.py | 55 +++++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/renderapi/layout.py b/renderapi/layout.py index 68039c82..05e5240c 100644 --- a/renderapi/layout.py +++ b/renderapi/layout.py @@ -1,4 +1,3 @@ - class Layout: """Layout class to describe acquisition settings @@ -104,4 +103,4 @@ def from_dict(self, d): self.stageX = d.get('stageX') self.stageY = d.get('stageY') self.rotation = d.get('rotation') - self.pixelsize = d.get('pixelsize') \ No newline at end of file + self.pixelsize = d.get('pixelsize') diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index fcf24cce..e4196ac1 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -5,8 +5,7 @@ from .render import format_preamble, renderaccess from .utils import NullHandler, get_json from .stack import get_z_values_for_stack -from .transform import TransformList,estimate_dstpts -from .errors import RenderError +from .transform import TransformList, estimate_dstpts from .image_pyramid import MipMapLevel, ImagePyramid from .layout import Layout from .channel import Channel @@ -77,7 +76,8 @@ def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, minint=0, maxint=65535, layout=None, tforms=[], inputfilters=[], scale3Url=None, scale2Url=None, - scale1Url=None, json=None, channels=None,mipMapLevels=[], **kwargs): + scale1Url=None, json=None, channels=None, + mipMapLevels=[], **kwargs): if json is not None: self.from_dict(json) else: @@ -119,7 +119,8 @@ def bbox(self): 'undefined bounding box for tile {}'.format(self.tileId)) return box - def bbox_transformed(self,ndiv_inner=0,tf_limit=None, reference_tforms=None): + def bbox_transformed(self, ndiv_inner=0, + tf_limit=None, reference_tforms=None): """method to return Nx2 transformed coordinates of bounding box Paramters --------- @@ -135,24 +136,24 @@ def bbox_transformed(self,ndiv_inner=0,tf_limit=None, reference_tforms=None): ------- Nx2 array ready for input to shapely.Polygon() """ - #start with closed Nx2 array of corners - xy = np.zeros((5,2)).astype('float') - xy[0,:] = [0,0] - xy[1,:] = [0,self.height] - xy[2,:] = [self.width,self.height] - xy[3,:] = [self.width,0] - xy[4,:] = [0,0] - - #recursively add points to the boundary - while ndiv_inner>0: + # start with closed Nx2 array of corners + xy = np.zeros((5, 2)).astype('float') + xy[0, :] = [0, 0] + xy[1, :] = [0, self.height] + xy[2, :] = [self.width, self.height] + xy[3, :] = [self.width, 0] + xy[4, :] = [0, 0] + + # recursively add points to the boundary + while ndiv_inner > 0: sz = 2*xy.shape[0]-1 - newxy = np.zeros((sz,2)).astype('float') - newxy[0::2,:] = xy[:,:] - newxy[1:sz:2,:] = 0.5*(newxy[0:(sz-2):2,:] + newxy[2:sz:2,:]) + newxy = np.zeros((sz, 2)).astype('float') + newxy[0::2, :] = xy[:, :] + newxy[1:sz:2, :] = 0.5*(newxy[0:(sz-2):2, :] + newxy[2:sz:2, :]) xy = newxy - ndiv_inner-=1 + ndiv_inner -= 1 - xy = estimate_dstpts(self.tforms[0:tf_limit], \ + xy = estimate_dstpts(self.tforms[0:tf_limit], src=xy, reference_tforms=reference_tforms) return xy @@ -181,7 +182,7 @@ def to_dict(self): # thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] thedict['transforms']['specList'] = [] if self.channels is not None: - thedict['channels']=[ch.to_dict() for ch in self.channels] + thedict['channels'] = [ch.to_dict() for ch in self.channels] for t in self.tforms: strlist = {} # added by sharmi - if your speclist contains a speclist (can @@ -226,7 +227,7 @@ def from_dict(self, d): self.maxX = d.get('maxX', None) self.maxY = d.get('maxY', None) self.minY = d.get('minY', None) - mmld = d.get('mipmapLevels',{}) + mmld = d.get('mipmapLevels', {}) self.ip = ImagePyramid(mipMapLevels=[ MipMapLevel( int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) @@ -234,7 +235,7 @@ def from_dict(self, d): tfl = TransformList(json=d['transforms']) self.tforms = tfl.tforms - chd = d.get('channels',None) + chd = d.get('channels', None) if chd is None: self.channels = None else: @@ -283,8 +284,7 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/render-parameters" % (tile) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -351,8 +351,7 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/" % (tile) - return TileSpec(json=get_json(session,request_url)) - + return TileSpec(json=get_json(session, request_url)) @renderaccess @@ -445,7 +444,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, "/z/%d/box/%d,%d,%d,%d,%3.2f/render-parameters" % ( z, x, y, width, height, scale) logger.debug(request_url) - tilespecs_json = get_json(session,request_url) + tilespecs_json = get_json(session, request_url) return [TileSpec(json=tilespec_json) for tilespec_json in tilespecs_json['tileSpecs']] @@ -477,7 +476,7 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/tile-specs' % (z) logger.debug(request_url) - tilespecs_json = get_json(session,request_url) + tilespecs_json = get_json(session, request_url) if len(tilespecs_json) == 0: return None From 3fdc9ca68d773e77604aaadb4c3a0ab6da53bcbd Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:26:25 -0700 Subject: [PATCH 608/766] pointmatch: flake8 --- renderapi/pointmatch.py | 43 ++++++++++++++--------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index fc926362..897907a5 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -5,9 +5,7 @@ import requests import logging from .render import format_baseurl, renderaccess -from .errors import RenderError from .utils import NullHandler, get_json, put_json, rest_delete -import json logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -41,8 +39,7 @@ def get_matchcollection_owners(host=None, port=None, """ request_url = format_baseurl(host, port) + \ "/matchCollectionOwners" - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -74,8 +71,7 @@ def get_matchcollections(owner=None, host=None, port=None, """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollections" % owner - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -110,8 +106,7 @@ def get_match_groupIds(matchCollection, owner=None, host=None, """ request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/groupIds" % (owner, matchCollection) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -158,7 +153,7 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, owner, matchCollection, groupId) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url,stream=stream) + return get_json(session, request_url, stream=stream) @renderaccess @@ -206,7 +201,7 @@ def get_matches_within_group(matchCollection, groupId, mergeCollections=None, owner, matchCollection, groupId) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url,stream=stream) + return get_json(session, request_url, stream=stream) @renderaccess @@ -258,8 +253,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, owner, matchCollection, pgroup, qgroup) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url,stream=stream) - + return get_json(session, request_url, stream=stream) def add_merge_collections(request_url, mcs): @@ -336,8 +330,7 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, owner, matchCollection, pgroup, pid, qgroup, qid)) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -384,8 +377,7 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, owner, matchCollection, pgroup) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url,stream=stream) - + return get_json(session, request_url, stream=stream) @renderaccess @@ -423,8 +415,7 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, "/owner/%s/matchCollection/%s/pGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -463,8 +454,7 @@ def get_match_groupIds_to_only(matchCollection, mergeCollections=None, "/owner/%s/matchCollection/%s/qGroupIds" % (owner, matchCollection) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -513,8 +503,7 @@ def get_matches_involving_tile(matchCollection, groupId, id, owner, matchCollection, groupId, id) request_url = add_merge_collections(request_url, mergeCollections) - return get_json(session,request_url,stream=stream) - + return get_json(session, request_url, stream=stream) @renderaccess @@ -560,8 +549,7 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, request_url = format_baseurl(host, port) + \ "/owner/{}/matchCollection/{}/group/{}/matchesWith/{}".format( owner, matchCollection, pGroupId, qGroupId) - r = rest_delete(session,request_url) - + r = rest_delete(session, request_url) # noqa: F841 @renderaccess @@ -594,12 +582,12 @@ def import_matches(matchCollection, data, owner=None, host=None, port=None, request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s/matches" % (owner, matchCollection) logger.debug(request_url) - return put_json(session,request_url,data) + return put_json(session, request_url, data) @renderaccess def delete_collection(matchCollection, owner=None, host=None, port=None, - session=requests.session(), render=None, **kwargs): + session=requests.session(), render=None, **kwargs): """delete match collection from render database :func:`renderapi.render.renderaccess` decorated function @@ -630,5 +618,4 @@ def delete_collection(matchCollection, owner=None, host=None, port=None, request_url = format_baseurl(host, port) + \ "/owner/%s/matchCollection/%s" % (owner, matchCollection) logger.debug(request_url) - r = rest_delete(session,request_url) - + r = rest_delete(session, request_url) # noqa: F841 From f5f6d662c82e21bb0be4796e857d353f0df23cc9 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:26:49 -0700 Subject: [PATCH 609/766] render: flake8 --- renderapi/render.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index f3f661f6..0f466901 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -3,7 +3,7 @@ import os import requests from .utils import defaultifNone, NullHandler, fitargspec, get_json -from .errors import ClientScriptError, RenderError +from .errors import ClientScriptError from decorator import decorator from six.moves import input as raw_input @@ -121,7 +121,7 @@ def run(self, f, *args, **kwargs): >>> render = Render('server',8080) >>> metadata = render.run(renderapi.render.get_stack_metadata_by_owner, 'myowner') - """ + """ # noqa: E501 # FIXME WARNING I think renderaccess can default to # another render if defined in args (test/squash) kwargs['render'] = self @@ -483,8 +483,7 @@ def get_owners(host=None, port=None, session=requests.session(), """ request_url = "%s/owners/" % format_baseurl(host, port) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -513,8 +512,7 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, request_url = "%s/owner/%s/stacks/" % ( format_baseurl(host, port), owner) logger.debug(request_url) - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess From 3414da056c41fee669d625cdfc2d24340f633521 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:27:11 -0700 Subject: [PATCH 610/766] resolvedtiles: flake8 --- renderapi/resolvedtiles.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py index 958c0814..e8f018ce 100644 --- a/renderapi/resolvedtiles.py +++ b/renderapi/resolvedtiles.py @@ -43,20 +43,21 @@ def from_dict(self, d): tform_json['transformId'] = transformId self.transforms.append(load_transform_json(tform_json)) - #def get_tilespecs(): + # def get_tilespecs(): """return a set of TileSpecs that include resolved tilespecs Returns ------- List(renderapi.tilespec.TileSpec) A list of tilespecs stored in this ResolvedTiles with the transformations dereferenced - """ + """ # noqa: E501 + @renderaccess -def put_tilespecs(stack,resolved_tiles=None,deriveData=True, - tilespecs=None,shared_transforms=None, - host=None,port=None,owner=None,project=None, - session=requests.session(),render=None,**kwargs): +def put_tilespecs(stack, resolved_tiles=None, deriveData=True, + tilespecs=None, shared_transforms=None, + host=None, port=None, owner=None, project=None, + session=requests.session(), render=None, **kwargs): """upload resolved tiles to the server :func:`renderapi.render.renderaccess` decorated function @@ -75,7 +76,7 @@ def put_tilespecs(stack,resolved_tiles=None,deriveData=True, list of shared transforms to upload render: renderapi.render.Render render connect object - + Returns ------- requests.response.Reponse @@ -86,14 +87,15 @@ def put_tilespecs(stack,resolved_tiles=None,deriveData=True, qparams = {} if deriveData is None else {'deriveData': jbool(deriveData)} logger.debug(request_url) if resolved_tiles is None: - if (tilespecs is None): + if (tilespecs is None): raise RenderError("need to pass resolved_tiles or tilespecs") resolved_tiles = ResolvedTiles(tilespecs=tilespecs, transformList=shared_transforms) - r=put_json(session,request_url,resolved_tiles,qparams) + r = put_json(session, request_url, resolved_tiles, qparams) logger.debug(r) return r + @renderaccess def get_resolved_tiles_from_z(stack, z, host=None, port=None, owner=None, project=None, @@ -123,5 +125,5 @@ def get_resolved_tiles_from_z(stack, z, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/resolvedTiles' % (z) logger.debug(request_url) - d= get_json(session,request_url) + d = get_json(session, request_url) return ResolvedTiles(json=d) From cc781799e0ff488a8af42033d71b591c58615c80 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:27:41 -0700 Subject: [PATCH 611/766] stack: flake8 --- renderapi/stack.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 947cb426..190a13a2 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -3,7 +3,7 @@ from time import strftime import requests from .errors import RenderError -from .utils import jbool, NullHandler, post_json, put_json,rest_delete +from .utils import jbool, NullHandler, post_json, put_json, rest_delete from .render import (format_baseurl, format_preamble, renderaccess) from .utils import get_json @@ -153,8 +153,7 @@ def get_full_stack_metadata(stack, host=None, port=None, owner=None, request_url = format_preamble(host, port, owner, project, stack) logger.debug(request_url) - return get_json(session,request_url) - + return get_json(session, request_url) def get_stack_metadata(*args, **kwargs): @@ -315,7 +314,7 @@ def delete_stack(stack, host=None, port=None, owner=None, """ request_url = format_preamble(host, port, owner, project, stack) - r = rest_delete(session,request_url) + r = rest_delete(session, request_url) logger.debug(r.text) return r @@ -346,7 +345,7 @@ def delete_section(stack, z, host=None, port=None, owner=None, """ request_url = '{}/z/{}'.format( format_preamble(host, port, owner, project, stack), z) - r = rest_delete(session,request_url) + r = rest_delete(session, request_url) logger.debug(r.text) return r @@ -379,7 +378,7 @@ def delete_tile(stack, tileId, host=None, port=None, owner=None, """ request_url = '{}/tile/{}'.format( format_preamble(host, port, owner, project, stack), tileId) - r = rest_delete(session,request_url) + r = rest_delete(session, request_url) logger.debug(r.text) return r @@ -531,8 +530,7 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, request_url = format_preamble( host, port, owner, project, stack) + "/zValues/" logger.debug(request_url) - return get_json(session,request_url) - + return get_json(session, request_url) def get_z_value_for_section(stack, sectionId, **kwargs): @@ -586,7 +584,7 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, request_url = format_preamble( host, port, owner, project, stack) + '/z/%f/bounds' % (z) - return get_json(session,request_url) + return get_json(session, request_url) @renderaccess @@ -617,8 +615,7 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, """ request_url = format_preamble( host, port, owner, project, stack) + '/bounds' - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess @@ -696,7 +693,7 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, """ request_url = format_preamble( host, port, owner, project, stack) + '/sectionData' - return get_json(session,request_url) + return get_json(session, request_url) @renderaccess @@ -729,8 +726,7 @@ def get_section_z_value(stack, sectionId, host=None, port=None, """ request_url = format_preamble( host, port, owner, project, stack) + "/section/%s/z" % sectionId - return get_json(session,request_url) - + return get_json(session, request_url) @renderaccess From d5558595998481534b4e4e521adedc939bf746c5 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:28:07 -0700 Subject: [PATCH 612/766] transform: flake8 --- renderapi/transform.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 84c90935..1093e225 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -103,7 +103,7 @@ def from_dict(self, d): self.tforms.append(load_transform_json(td)) return self.tforms - + def load_transform_json(d, default_type='leaf'): """function to get the proper deserialization function @@ -193,7 +193,7 @@ class InterpolatedTransform: lambda_ : float value in interval [0.,1.] which defines evaluation of the linear interpolation between a (at 0) and b (at 1) - """ + """ # noqa: E501 def __init__(self, a=None, b=None, lambda_=None, json=None): """Initialize InterpolatedTransform @@ -1384,19 +1384,24 @@ def estimate_dstpts(transformlist, src=None, reference_tforms=None): dstpts = src for tform in transformlist: if isinstance(tform, list): - dstpts = estimate_dstpts(tform, dstpts,reference_tforms) - elif isinstance(tform,TransformList): - dstpts = estimate_dstpts(tform.tforms,dstpts,reference_tforms) - elif isinstance(tform,ReferenceTransform): + dstpts = estimate_dstpts(tform, dstpts, reference_tforms) + elif isinstance(tform, TransformList): + dstpts = estimate_dstpts(tform.tforms, dstpts, reference_tforms) + elif isinstance(tform, ReferenceTransform): try: - tform_deref= next(tf for tf in reference_tforms if tf.transformId==tform.refId) + tform_deref = next((tf for tf in reference_tforms + if tf.transformId == tform.refId)) except TypeError: - raise RenderError("you supplied a set of tranforms that includes a reference transform,\ - but didn't supply a set of reference transforms to enable dereferencing") + raise RenderError( + "you supplied a set of tranforms that includes a " + "reference transform, but didn't supply a set of " + "reference transforms to enable dereferencing") except StopIteration: - raise RenderError("the list of transforms you provided references transorm {} but that transform\ - could not be found in the list of reference transforms".format(tform.refId)) - dstpts=estimate_dstpts([tform_deref],dstpts,reference_tforms) + raise RenderError( + "the list of transforms you provided references " + "transorm {} but that transform could not be found " + "in the list of reference transforms".format(tform.refId)) + dstpts = estimate_dstpts([tform_deref], dstpts, reference_tforms) else: dstpts = tform.tform(dstpts) return dstpts @@ -1436,7 +1441,8 @@ def __init__(self, dataString=None, json=None, transformId=None, if labels is not None: self.labels = labels self.transformId = transformId - self.className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' + self.className = ( + 'mpicbg.trakem2.transform.NonLinearCoordinateTransform') def _process_dataString(self, dataString): @@ -1580,7 +1586,7 @@ def flatten(l): if isinstance(i, Iterable): try: notstring = isinstance(i, basestring) - except NameError as e: + except NameError: notstring = isinstance(i, str) if notstring: for sub in flatten(i): From e01b2ffe40f612fa02f111e55d3470e593f4c0b7 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:28:35 -0700 Subject: [PATCH 613/766] utils: flake8 --- renderapi/utils.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 07ac9372..b015c8da 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -9,7 +9,7 @@ import json from .errors import RenderError import numpy -import requests + class NullHandler(logging.Handler): """handler to avoid logging errors for, e.g., missing logger setup""" @@ -42,7 +42,8 @@ def default(self, obj): json encodable datatype """ - if isinstance(obj, numpy.integer): return int(obj) + if isinstance(obj, numpy.integer): + return int(obj) to_dict = getattr(obj, "to_dict", None) if callable(to_dict): return obj.to_dict() @@ -94,13 +95,15 @@ def post_json(session, request_url, d, params=None): headers['Accept'] = "application/json" r = session.post(request_url, data=payload, params=params, headers=headers) - if r.status_code not in [200,201,204]: + if r.status_code not in [200, 201, 204]: raise RenderError( - 'cannot post {} to {} with params {} returned status_code {} with message {}'\ - .format(d, request_url, params,r.status_code,r.text)) + 'cannot post {} to {} with params {} returned status_code ' + '{} with message {}'.format( + d, request_url, params, r.status_code, r.text)) return r - -def rest_delete(session,request_url,params=None): + + +def rest_delete(session, request_url, params=None): """DELETE requests with RenderError handling Parameters @@ -115,9 +118,9 @@ def rest_delete(session,request_url,params=None): server response """ r = session.delete(request_url) - if r.status_code not in [200,202,204]: - raise RenderError("delete of {} returned {} with message {}"\ - .format(r.url, r.status_code,r.text)) + if r.status_code not in [200, 202, 204]: + raise RenderError("delete of {} returned {} with message {}".format( + r.url, r.status_code, r.text)) return r @@ -154,14 +157,14 @@ def put_json(session, request_url, d, params=None): headers['Accept'] = "application/json" r = session.put(request_url, data=payload, params=params, headers=headers) - if r.status_code not in [200,201,204]: + if r.status_code not in [200, 201, 204]: raise RenderError( 'put {} to {} returned status code {} with message {}'.format( - d, r.url,r.status_code,r.text)) + d, r.url, r.status_code, r.text)) return r - -def get_json(session,request_url,params=None,stream=False,**kwargs): + +def get_json(session, request_url, params=None, stream=False, **kwargs): """get_json wrapper for requests to handle errors Parameters @@ -186,18 +189,18 @@ def get_json(session,request_url,params=None,stream=False,**kwargs): RenderError if cannot get json successfully """ - - r = session.get(request_url,params=params,stream=stream) + + r = session.get(request_url, params=params, stream=stream) if r.status_code != 200: message = "request to {} returned error code {} with message {}" - raise RenderError(message.format(r.url,r.status_code,r.text)) + raise RenderError(message.format(r.url, r.status_code, r.text)) try: return r.json() except Exception as e: logger.error(e) logger.error(r.text) raise RenderError(r.text) - + def renderdumps(obj, *args, **kwargs): """json.dumps using the RenderEncode From b5e2d149cf4fbe9194cb71a741135fb50698cefb Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:29:57 -0700 Subject: [PATCH 614/766] test: flake8 tests --- test/rendersettings.py | 12 ++-- test/test_channel.py | 22 +++---- test/test_client.py | 13 ++-- test/test_resolvedtiles.py | 25 ++++---- test/test_transform.py | 122 +++++++++++++++++++------------------ 5 files changed, 103 insertions(+), 91 deletions(-) diff --git a/test/rendersettings.py b/test/rendersettings.py index 19e9a035..c96520f5 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -33,13 +33,17 @@ TEST_TILESPECS_FILE = os.path.join(TEST_FILES_DIR, 'tilespecs.json') -INTERPOLATED_TRANSFORM_TILESPEC = os.path.join(TEST_FILES_DIR, 'tilespec_interpolated.json') +INTERPOLATED_TRANSFORM_TILESPEC = os.path.join( + TEST_FILES_DIR, 'tilespec_interpolated.json') -REFERENCE_TRANSFORM_TILESPEC = os.path.join(TEST_FILES_DIR, 'tilespec_ref.json') +REFERENCE_TRANSFORM_TILESPEC = os.path.join( + TEST_FILES_DIR, 'tilespec_ref.json') -REFERENCE_TRANSFORM_TILESPECS = os.path.join(TEST_FILES_DIR, 'tilespec_references.json') +REFERENCE_TRANSFORM_TILESPECS = os.path.join( + TEST_FILES_DIR, 'tilespec_references.json') -REFERENCE_TRANSFORM_SPECS = os.path.join(TEST_FILES_DIR,'transform_references.json') +REFERENCE_TRANSFORM_SPECS = os.path.join( + TEST_FILES_DIR, 'transform_references.json') NONLINEAR_TRANSFORM_KWARGS = { 'className': "mpicbg.trakem2.transform.NonLinearCoordinateTransform", diff --git a/test/test_channel.py b/test/test_channel.py index 3c536e80..04fc1c9a 100644 --- a/test/test_channel.py +++ b/test/test_channel.py @@ -1,19 +1,19 @@ import renderapi -import pytest import json d = { - "name":"DAPI", - "maxIntensity":255, - "minIntensity":0, - "mipmapLevels":{ - "0":{ - "imageUrl":"file:///not/a/path", - "maskUrl":"file:///not/a/mask" + "name": "DAPI", + "maxIntensity": 255, + "minIntensity": 0, + "mipmapLevels": { + "0": { + "imageUrl": "file:///not/a/path", + "maskUrl": "file:///not/a/mask" } } } + def test_channel(): mml = renderapi.image_pyramid.MipMapLevel(0, imageUrl='file:///not/a/path', @@ -24,6 +24,6 @@ def test_channel(): minIntensity=0, ip=ip) - a=json.loads(renderapi.utils.renderdumps(channel)) - b=json.loads(renderapi.utils.renderdumps(d)) - assert(a==b) + a = json.loads(renderapi.utils.renderdumps(channel)) + b = json.loads(renderapi.utils.renderdumps(d)) + assert(a == b) diff --git a/test/test_client.py b/test/test_client.py index a50dcb69..42d32505 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -13,7 +13,7 @@ def test_render_client(): - r = renderapi.render.connect(**args) + r = renderapi.render.connect(**args) # noqa: F841 def test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER, **kwargs): @@ -94,7 +94,7 @@ def checkexpected(expectation, values): newargs['client_scripts'])) with open(renderapi.render.RenderClient.clientscript_from_clientscripts( - str(tmpdir)), 'w') as f: + str(tmpdir)), 'w') as f: # noqa: F841 # test that renderclientaccess decorated funtion works with Render # objects missing client_script assert checkexpected(expected, renderclientaccess_decorated( @@ -124,11 +124,12 @@ def test_renderclientaccess_decorator_fail(tmpdir): with pytest.raises(renderapi.errors.ClientScriptError): _ = renderclientaccess_decorated( - 5, render=renderapi.render.RenderClient(**newargs)) + 5, render=renderapi.render.RenderClient(**newargs)) # noqa: F841 with pytest.raises(renderapi.errors.ClientScriptError): - _ = renderclientaccess_decorated( - 5, render=renderapi.connect(force_http=False, **newargs)) + _ = renderclientaccess_decorated( # noqa: F841 + 5, render=renderapi.connect( + force_http=False, **newargs)) def test_renderclientaccess_override(tmpdir): @@ -147,7 +148,7 @@ def checkexpected(expectation, values): newargs['client_scripts'])) with open(renderapi.render.RenderClient.clientscript_from_clientscripts( - str(tmpdir)), 'w') as f: + str(tmpdir)), 'w') as f: # noqa: F841 # test that renderclientaccess decorated funtion works with Render # objects missing client_script assert checkexpected(expected, renderclientaccess_decorated( diff --git a/test/test_resolvedtiles.py b/test/test_resolvedtiles.py index f932eb84..e4da011d 100644 --- a/test/test_resolvedtiles.py +++ b/test/test_resolvedtiles.py @@ -6,25 +6,28 @@ @pytest.fixture(scope='module') def referenced_tilespecs_and_transforms(): - with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS,'r') as fp: + with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS, 'r') as fp: ds = json.load(fp) tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ds] - with open(rendersettings.REFERENCE_TRANSFORM_SPECS,'r') as fp: + with open(rendersettings.REFERENCE_TRANSFORM_SPECS, 'r') as fp: ds = json.load(fp) transforms = [renderapi.transform.load_transform_json(tf) for tf in ds] - return tilespecs,transforms + return tilespecs, transforms + @pytest.fixture(scope='module') def resolvedtiles_object(referenced_tilespecs_and_transforms): - tilespecs,transforms = referenced_tilespecs_and_transforms - resolved_tiles = renderapi.resolvedtiles.ResolvedTiles(tilespecs = tilespecs, - transformList=transforms) + tilespecs, transforms = referenced_tilespecs_and_transforms + resolved_tiles = renderapi.resolvedtiles.ResolvedTiles( + tilespecs=tilespecs, transformList=transforms) return resolved_tiles -def test_resolvedtiles_from_dict(resolvedtiles_object,referenced_tilespecs_and_transforms): - tilespecs,transforms = referenced_tilespecs_and_transforms - d=resolvedtiles_object.to_dict() + +def test_resolvedtiles_from_dict(resolvedtiles_object, + referenced_tilespecs_and_transforms): + tilespecs, transforms = referenced_tilespecs_and_transforms + d = resolvedtiles_object.to_dict() resolved_tiles = renderapi.resolvedtiles.ResolvedTiles(json=d) - assert(len(tilespecs)==len(resolved_tiles.tilespecs)) - assert(len(transforms)==len(resolved_tiles.transforms)) + assert(len(tilespecs) == len(resolved_tiles.tilespecs)) + assert(len(transforms) == len(resolved_tiles.transforms)) diff --git a/test/test_transform.py b/test/test_transform.py index 179a9535..ad6d7a28 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -6,10 +6,11 @@ import importlib import pytest + def cross_py23_reload(module): try: reload(module) - except NameError as e: + except NameError: importlib.reload(module) @@ -367,57 +368,56 @@ def test_estimate_translation_transform(): def test_non_linear_transform(): - lens_tform = renderapi.transform.NonLinearTransform(dataString=\ - ("6 28 535.9261337198133 -0.07156495284083819 " - "1.5429761616475304 482.5391851962856 -1.3964665434772456 " - "-6.442573400985017 -2.9852709783360574 4.123111145658342 " - "-11.540309826482599 5.014980246618293 13.616409494909078 " - "6.8938038217739255 -2.580163142352532 -5.848702515778433 " - "11.520166623181623 -0.26414636893418164 17.725654626324285 " - "-3.6910106603259862 -5.747999298096374 6.850067634843171 " - "16.792611164648633 7.81339608279572 -1.6748576247746882 " - "7.294185157592906 -8.213479728551924 -5.181177837004834 " - "-2.363460931085683 2.9307897710181123 -19.60977878575318 " - "-5.626357722731385 -3.8857454141075323 -15.21120604177655 " - "-21.23967741658828 -7.433492911261084 5.986054967656173 " - "-0.2472470831304605 -4.252500955671167 6.285397956497022 " - "-10.087631670227893 -2.936382384586807 13.850132289055523 " - "-1.6230114467674603 -6.137660325940608 12.358797331540874 " - "15.361852639613971 -7.395509537102164 -3.4680250777444144 " - "11.543534617091353 2.1153010560743724 -6.1108581863128535 " - "2.4532196427825284 -1.6047644982741716 5.190795110418094 " - "0.40139303958456196 11.96302944378589 12.660577579061767 " - "1196.3041637890071 1266.0596612286872 1719846.5445062846 " - "1485783.6202875 1838252.7229766874 2.714668570418218E9 " - "2.106666890772152E9 2.1421214796751451E9 2.8349699962925887E9 " - "4.528468416208078E12 3.2949497103892886E12 3.0161488655484473E12 " - "3.2930355498080786E12 4.550261299826646E12 7.824367950003595E15 " - "5.464560367850104E15 4.686502506566912E15 4.616510470870998E15 " - "5.276444951800235E15 7.530553575860649E15 1.3844135530148071E19 " - "9.408014352057864E18 7.7275759384866017E18 7.1369460697416591E18 " - "7.373674200122966E18 8.7221483757320745E18 1.277368891891578E19 " - "100.0 537.4423453941299 485.2431449404924 1253541.1543424227 " - "899439.7598253534 1082433.2436210443 2.5451681382231455E9 " - "1.814150348079708E9 1.6692374367919033E9 2.1769137889095693E9 " - "4.992425732284977E12 3.5258872445513765E12 3.074020072484778E12 " - "3.1163128442286265E12 4.266956980967905E12 9.700243843041522E15 " - "6.781968704541316E15 5.745611297869626E15 5.443791159762913E15 " - "5.856309886603154E15 8.280136088463922E15 1.8821205899653665E19 " - "1.3042150458228838E19 1.0829624014456685E19 9.899910593314652E18 " - "9.885988268659214E18 1.1051253229808925E19 1.6002173610912907E19 " - "0.0 2048 2048 "), - transformId="testing") - - ticks = np.arange(0,2048,64,np.float) - xx,yy = np.meshgrid(ticks,ticks) + lens_tform = renderapi.transform.NonLinearTransform(dataString=( + "6 28 535.9261337198133 -0.07156495284083819 " + "1.5429761616475304 482.5391851962856 -1.3964665434772456 " + "-6.442573400985017 -2.9852709783360574 4.123111145658342 " + "-11.540309826482599 5.014980246618293 13.616409494909078 " + "6.8938038217739255 -2.580163142352532 -5.848702515778433 " + "11.520166623181623 -0.26414636893418164 17.725654626324285 " + "-3.6910106603259862 -5.747999298096374 6.850067634843171 " + "16.792611164648633 7.81339608279572 -1.6748576247746882 " + "7.294185157592906 -8.213479728551924 -5.181177837004834 " + "-2.363460931085683 2.9307897710181123 -19.60977878575318 " + "-5.626357722731385 -3.8857454141075323 -15.21120604177655 " + "-21.23967741658828 -7.433492911261084 5.986054967656173 " + "-0.2472470831304605 -4.252500955671167 6.285397956497022 " + "-10.087631670227893 -2.936382384586807 13.850132289055523 " + "-1.6230114467674603 -6.137660325940608 12.358797331540874 " + "15.361852639613971 -7.395509537102164 -3.4680250777444144 " + "11.543534617091353 2.1153010560743724 -6.1108581863128535 " + "2.4532196427825284 -1.6047644982741716 5.190795110418094 " + "0.40139303958456196 11.96302944378589 12.660577579061767 " + "1196.3041637890071 1266.0596612286872 1719846.5445062846 " + "1485783.6202875 1838252.7229766874 2.714668570418218E9 " + "2.106666890772152E9 2.1421214796751451E9 2.8349699962925887E9 " + "4.528468416208078E12 3.2949497103892886E12 3.0161488655484473E12 " + "3.2930355498080786E12 4.550261299826646E12 7.824367950003595E15 " + "5.464560367850104E15 4.686502506566912E15 4.616510470870998E15 " + "5.276444951800235E15 7.530553575860649E15 1.3844135530148071E19 " + "9.408014352057864E18 7.7275759384866017E18 7.1369460697416591E18 " + "7.373674200122966E18 8.7221483757320745E18 1.277368891891578E19 " + "100.0 537.4423453941299 485.2431449404924 1253541.1543424227 " + "899439.7598253534 1082433.2436210443 2.5451681382231455E9 " + "1.814150348079708E9 1.6692374367919033E9 2.1769137889095693E9 " + "4.992425732284977E12 3.5258872445513765E12 3.074020072484778E12 " + "3.1163128442286265E12 4.266956980967905E12 9.700243843041522E15 " + "6.781968704541316E15 5.745611297869626E15 5.443791159762913E15 " + "5.856309886603154E15 8.280136088463922E15 1.8821205899653665E19 " + "1.3042150458228838E19 1.0829624014456685E19 9.899910593314652E18 " + "9.885988268659214E18 1.1051253229808925E19 1.6002173610912907E19 " + "0.0 2048 2048 "), transformId="testing") + + ticks = np.arange(0, 2048, 64, np.float) + xx, yy = np.meshgrid(ticks, ticks) x = np.ravel(xx).T y = np.ravel(yy).T - xy = np.vstack((x,y)).T + xy = np.vstack((x, y)).T - xyp=lens_tform.tform(xy) + xyp = lens_tform.tform(xy) dv = xyp-xy - mean_disp= np.mean(np.sqrt(np.sum(dv**2,axis=1))) - assert((mean_disp-0.7570507)<.01) + mean_disp = np.mean(np.sqrt(np.sum(dv**2, axis=1))) + assert((mean_disp-0.7570507) < .01) @pytest.mark.parametrize("transform_class,transform_json", [ @@ -457,29 +457,33 @@ def test_load_json_transforms(transform_class, transform_json): assert (tform.className == tform_d['className'] == transform_json['className']) + @pytest.fixture(scope='module') def referenced_tilespecs_and_transforms(): - with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS,'r') as fp: + with open(rendersettings.REFERENCE_TRANSFORM_TILESPECS, 'r') as fp: ds = json.load(fp) tilespecs = [renderapi.tilespec.TileSpec(json=d) for d in ds] - with open(rendersettings.REFERENCE_TRANSFORM_SPECS,'r') as fp: + with open(rendersettings.REFERENCE_TRANSFORM_SPECS, 'r') as fp: ds = json.load(fp) transforms = [renderapi.transform.load_transform_json(tf) for tf in ds] - return tilespecs,transforms + return tilespecs, transforms + def test_estimate_dstpoints_reference(referenced_tilespecs_and_transforms): - tilespecs,transforms = referenced_tilespecs_and_transforms + tilespecs, transforms = referenced_tilespecs_and_transforms - ticks = np.arange(0,2048,64,np.float) - xx,yy = np.meshgrid(ticks,ticks) + ticks = np.arange(0, 2048, 64, np.float) + xx, yy = np.meshgrid(ticks, ticks) x = np.ravel(xx).T y = np.ravel(yy).T - xy = np.vstack((x,y)).T + xy = np.vstack((x, y)).T - xyt=renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy,transforms) - assert(xy.shape==xyt.shape) + xyt = renderapi.transform.estimate_dstpts( + tilespecs[0].tforms, xy, transforms) + assert(xy.shape == xyt.shape) with pytest.raises(renderapi.errors.RenderError): - renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy) + renderapi.transform.estimate_dstpts(tilespecs[0].tforms, xy) with pytest.raises(renderapi.errors.RenderError): - renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy,[transforms[2]]) + renderapi.transform.estimate_dstpts( + tilespecs[0].tforms, xy, [transforms[2]]) From caba3620fa5f2be6a0924d7e86809799c22089bd Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 07:31:54 -0700 Subject: [PATCH 615/766] tests: flake8 integration tests --- integration_tests/test_client_integrated.py | 85 ++++++++------ .../test_coordinate_integrated.py | 2 - integration_tests/test_data.py | 37 +++--- integration_tests/test_multichannel.py | 18 +-- .../test_pointmatch_integrated.py | 20 ++-- integration_tests/test_stack_integrated.py | 107 +++++++++++------- 6 files changed, 153 insertions(+), 116 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index ab4cbeeb..909e298e 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -7,7 +7,8 @@ import json import numpy as np from test_data import (render_host, render_port, - client_script_location, tilespec_file, tform_file, test_pool_size) + client_script_location, tilespec_file, tform_file, + test_pool_size) import PIL root = logging.getLogger() @@ -62,9 +63,11 @@ def validate_stack_import(render, stack, tilespecs): ts = renderapi.tilespec.get_tile_specs_from_stack(stack, render=render) assert len(ts) == len(tilespecs) -@pytest.mark.parametrize('call_mode',('call','check_call','check_output')) + +@pytest.mark.parametrize('call_mode', ( + 'call', 'check_call', 'check_output')) def test_import_jsonfiles_validate_client( - render, render_example_tilespec_and_transforms,call_mode): + render, render_example_tilespec_and_transforms, call_mode): stack = 'test_import_jsonfiles_validate_client' renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -76,9 +79,10 @@ def test_import_jsonfiles_validate_client( validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) -@pytest.mark.parametrize('call_mode',('check_call','check_output')) + +@pytest.mark.parametrize('call_mode', ('check_call', 'check_output')) def test_failed_jsonfiles_validate_client( - render, render_example_tilespec_and_transforms,call_mode): + render, render_example_tilespec_and_transforms, call_mode): stack = 'test_failed_import_jsonfiles_validate_client' renderapi.stack.create_stack(stack, render=render) with pytest.raises(renderapi.errors.ClientScriptError): @@ -86,11 +90,12 @@ def test_failed_jsonfiles_validate_client( stack, ['not_a_file'], render=render, subprocess_mode=call_mode) + @pytest.mark.parametrize('use_rest,stack', - [(True,'test_import_jsonfiles_parallel'), - (False,'test_import_jsonfiles_parallel_rest')]) + [(True, 'test_import_jsonfiles_parallel'), + (False, 'test_import_jsonfiles_parallel_rest')]) def test_import_jsonfiles_parallel( - render, render_example_tilespec_and_transforms, + render, render_example_tilespec_and_transforms, stack, use_rest, poolsize=test_pool_size): renderapi.stack.create_stack(stack, render=render) @@ -103,35 +108,39 @@ def test_import_jsonfiles_parallel( validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) + def test_bbox_transformed(render, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms ts = tilespecs[0] - xy = ts.bbox_transformed(ndiv_inner=0,tf_limit=0) - assert xy.shape == (5,2) - assert np.abs((xy[2,:]-np.array([ts.width,ts.height])).sum()) < 1e-10 - xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=0) - assert xy.shape == (9,2) - #xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=4) - #assert xy.shape == (9,2) - #xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=None) - #assert xy.shape == (9,2) + xy = ts.bbox_transformed(ndiv_inner=0, tf_limit=0) + assert xy.shape == (5, 2) + assert np.abs((xy[2, :] - np.array([ts.width, ts.height])).sum()) < 1e-10 + xy = ts.bbox_transformed(ndiv_inner=1, tf_limit=0) + assert xy.shape == (9, 2) + # xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=4) + # assert xy.shape == (9,2) + # xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=None) + # assert xy.shape == (9,2) def square(x): return x**2 -#this test was added in order to validate that multiple WithPools would work -#pathos was breaking when we did this before. Should now be not relevant, -#but who ever deletes a test if you don't have to. + + +# this test was added in order to validate that multiple WithPools would work +# pathos was breaking when we did this before. Should now be not relevant, +# but who ever deletes a test if you don't have to. def test_import_jsonfiles_parallel_multiple( - render, render_example_tilespec_and_transforms, poolsize=test_pool_size): + render, render_example_tilespec_and_transforms, + poolsize=test_pool_size): stacks = ['testmultiple1', 'testmultiple2', 'testmultiple3'] mylist = range(10) for stack in stacks: with renderapi.client.WithPool(poolsize) as pool: - results = pool.map(square, mylist) + results = pool.map(square, mylist) # noqa: F841 test_import_jsonfiles_parallel( - render, render_example_tilespec_and_transforms, stack, - use_rest=False,poolsize=poolsize) + render, render_example_tilespec_and_transforms, stack, + use_rest=False, poolsize=poolsize) def test_import_tilespecs_parallel(render, @@ -153,7 +162,8 @@ def test_import_jsonfiles(render, render_example_tilespec_and_transforms, render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles( - stack, tfiles, transformFile=transformFile, poolsize=test_pool_size, render=render) + stack, tfiles, transformFile=transformFile, poolsize=test_pool_size, + render=render) validate_stack_import(render, stack, tilespecs) @@ -184,13 +194,13 @@ def test_tile_pair_client(render, teststack, **kwargs): ({'maxX': 2000, 'minX': 1000, 'minY': 1000, 'maxY': 2000}, False), (None, False) ]) -def test_renderSectionClient(render,teststack, bounds, raises, scale=.05): +def test_renderSectionClient(render, teststack, bounds, raises, scale=.05): root_directory = tempfile.mkdtemp() root.debug('section_directory:{}'.format(root_directory)) zvalues = renderapi.stack.get_z_values_for_stack(teststack, render=render) if raises: - with pytest.raises(renderapi.client.ClientScriptError) as e: + with pytest.raises(renderapi.client.ClientScriptError): renderapi.client.renderSectionClient(teststack, root_directory, zvalues, @@ -208,16 +218,19 @@ def test_renderSectionClient(render,teststack, bounds, raises, scale=.05): format='png') pngfiles = [] for (dirpath, dirname, filenames) in os.walk(root_directory): - pngfiles += [os.path.join(dirpath,f) for f in filenames if f.endswith('png')] + pngfiles += [os.path.join(dirpath, f) for f in filenames + if f.endswith('png')] assert len(pngfiles) == len(zvalues) if bounds is not None: for f in pngfiles: img = PIL.Image.open(f) width, height = img.size assert( - np.abs(width - (bounds['maxX'] - bounds['minX']) * scale) < 1) + np.abs(width - (bounds['maxX'] - bounds['minX']) * scale) + < 1) assert( - np.abs(height - (bounds['maxY'] - bounds['minY']) * scale) < 1) + np.abs(height - (bounds['maxY'] - bounds['minY']) * scale) + < 1) def test_importTransformChangesClient(render, teststack): @@ -264,14 +277,16 @@ def test_transformSectionClient(render, teststack, for ts in output_ts]) renderapi.stack.delete_stack(deststack, render=render) -def test_point_match_client(teststack, render,tmpdir): + +def test_point_match_client(teststack, render, tmpdir): collection = 'test_client_collection' zvalues = np.array(renderapi.stack.get_z_values_for_stack( teststack, render=render)) tilepairjson = renderapi.client.tilePairClient( teststack, np.min(zvalues), np.max(zvalues), render=render) - tile_pairs = [(tp['p']['id'],tp['q']['id']) for tp in tilepairjson['neighborPairs'][0:1]] + tile_pairs = [(tp['p']['id'], tp['q']['id']) for tp + in tilepairjson['neighborPairs'][0:1]] sift_options = renderapi.client.SiftPointMatchOptions(renderScale=.25) renderapi.client.pointMatchClient(teststack, collection, @@ -280,8 +295,9 @@ def test_point_match_client(teststack, render,tmpdir): sift_options=sift_options, render=render) tp = tilepairjson['neighborPairs'][0] - pms = renderapi.pointmatch.get_matches_involving_tile(collection,tp['p']['groupId'],tp['p']['id'],render=render) - assert(len(pms)>0) + pms = renderapi.pointmatch.get_matches_involving_tile( + collection, tp['p']['groupId'], tp['p']['id'], render=render) + assert(len(pms) > 0) def test_call_run_ws_client_renderclient(render, teststack): @@ -293,4 +309,3 @@ def test_call_run_ws_client_renderclient(render, teststack): render.DEFAULT_PROJECT, teststack) + [zvalues[0]] assert not renderapi.client.call_run_ws_client( test_class, add_args=args, subprocess_mode='call', renderclient=render) - diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index 5600c724..a53548bf 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -1,7 +1,5 @@ import renderapi import pytest -import tempfile -import os import logging import sys import json diff --git a/integration_tests/test_data.py b/integration_tests/test_data.py index ee7b2aaf..7015a638 100644 --- a/integration_tests/test_data.py +++ b/integration_tests/test_data.py @@ -2,38 +2,43 @@ from jinja2 import Environment, FileSystemLoader import json + def render_json_template(env, template_file, **kwargs): template = env.get_template(template_file) d = json.loads(template.render(**kwargs)) return d -example_dir = os.environ.get('RENDER_EXAMPLE_DATA','/var/www/render/examples/') + +example_dir = os.environ.get('RENDER_EXAMPLE_DATA', + '/var/www/render/examples/') renderapp_example_dir = os.environ.get('RENDERAPP_EXAMPLE_DATA', example_dir) test_files_dir = os.path.join(os.path.dirname(__file__), 'test_files') example_env = Environment(loader=FileSystemLoader(test_files_dir)) - -render_host = os.environ.get('RENDER_HOST','renderservice') -render_port = os.environ.get('RENDER_PORT',8080) -client_script_location = os.environ.get('RENDER_CLIENT_SCRIPTS', - ('/var/www/render/render-ws-java-client/' - 'src/main/scripts/')) +render_host = os.environ.get('RENDER_HOST', 'renderservice') +render_port = os.environ.get('RENDER_PORT', 8080) +client_script_location = os.environ.get( + 'RENDER_CLIENT_SCRIPTS', + '/var/www/render/render-ws-java-client/src/main/scripts/') render_params = { - 'host':render_host, - 'port':render_port, - 'owner':'test', - 'project':'test_project', - 'client_scripts':client_script_location + 'host': render_host, + 'port': render_port, + 'owner': 'test', + 'project': 'test_project', + 'client_scripts': client_script_location } -tilespec_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_tiles.json') -tform_file = os.path.join(example_dir,'example_1','cycle1_step1_acquire_transforms.json') -test_pool_size = os.environ.get('RENDER_PYTHON_TEST_POOL_SIZE',3) +tilespec_file = os.path.join( + example_dir, 'example_1', 'cycle1_step1_acquire_tiles.json') +tform_file = os.path.join( + example_dir, 'example_1', 'cycle1_step1_acquire_transforms.json') +test_pool_size = os.environ.get('RENDER_PYTHON_TEST_POOL_SIZE', 3) multi_channel_dir = os.path.join(renderapp_example_dir, 'multichannel-test') -test_2_channels_d = render_json_template(example_env, +test_2_channels_d = render_json_template( + example_env, 'test_2_channels.json', multi_channel_example_dir=multi_channel_dir) diff --git a/integration_tests/test_multichannel.py b/integration_tests/test_multichannel.py index e30c5e28..cbb7ebf2 100644 --- a/integration_tests/test_multichannel.py +++ b/integration_tests/test_multichannel.py @@ -14,23 +14,27 @@ ch.setFormatter(formatter) root.addHandler(ch) -render_params['project']='multi_channel_test' +render_params['project'] = 'multi_channel_test' + @pytest.fixture(scope='module') def render(): return renderapi.connect(**render_params) + @pytest.fixture(scope='module') def multichannel_test_stack(render): stack = 'multichannel_test' - tilespecs = [renderapi.tilespec.TileSpec(json = d) for d in test_2_channels_d] - renderapi.stack.create_stack(stack,render=render) - renderapi.client.import_tilespecs(stack,tilespecs, render=render) - renderapi.stack.set_stack_state(stack, 'COMPLETE',render=render) + tilespecs = [renderapi.tilespec.TileSpec(json=d) + for d in test_2_channels_d] + renderapi.stack.create_stack(stack, render=render) + renderapi.client.import_tilespecs(stack, tilespecs, render=render) + renderapi.stack.set_stack_state(stack, 'COMPLETE', render=render) return stack -def test_section_image_channels(render,multichannel_test_stack): + +def test_section_image_channels(render, multichannel_test_stack): section_image = renderapi.image.get_section_image(multichannel_test_stack, - 1.0,channel='DAPI', + 1.0, channel='DAPI', render=render) print(section_image.shape) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index 62cb6028..e8b45edf 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -1,13 +1,8 @@ import renderapi import pytest -import tempfile -import os import logging import sys -import json -import numpy as np -from test_data import render_host, render_port, \ - client_script_location, tilespec_file, tform_file +from test_data import render_host, render_port, client_script_location logger = logging.getLogger() @@ -43,7 +38,7 @@ "w": [1, 1, 1, 1] } }, - { + { "pGroupId": "0", "pId": "0-1", "qGroupId": "2", @@ -181,22 +176,23 @@ def test_get_match_groupIds_to_only(render, test_pm_collection): def test_delete_point_matches_between_groups(render): collection = 'test_delete' - owner = 'test' + # owner = 'test' renderapi.pointmatch.import_matches( collection, test_matches, render=render) - group1 = '0' - group2 = '1' + # group1 = '0' + # group2 = '1' renderapi.pointmatch.delete_point_matches_between_groups( collection, '0', '1', render=render) groups = renderapi.pointmatch.get_match_groupIds(collection, render=render) assert len(groups) == 2 + def test_delete_collection(render): collection = 'test_delete_collection' - owner = 'test' + # owner = 'test' renderapi.pointmatch.import_matches( collection, test_matches, render=render) - renderapi.pointmatch.delete_collection(collection,render=render) + renderapi.pointmatch.delete_collection(collection, render=render) collections = renderapi.pointmatch.get_matchcollections(render=render) assert(collection not in collections) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index c60b99c6..7b44ca5d 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -1,14 +1,12 @@ import renderapi import pytest -import tempfile -import os import logging import sys import json import numpy as np from test_data import (render_host, render_port, - client_script_location, tilespec_file, - tform_file, test_2_channels_d) + client_script_location, tilespec_file, + tform_file) root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -202,8 +200,8 @@ def test_remove_section(render, simpletilespec, tmpdir): assert simpletilespec.z in stack_zs_before r = render.run(renderapi.stack.set_stack_state, 'test_insert', 'LOADING') - r = renderapi.stack.delete_section('test_insert', - simpletilespec.z, render=render) + r = renderapi.stack.delete_section( # noqa: F841 + 'test_insert', simpletilespec.z, render=render) stack_zs_after = render.run(renderapi.stack.get_z_values_for_stack, 'test_insert') assert len(stack_zs_after) == (len(stack_zs_before) - 1) @@ -218,7 +216,8 @@ def teststack(request, render, render_example_tilespec_and_transforms): r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) render.run(renderapi.client.import_tilespecs, stack, tilespecs, sharedTransforms=tforms) - r = render.run(renderapi.stack.set_stack_state, stack, 'COMPLETE') + r = render.run( # noqa: 841 + renderapi.stack.set_stack_state, stack, 'COMPLETE') yield stack render.run(renderapi.stack.delete_stack, stack) @@ -306,10 +305,10 @@ def test_bb_image(render, teststack, **kwargs): data = render.run(renderapi.image.get_bb_image, teststack, z, x, y, width, height, scale=.25, img_format=fmt, **kwargs) - dr = data.ravel() + dr = data.ravel() # noqa: 841 assert data.shape[0] == (np.floor(height*.25)) assert data.shape[1] == (np.floor(width*.25)) - if len(data.shape)>2: + if len(data.shape) > 2: assert data.shape[2] >= 3 @@ -322,7 +321,7 @@ def test_bb_image_options(render, teststack): def test_tile_image(render, teststack, render_example_tilespec_and_transforms, **kwargs): (tilespecs, tforms) = render_example_tilespec_and_transforms - fmt = 'png' + # fmt = 'png' data = render.run(renderapi.image.get_tile_image_data, teststack, tilespecs[0].tileId, **kwargs) if kwargs.get('scale') is None: @@ -361,7 +360,7 @@ def test_section_image(render, teststack, **kwargs): def test_section_image_options(render, teststack): test_section_image(render, teststack, filter=True, - maxTileSpecsToRender=50) + maxTileSpecsToRender=50) def fail_image_get(render, teststack, render_example_tilespec_and_transforms): @@ -378,12 +377,15 @@ def test_get_tilespecs_from_z(render, teststack, tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] assert len(tiles) == len(tsz) + def test_get_tilespec_raw( render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms - ts = renderapi.tilespec.get_tile_spec_raw(teststack, tilespecs[0].tileId, render=render) + ts = renderapi.tilespec.get_tile_spec_raw( + teststack, tilespecs[0].tileId, render=render) assert ts.to_dict() == tilespecs[0].to_dict() + def test_get_tile_specs_from_minmax_box( render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -417,61 +419,78 @@ def test_get_tile_specs_from_stack(render, teststack, ts = renderapi.tilespec.get_tile_specs_from_stack(teststack, render=render) assert len(ts) == len(tilespecs) -def test_get_sectionId_for_z(render, teststack, render_example_tilespec_and_transforms): + +def test_get_sectionId_for_z(render, teststack, + render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms - sectionId = render.run(renderapi.stack.get_sectionId_for_z, teststack, tilespecs[0].z) + sectionId = render.run( + renderapi.stack.get_sectionId_for_z, teststack, tilespecs[0].z) assert (sectionId == tilespecs[0].layout.sectionId) + def test_get_resolvedtiles_from_z(render, teststack, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms - resolved_tiles = renderapi.resolvedtiles.get_resolved_tiles_from_z(teststack, - tilespecs[0].z, - render=render) + resolved_tiles = renderapi.resolvedtiles.get_resolved_tiles_from_z( + teststack, tilespecs[0].z, render=render) tsz = [ts for ts in tilespecs if ts.z == tilespecs[0].z] - assert(len(tsz)==len(resolved_tiles.tilespecs)) - matching_ts = next(ts for ts in resolved_tiles.tilespecs if ts.tileId == tsz[0].tileId) - assert (len(matching_ts.tforms)==len(tsz[0].tforms)) + assert(len(tsz) == len(resolved_tiles.tilespecs)) + matching_ts = next(ts for ts in resolved_tiles.tilespecs + if ts.tileId == tsz[0].tileId) + assert (len(matching_ts.tforms) == len(tsz[0].tforms)) + -def test_put_resolved_tiles_scratch(render,render_example_tilespec_and_transforms): +def test_put_resolved_tiles_scratch( + render, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms out_stack = 'resolved_test_stack' - renderapi.stack.create_stack(out_stack,render=render) - resolved_tilespecs = renderapi.resolvedtiles.ResolvedTiles(tilespecs,tforms) - r=renderapi.resolvedtiles.put_tilespecs(out_stack,resolved_tilespecs, - render=render) - tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack(out_stack, - render=render) - assert(len(tilespecs_out)==len(resolved_tilespecs.tilespecs)) - -def test_put_tilespecs_and_tforms(render,render_example_tilespec_and_transforms): + renderapi.stack.create_stack(out_stack, render=render) + resolved_tilespecs = renderapi.resolvedtiles.ResolvedTiles( + tilespecs, tforms) + r = renderapi.resolvedtiles.put_tilespecs( # noqa: 841 + out_stack, resolved_tilespecs, render=render) + tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack( + out_stack, render=render) + assert(len(tilespecs_out) == len(resolved_tilespecs.tilespecs)) + + +def test_put_tilespecs_and_tforms( + render, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms out_stack = 'resolved_test_stack2' - renderapi.stack.create_stack(out_stack,render=render) - r=renderapi.resolvedtiles.put_tilespecs(out_stack,tilespecs=tilespecs, - shared_transforms=tforms,render=render) - tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack(out_stack, - render=render) - assert(len(tilespecs_out)==len(tilespecs)) + renderapi.stack.create_stack(out_stack, render=render) + r = renderapi.resolvedtiles.put_tilespecs( # noqa: F841 + out_stack, tilespecs=tilespecs, + shared_transforms=tforms, render=render) + tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack( + out_stack, render=render) + assert(len(tilespecs_out) == len(tilespecs)) + def test_put_tilespecs_fail(render): with pytest.raises(renderapi.errors.RenderError): - r=renderapi.resolvedtiles.put_tilespecs('fail_stack',render=render) - + r = renderapi.resolvedtiles.put_tilespecs( # noqa: F841 + 'fail_stack', render=render) + + def test_bad_delete(render): with pytest.raises(renderapi.errors.RenderError): - renderapi.stack.delete_section('not_a_stack',0.0,render=render) + renderapi.stack.delete_section('not_a_stack', 0.0, render=render) + def test_bad_get(render): with pytest.raises(renderapi.errors.RenderError): - renderapi.stack.get_stack_metadata('not_a_stack',render=render) + renderapi.stack.get_stack_metadata('not_a_stack', render=render) + def test_bad_post(render): - sv = renderapi.stack.StackVersion() + # sv = renderapi.stack.StackVersion() with pytest.raises(renderapi.errors.RenderError): - renderapi.stack.set_stack_state('not_a_stack','LOADING',render=render) + renderapi.stack.set_stack_state( + 'not_a_stack', 'LOADING', render=render) + def test_bad_put(render): with pytest.raises(renderapi.errors.RenderError): - renderapi.stack.clone_stack('not_a_stack','not_getting_here',render=render) - \ No newline at end of file + renderapi.stack.clone_stack( + 'not_a_stack', 'not_getting_here', render=render) From 1a63e6f0c85f0e344178d35cadebce616d241fdc Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 07:35:37 -0700 Subject: [PATCH 616/766] added fail load transform test --- test/test_transform.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/test_transform.py b/test/test_transform.py index 179a9535..ad6481ed 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -6,12 +6,32 @@ import importlib import pytest + def cross_py23_reload(module): try: reload(module) except NameError as e: importlib.reload(module) +def test_TransformList_init(): + tlist = renderapi.transform.TransformList() + +def test_simple_TransformList_init(): + aff = renderapi.transform.AffineModel() + tlist = renderapi.transform.TransformList(tforms=[aff]) + +def test_fail_TransformList_init(): + with pytest.raises(renderapi.errors.RenderError): + aff = renderapi.transform.AffineModel() + tlist = renderapi.transform.TransformList(tforms=aff) + +def test_fail_loadtransform_json(): + with pytest.raises(renderapi.errors.RenderError): + d={'type':'not_a_type', + 'dataString':'junkstring', + 'transformId':'badtform'} + renderapi.transform.load_transform_json(d) + def test_affine_rot_90(): am = renderapi.transform.AffineModel() From 8f28249d5dc67af9efd2334fcbd47dce90cf304a Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 08:43:07 -0700 Subject: [PATCH 617/766] some improved bug fixes and test coverage on transforms, stack --- renderapi/transform.py | 10 +++++--- test/test_transform.py | 53 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 84c90935..ef6895fa 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -773,9 +773,13 @@ def __init__(self, *args, **kwargs): def _process_dataString(self, dataString): """expected dataString is 'tx ty'""" - tx, ty = map(float(dataString.split(' '))) + tx, ty = map(float,dataString.split(' ')) self.B0 = tx self.B1 = ty + self.M00 = 1 + self.M10 = 0 + self.M01 = 0 + self.M11 = 1 self.load_M() @staticmethod @@ -864,7 +868,7 @@ def __init__(self, *args, **kwargs): def _process_dataString(self, dataString): """expected datastring is 'theta tx ty'""" - theta, tx, ty = map(float(dataString.split(' '))) + theta, tx, ty = map(float,dataString.split(' ')) self.M00 = np.cos(theta) self.M01 = -np.sin(theta) self.M10 = np.sin(theta) @@ -993,7 +997,7 @@ def __init__(self, *args, **kwargs): def _process_dataString(self, dataString): """expected datastring is 's theta tx ty'""" - s, theta, tx, ty = map(float(dataString.split(' '))) + s, theta, tx, ty = map(float,dataString.split(' ')) self.M00 = s * np.cos(theta) self.M01 = -s * np.sin(theta) self.M10 = s * np.sin(theta) diff --git a/test/test_transform.py b/test/test_transform.py index ad6481ed..5269585e 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -6,7 +6,7 @@ import importlib import pytest - +EPSILON = 0.0000000000001 def cross_py23_reload(module): try: reload(module) @@ -26,12 +26,26 @@ def test_fail_TransformList_init(): tlist = renderapi.transform.TransformList(tforms=aff) def test_fail_loadtransform_json(): - with pytest.raises(renderapi.errors.RenderError): - d={'type':'not_a_type', + d={'type':'not_a_type', 'dataString':'junkstring', 'transformId':'badtform'} + with pytest.raises(renderapi.errors.RenderError): renderapi.transform.load_transform_json(d) + with pytest.raises(renderapi.errors.RenderError): + renderapi.transform.load_leaf_json(d) +def test_load_unknown_tform(): + d={'className':'mpicpg.not_supported_transform', + 'dataString':'crazy_parameters'} + tform = renderapi.transform.load_transform_json(d) + assert(isinstance(tform,renderapi.transform.Transform)) + +def test_fail_bad_tform(): + d={'type':'leaf', + 'className':'mpicpg.not_supported_transform'} + with pytest.raises(renderapi.errors.RenderError): + tform = renderapi.transform.load_transform_json(d) + def test_affine_rot_90(): am = renderapi.transform.AffineModel() @@ -267,7 +281,7 @@ def test_reference_transform(): assert (dict(ref_args) == ref_args.to_dict() == dict(ref_ts) == ref_ts.to_dict() == dict(ref_dd) == ref_dd.to_dict()) - + print(ref_dd) def test_transform_hash_eq(): t1 = renderapi.transform.Transform( @@ -503,3 +517,34 @@ def test_estimate_dstpoints_reference(referenced_tilespecs_and_transforms): renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy) with pytest.raises(renderapi.errors.RenderError): renderapi.transform.estimate_dstpts(tilespecs[0].tforms,xy,[transforms[2]]) + +def test_fail_convert_points(): + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float) + with pytest.raises(renderapi.errors.ConversionError): + renderapi.transform.AffineModel.convert_to_point_vector(points_in.T) + +def test_translation_transform_init(): + d={'type':'leaf', + 'className':'mpicbg.trakem2.transform.TranslationModel2D', + 'dataString':'10 0'} + tform = renderapi.transform.load_transform_json(d) + assert(isinstance(tform,renderapi.transform.TranslationModel)) + assert(tform.M[0,2]==10) + +def test_rigid_init(): + d={'type':'leaf', + 'className':'mpicbg.trakem2.transform.RigidModel2D', + 'dataString':'{} 10 5'.format(np.pi/2)} + tform = renderapi.transform.load_transform_json(d) + assert(isinstance(tform,renderapi.transform.RigidModel)) + assert(tform.M[0,2]==10) + assert(tform.M[0,0] Date: Fri, 20 Apr 2018 08:52:09 -0700 Subject: [PATCH 618/766] had to back off epsilon for python27 --- test/test_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_transform.py b/test/test_transform.py index 5269585e..8ba67bb0 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -6,7 +6,7 @@ import importlib import pytest -EPSILON = 0.0000000000001 +EPSILON = 0.0000000001 def cross_py23_reload(module): try: reload(module) From befb1c3045d81e19980d722fada5210983b3cbaf Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 09:00:14 -0700 Subject: [PATCH 619/766] integration_tests: remove unused tests and variables, fix some noqas --- integration_tests/test_client_integrated.py | 4 ---- integration_tests/test_pointmatch_integrated.py | 3 --- integration_tests/test_stack_integrated.py | 7 ++----- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 909e298e..c5cf9b8f 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -117,10 +117,6 @@ def test_bbox_transformed(render, render_example_tilespec_and_transforms): assert np.abs((xy[2, :] - np.array([ts.width, ts.height])).sum()) < 1e-10 xy = ts.bbox_transformed(ndiv_inner=1, tf_limit=0) assert xy.shape == (9, 2) - # xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=4) - # assert xy.shape == (9,2) - # xy = ts.bbox_transformed(ndiv_inner=1,tf_limit=None) - # assert xy.shape == (9,2) def square(x): diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index e8b45edf..a57f21ce 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -176,11 +176,8 @@ def test_get_match_groupIds_to_only(render, test_pm_collection): def test_delete_point_matches_between_groups(render): collection = 'test_delete' - # owner = 'test' renderapi.pointmatch.import_matches( collection, test_matches, render=render) - # group1 = '0' - # group2 = '1' renderapi.pointmatch.delete_point_matches_between_groups( collection, '0', '1', render=render) groups = renderapi.pointmatch.get_match_groupIds(collection, render=render) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 7b44ca5d..623fa723 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -216,7 +216,7 @@ def teststack(request, render, render_example_tilespec_and_transforms): r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) render.run(renderapi.client.import_tilespecs, stack, tilespecs, sharedTransforms=tforms) - r = render.run( # noqa: 841 + r = render.run( # noqa: F841 renderapi.stack.set_stack_state, stack, 'COMPLETE') yield stack render.run(renderapi.stack.delete_stack, stack) @@ -305,7 +305,6 @@ def test_bb_image(render, teststack, **kwargs): data = render.run(renderapi.image.get_bb_image, teststack, z, x, y, width, height, scale=.25, img_format=fmt, **kwargs) - dr = data.ravel() # noqa: 841 assert data.shape[0] == (np.floor(height*.25)) assert data.shape[1] == (np.floor(width*.25)) if len(data.shape) > 2: @@ -321,7 +320,6 @@ def test_bb_image_options(render, teststack): def test_tile_image(render, teststack, render_example_tilespec_and_transforms, **kwargs): (tilespecs, tforms) = render_example_tilespec_and_transforms - # fmt = 'png' data = render.run(renderapi.image.get_tile_image_data, teststack, tilespecs[0].tileId, **kwargs) if kwargs.get('scale') is None: @@ -447,7 +445,7 @@ def test_put_resolved_tiles_scratch( renderapi.stack.create_stack(out_stack, render=render) resolved_tilespecs = renderapi.resolvedtiles.ResolvedTiles( tilespecs, tforms) - r = renderapi.resolvedtiles.put_tilespecs( # noqa: 841 + r = renderapi.resolvedtiles.put_tilespecs( # noqa: F841 out_stack, resolved_tilespecs, render=render) tilespecs_out = renderapi.tilespec.get_tile_specs_from_stack( out_stack, render=render) @@ -484,7 +482,6 @@ def test_bad_get(render): def test_bad_post(render): - # sv = renderapi.stack.StackVersion() with pytest.raises(renderapi.errors.RenderError): renderapi.stack.set_stack_state( 'not_a_stack', 'LOADING', render=render) From 889f716f66e7fba2e44457622581bf46c265ce08 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 12:06:08 -0700 Subject: [PATCH 620/766] transform: manage imports --- renderapi/transform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/renderapi/transform.py b/renderapi/transform.py index 1093e225..7e5383a4 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -7,7 +7,9 @@ import json import logging from collections import Iterable + import numpy as np + from .errors import ConversionError, EstimationError, RenderError from .utils import NullHandler From eb0b02760d67220f543e26e431c487257c5d2d9d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 12:15:32 -0700 Subject: [PATCH 621/766] added stack rename call and test --- integration_tests/test_stack_integrated.py | 25 ++++++++++++++- renderapi/stack.py | 36 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index a1a164fe..b7a00fb1 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -474,4 +474,27 @@ def test_bad_post(render): def test_bad_put(render): with pytest.raises(renderapi.errors.RenderError): renderapi.stack.clone_stack('not_a_stack','not_getting_here',render=render) - \ No newline at end of file + +def test_rename_stack(render): + from_stack = 'from' + from_owner = 'from_owner' + from_project = 'from_project' + to_stack = 'to' + to_owner = 'to_owner' + to_project = 'to_project' + + renderapi.stack.create_stack(from_stack,owner=from_owner, + project=from_project, render=render) + renderapi.stack.rename_stack(from_stack,to_stack,to_owner=to_owner, + to_project=to_project,owner=from_owner, + project=from_project,render=render) + stacks_out = renderapi.render.get_stacks_by_owner_project(project=to_project, + owner=to_owner, render=render) + stacks_in = renderapi.render.get_stacks_by_owner_project(project=from_project, + owner=from_owner, render=render) + + assert(to_stack in stacks_out) + assert(from_stack not in stacks_in) + + renderapi.stack.delete_stack(to_stack, owner=to_owner, project=to_owner, + render=render) diff --git a/renderapi/stack.py b/renderapi/stack.py index 947cb426..ded45646 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -446,6 +446,42 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, raise RenderError(r.text) +@renderaccess +def rename_stack(stack, to_stack, to_project=None, to_owner=None, + host=None, port=None, owner=None, project=None, session=requests.session(), + render=None, **kwargs): + """ + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + inputstack : str + name of input stack to clone + to_stack : str + name of destination stack. if exists, must be LOADING + to_project : str + name of project to rename stack to (default = leave the same as inputstack) + outputOwner: str + name of owner to rename stack to (default = leave the same as inputstack) + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + requests.session.response + server response + """ + + request_url = format_preamble(host,port,owner,project,stack)+"/stackId" + d = { + "owner": owner if to_owner is None else to_owner, + "project": project if to_project is None else to_project, + "stack": stack if to_stack is None else to_stack + } + return put_json(session,request_url,d) + @renderaccess def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, zs=None, close_stack=True, host=None, port=None, From 0552e36481a5d3642b71b0427155756a32d4a077 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 12:40:08 -0700 Subject: [PATCH 622/766] transform: fix warning --- test/test_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_transform.py b/test/test_transform.py index 8e093811..3988b29d 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -52,7 +52,7 @@ def test_fail_bad_tform(): d = {'type': 'leaf', 'className': 'mpicpg.not_supported_transform'} with pytest.raises(renderapi.errors.RenderError): - tform = renderapi.transform.load_transform_json(d) + tform = renderapi.transform.load_transform_json(d) # noqa: F841 def test_affine_rot_90(): From 2061dec1a1f718cdca0a46eb9a67e793b665a5ce Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 12:50:33 -0700 Subject: [PATCH 623/766] stack: flake8 on new stuff --- renderapi/stack.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 119a0868..9e57ecb3 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -447,8 +447,8 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, @renderaccess def rename_stack(stack, to_stack, to_project=None, to_owner=None, - host=None, port=None, owner=None, project=None, session=requests.session(), - render=None, **kwargs): + host=None, port=None, owner=None, project=None, + session=requests.session(), render=None, **kwargs): """ :func:`renderapi.render.renderaccess` decorated function @@ -471,15 +471,17 @@ def rename_stack(stack, to_stack, to_project=None, to_owner=None, ------- requests.session.response server response - """ + """ # noqa: E501 - request_url = format_preamble(host,port,owner,project,stack)+"/stackId" + request_url = format_preamble( + host, port, owner, project, stack) + "/stackId" d = { "owner": owner if to_owner is None else to_owner, "project": project if to_project is None else to_project, "stack": stack if to_stack is None else to_stack - } - return put_json(session,request_url,d) + } + return put_json(session, request_url, d) + @renderaccess def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, From 6e149c825200ae2fdf12fd5935941d4bc8e07212 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Fri, 20 Apr 2018 13:08:04 -0700 Subject: [PATCH 624/766] tests: cleanup comments --- integration_tests/test_coordinate_integrated.py | 2 +- integration_tests/test_pointmatch_integrated.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/integration_tests/test_coordinate_integrated.py b/integration_tests/test_coordinate_integrated.py index a53548bf..c515347b 100644 --- a/integration_tests/test_coordinate_integrated.py +++ b/integration_tests/test_coordinate_integrated.py @@ -16,7 +16,7 @@ formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) -# + logger.addHandler(ch) diff --git a/integration_tests/test_pointmatch_integrated.py b/integration_tests/test_pointmatch_integrated.py index a57f21ce..0981b40e 100644 --- a/integration_tests/test_pointmatch_integrated.py +++ b/integration_tests/test_pointmatch_integrated.py @@ -186,7 +186,6 @@ def test_delete_point_matches_between_groups(render): def test_delete_collection(render): collection = 'test_delete_collection' - # owner = 'test' renderapi.pointmatch.import_matches( collection, test_matches, render=render) From a62bf538f2f7f82ff2cbfa52d353464b04190a67 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 13:28:24 -0700 Subject: [PATCH 625/766] changed max line length to 79 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 30bc29fe..4bba92fa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,3 @@ [flake8] ignore = E226,E126 -max-line-length = 200 +max-line-length = 79 From 8e19a3c06b20804d7ceab61358dc2e249a1acd24 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 13:30:31 -0700 Subject: [PATCH 626/766] fixing flake8 on new stuff --- integration_tests/test_client_integrated.py | 6 ++--- renderapi/client.py | 28 ++++++++++----------- renderapi/transform.py | 6 ++--- test/test_utils.py | 13 ++++++---- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index c5cf9b8f..b8ea89e5 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -222,11 +222,9 @@ def test_renderSectionClient(render, teststack, bounds, raises, scale=.05): img = PIL.Image.open(f) width, height = img.size assert( - np.abs(width - (bounds['maxX'] - bounds['minX']) * scale) - < 1) + np.abs(width - (bounds['maxX'] - bounds['minX']) * scale) < 1) assert( - np.abs(height - (bounds['maxY'] - bounds['minY']) * scale) - < 1) + np.abs(height - (bounds['maxY'] - bounds['minY']) * scale) < 1) def test_importTransformChangesClient(render, teststack): diff --git a/renderapi/client.py b/renderapi/client.py index e1dbba59..ce0f1dfe 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -940,7 +940,7 @@ def get_canvas_url_template( default=False """ # noqa: E501 request_url = format_preamble(host, port, owner, project, stack) - tile_base_url = request_url+"/tile" + tile_base_url = request_url + "/tile" url_suffix = "render-parameters" if filter: url_suffix += '?filter=true' @@ -1014,7 +1014,7 @@ def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, class MatchDerivationParameters(ArgumentParameters): def __init__(self, matchIterations=None, matchMaxEpsilon=None, matchMaxNumInliers=None, - matchMaxTrust=None, matchMinInlierRatio=None, + matchMaxTrust=None, matchMinInlierRatio=None, matchMinNumInliers=None, matchModelType=None, matchRod=None, **kwargs): super(MatchDerivationParameters, self).__init__(**kwargs) @@ -1113,18 +1113,18 @@ def pointMatchClient(stack, collection, tile_pairs, argvs += sift_options.to_java_args() canvas_url_template = get_canvas_url_template( - stack, - filter, - renderWithoutMask, - normalizeForMatching, - excludeTransformsAfterLast, - excludeFirstTransformAndAllAfter, - excludeAllTransforms, - host=host, - port=port, - owner=owner, - project=project, - client_script=client_script) + stack, + filter, + renderWithoutMask, + normalizeForMatching, + excludeTransformsAfterLast, + excludeFirstTransformAndAllAfter, + excludeAllTransforms, + host=host, + port=port, + owner=owner, + project=project, + client_script=client_script) for tile1, tile2 in tile_pairs: argvs += [canvas_url_template.format(tile1), diff --git a/renderapi/transform.py b/renderapi/transform.py index 38d041b2..5dd016cc 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -775,7 +775,7 @@ def __init__(self, *args, **kwargs): def _process_dataString(self, dataString): """expected dataString is 'tx ty'""" - tx, ty = map(float,dataString.split(' ')) + tx, ty = map(float, dataString.split(' ')) self.B0 = tx self.B1 = ty self.M00 = 1 @@ -870,7 +870,7 @@ def __init__(self, *args, **kwargs): def _process_dataString(self, dataString): """expected datastring is 'theta tx ty'""" - theta, tx, ty = map(float,dataString.split(' ')) + theta, tx, ty = map(float, dataString.split(' ')) self.M00 = np.cos(theta) self.M01 = -np.sin(theta) self.M10 = np.sin(theta) @@ -999,7 +999,7 @@ def __init__(self, *args, **kwargs): def _process_dataString(self, dataString): """expected datastring is 's theta tx ty'""" - s, theta, tx, ty = map(float,dataString.split(' ')) + s, theta, tx, ty = map(float, dataString.split(' ')) self.M00 = s * np.cos(theta) self.M01 = -s * np.sin(theta) self.M10 = s * np.sin(theta) diff --git a/test/test_utils.py b/test/test_utils.py index b7963724..dbcd5192 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,19 +1,22 @@ import renderapi import pytest + def test_jbool(): assert(renderapi.utils.jbool(True) == 'true') assert(renderapi.utils.jbool(False) == 'false') assert(renderapi.utils.jbool(0) == 'false') assert(renderapi.utils.jbool(1) == 'true') + def test_renderdumps_simple(): - s=renderapi.utils.renderdumps({'a':1}) - assert(s=='{"a": 1}') + s = renderapi.utils.renderdumps({'a': 1}) + assert(s == '{"a": 1}') + + s = renderapi.utils.renderdumps(5) + assert(s == '5') - s=renderapi.utils.renderdumps(5) - assert(s=='5') def test_renderdumps_fails(): with pytest.raises(AttributeError): - renderapi.utils.renderdumps(pytest) \ No newline at end of file + renderapi.utils.renderdumps(pytest) From 59fd23f392fe46ba7736b464f903610a8ebb4d8e Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 13:32:45 -0700 Subject: [PATCH 627/766] flake8 --- integration_tests/test_client_integrated.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index b8ea89e5..5adc9cfb 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -222,9 +222,13 @@ def test_renderSectionClient(render, teststack, bounds, raises, scale=.05): img = PIL.Image.open(f) width, height = img.size assert( - np.abs(width - (bounds['maxX'] - bounds['minX']) * scale) < 1) + np.abs( + width - (bounds['maxX'] - bounds['minX']) * scale) < 1) assert( - np.abs(height - (bounds['maxY'] - bounds['minY']) * scale) < 1) + np.abs( + height - + (bounds['maxY'] - bounds['minY']) * scale) < 1 + ) def test_importTransformChangesClient(render, teststack): From 2f1e530624b9a6a1fbdb324200b4d44993aaffba Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 20 Apr 2018 13:34:29 -0700 Subject: [PATCH 628/766] fixing bad pytest dump test (#100) --- test/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index b7963724..6d3419ba 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,5 +1,6 @@ import renderapi import pytest +import numpy as np def test_jbool(): assert(renderapi.utils.jbool(True) == 'true') @@ -16,4 +17,4 @@ def test_renderdumps_simple(): def test_renderdumps_fails(): with pytest.raises(AttributeError): - renderapi.utils.renderdumps(pytest) \ No newline at end of file + renderapi.utils.renderdumps(np.zeros(3)) \ No newline at end of file From afec24f7fd73cc1ec51f6165d9d014e48c7f0b62 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 8 May 2018 13:20:36 -0700 Subject: [PATCH 629/766] utils: allow ujson for requests calls if it can be imported --- renderapi/utils.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index b015c8da..756d49cf 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -7,8 +7,18 @@ import inspect import copy import json -from .errors import RenderError + import numpy +import requests + +from .errors import RenderError + +# use ujson if installed for faster json +try: + import ujson as requests_json +except ImportError: + import json as requests_json +requests.models.complexjson = requests_json class NullHandler(logging.Handler): From ab0d730eb1eb138d00e7340c8ebdc8a7c1d0ef00 Mon Sep 17 00:00:00 2001 From: RussTorres Date: Wed, 9 May 2018 08:28:23 -0700 Subject: [PATCH 630/766] test: add test for ujson defaulting (and run integration tests using ujson) --- test/test_utils.py | 34 +++++++++++++++++++++++++++++++++- test_requirements.txt | 1 + 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/test/test_utils.py b/test/test_utils.py index 9a09e77d..a2500e62 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,6 +1,39 @@ +import importlib +import json import renderapi import pytest import numpy as np +import ujson + + +def cross_py23_reload(module): + try: + reload(module) + except NameError: + importlib.reload(module) + + +@pytest.mark.parametrize("use_ujson", [True, False]) +def test_json_load(use_ujson): + if not use_ujson: + try: + import builtins + except ImportError: + import __builtin__ as builtins + realimport = builtins.__import__ + + def noujson_import(name, globals=None, locals=None, + fromlist=(), level=0): + if 'ujson' in name: + raise ImportError + return realimport(name, globals, locals, fromlist, level) + builtins.__import__ = noujson_import + cross_py23_reload(renderapi.utils) + assert (renderapi.utils.requests_json is ujson + if use_ujson else renderapi.utils.requests_json is json) + assert ( + renderapi.utils.requests.models.complexjson is ujson + if use_ujson else renderapi.utils.requests.models.complexjson is json) def test_jbool(): @@ -21,4 +54,3 @@ def test_renderdumps_simple(): def test_renderdumps_fails(): with pytest.raises(AttributeError): renderapi.utils.renderdumps(np.zeros(3)) - diff --git a/test_requirements.txt b/test_requirements.txt index 2f722abe..d54f0207 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -8,4 +8,5 @@ pytest-xdist>=1.14 flake8>=3.0.4 pylint>=1.5.4 scipy +ujson jinja2 From 88c8db97d6041d1744c29b04edba253dda708377 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 11 Jun 2018 11:56:46 -0700 Subject: [PATCH 631/766] adding estimate method for nonlinearcoordinate transform --- renderapi/transform.py | 79 ++++++++++++++++++++++++++++++++++++++++-- test/test_transform.py | 36 +++++++++++++++++++ 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 5dd016cc..7085a281 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1485,7 +1485,7 @@ def _process_dataString(self, dataString): "incorrect number of normVar coefficents " "{} != {}".format(self.normVar.shape[0], self.length)) - def kernelExpand(self, src): + def kernelExpand(self, src, normMean=None, normVar=None): """creates an expanded representation of the x,y src points in a polynomial form @@ -1510,11 +1510,84 @@ def kernelExpand(self, src): np.power(x, j) * np.power(y, i - j)) pidx += 1 - expanded[:, :-1] = ((expanded[:, :-1] - self.normMean[:-1]) / - self.normVar[:-1]) + if normMean is None: + normMean=self.normMean + if normVar is None: + normVar=self.normVar + + expanded[:, :-1] = ((expanded[:, :-1] - normMean[:-1]) / + normVar[:-1]) expanded[:, -1] = 100.0 return expanded + def fit(self,A, B): + """function to fit this transform given the corresponding sets of points A & B + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + beta + a self.lengthx2 matrix with polynomial factors + normMean + a self.length vector of expanded means + normVar + a self.length vector of expanded standard deviations + """ + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + normMean = np.zeros(self.length).astype('float') + normVar = np.ones(self.length).astype('float') + src_exp = self.kernelExpand(A,normMean=normMean,normVar=normVar) + normMean = src_exp.mean(0) + normVar = src_exp.std(0) #poorly named variable + src_exp = self.kernelExpand(A,normMean=normMean,normVar=normVar) + + xcoeff,xresiduals,xrank,xs = np.linalg.lstsq(src_exp,B[:,0]) + ycoeff,yresiduals,yrank,ys = np.linalg.lstsq(src_exp,B[:,1]) + + beta = np.zeros((self.length,2)) + beta[:,0]=xcoeff + beta[:,1]=ycoeff + + return beta,normMean,normVar + + + def estimate(self, A, B, return_params=True, **kwargs): + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the dataString + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + dataString + """ + + beta,normMean,normVar = self.fit(A,B) + self.beta = beta + self.normMean = normMean + self.normVar = normVar + + if return_params: + return self.dataString + def tform(self, src): """transform a set of points through this transformation diff --git a/test/test_transform.py b/test/test_transform.py index 3988b29d..69263c52 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -461,6 +461,42 @@ def test_non_linear_transform(): mean_disp = np.mean(np.sqrt(np.sum(dv**2, axis=1))) assert((mean_disp-0.7570507) < .01) +def test_estimate_non_linear_transform(): + dataString=("5 21 1134.6315 27.041916 49.84044 1069.9268 8.754167 " + "-8.966886 -67.81902 -26.337778 -42.44015 47.329388 " + "-57.457996 -19.826283 12.059551 16.236008 25.214308 " + "-6.14843 13.517356 -69.09145 59.978294 9.0963745 2.8096974 " + "17.50649 1.8950082 -16.095743 -6.6861115 5.4056945 1.330886 " + "67.93751 -21.105675 -1.9647278 -2.1779869 -8.596665 " + "0.25304613 0.07506427 -2.4152899 10.851463 5.036614 " + "-8.608638 -2.1753247 -22.031227 18.718458 20.106474 " + "1871.7964 2010.6288 4716919.0 3762185.0 5205234.5 " + "13463128000.0 950763500.0 9750733000.0 14992724000.0 " + "41204630000000.0 27199392000000.0 24678496000000.0 " + "28117224000000.0 45919358000000.0 1.3180739E+17 " + "8.3419896E+16 7.0674915E+16 7.122505E+16 8.621747E+16 " + "1.4640968E+17 100.0 1101.5017 1078.2463 4353785.0 3244154.8 " + "4338782.5 15909609000.0 11250277000.0 11111349000.0 " + "15978368000.0 57421630000000.0 39513703000000.0 " + "36268440000000.0 38755805000000.0 57893900000000.0 " + "2.0705793E+17 1.4025713E+17 1.2428827E+17 1.23013825E+17 " + "1.3686534E+17 2.0921157E+17 0.0 3840 3840 ") + + n=500 + x=np.linspace(0,3840,n) + xp,yp = np.meshgrid(x,x) + src=np.transpose(np.vstack((xp.flatten(),yp.flatten()))) + + tf=renderapi.transform.NonLinearCoordinateTransform(dataString=dataString) + dst=tf.tform(src) + + tf.estimate(src,dst) + + fdst=tf.tform(src) + + assert(np.abs(fdst-dst).max()<1e-5) + + @pytest.mark.parametrize("transform_class,transform_json", [ (renderapi.transform.Transform, { From 74b969d45ed58ec8bd39d6706b9a6ad28d040f4e Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Wed, 13 Jun 2018 09:54:09 -0700 Subject: [PATCH 632/766] applying flake8 formatting --- renderapi/transform.py | 33 ++++++++++++++++----------------- test/test_transform.py | 26 +++++++++++++------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/renderapi/transform.py b/renderapi/transform.py index 7085a281..002bd299 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -1511,16 +1511,16 @@ def kernelExpand(self, src, normMean=None, normVar=None): pidx += 1 if normMean is None: - normMean=self.normMean + normMean = self.normMean if normVar is None: - normVar=self.normVar + normVar = self.normVar expanded[:, :-1] = ((expanded[:, :-1] - normMean[:-1]) / normVar[:-1]) expanded[:, -1] = 100.0 return expanded - def fit(self,A, B): + def fit(self, A, B): """function to fit this transform given the corresponding sets of points A & B Parameters ---------- @@ -1531,7 +1531,7 @@ def fit(self,A, B): Returns ------- - beta + beta a self.lengthx2 matrix with polynomial factors normMean a self.length vector of expanded means @@ -1545,22 +1545,21 @@ def fit(self,A, B): normMean = np.zeros(self.length).astype('float') normVar = np.ones(self.length).astype('float') - src_exp = self.kernelExpand(A,normMean=normMean,normVar=normVar) + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) normMean = src_exp.mean(0) - normVar = src_exp.std(0) #poorly named variable - src_exp = self.kernelExpand(A,normMean=normMean,normVar=normVar) + normVar = src_exp.std(0) # poorly named variable + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) - xcoeff,xresiduals,xrank,xs = np.linalg.lstsq(src_exp,B[:,0]) - ycoeff,yresiduals,yrank,ys = np.linalg.lstsq(src_exp,B[:,1]) + xcoeff, xresiduals, xrank, xs = np.linalg.lstsq(src_exp, B[:, 0]) + ycoeff, yresiduals, yrank, ys = np.linalg.lstsq(src_exp, B[:, 1]) - beta = np.zeros((self.length,2)) - beta[:,0]=xcoeff - beta[:,1]=ycoeff + beta = np.zeros((self.length, 2)) + beta[:, 0] = xcoeff + beta[:, 1] = ycoeff - return beta,normMean,normVar + return beta, normMean, normVar - - def estimate(self, A, B, return_params=True, **kwargs): + def estimate(self, A, B, ndim=None, return_params=True, **kwargs): """method for setting this transformation with the best fit given the corresponding points A,B @@ -1577,10 +1576,10 @@ def estimate(self, A, B, return_params=True, **kwargs): Returns ------- - dataString + dataString """ - beta,normMean,normVar = self.fit(A,B) + beta, normMean, normVar = self.fit(A, B) self.beta = beta self.normMean = normMean self.normVar = normVar diff --git a/test/test_transform.py b/test/test_transform.py index 69263c52..0b2cae58 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -461,8 +461,10 @@ def test_non_linear_transform(): mean_disp = np.mean(np.sqrt(np.sum(dv**2, axis=1))) assert((mean_disp-0.7570507) < .01) + def test_estimate_non_linear_transform(): - dataString=("5 21 1134.6315 27.041916 49.84044 1069.9268 8.754167 " + dataString = ( + "5 21 1134.6315 27.041916 49.84044 1069.9268 8.754167 " "-8.966886 -67.81902 -26.337778 -42.44015 47.329388 " "-57.457996 -19.826283 12.059551 16.236008 25.214308 " "-6.14843 13.517356 -69.09145 59.978294 9.0963745 2.8096974 " @@ -482,20 +484,18 @@ def test_estimate_non_linear_transform(): "2.0705793E+17 1.4025713E+17 1.2428827E+17 1.23013825E+17 " "1.3686534E+17 2.0921157E+17 0.0 3840 3840 ") - n=500 - x=np.linspace(0,3840,n) - xp,yp = np.meshgrid(x,x) - src=np.transpose(np.vstack((xp.flatten(),yp.flatten()))) - - tf=renderapi.transform.NonLinearCoordinateTransform(dataString=dataString) - dst=tf.tform(src) - - tf.estimate(src,dst) - - fdst=tf.tform(src) + n = 500 + x = np.linspace(0, 3840, n) + xp, yp = np.meshgrid(x, x) + src = np.transpose(np.vstack((xp.flatten(), yp.flatten()))) - assert(np.abs(fdst-dst).max()<1e-5) + tf = renderapi.transform.NonLinearCoordinateTransform( + dataString=dataString) + dst = tf.tform(src) + tf.estimate(src, dst) + fdst = tf.tform(src) + assert(np.abs(fdst-dst).max() < 1e-5) @pytest.mark.parametrize("transform_class,transform_json", [ From 4765f34d323d53523bbc761c9975ca922fba90ff Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Fri, 22 Jun 2018 17:15:33 -0700 Subject: [PATCH 633/766] Add thin plate spline transform (#110) * added base64 encoding/decoding * starting framework for ThinPlateSplineTransform * updated * changin encode/decode to deal only with numpy arrays * adding from_dict method to thin_plate_spline * unflattening matrix * fixing np -> numpy * adding to_dict to thin plate spline * apply flake8 * commenting out thinplate split to_dict * adding whitespace to start of datastring * adding textto start dataString * commenting out _process dataString * modifying _process dataString * modifying _process dataString * modifying _process dataString * fixing up ThinPlateSpline class * fixing * flake8 * fixed test/transform/thinplatespline * removing trailing L's in hex from encode/decode * increasing coverage on affine and b vectors * removing trailing L in hex from encode/decode * one more hex change * adding cast to byte for python 3 for zlib decompress output * adding cast to byte for python 3 for zlib compress input * casting utf-8 * fixing... * think python2/3 zlib stuff fixed * one last python 3 bit * trivial change to retrigger failed build * another trivial change to retrigger failed build * another trivial change to retrigger failed build * adding decode support for '@' * major simplification from Russel on encode/decode * getting rid of leading b'' in encode for python 3 * catching and testing for '@' in decode * removing bitstring requirement * removing n argument to decodeBase64 * fixing multiline string * removed documentation referring to n, retrigger failed build * accidental deletion * first try with pytest.raises() for code coverage * changing IndexErrors to ValueErrors * last flake8 indent fix * trivial change to retrigger failed build --- renderapi/transform.py | 81 +++++++++++++++++++++++++- renderapi/utils.py | 43 ++++++++++++++ requirements.txt | 2 +- test/rendersettings.py | 3 + test/test_files/thin_plate_spline.json | 6 ++ test/test_transform.py | 37 ++++++++++++ 6 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 test/test_files/thin_plate_spline.json diff --git a/renderapi/transform.py b/renderapi/transform.py index 002bd299..611e82fb 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -11,7 +11,7 @@ import numpy as np from .errors import ConversionError, EstimationError, RenderError -from .utils import NullHandler +from .utils import NullHandler, encodeBase64, decodeBase64 logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -1626,6 +1626,85 @@ def dataString(self): shapestring, betastring, meanstring, varstring, dimstring) +class ThinPlateSplineTransform(Transform): + """ + render-python class that can hold a dataString for + mpicbg.trakem2.transform.ThinPlateSplineTransform class. + Parameters + ---------- + dataString: str or None + data string of transformation + labels : list of str + list of labels to give this transform + json: dict or None + json compatible dictionary representation of the transformation + Returns + ------- + :class:`ThinPlateSplineTransform` + a transform instance + """ + + className = 'mpicbg.trakem2.transform.ThinPlateSplineTransform' + + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + self.labels = labels + self.transformId = transformId + self.className = ( + 'mpicbg.trakem2.transform.ThinPlateSplineTransform') + + def _process_dataString(self, dataString): + fields = dataString.split(" ") + + self.ndims = int(fields[1]) + self.nLm = int(fields[2]) + + if fields[3] != "null": + try: + values = decodeBase64(fields[3]) + self.aMtx = values[0:self.ndims*self.ndims].reshape( + self.ndims, self.ndims) + self.bVec = values[self.ndims*self.ndims:] + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + else: + self.aMtx = None + self.bVec = None + + try: + values = decodeBase64(fields[4]) + self.srcPts = values[0:self.ndims*self.nLm].reshape( + self.ndims, self.nLm) + self.dMtxDat = values[self.ndims*self.nLm:].reshape( + self.ndims, self.nLm) + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + + @property + def dataString(self): + header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) + + if self.aMtx is not None: + blk1 = np.concatenate((self.aMtx.flatten(), self.bVec)) + b64_1 = encodeBase64(blk1) + else: + b64_1 = "null" + + blk2 = np.concatenate((self.srcPts.flatten(), self.dMtxDat.flatten())) + b64_2 = encodeBase64(blk2) + + return '{} {} {}'.format(header, b64_1, b64_2) + + class NonLinearTransform(NonLinearCoordinateTransform): className = 'mpicbg.trakem2.transform.nonLinearTransform' diff --git a/renderapi/utils.py b/renderapi/utils.py index 756d49cf..8915d25a 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -7,6 +7,8 @@ import inspect import copy import json +import base64 +import zlib import numpy import requests @@ -358,3 +360,44 @@ def fitargspec(f, oldargs, oldkwargs): logger.error('Cannot fit argspec for {}'.format(f)) logger.error(e) return oldargs, oldkwargs + + +def encodeBase64(src): + """encode an array or list of doubles + in Base64 binary-to-text encoding + same as in trakem2...ThinPlateSplineTransform.java + + Parameters + ---------- + src : 1D numpy array + floating point values to be encoded + + Returns + ------- + encoded: string + """ + return base64.b64encode( + zlib.compress( + src.byteswap().tobytes()) + ).decode('utf-8') + + +def decodeBase64(src): + """decode a string + encoded in base64 binary-to-text encoding + same as in trakem2...ThinPlateSplineTransform.java + + Parameters + ---------- + src : string + encoded string + + Returns + ------- + arr: length n numpy array of double-precision floats + """ + if src[0] == '@': + b = base64.b64decode(src[1:]) + else: + b = zlib.decompress(base64.b64decode(src)) + return numpy.frombuffer(b).byteswap() diff --git a/requirements.txt b/requirements.txt index 1ec45f7d..3f22a4bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ numpy pillow sphinxcontrib-napoleon decorator -six \ No newline at end of file +six diff --git a/test/rendersettings.py b/test/rendersettings.py index c96520f5..613104af 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -33,6 +33,9 @@ TEST_TILESPECS_FILE = os.path.join(TEST_FILES_DIR, 'tilespecs.json') +TEST_THINPLATESPLINE_FILE = os.path.join(TEST_FILES_DIR, + 'thin_plate_spline.json') + INTERPOLATED_TRANSFORM_TILESPEC = os.path.join( TEST_FILES_DIR, 'tilespec_interpolated.json') diff --git a/test/test_files/thin_plate_spline.json b/test/test_files/thin_plate_spline.json new file mode 100644 index 00000000..4718f574 --- /dev/null +++ b/test/test_files/thin_plate_spline.json @@ -0,0 +1,6 @@ +{ + "type":"leaf", + "id":"", + "classname":"mpicbg.trakem2.transform.ThinPlateSplineTransform", + "dataString":"ThinPlateSplineR2LogR 2 995 null eJx8u3k4VV//Pk6SWYbmUpIGpaIkVNurJJREMkSJDJGxjMk8z8Mxz2dyznHOUZQ5bcmQZI6ERKYMRSEZwve89ea5nufzu37+sK591t5rr+E13Pe91mZi+v/+gyf//+Xafen/Xf6f32P+NG9OVXGG6KbKUBE2DET8T31AXuMZsuMCRA0UiHYf3wlRSWX0IXcXSOG5/0BX8gcEeP29Lxn+lt7FmB3HPG0hKXZfNz9qA1H4Q7fu9fRDwoN2QqKwEESldNPHe6chyXz5GbUvAYLT9Rb5ZU5D9L1Wh47qHAj4ddS209cWMNjKxstakRB0LlzOUG03xNfL3m38+BMiymWWFxouQMzU0S0nlcXXxhOxVmpwCnrKQpJgu+sMmECY8N/fUyT+lmFbfzxk198GUY/Tl/sIjZB829UPhQvg/vZByeseHvDD/r0vwe5vGTKlsTR2lBPiHuowhVg8hhCTJ/c0lH4CZkg36KHYCPgo8PeTOzMhzoxXSWv5K8RsM9CkCJVDSgjPw0fJwhBc8+/8rP9benl0qpwNngasuOrl6vqnYKf3mKhOL4ekDaXnu3eIQDr6my395hTgLp/M7ts/D+6Izczsj/MQk0o18fx0E2LtdkU6alpAQtuNPSYq2hBf4DFUJGEA8Z58cxyWJRBRy9/mOZcPqWrXb3lz6kCssOkeJDgL4uZE8vE8nyCgao/uq1AniC0Q3uhspQwJnHwGg+aZkHhGdX1NaRqk7hT+Kr0+AxwP7xcrGbCGuO+3RMbPYyCmuLN6vf5TwCApnOJZdEhSsDq2qBD+f+zKm3mrS7eRHSTs+ZW4tVgPYl92LzmWKgK2sldkmT0YEuvXBRy/GQKJu4Tcy5S9wOd2JW2IIANp62cLBz6fhTDuDz5zF0Uh4eYC8cOAB4Q3bp+UMr0LGFGP3UeHH4DrCZ3WtMckSKSe6eHYaA8Bs3/fm5j63/2I/9cu43DS6WfjgyHpSuA4F+cMRLvt/XKlpwXi9M9U+OAkwBc/fv1z7X6I2c8WFhCKBQyO2XdutwkkqFaRI/fjIeZq/kuNY28gzTRc2CpQAJJcKxc6269D1EXXOdNn3BD7vKNkwGoR0vGdW3Zfc4K4fJwo2yVFiAyRUHe1bIeIagWRpzoPIG7sblua1leIPif/YC/tCkRWtpoHUj5AzPjNRsGGUUga5/NNJGlB0M+vopKncyDCZEtX7MbvEJ354FFZjyfE7LyzoMypD8k31Hp2s4tBzNydN4ZuF8DXQF3WvF0PUs5W7xe+PAdxKaJ143qlEKeu6pO62AKJwpxfrx0XgJR9HId633Yy7EDUyL7oDkS/vPxT8foOiBFSkZp41waxOmWB2excEG0rH6kkAZBm0T9gXlsFiRKnNGLsQgFT08G2ES2BhIBfJJ+CQxAVeK8m4hENElnPYs9e4YTguBujbHanIVEpVv1AMDOEO8DZ5vNkSCkmHXg2KQEhkTnCiPEIJJ8oXgrwZazzjyKf7MpaSDuUf2CjwQWIenb28bq2NEgxTKCaTxdA5O/XRW9uXWH4kbXOcoo2YA59dCMp1AHF8L/XOc1AYdsfdWWIzZmW22u8BCkLPvPqMfEQU7nHst99BGLvkKe1yC0Qc5s1gFnaCGJcDh27WDACkQqOi/UziZAkVpqIF2kDv+UGl3TnaxC/sSqPmrANAgveaD16uh1iafjLBy09Iax6f3/n8TCID0xdOMmfCaFTPe+411kDQYIp9ERDMDzeJZ/nlZkNGLnXRzUe9EO0qUOWUlggRFUI3vriYwYZv380qGVzA6bydffL/A7IMB+89H5uGDIShr5tHXsK6UJH5uQnDeAx90/Sh0TGOHzcNnO4eYO917vgD9m/Ia5VrJe44wAkjmbS6zVZIa7e2FOmSgESTPRvyZaWQGwDLnz/9yZIMWUVES6VgfjzXX0PHr2CpDeXZ6pT9ACzLv/51okvgO397/lLXH95aufm4+DdFhFrzvwAkmVO+JuxjoOP2qZ+6ZQ08DVnuMnxKAg/apMjGRoDMb9qKqwprhCkLBJlnmgH7np/ulBmXQjh/WbGKbwJIvad+u17sxRCYWZa5gqOEWe2SFu76oCf+lvL4jAPSBHah+NoMgZ/WT/pP5ujIPhks+FGKXXAHRbTu9woCZmtFLVq6VzA1SgctWnYBAlnM047FglCWi4z96/cGogzcNXY3sAGqbIeB9Pev4D4CaGy0BQqpB4M4C47IgpeDbmHY5eZAN/FtKejZCdgyeTWMo54wGYFEWYTj4ITL9MbudJYSBa/6po0LwBWXk/MbxdaQpq8dV7dfra1eVnNZw7wqNBtXQ6kSk7w+F1lBmd9z8aKWQFIHwymP7I7Bu5Gog/lni5DMsce/H2RPHC9F9oZeyob4ovmcmjHtoLDxfzAD3NOkIC5gfvhJQGP/OuE2+6mQxyHVLOxlC/YmcqONI1gABPou+miHgHS/rBjaOaNkOCR/F0wVnI1jq31K2lu46fUq1kQFHF1NlfnwGreW6sPC6l0r3BFACv9gC59Qx2SCRuwWo7ukPZAbCJOLRcwabx41WofINQ2DfAVSIKn4I7NNxa6IP0+ro6P13KtHdK/8TO9v3Jr7d69EHDVZ0KNeTOkLn753X4WA/HINd/Hl/cBhtvChJGIIGT3/vjzM4qrz621Exm8aZfPlmrA1oT43Yx3gMRlUcddfZGQwWv5lG+fCWDO96TZ8ksAjml0+JUyw0739PVmDJsD/jtzoPz8awgIGUv0qyZChmuCDCNjgC+TwzLT/leQse7FPU/LHZDYx9a464QZpLZtYGlzjodMpmS/La7xgEuoQjDcbICpyGIZtVMFYrrjzhgW97V+4V/9LTFyZ6tOEBMhXIk/aPr4MAQLqyRe5j0PWDVVrdFxYQiRfdP0pCMN0p9gLt8PvA2BZ7qsa9lmIBXd4OI2YgihW/xX4mHqN3ne9x/1INMZb02nVQG2qEp76sgncFywzcGwEAC3rWoouVUcUge/DYw2VULgadGA7KPvIO3uIY7oR3LgczVk+sdHRbDZsS9m3GAcMgxE8GeWQ9f6+6/fwh29hYkmtk2Q7jyktf00BzjY/NJJ/hkEWEStx0iUDcxUNP8xaMBOJ2s61nSuPZ/x7/Npew/yigs8AXeOXU7hC+2ArWt3dPi9D9K+eA+F8JyCjOkbwS1usuB5x0Y8cVAebKR468NLvwP+DYuocVgC3MDib4c/agZc09/2LO7taBG8TQPcF+fQz1XTYCAXcsNH5gbg80P/Wbe19xOxf0t1O39PCaMaIOT8a1//1q/iV7PvmbtP7DgGxLqlNwrCfHDr29hS+ZkMwIphC+wrpyFj7/k+3Wuq4NPgMtSs1wxYDy8JbG4p+FTf69xd+RVw5ywM7T/vBvfNihg+mVuQnu1belXkLdwrK8Wxpmeurvtav3Bb/Y/3T+lB8Bd9RvrRBqwtz6On2wLBZaPD6/ydHyC9Vu/e8CUSuHC1yn5W3gN4ZvmIV7O+YH3giDVTnj6kH0utSeEdhIec1Wn2F8YhQf/ngRGXNvA/KWSWnccCOMMf/SEeDuDInOkoh7wC/DFZ5cSZDeDKVJ748z1jXm2N9cibfcHP/zH3u1e3Ac++8X7MZDR4jSAlP9dJA86206njsST4iXL67BGLBuyDJxs0mXQhoG538QCmFLCe7gsfMmPBauhBlbxsC3h3iqPEsW6I6pfyrV04A/GDQ2ZZPorwcJPG1HBy8Kod/MeuzP+W+myXOEmqDD90tV5Ho43BLWNGKydVV9f5P37jLFD/JaMCDK78WcEX/67rWj3BrLCn//AGsJ0hav++nby6zmv1XpW3JbIvzQDG/IBespcKuFSd2F+o8gSiKqPEfloy8JXH7aLFvgmw7wChW5I3AY/7vrI+9uUzO3PuPAWKZ4DDlSPOcO/AritRe68A8dDfdo3uPlzxe4JqY/FWHmO4q9mB/ZH5AzKj/tYb1mzokNm8A4gdFarY176gUypF8ZAZA+cwl+d26x0Z8ZekVJKVDxajMryNpFSIqJiaOSvZAx6BxxfWj7tBhDaXwFWOdHBM7Uyy7/gG4TJ6zOPfg9fG5cH03+Nc5UXWR5Lvfj8lAiHSWoNPJ3HgeK06z6/dEoI/TK/EQcdmWfVirVkIcN5zavtLgFuKNgHVuvIQ2l8tqv3bYK29EPhb3onRVOcQb4TAitnP1j3Ma/U+f+0akp/lCiyeqAbnl+cbXVyKIPEaJ37HgxR4vPXR8SNmBRCXxGx4Xf04uC+ksDrksUE8HiVSyrXg7p5S1KhxGYgvo05WMx0G5+utt1xIXUC8bdgt/yUB7GSKZcnlj1fnc+29q3676scP4ryRSsNxiCof9Tz7mxkCXD6r+zRqgtvpoOzxF1Jw/yvr4BSvF3j9tUPQbSw3jXKPXrt2sfvmNzubCdaXVb493yUCxmT7KxqnD4PfAd2VdbHrviF+Nv07+P5SCrGm0//PvLv8uHV43QkO8HnosqDwMQ2sHlKbm87XQcBkHVx+OAYOuXwtXvenwb2zUFf/CwV8KhvzzgU3gK+iZFdtPAasnG9/JSuQwDWgX9l2fQrY/E/7Dq++aEp/LgbHet9WywOvwDNo0erhdhEG72VLWDdkAj6Pjb61UvnB3WlqyDurGDyWSxodLs2D/3dkj5zOGPhasOdu4sCDjXHzm8/qDL9ne/tpNmYBgsReLDAIDcTnNXSHGk6CS9sW+W5TBn7aODUwsc4HHKTXL/D7FkD8mTsxy/sHwcdgQaRp0BASDfZ72ruTwLrrZs3bE7yruOU//thBXMEbTnL0G6U/CoEU5HX1ZKQr3NkX7W5pSgWS7AMP5zZRsDMJMbfxMIbMK7yD+4MPwgOFT4VRC6bgVp3w4/u5AHjsFj3mv7sIfBfPy19OpIFT0rcLI8GpEMJ++MXFQW6wG2P9i5ut+fWaN4yDY0CD6AvcEQijqSx8tQoFN5ErNa8aMGv9+ndewUrt5sn+W/pgH/47123wGOg4KfIuvq0CZ5FvBuKfmMG7JT9J9FArhNjamDyTBAi2r34zOvgS/A6OubBcYvDZ6raM6UVrCOUO6GUfyYPAp1Lv0htNINDWrSNTxARCHD8N5onegVD28AMO7HwQFvSm+RS/HiP/etyR+LoM4cuCcedGL4FPVmzhs2AtiFLlsrPYTYHAH/PklJJPgDmlImj0KRb8nrb/uMf6FpKbNr7isB0Dk6U/Dbt090Nq/kAzSd0ILDzCVvJ6qlIg+e1nVrge6BLdt/kkJBr+Hae6yZXUAr8wiIPbol8MO8AqXPdCoVQ+xAZey5L7fRW08Sw6BK31gHn13+sX9W17UlJSMjjMm6zgEQyvJFuhnxmYxo/6Hknigpii/IN41kdwT2Az9XfkBOiJOemTJirAzpkiyuXCAlGPkm7sfXUR/NxavBdT8BC18GaBnicGnpt2X/hyQBiSR4a2lr/MBLv+RmOvddshSS6BtpDyHYy5Hix+fhkFmIcq+cb7SuFBwl0Jp1caENF8Avnycg+Y3/bHohocEPWv36olNl+3Dt0NkfV87TKGyeC613Ql32EwpAhk0g8e9bC8HX+3AFEfMKdmOaLB2oERdh06V+PW2njDs8MHryi3gn3aEMYuxwpicI6VIiyG4EXwe1bo3Azh+a0vncYd4KaQ0T9C0GqcXXt+Vb+5rd4x/d5NEDAH+tTzReXAO879UNO6PvCsTa3RmdoApnZx/qf7qiD4+M4n9Wz+cPvxvYFNXydW4+xae6GkWwrPuk+D2Qevzh0xYhAw0mSl8GsO7p+c2RbKfx4y2oreW6p6AiZS+L7NiQEwMfDHtx9fhPgp840Cr13h9tSlgrSCO5DEty7zq38BmG//XfjGhxNSOMSLzdDHa+9ZtRPtZDr33YEWSLU6B6LPEUgOjqpgX6JDQl306NLQAljciJS2UE6CuAtKE3rx19ae/9dugHJMo75SIRrsFRS6uwQY74nitpWe8IKks4Ki55cokGybJbj7VT6kSJ4N7DoiBUnqB5fYnyxAUtOB/gc3/sGHYtpyGTKQYveQAbNsIOW5ru+iugikSmfArmuvIS36ZN6z+VhIctm636uCA7DlQA1Q/8rg7d9GdkXQgfyvbnZVmIFuN5kBXpCNaGRfBHGnPr4tUtSG9MDAFTydaqL08BolH7Bzr4NCb9yCjEymb/MWKPgFdWYb7lkAbLvphk1bTkOqxBJhy47dDB63V+KmyR7ABkSpNgzSIRHKGO6rC1i5oNtx3aoQvyMnZ/Lxc4hPNT3DLvgOMvquBKs7yUE6qhfPZYuFuKlrpolfeACH4/HfcuMDROVgvkzYlwP+akJ++OaDkHDNQN7lRStkwEbnpyYbIPHd7osSuY+AqHLgtmz8AiTrI2dCttdCfHMcf8+fG4CTaZU6YzcBxP3dlJf7uCH17hsxW1NBILIw0IsaMyQa+TDC8wXIPKukw62tDrFz8ZIPzd8zeK3qNazGRUgqGRi85ooBgo7XieKoHIjvctNDZH8DURQdZFk4DdEFTnc26vEB/rbLzQvRDHxSZivgaeAPBLbJTy2tapA4wLxk8usq4C6FHGw6hIf4JxFE00wnwN2M0OtOPQSJmmd2HL6aC4RdL1b0p9jao1NzRl6QGTvcs5mYDQm+hTvXmysCsUhpS2SEB8QvmT+RbBcEQpcbOUbHEGIiNx/2k+EA8tvoXp+w03D3arr3u0svIFk7p8pkzhOw+rErfkS+845+9EcrWJWlnhesDmDg1a3XT7S8AJxAmDnzg+1A/R9/oqjlf7ekVEPiHa88vSEmyJLjpOydTIJEVnut9YgmhFyl2Evr+gD+RelWhRETSCOIRnYsnoaMyT/ApsMN0TWtEkdgBvCc7GY7vnJDlNbR2ohj4oBvwqSWFFyGjBS2lfiVUbAjaGLpMJAejK7E8QSm0YZ77YzxMwkWNyazQWL/kxt54WeAtO/le+cvxRCTEmf7h98AMoUuruhZmGqL5meDKUBAqlbiKubw4FYjxTgg2VVLHzZwgeT9jqczsjyAvMsl5aPMGYhzGXewKr4HGdeWuZ27AyG5//cDJqFbgO2xTxIPY6y/8tyR138sIONIoTd0A+May6CF+8DF1yHwT7Eo4NXrTAQa+SBrnOsV1gYLaVfxzEoHu4G4h66H76+CuPslJWj2NcDqELKaZ5gh6au0bNsAEfzT/XpvWPACfu/tv/PqWC/5p0UNsCl9XV8nK4Bo28eAQww+LSe+7srJLsAGqXN+WY8B/D0XHoNwhh3n5ml0ogchPUpp372BzUCQMtshVOQCYaJe+rJ9rmv4blVPJTg5ahyrfglRuw6zdkfGAGnbtDID2EGcoUTVmORZyLzLoXR/NhVS+Hzb30dzAz5WYKuqai9ENUancD77ApmBt2TY/Tkh9JflOm6VkrX2V/V78oF1RedFwwF7oKXMCf+LwdsxQRbV1hDJ7V4SFXIASKIba03OvoKoLL71hb8ygez2/DSFex4SXFp4tMU+AsnnvLDciSOQpi/RX598HTLb3VfmAyMiGcAgzED5eefojjEEkspuznqukweyRZVql+sgJPmeYsAAPSAdDkqS3RsCKW+4HnVwTAJ2opZk/dIeMLKChhua2oBUzNpnseUiJMYkymkWDQKluDwqQlcI0rbpmU8XCEPYE++sEzEMfqxqb/qtoh+y4vv6keXt8LCIs2hPfDpQlNkrcipvQvLS+ePvxe8C+YKW5cWdxZBSqdiv+bsRsj5WT0X5/4JknWg3vAAvkAmvFgKmH0Pa4rrSuR0ngXa4+EeF6k7wU86zaVF8DrT15xZ4NGYh/VbG9xF3NsjayRT2lSUHUoLZRL7QdYHi37XCU+OIv8ru87wArOiQ71lDFkilTymGtXAB9vi8LeFPLUTMMheZDZwBHCYVryW2BDHakrM7dB8Atj5tXDtEHaLi5wel1XcDTpGfOlTzG8JfdJ8ac9AFvKiXtOmWLgjdVlw3P8HwywPON67kPoWIuId2h6UCIV3txkq+Dn2qFcZeeAdwrfov0bN8EBje5av86yzgv3wY9b/oAkFn5K5OeyYBdudEE5vgDwgJr31auqMBiBIfKew1B8DXanNKhTwj/ulVPH5Szw7+7J909+uzAIHl1uaWpl3gbbw5bqnTDIg6t/XNWQkQNHlbYcNgPSTuD5JSPf0V8EwhU57oIqTqyrn1/jkOaeXDMZLYMMCTBGNY3I0h/Vkx5ymeQ4Cvyw1qwc1BSmaX4qFD5yDTcSihyRKFyOuVBmoWDD55/kPLPfCGtKhryY3znkDNsLiqPkYDQp0NsfbaRaCQ6yaCil0go12PxTv2FVB7ilZwQerxUS12O35ILXvBX938BcIxBKMkF26g79h+64KwNKTORhWpBHFAVs69s9fOkMEoW3fxVmE0pMcr7HtRPQaRsoz/XG6QJUSA6FeBELSt/mabgRVkFuxLMZs7Cf7h8+sj1ssBsTB7rIM5AkLfH9h6xn8YwkN0JHy1QgDvtc+MV6MFgtzfCAywazLslp+f9Us5RPrv4hK8+QhwzcVn+o4ZADngm0XsNnXAuyhfuV1nArjKJdzwrjxIJbeXEat3Q3JMoZi4ihTg4t5tVq8VAnyAhnx4XjCkiUhwuulVAp5np8v4oXuQciI1mm0eCwTLhBVdPNlOR0mjQQwyZ91YHklehIyY3R6Zs1eBuKlvAbt7AjLUDYf9I39D1h+D2v3XsQxc5b50AmsGWYqnyqvZwxm4Yb3e4DIRKALZHGWpxoA7gK7Xf6oJ5E/eX+PdnkJGSOfQhbJEoP/S1jZ/bQTpR+QmNcpjgS708dBO8h9IJOCOR/H5AeXbtSfdDe6g1dLbwBV9bC3+eKzmp1ltzYDIFvAK4h2zO8ACmWfuGrxwloOgWC2TgrtLQLhW0zDzsBhwvg1nWPp4wHMqNkffbQlwzkb9ChLtkHzOWmpz7gXA+/e/OlPZAjHvW0SiFwAIsq1XOEYigFR42aq8loFLvjumqjhfAKITcoU5QADSvuAu7bS+CJlsfCv7bxlmRcFn3Bn2LvFo882jDL+4Lv1W7sFmSN1/4bH4G3bAB1t03z73FrKu38zd2D4AWNWLQV+Tn6yNZ1XfpPI7CGop6QG+9eeWsdamtfpVfTdrsMLQ0hUPuJKlOx+z6ZAltc9I+9ZRwAv3MhLQIaBVRUranN4MyXXj2/8cDgJqa9iTmrY3kHz6s1HjUs5ae6t6ebaphLRC0DdIFf57TY9Jz8UTuyDFky9NRcMXqL0jxzKPzUGsY/p1XYw5UN19/hG4wPo1uMQLvge68P/ghbESq37dw+AyJN7s9/ohUDfMb/pxnQfcoqT5pdaPQmLlt+uGvwaAsJedr/o3DoieD7OF1gsBdrjUgEtNAGJunNh45Ns4YG3PrcQPD1PmhEETe8gcHkuWWaiFrLc9OGzjNoh7tXfPNXUsxJqmxh5VYeRT11q3rZQYoO6Mt5Nb8ATfP9bZRVKRQLKYtCneKQ9Yu7d/47cwm/FO3SuA244mX532Avp9/0GxkgHwCB8u3ZIqCnHNDa8bLl0GvNDblbhCnfd5bNv3FXDUT1EImRVo2qJD2xqOgt3Qh9aKMzcYcWqDbuVLhn+VHfYZdheHFJL84FuhzZCh9Ej95G4Gj3nK9rCt9yEQhhMsdxxWgyykchqeCkG09dQPqZw+oE590K63Og4JklMbw0yWgKpyYr2YnhUkpnzcK6boADRlJUO95zQIFkxmYyRCyHrJFP8isQa8T6mm+GmZAvWARCou9ANgZzP07+APgn9eubFzehoQAjpW9lUyCsRXeC6WfKzYQG8Z8FvqA2qD/wD28W8OByF28ON4cp/BjAHfVhzgRGbE7ZNHvn1v9WPEC9JsaIILZH3QOho4Gwgx1m9HwuufAyVfNuGOeAHE3A/zeiP3Emhy7MeyDAshyuZd1Gi6LZC79tO2C32GWKVj32QfpQMZd2d73qdvEK2oPZu9n2E/5Vzz1hM1gNGpXlA9ux0o1vyz/XQGTnxfrxlSHgUUz1eHKqwpEF5gU+d9lWE/ki850nntISopXsRbSnnNflf5U5apy5Hh0q0Qyf/+8+6jw0BNNEGH2UohVOfvuQQaTunzrLIbxJx6zql2OgXoipM39V8MQ9ybGcNBoQNAzdp0d6CZBBGRJBVGQgFaaYfSk02PISzBggHbHgF9/5GICEIq+OCPy1//cwDoQfgxr2/zEGVQXLG+bhyoj0VPhfxRhsAAopXT5V6gXSas7PM5/zoreq5uFKiR/IMZR6gQ1002XQzpApqiyleH8TKGH6Su6DHU3Kvsar47wWtpL07rGgpEwVaeqD5GXg/rkGdzq4NUty8CTgscgEusbJtLlYXw4hpFweNBQPANOHNpcQxoNWJHFy2uQsLuNJuQ2S1AG+geeSWSDNFhS/sOzU0z8tXvT65lJRBYpEjoz3AGenMo+USPOsTw+BoUKrpBFBsnTmM3E2TaBNHlAnjBTeKpYe+YOWArBY2kGhCImrW99F1UHwgjd6altvdD9Ga9lMbWMiBWHOy3FrkASbxqfQEPHRk8hif3gvg7wGMUBI48fgq4g/zP2Q6NgE91z4r+QpwcO/qLtRlCNZ6DGjELiDeiw7Z7aK+t56r+SouRM5JZ/gOeJ/XXm18MX6tf1Z+olm+XKPzi4CRAOCaz9ylkq9fNyDCJQLRd8g51njuQ7c+1iRmphFjsv/Hsgs57zlZbiJXgKeG8bgFBvea7lQQGAPfn7YcNkg/BJVD7qidvKxBbePdqSvdATPOQ97dWKuBK9xwLWx8A9GHjDCkeNQi7gtMQWq5a68+qvkufp+8rjDsK/sq4qT2XvkO2EP2N2K808D5C27eJWR7oPz4uJ4g6QFAt695urN7a86v6L533w/Ge8lSwIDPg/KZh8JQxtmIAJCAmNey7HDgA9rLiQbuVBAH34fSXhDtHIVj5VPM5hw9ASCioynHpAOf1XIeuH8MBIdopwbL8B7gnOv2qnfwIBJ+7bWj/B/B/HLqy30x61OUVpJ4KD7o1nDbd1ASSuFQdDfkMD7myRfXFdgHxqo2rg5wJ+LRMFvxMOgWZXfeuxZRygjPpt2RkFStkPg0J7I/IgQfvo58NJs9D5nH3K41clqCZMK9q9PHumm5/r4vV+PtGxvVTGtNyJ0DEImcdZtN9iHiTPdjsFQ03XG65fUx8Dv8q5bCmmK/age7HetJXQ/D+nnQ0scwTguOiMi1yHcFrz5U5X95zkFY11xEl9gQw1Ar+PdynIL10U1643FeIVsGF9TVPAlFMSTZHdAY8THpu1s0z8J9nn4TWCSPwOlow++P8Jcj8nCqPz5YHF1Ois0u3EZC3hwo7jEaA8/y+k07XBIA8SzdnEFVw17fuP2HRC+S+HU6aCg7gRxu98KRZFkh1E1l9pXPgLTp6zo7QDhSZxbH5SA/wRlOVBlKEIZNPavHwXSr4njKtb+MMBpJ3np/GiVlwu7azZZ7HBUgNbU7BtdvBP/BpiuCW20COO7lix8GdfWHSnUFAUkBTZD5wQjAP6w9bi4tA2ojXd0juBp9Y5dGyP11AufDHeXnzbgh8NfDPAQMgmyVfCzzQBBGnTW8UoXNA+kEVWogJgShbgRtDfSJAst1zuNdDD8I9nux7KpkLBI/3bSXPtkHaO+kVnS97vRMvb4EbJOFfuruJ4yCbzQ6HHlGBBGen9rvXWdfsdVXvyuZNW8kHcYPqlws2FEHWwbfY0D8i4KcUIdkugIMshZKNtux1gJNOXhkPnTnztSow8um6TZMqBtvW2lvdH6WdElXy/Hkd8C1bD2yYrwZKRMTKvhZR5/rKuTTatZy+jNQRwFo3ZfiiZpAdLr9je0DJ2v7oanur+5WUpUwX7527gXAxaxkzWQD0PMLJPwGNgB1j6h27VwE00qnJrA+HgFBGvX359iTQ2UXVQ080MvIfU9CsEQnoy7V2OVYXAW/cvMlGlReyUtk+HrTvA+L+bdnmvs5A/rHLkv77IoNXOxTvw6cCxfJTu8rBaEbc+6XnLroNsk9GeAcd/sTInwl6fFsa1vq3uv+a+SVWVtqHgWeiC54d+pQIJF4aySI0HwiU9j8Jcc+BviB7NT/uERAqpV0ldwcA6fe3QltrJyAoq8vPRG0G8obX900bzYFYcC63xoRhLxJPrFqiTwA+Z/IwZ5w/ZB4qiX9+ShUIXCw5lbohjP4nsH1xbAbypr/xhVwSOyyZvReIF61X/JTCnv1U7Go7ZAbUbDPa8wGITY+uhepGANFn3RUtc0Y+zT4SJOLSDJmp91l1dD5ApqJ5lN3CNyC0ZybYTj0BqkGi4adiKqM/YYnnHjH4rN6fgNCMT5AZSVbJrvkD1Fdtj4UlGyGj/k2FuqoK0Ne9W9mHyThVFFFaXQX0nceqdRF/wJoKCrW9716br9X9ZlL5yVyxdbqAm5o6/FPxOpBmfKml7WGQ2bJt/iW2F7DralIHyPlAVTQPz9G5AuSnt9guKd4Bkp7aXSmKB1AKHVueRAgBKaiJU6+uDXAmt39uPfYTqDK/7vZmDAPpUvqKfkK6PqSVYVMNxOdpvxjEF6iNKiYwKcvA9wnyG3uVgeTxckXfpV8Jq6RbdTHmweHuTg8uoN3uEczWHwdiD8mu/ORZyFpsdqisXwDSmR/c14U6gT4ZPdD2kXdNZ6VWjVirnswH8ta+uJEvuUB72dHiJnsaSPpcWvXRE5CVzvsu3pIx77yRMoR8N6AHnFv2OVIGWZ7DeWWR9Ws8YHWeKHkF8skqFZApdN/wWY08ZE1wm9dW5kMW857yl5kCQFjPdSMjlmEX+YgFq/s2IPZxPYobF4RMb4caadOtQGT+lCx4ljE/orsGFLi2AaGfZftXyfeMOFb5l4coJj7WLrIHwtvnxWEeDL/JRhlhYwsQaMqP8L5XAf+1gs2KlRuImcEKW+lVQFTtO/x+ZC8QBO+s7O8TlGJn27e8Bzx28vhBqVnAsQgPWeUyeJqR2hddR27A/4zlGLzFBwS9Z49eZJcDfmtJUWaDPuCrhutONWwDXFePa3R8PBDLLetf9G4G3I70kLx5ZSC4hLi/6y4BnOH57h17RwF/ndVy7Isp4ObYhISUGOv2ru7khzcigP11WbPzIQWIlrkr53ZSF9/YGG2gQSaneV4682dI/7Gl2HN4HAh5xZejlLggA1OhlJmzDMTfbwe3TX1n8DjXz0zNhUD4cIHL4CNjHBmTu/Y3OQBZdaPRu7opwIdr21dOMfKr7esTqr5mgFO/6nIn/BiQ5h5ptf26D3Q1vHxdOsM/DI67UR1DIHuHtrVuzQQQWyNiaULqa+u4mj+zOUdy54ttgcT+P+v8b/ylziiyWqmHAklbgnJlYD1k/KlGXecfAk3k6cEXx78BdaTH5VRKBVCfniMqqyhCukz1PsYCAdEZc7XlJxVSRrYcetcsCoSPIn6PRU0g2WYXnUHUgGAsvbIPlXShR/Tj4m3IFMudqa3fBknceiM0sUEginYK3qZ+gOQi3OON99uBiNFYqPqcCvEcVMXWPWRG3PhtIzRaDwkVimMOOuVALB6wDzmjCwlHeSVqCQeBtFVMWUpVBgjusRX92+WB9N23k2fxB6R7ZZ/zTw0H0i9aOeOGVf35/5yTXv2d+sSX3ElchKwas+un1z+CNM3ZQ+dOjQFB/sWeEA4WSD/64Km7RDbgrzwaTT53DDKyZ/4RJgA3883BcVc/xG4JIhutewSk9fzfy0JTASMwWp/uOwCZz4R3bL7xB+Ke7/JnJCwgp/UUy6RcX9M513hzRjVt0zdhIM37hOmaO0O6kZmX3lFGPO/dtYdsKgVYlQf/+rOvzQdnO0gNfykndAKA5OKtq+khDsmoAN5zpwVkjj3h+DQbC0nss4+8fhkA6d4VHUPZAxCDV/1az6IJZG+1wnfPayDh8p4DlsRYIDsajTzVJwC58KMSp1c4ZB2/EC35jQvofp0FT9y1gfT8Nf/Fd31r/VyNN9g+GxsFqitkr0M93/8qgKwwAep2hXSgbLosz1hooDSx3N3z9C1Qvi8Inh4TA2z5PI9UyghQ3yAlG4RpQAt1WNlnzRK5p8a86T1QAoR97lPGgHr4hbqq8hOg5Xgo3LvdDdQjrmUnAxn8o8PHie9gHNBCflwM+/0O4mj2rxPKMiEzi3haiOAJES53h0b9GbhGhGmF/4fw5fH2WsdCZog1ii3ZAwFy/XZKUUWQaXLtSfKzdPCfsfox18LgG47Jz+9h5yDqYcc7c+UyIC0XSN1IMQasgniw6kU3yIzt0nC9tBVSPF7+kwYgE5H7zVEoC0lYyX8OjADFyFAo46oMYLwXFDaqqDDyyoTLJp5WiIvczmv2jg9IjWIxXL09gKvVn7QVqgLypwrz0M5YoE+YHYg2OQ8Un3eaDxJ2QXbsrmdxI32Qpf4vfsm6v2I/lG221x3Hk4HSOLGiV9PueSkYzBtB2E9p/0S9Nsjc9I4vsdUFInqTjAWmG4Go62WrVOACQcPez5+MxzHshb9FVkgXotqFibfPzwPptEYci+QABNVrD4upZDL860hSq8YfCGw5/jW99zCQAxOCEkbiIbpj2PpYux+QazVtZlkNIb0paRe7gwLQ2sDuxv5JoOGXh+u6GfYjMzfDCARr9vGvbgMUUb0Dg0pbgRK47ceeInGgJkcWiR/5CJRjNyW2KJ8H8u+mosayaiBX/Nn0dTwaKLUne3WT64BstRF1qsmD2BIOoc+ndYGi8AG3++128PsuzRPkwchXlw59xx+/D9TY+x8HTlwEWjjat7eGAiGVVec/DpswcEdHrNv8QXBz8U39KPAYKI882AbJkRBxePocS3w4UK5pFxXEaYLP3adR8S+SgJwo02sqsQH8z2xipDMDoMhlcJ6zI4L3SYI/XDoPpLz9hZZnisCn/FD0tvJEhj3GiOGK5yFMcWPPGRM9IB85cZuRICD0ctiWm2wdQO1oIF7cGgherf1vzWMRyLp4NNr8YQ4E3swu2JljyIg386LUTqk1Xro6b6Eek2cvkGwgi0z+Rydd4y2r8WlVl1y933/dZmDZrwb0kMcr5/BD84veRbBbAaW9DT/78htgdIto14i+QNU7kWJ5+Cf4Vd2x09XJAdrRBrVQDzUIok+yqVE6gdZUYEmMeQuRpXqYXfXxQFXNuLDkowGu3MFc593wQPcsXDnPGulnYaNLOA50RaPfGuvbwM2c55iG6j6gXn6XnJaZCF5VvhcrPBi4mvu2QG13D4TdZ0G/yboCDX1R9WhSCiLHqPYP+AaAphKwSzrBCsLYelZ0K3pRPv8WSjFghqgzYJwGtAn9FZwQ/z/zEy2Tem9LZAQjDjBYj2szRMaSVe5nC0H2sTM129mH1nj86v1evS0YRN8faIt/7Ot1pMEVe6dBUr0LsjSv182gMuCYXNxruKQMtCh+JkMXvjXevPZ87bYzQ4YMv+QqOv/9yw5wFN+5ci4nq++e4EFRSbAbFR4Si9sPlBBpk19X94CTb9ABakQeUIsEGbDm0dr6rfqDTXmqgnimK9Ap2tWlw5oQfSpqw+XUT4z4p2B2vOc1RGOSCmWGmhk8wa7+wjnM2nmK/9UFsrHeVlULswx7OjMd+KcYso8cTD4k1QKODvLvMHKHIVtfLiCWRRfMlHsVS35JARXDHlhrcg6iRef3lc1mQdaAgPosbIOIBYee9d0MPHf/skXq4zqIej5w/kXYSaDc+jNeZvoJMANLG88+HwQKdv3K/nZsRcXhDcZvIItC8D147yrEdH1SWXTdBNlMW1fy2Srvzr7i7eblt2dt/Nl8f8s76VOvO7t8GThXSihn03G48WrhNgufNtDUuPSEu6sBsyz1bD7mKWT5lfnnF7JBwu4tn/pTFIEyuX0ljsUXamw99oORF4wFLVU9T0Gcf5bZ4zu7gDLdp/K6PBcS5ka9P/E8A4qs4/JQWzjEratIM7gsA3QWzQTtt3IQu53r4LsZhv2xFL8PojDu15VuPD8pCPR8W4LWegTil3hvTpakMfz77tlRqx1rOuDq/KcealJ633oIaHZi2zzc2NZ07tX6uKemTJi630AtkMdqKJRBkpzVMnnMisGTZspMuxYhySrXW4/tOdAqTLa4xtX8h/cmBCYE794LyXvbVTkqGoDe+YInTnAOUthPy5XG2jF4DJtcygZvSEut1nGU7AJ6RvwFWUMHSOX8qxtnGx1DmmYOQhr5QB2+6y5kad/1llLeAAmf2joGuP2ANvl9+0swh0QnQ8ejWQFA/RyfppG3BEQeh5VzYFkP2lfO66bcOrlvqa+Wwbv5/vCIy0BKT11V0SctIC9ms0j2UyD16HLADsNWoF44LWOYlA6p48d37p+RAvK1uGfPLwlCOk2t723EDaCcDE9QMy2DdNUPK+coqTd/3L3aMgnJXu9V+z8bAbVdfwUXYMPVChXduIA8ncKtGIQFLERNBYk8giy7/kP2pVFACC5iwMnLQDnHVl9H5oG0p3teZjtuB9pujWJ7zXxI26ey7jNTC8P/jOualkIhg5b/cNOP60AOvbpOaZ0QZOzjn7PoZ+CPP+03Du97BTjkynhFAiOPfpZqsSUsAv7xi8lnskSgDGw525EpAkSPyvKxmjNADjue9kv5OxDdhkZrDHWAElp3rm5UDEiHrrf1PqgCysadUkTVx5Ap/31B3bodKDzmlZ/LX0PmC5l72REKkJXHZtVwxwGI474r58my1LsuZz3wYODA55PizF+B0izDCLQNa/sya/swxdxMjnaM9sfcP1XLbwLyFqMM3I4EoPC5y8bP+wIJm9xvGM7ByLfiShU8EoA3Zj/xh86Ih8v3/+5D9CQYD9C+Ap3VY4s68Tbgnkcf0sMIAe30oZXvNPBdASllPG+Aes3nyRXmAMDHXNV7Jf8baKJ7jp+sPQEEkugpeshpoHGOqN+gH2bw5KG/+zxS3MLcG9cBmXtYW0iBwcNzBZJ3ZiQB8a59a1jeU6D1yIbT9zNwtgvzYd43DHwwarLV1FlzTTdZ4yGWQRxqt/WBfmpoJT9mPnPsZhBooB3n7WYVOwWkE569IRPPgT5zZfFBjhJkmv+gYV7dBaotioR3qjDe/+oGXawMqIEFr3hwWWu8ZW3+PC+unDui7Rq28bj7icE7lre9XTBi+L887y8zFSCmCRPaOc8B/Ztjg1vTqTVe9L+4dm3frGdCPPa5N9B/uh0NO6y4pgtle/Ka908w+CoqKRUqwg70pt/3H5rggP4+LcC09zhQX5zN3NyQzsjbcz779K8BPc25O/A4A7/7Lx9rFDYBsrDdzgCbi4AvIWy7EZIKlF3cMaNHJYBwzWDinJwQZHFblSr/5AJcp8jgI0sGH3GLYtobPgHkB3cn7a/5Almk2Xl0l8Wa7pOtFxAZ4d69pnutjWfi9cr3O9TpV3skp22BPHT5H8GXEU+SNoxLvIOsQ7Gd7/fUA71ulxYTdxyQ91au6MF0s/iV73/IFxu9BHragebTtnJemaIsIhXw4zLQv9pSfnfZAOWJkJaPAAPHHj0jrHkrHTLNroQTXgtA9vGpmEtJF4C8jvfkgbQ5yIZWbr+H5YDlPv3Ojn4G6KHBxNM+V9f6Sfv3e0z6N/kRBwIArV2roMKJe61+NX+s5tG152R2LmzOdwA6Pye7G+ni2n3ZQX/LLKqgbbnuAtA6HQhfm09BVkLRz6v5cYz5aPU107ABGvKwtXXdFcg+87lF8RkGshq0fLRypCFbzTLRWCYPsk87zRvpRQCd+n0ad/Ao0GWH6AGs5gzcXvT+ibkBkLdWljYeqgHM7LkiZ9oJyMLvXfHXUJ29XYGuDDs2CtkaaY6Co/em/CSqEWQzn7VK+NgINvFpOLMt+av9/U+eDL7Wf3iDCTh5/fc4VvPqv/O0dj+N8re8KcC9kXmBkVcTv/grZ70EPcmvNEcTCYi/lqpkYGwMBO1SXr1THyB24tYfn1RG3Gc9lntugQfu/RTI9s5m8Nvsik2mwxiwpVxI5zyPB+OsUml+Kda1fmhcOmHRk0AD7RcO9ZJOKWu4xNC12iBYRBMwbbUxYuf4QC0YQcQkHq/tj5D4ec09FfWAmBHLcS7eBDJCfgR9eF8I+M97qfIsVyBpt/sBiW+SQPbaFrDl+BaIS7o+EpwyA/SPuG8zMkxr7ayut5NKPJnu3QpkY4ErXdhMeHxEpVijqgpIReIfE5+NglMMy+CJ3YYMfsFvIPvgDLj70U8p9VoAOeYdAz44gGGKrN8OOiM/5ekwd0y/B+vKTFG3Qi4G/ln6Jw+tzev/flf4Lx9bu16NE2Y3S3ZuN2TwKUJ0j9Oul2B96Up6o7EXkD3LPBnAFm4ffPj2igwPIx8cidzUdRNua/4DbxyBbHnfMqauGe4PY2vqjeuAFnag3KPkJ2TS/btVnZqBfKLsBntc85pOuabzEk7KfOA0Brprp31byTOg0ruORt1igVued1fiuaejfpvANyuwt145Ry/PnHtpIwiQkXydSVZZB0/kzWjlJLemShmvNp/olVtl6FvOCqv3b7egj4uXJmpI+1Aq16TxLss6ZGBYXiuE7zw6se/43sQccWTAFjE9fv8wErJltjG63hqZbhfAb5guR78UTs+I1rSgbWITZkyFgPwc41Dtq+JGJouG9j68ZIRMXW/R5arlQ7qcAmsxy6/lN/mIpmjNx6IT+7EBPU6GyKjjbMefg23ojMZNWw0cBkFtso85dfMjbbI5L5w7lpCOukd9Ga+V0L5KcaxcwDDyKWGgWOlIOVLF+Sj8m0Q3+nPvYSWxUwPoe2pUkc96STT3ZI5g2sNFpC7B+pt+M3sZ0yJnzzHhXCSJh8v11zZdZIgpFK/G14D2Ti6wtVzEI2MyblWt9Rh05Pf9qTbeCHRIsfjmuvOLaPQxthrTX7JIPz45Z6NSLzLQ/DLv6sQO5FOp6EWN1iX0C63czj29D33fv+S42FyHfI+ntBV+HkUn+Z+pm93qQD/+OsqrJaeLvpQG8cahBfRrflpfzMbfyPMEt6gG4Qyk7VTKRiXndYj3NkPKNh41dK7+zgOMFAv6q+jZlbp7Gmi7GbW22FUUbZdgCpL74oGOa4dJxxfkoU1sGrsFbOzQQdESDkelc8ikQoS+8QcvtJmvRfjalBji+WxB/paUEvI6TC43W/4t0h82Y/z6y2a0d6ZWZlfuE/TT4bCT0UcT0SEf1f1pW6qQFvPWbptjZHTogUTryyZBpH66yeBhmjTSNskxfJqJG3npjPaFE2eR8Mmcqyc3tCEDcNQpiCcWwU4EdeCn3ZHWUe2SLNMy9IOS6Uf/4BG0Yi4r8tJyODLs2agf4YogXQIzkj4R95HvSq/85nK0kTfLCSOJznno190bnAWSCejcSBJr1HAOMilu73lE8yMy2lXVctvxBFqBP7D8kJSK9A86DKd92YyUQQTH7mA2dFBVHRSa/ZDh9NbP+Wki6GuDFwJvs62RVltd97xFA6SCyr2ep5uKzIgdHpZSsUda7xo9Hpa/i9Qqnd0YvM0XacHwjAb7jyBf2lk13l/LR8a17m9MUnogzyRau/n3hh5kKHrfzVMJquiXeD6K275ltGORw/r10T6ksJ0vhCm4DZ3mlAlZ+APIEBp1PVbJAE3aokBXw+Wi9d0LNidVriIjl5TViqIM0Bb7C+9vnShFYi1lkV2Gvmhm46Jra1Qhgv1pOnak2QyZNMjH+LzZjv6gPz+sa1mJzkYKbubvd0V9AlMdngvuRD6/mG3ybRFHJiUdlWae70bG4kN3nHtligxNnmH6WX8H7c18WK+VUoPWJPE73NoviAzHkdSjHG4gy8sLhjHTCWjnxWN64750pDDa15Hp9Sa0+4TvniJfH3m2o3FHsvQ7kDqW577WpdpI8z6vZ+eW96OVdjy40Rg99Edu6PVd46eRlgRvtPI6J9I85R3ru74H6WdJ6kqdE0AnDW3Ks7WK0EmLJz+4Pj5GB10uq2m4yiK9pQZn0vV7EZLCiNOcoiXSfHtzQWLwC7ThF/z65XUb6XM21PN+F4728n4kCPQmoW1Zpy46vjNBuvqDbfDtcfLMG6rurrO7gQ4eKxbbyuqKfHB2s3h+4Q3yyajyScM1Ejqsv2Oc3pSPvrbX2m0dZYF2YB5lVrYeQD8Znv+EsVlGUbDleZ55Hak7xzJdc6AEWdyQ+fjb0DlkULntLe9OE3QIA3biegR0Rt3x8LGJRGRgTHPxKZGG9vqdP5t8mRUtlxy8z935C+3tKz7wZPoW0mkbtIWAbkXfMD2DtnsjyMDhu7vTpBfQkryARo202rJ117J/eaLMZUzmF7Y7sGii0y+55nM7iehI/LPznjqh8izZS0FWNg+QNm0ZgcbaNrTmZUniS+VDCG6rz4LGwwmkTS7iefb0BNL1fKemgm420tcWKC5GPoV+KRvLf80ljVaWW7RkZfKiExW3km59uom2Brzli3DbjYxrN8ZIB9egXZEBVCn2U2hVv0A0q4cGMsRRbsJVa4p0auRdnqTcROsxeo0tDqfRNKenYvpW9sgTr8ST0QZU9Mtyzpafh2zRpo37tqcpv0F/tFsqsHC/QqfCMY0sP/EoXTSw641kE9q8Iaw0MpkRP7PcAvbIJCCTGUctXm6OLGMWDpVMW7RFfk7ZfXwavA/p5ItCYl7rIzOv9puKJG5BZicHn7Xh7qIjR0yZsYl1KGW2bqdvmzUyqoAtzwucRltzNU9v1GpGp0b5Q9VfN6Gzz8YFbS2W5FmJaUbflZWQLpAouVd/EF2spXuNvOREfp7uOlCmE41OiX8XTDEqRweDuLkqDo2jSxZIbHXkM3S67IGXTEYs2npo+f2u49xlHL1aNAv+AWQ64QDt8SQFXfhcqTBex4L8/LrpNKZ8Vp45CFeW8JOCtIvdeMc9PIIm8L+7pallhLayqk69HuxDPxC7XIrnd6Ij0zTaR10fdEThvvqo8hFk5lz81ymbOOST+D6z2HINdCy/htxc9ofRD4mcgCRp9OuL8pCu+m1It3tm7bv6WHmmzdl9lj6zZTxsWBKfcI88y3ez/EkDHLLU3z9+i0VFnl/jSWTa5WF5gTIhg2CBkTI2k4lxyec66Njn8TF/r5fy7NmHzxTG26C9LHyd25Nd0F9boqtLte+gM4O39zv2oejMQ+xxtQvcZWzzScq8ZkXIb1ypYlu3FFo1VGxOUuxG+u2UT+CPxqNfku6dvLXzPjqo3NfSxYoiC23x1DHRm8hHylRzmO5PZMYCynsqC5AvhicUcNt3IS2Cr+IaH+5Ffp5T8HCTnJRnyddi73t1XX59/abt1aVT6KLDoLFmpHoZq3Ji/+mpKKSFZWJHdN1d5Jms/f6F7zbI+9pt68uYSWUsTBtjc0aK5Nmy1hlM3PNEBml50zeNf6FNF1Ed23dh8szN3tMcu2+iP06WOLrLmaDzH+gfYzaWIb+d5KfFXjAhk1GnLTcI/UL6dZJ4g/T2oO89E991qjDicqDjHR1tJbRgs9BXhQRupAlPwlRXT6GfZjIbL7yqRd5et8FdS6tD+4NM5NVJM0hjgqTUtmYdpG/p3BNF7rfITC4mR2eChsy2KSl/3XAJnbtXf3XTYAo6mGQPu1LKkeZg0SK7bCO0zwATvQfLggwmPiPb879Cm/aQTBcGTcvY43a3G3WgyNStI4WB7VnoABfHTOiCN9I//ziicYs9+idkotH1XiHS7p0gmNN+BfmqMeW8pOxQtl5597Ee5zq0U87AeCJcGh0IFLj70NwLfX9aM2fJLBddvuR5Nl78HVol5HO89wMTMleXXLjx5CjSiTFf0FWsk1/ffWww5qwA0r/3yxObSU5kqudbwWTgLNIZxPHq6fPL6Gjw9r2FphNo4xzz6B4vJ/RjatIZu13X0e+IcG6j7T1k4A4Hn6ojHZ122WPMecQA6bS6H7EJyUXa3EKfcP/sRT8fDSglHh9DnocWCOxwOIL0lGWzBBe6IkVH4Bn/gw601/Qc/f23i2iqBrt47QsiMnN6w3Yzn7dof97mjnbWa2jzXSVt0RxZZLg7GtM/ZYWU3Ph5o35+I9LwuaHk4Pa3aMu6P/WzShZILzaas/oMDQky2L7LQH0a7f143jD4+XWkXOzTuJhGB/LJQuXsgmAT8lEH/SqWV4c2jjlbCd2noa0Rb0U+l5xDc7STeLX3N6CT9/bHL17vRTuUkCf+BWeQ9xxzzYeSHyPjDdajTZ7daP98p/mgrifS41/P3Smji2ZdYLXzIuWhr288endjXhNx85iJfxnOjL6f4mvIsGNG27Hvds6dzEFahS/9MvihibxvzcmkvrVAl+p/JyUIPES/CL6v7vP5fw2Xhz8VbBiGSXYhFVIiadkZlfUoKk1RiYRkJEl2ShlZURriE5khM3tHj7333uNwrGMee/v8Ce/vee/7vi6DHY56tcYYwwSFYRKdg9u1MJdpeTBZ+SI0FiCt0NfDMGg2cFLmrTKkBJVLvkZFTHebfDBm8RrHY14ds9a5jIOlI68gNh5eEbj2KnOZYqXQl8jQ9G0oy645KbJkD4000zNdS0kw6Hn27l0PVxzWCY/Kb92FFX5nfy7eOI6kmhfgEHAOUv/w8d4leGAlm46bxvVg+C3QT/JV+AbEhdbUY6uM0P7tjlWA31Mo75NzMqCcQgsGzm7NFTZonEqleM7bDoS3uyap+Ddwxnuv2q9MRVh08XdXDNyH/xGPhYh+yYSqc2Lbh0q0cO3q55QB70mI4ShcajN9Bt2Xs6a1CuqxjjZP5fyQHHR8ebUan/gQ60WfWLx9VwXe2Zxmr224cdqDTUHXMBgm+P/RMpjtxg4Jm/KhV/ehcOSZzZ7Dh3BoD8UapwsjEi/rSh244wLt584XPw2RgbIHvoNtPZkweKry9tNQMex92S5/ogOx5eyavgzXOUh+9N6XINWDfXs32Q58UoU65tG49xWiUH/yyMS20THon/nz31xmJLalWQmJnT0FLS3n1KbWabDw89rFLyaVSByuLRWjU0TCNPduURdfLPdZeNLQmwGVFK2sA4JuGH9dKChJQheaPM4Zni/cwvdnKZyTZn9Awf1vMRZadyG1gL6rui0eW428/vJzVmDXRON5Vg4zTBzN+bwiKY0FwmetqR8ToFN4r8Qrzx9Ql6UxTeZ9idOyj33+ZehA+yGjpPclgzA1QaXAPhANo8xd9RuzbVg55EPUk6jHwSxVId9jM9CiUUqILd2EHqHjZ+d8XYBYcJfD2sMO+sLon4Rf84dyA79eZXYdbLU8eJVxXQnLi4S4rx76D4vGh85eTefCpuDxm/MRnkA4uPtwilE6FMySj7oLseMApXmRUbcozp2i4dUcXcemOQtmatF0mI+0KLk5qwCzmrkfvkTXQVv10e5eBwro5yp4MVnRiH50JjmO+R+wYvxrjUw/4rTaIjk0/jn0drhysm/uhz+OxeeySU3Y3fzm5ONAPxxLspg1LVZBwhElNVLLDJQmHbjRuL2MVUzKvxNHz0DfWrv098FCIBUEShmfPooTz4l+loZKGOaiES9xcg5mhYOm3u5WwivWuj3Mft9x8LVuTlurD0zajFPePfoG/rNSFZfp9oCm9OTERCMmqBqo5GkZJuGQ94EzNx6ZwrQid2L+zRCYsn3rt6r/CxoYk50rpT5Ck4z1oP2DBhxJI93o4PyBaRcFpp/kjcCCdceGy6tdWOB2KVLUyQtq8ltEvOJVseFv8jxXwEUI6X/NqjZyBzcuaZ8Q6MlDokS8ZyJDEQ5YWuR0ATesBi39e733ABYmrsU6v4/EbdmfpPWMJeyrTqhw7lmA5bZHZfBAGJpUzErmYAHybz2uMVsrwjIZ1eIaQSkYNGQ5+NpKFWY043pOnddAwvOzBZWug5Bx4kJSA0MnBi6x30t0NYbGN0Xm382IMPu+XT/nVA42vLL8umLFBYUPtOj7BMnQ21M7Phv+B+Zl7irV3fUD1x+d87dHOnHy2rcEI9oG6Lu860BxWPwO31yKePiSD/K+lybVqtJiz2jE+fi6bVzXcM2s/BEC7fXOG1v2p6CZ0e5IiWkl9Dd2dejV3ccBbOt+VMWD43nDrmIfOqGpv7pOUNMf5xeXPUzUb+C0/a48/40uqNyvLrOWcx5ixP86P//vGU66+f9xYSuECrMj64ESQkD2u2fSdWMdevbW+d3ycMRJlhLnlQuDmP+1W9fmgzK0ZL17/PNiJYxEO1IQzB7g2Glvrk9y+jgq4HJK2s4Ppwz2nlPk1kSyfENIkZogBmRvbTTP58Ewje1Y28QLIOwl8lTbhEG78J5Uj3pFrPYfvG7PFQejGqceGvyqhMUTP79Z/sqF8Q2NfUiwA9+jKVV3DLdhfFe2W5vxDRwS4Y3X//0cRh/oqwu07sXJ8F3IuP85dCSOmXiadkFb2BEhc/NJnHd17ly71QCkdy33KNOu42JNXt5kohH2Pjie5SRxHuroAideKtRDF1V0EW8jG84ez6DiniqRp3h4wZPopZhP+XP6NVuXHXY6aU3/XovBWdfNPI9ORxy5w+QmSt2A4ZYl3yYZOGEwtLRqSmcLh+LmC5rddvhKU7D/QC8HTJ+m65FQkYIt8Z+CCRd8cHOqtXwoWRyGFFveWWdkw6yIm3rnzTWoNMuR3ce9O39X+OPqYRMu7H+4kLOn7BwM0+oVsFX64pKUA4xeiMRCdddXq7L7Yd1us36GKhMH85tqBnT9oYD5cquC9Cb2EYdcVK6fg2XVqKdxH7SRTMtcx6U5B4s0PcYRT5dwkSdifT+TBzSfrq5TNFyEQmFvhmd9b4A0uM+K16sin3o49KDxuhMuj/AJ5zC7QNf+96Nn01ugp+viZQNJJczZ716a/vIEjhU+Za56z4jj3Dnz3P90YcSS+bhLpSEuBrSYzmj6YHmzVvls6V8YZCusOE2rD6Ox4QPntWswvMbnlmSCJOZlMK05n3GGCtfjYhIbTPDhadF/+TOm2FNyO5VCUBY6+mmGRc5LQWFRd+dDws7eqT5i8jnQhL3pjHwSZks4QVupmqM+Dn13BqTUlL5AJ1+Nm51uCHQ+flT/u/8qEBopRw9FiUFD3daFq+f55RkF0pb1Nl/gWPkF59OljDgSx0PYyrDF6j1UFz/fqoAPbdKOrL8DkMzHn97r0IQ9J99oXaTygtmP1rFvqc/j0Kx9v2mLL4TfnFpJPFGM/ZbFAovRFyFpz/0VNaZS8H/w7xud7xIsx+EVDt05JIy4ND/82wetdrIdzflJSErf7nOhv4aEk3YaNplD8M2qMKo5ORVbSr7F656bgdW8Ju9vR25BC2/yBv+4Mow7xPyJdq8GQkXqsYKSTuyMVpuxuJeMBBoJH3HyXSi/BxcaVdSxITRvhcqQC5dOd0Q/i3BEbzuuADZXaajgT0xbUg7C4gLFDePJKpx/w7geZiMPhIAg/lO/+GE5h2dU+lgOdrfJ+dEYysOS9CnLecE4KNnLE6CUIgCNDefSTvsP4NRvJ38OmauY4JVSeeTnJyBa2r6/+OczLMu1mIiX0WKLlZ6I0YYYzCQLXpk59geG5rle9LQFY4PxQ/+H0sq4Xv825MetAmy2arm/7pWP09MWd/iodvYpaFPT5x43zD+/Oe2bdRYWh02oJLpu5NMYM724ceNV/q5/wj9kFcSB+Ot+bjh5FKu4fzE5DlHCMqXURtrVh9guO5nUw1IKixNDX98828TOtNySPpIGlO4S194c44ORDi3SmMBx7KOmjZk5vYHNTFJavl8LcOT7HuP4/npo8lNS+DmhAmMlCmtr2ZpY0zmuryvSAUX6zS8ar7+GUb23agXPnyMx/mOH2CdBaCnSr3qu+QS6FQ+l+UrE4OQvi+z3u4pxUJxl2OufEXbqBfLU9IYgoSA0f6liEeok4t57y9dgt5vT0RXFdsicKczVfpuG7aP3Jh3rvsKYHE+AR9RtDEl2lNp70waJrnNNIYwbSKILT9bq7pLfFdgu/rw1DXNHFt6orvVD+YWOz8W4FxZjXOLzjEohzonisPDQSRwjcam8V9sNBOlYwnfxnf3QJZx/lvMcpyJYwt5tGODwaLX/kals6FSLbtT55wZDA0JXzN5UYlnfH9HLQcXw5+5XDv6mdIgRi1n1V+CGoiuGdCcLTLHRfVQ+S1oKf5yJ3ZTiPQ4LcTduxRyjhQ79FfMJVnvsCr3KPxVcAxXTR7/Y2uTg7L2lCtdDgtjU8NbvzB0VIKWfl2N+fhYnVfea6b5ogYkjbz3jM6awtNv6TviKJlRusBweVKKDwUaPh1wXz8DYkyfl5+/X7Hia23P6BC5sO5qRr2w3j2Fzoa/9it2wYEQ3V8ChG0fUpTejNA3lKd6dOUgIYMqn4TtHmPf/i2Pu2gSf6A+4uM/89H+qejDLP9WsdnoXjv/9+qsl5T8YNPXw6OP/CduxUOwaV4ub8oytNDE0ME/UqXggpQ7Nh//aFn7pwqEoZcrRd8+wtKru7ZPrv7Akzd9768sXSGIzNrV6WwpDF/he6LGtY0n+ETtxkgeOmbP/8B1gxOkA9UWvmnb8nWeWt+zAj3VsiZ8TdvYhOfk6LwttI5LKbuwWLYjEpXqTynMZbTt8aXG6785D2PKiGLBWVIBGb6c/d1kcsK743eBj3XH8Q1J+f42/CEpsZ0t+cUYCaV5Q3eK2KA4wdl1W1zDGjunQqQOlh5BAmLOqm++HdSrK8DkpdihJkaPRDDLOp3/0ok8i7QZsaLUfUPX5CFv+IxfWR4SwNlL2aniHJDQluXdUpnJA96dNymVvehitj3C/WOECUxsagcp6bTjSRbwY8TcbRo7iogQDM7bJC+orx9bi2CO2r3bF2TDRrnznSmAXEFiO7j8g9wua3ML22rzvxEGL5Fwhozs4cs72o/QDIrbu19Ud2c7CycfEe/xakUD+hprPPxTBmP2YcDmfHgze5Xw/lMsFBOqAr3tfa+FCHpHzbfVLnFIdmDuyZoNDHtals47GOKjM8UJb5TouNUn8nI7a8YSWcTlHnlMYQVx4M5fJgpGMZdMJ6Zyw+ZsjeFLYAKaMp+aDP63jKKt7Q+eZqzvvgSBKOTskvQs7X4AO2Ch246vwi8cwr6DzcFK3BR6I9m4P9B2EdoUmq7mkZFx2vLWr+j0FzDzbW/9EdRnWrZfU6jtlgFjHtiEgZI8tFuq8Ed2vcPXjAdYHxuxYu0f072SCD46+G7dlCHWGUdNv6n9HtSHiwLTGJ0k7JDccoVE364a+sazy/NAI0DancS4q28IxgfM3h717YIKdXJNhloPk4DaL6VAtaDY8OaHiSpCnAt1ueseDOFh/z2vm91Esl4v56STYjD29EaNbScVQK0lJbl6bhayQv00nOkax4o7hhTtzr7CZleH4uGMNjImYP7FbeozL/bEN6Xb6WOqet07x3AWHx/bMt701xYZ83eRmhwEYiq3Sr24zAFIKG8VhBcAq9rt/2MIPwtpd3tpQM3Yctz7P5uc7AKOkj4WKFVRYa6cV01zth51Nb+/+/krEYmrFFz+b1nCB8LGJ5+IFqAl88NIR5KG30bek8NRzmHVjPCT42Asnf/fSeRiWYuFlkS3Rw38hBc8Oh2kFw0xHxxJ/iweMu8l7nw+lg8nVL/FHU79jcUvzUKJ5BJLTtJcu3neGUAfeS4zxL6DUMDTp7lF1MGlje9enooOEsILrVNMpONxy8mZnZAcuLZv/N89Xh017j8ktU2fDaNYTTwbewzgjNe95tTcLSXXCMUoy0UB+w1flmuOF351H5WRqHXdy9t7C4E4uvhbqcmmuPInJShRN2w+ssPV7MnM93WHs65Muav1wHIqc1mS9icEwULHAs+aXBJ3+5qaX1S7gRhlJi71MHaY8RGsVNT5DgcRNniOKk1gjUn6tjWoMUnPfWIZO9kKCfs+LF247d730WU/bNgjCx/338fG/hAyNlYjEZAkgHZEyWVHwwqIO1aTNTwE4zqdN2U4kYG543SW1xRBY9am0Gz7ogIVrq98NL/Fi94ZNqcvRCZg7YlfQVqKJCezxsVsrb2C68n6e6r0kUJqRwghtH+y7dKL9CNLALE9Yphm3DbTx7TNhcfwNw26HOD/0fcH57t1EzUPzEHliTqOOJw/ahNLjevlE83f5LAX0UDZin/jp5GuEdGwPo+ExcvWBTvI94c+1NFh+JFS64Yst9nyvaG5buoLxRR8TtkaZcO7PR8l3IUkwq+Qm+SaQBJNCC3pxVxlw4FNT6PaAAmT4OGI5dw8M6o7XyVmcgd+FXrmdKiewTdqJpF/JBGOOR/aNetfB8sEGCWLaVewoS5GzDw6GpT+7lwbPGuG8082Us+8wnzIpyNdD0XmHY94w6LO8QnKeSzb3rp/QJls29niAhM0lVx1j3Tmhb9+Z/aE+etDfQzWe4N8BxJJkmpLvNjgrZ0hIm92L5BQjG46wHJy7S8dZf8gR1/RYc1i1iqFg31zvkRuf4W8Pw+zW4H7svMTDECY9AYVSlQ+EGD1hcM76Iq+9M3jWOR5t//4Xu239f9NVX8QpPDAgvXxR/mDi8n5PCSKMB3/2cadnkN8nJPO40+I1zvLd+Jj5akOeqlrQXjxgIZ9FxY4yUDgIX7ze1C7TOIMNJD/jl6z8OMDFoGRSQIC8AnNz3+yj2N4+Qplssy9/D/WJA3PT+6BZI7lP4ugWLkSyjkYH6OOIghW9dNM/7B08JuKRdg8WDxtsG5P4QE/+svHAkiqWrtImkas/QYfGs7ros2q4zKxWhVGRWDCwFR0poAZTFotBmUYi2NKRpnTUywjWr1Hbd72LxPF0G1/IrcRiU86hmS8jsBF/+v5/Z6WAQKPHHnd4DuKK/kpVpe9wzFPp5m0GVpz1N2CmLq2Afx+DP2mKOOL31R+GvAJvgCyn1Sf8KBLblt9YHZK7jUNnth28Am5g2xXjDostTRh7sBJr9PYqkMtFFN/TKmNrSE0z746Pkpgt33bwDcFYQWhFpMkA9MbnjNNeOA5Ze++3T2fw4bz2eNHKMj+2WrOvPGYKxyojASnp3puQuf7zhI00FwwS5rjv3+eF9u5kdoGpUSw/Xd4nnBaIVVHyOdcidWEgRcJiSSoUJsxzKk90EGDyVqLJlQ+a8hRefumcIpIYY2puJdXWB6tHCdVxt2qgtfJXpd3DdFg029dI9U8IUyvCbwarfkBMfy2Y/UEfO+4cXWeipcKh27P665R3sPsgDXuBuiSuZSgwNQ6EQFZuo8QWFxmXWSYvN9PIY+YzQUZdvUu4Jk7V+Op7J9bX2l1uTEvFqaHuOIJKPPQajfzm8veE5R8X9H76UULVaOD72OwSWFyyY3mYmwIFFwnEJIdz2PZC5v7bcFdoklsrSWyRhDRmAcXbjxihg526lFJAEPqtL4srP9wHVd76Ueu+OxzJHqc/80wCh+uc3gT63sHG2trDhfmUOOQwNpqx5yd0lCpEfNN9DGP6PYf7zfiQnFVCTRkAOHeK0VPaRwlHWGBDJrkB63WFSPn0/EAonbXwCODD5WtcldtDz7Df5IBrqFIqjqZaPk5/NAnpB32lb9WEYEPPVAdViz0QEy7xGzxNgibJlWXRDRNssVcfVSyggTKmdYXHUnLYlHncYGzTHGeYcijzOouxlhAo/CWKH7r7khimHFfRKL5I6cOlVEwKuKVVpXoCF4LFg8fff8DXvvcbO3lHYfab9NEqGiJMvXa9fc/sFI4mdvzHHN2L87UlPU8tq3GyU/LWWxExWCpeuJdfVg4NnLyuURpVaF//4N+lJAkY0B+bWt0liO3Bu6PuUBzBqVRnzUNOdkDQvXpYPuoeTE5S5vz0sMD2P4kyFARmmBXdeiPy4gFO/Cf74+h7X5z1TmXd5jgHZEWiW0j+HPSXuoYrbMdhVj8Xzyphpyd7VGfeBX2HuumAl9y7VHFx4+m11wP+2LCQWna2jEKe4pwK5Y8TY5A4Ys4YzXt2Z1+aBRzOK+CCgJTuncYnSHbR6P0cpQxN84XXU9qeYCQtQYzFtB8Ttl5afWfd2Q+GNzZ0L0nYeIklzOuTEKbxB1He0qWHYtHTf1YekaB/8obSsy17yO3Juvr1BSsWVUd+E8qNwOwXhJGyzKswshIDlAcYsCOmurNN8Ao2BGt57GtehLkf0y/jA29A18L09dxaPiS5+aRt8U1jTyN7WObSAIzBtar02m/Ym2T59Uv7eyzqPMnY4BQF82W/uO9trcHcvUfnR9S/wKyy1oFiP34sUBb2ZNcQgHnvzmbxvFRsMqxQ/a2pKc9ybf7zVa0I7LWfNBy7GorEVqppCe0KIDIwVt0P+Y3E6CgJ95+0MLHQXCwenIfzJczHBC43IZmL7/P2p1VMkPm2+NvTHtuJKZ5+7QiZqXnfq7cJ6Kbd9IiPgQzDd0gUplfXYPBSvrzPsgrOcJ5LvvqGChaeVdt1R9+DVnvG/9xSFOFP+KVHU09dceLESotQ4TSuhWZKk2OP4vDUP2Mni+/Q46Fpv9f8NUyZt8oFFVHCiK3w5yvenUhwWDSIW2XCDF61NL1pgKlk+421TD0k5L835PUOBcJbX/Mr1v2YrPZu8OIAFw4rrXnnOrJht1zjwIyTDS5/D6kqtQeoLTA+TrJ2hT634xHx5fdg1J8+219oDAtulh4/d2gAcppt6JaYuXf4aLGINssKv1n3fE6YuIbDsUHjNLfiYU5dzvxT7BmYDeVW7GZ4CXNvJOSWtwyxOeJ8eHpjIXbPK8+PvPyEOfu+WnERH+NQ53lN6uESHAleqLKLfoXTveQLx+xcIXXWyOHvQTsYP2Tj2xL9H1TK+nB4Tmng+G0WuTnnC7BoeJLzSEcGVuZ2OOkq7vS82FCC6Y5fTj37rUaRNAfRrvyHmFB2h/MTbzZXFMGC5vLaQUUf+LdIty1brYOhKaLJPfx/sErG6ctVdlp5ihedy1Ovb+OEO4lK1ogbp703iyZf6kDvgRvVrom8uLZ5lirlex+szHBfX2V3gg1SmrxlQgEu5CYH+qeHyO+1eRBDXtTG9fJptitVJNwCMzVdgVrYXKa9eLTrMpDt2T6N/HwHXYczQnIia2HgyIu8dHVG6Psm25ybyIyLZ06bXNBgh9Vyf7HV92swdvdZT9Grctz8tovV6u4rrFS/MxcTlYSzf79R7xOJBqKKoXvKU28c/iuX7VNaCEu7fw+FW0dAAuezMzUW9TgVWh7W1tuCQ1JFTvYRXbiod7niLrrAgpYAbWTWO5yq1hynmaLCYgdyAk/ZGk5xSgia3nbErf1iQXNXu3FUmmrGO0sXicc/KSlo0uG0Ce0wTdg6rBwf4wgR2YNh3nQempb78ymrM+KaCzKw+epc5y3+zxjhq621q2Uyn6KGpGNgQIvla9k8S0bHoUGEWOh/QQmWeZmkNpltcHNJ1muD5h1syTNJLvgEylOK/BThv0KEaRF68mUKR3naMdexP6f7cOZLUOKNj4Y4qcxpcM1iD0yY3lGhzX+VT3lk9q8e119cE7XqMs29iqP2XDPCF87g5viVe/JPzuDabp2PURkMMB/1VpMcsx/FP07Y+DzShaXmiAe1VGxIeDZk8dSIBQfkyve6prPDJsXsylTOcxiJGpZqNDTGRg4uvh8X6qCQwT85r8wQFglGfV1zFzDucY126JK8PANzq6wmg6A8BYnFbterx/lMC5EVNA3q+VQ3T4wecDgN7Y8UfNhZjmFdbO/r548cYONjZu2gEAtuP+4t76xMhjXFP92H3tDCwuE6nQ9C8VhEmp4TDgyHxQWzKxM8dDgmOXSNQU0Bp0TuhtfJmEABpfv33doWMHDTRspwUB4wV+9vqFYZzE3rfX6tn4+EFy2zu/KZcHw2/55grjp08D1p6tbWRKKDILtJZSzMhzFNGd3izqcR9VQFeWUY7KY8Rx1oCRPcufN8HbdhddJ8X/1BSuh7b8aRlcqKA+Zud+cOiKG/ClXgq+EveJfOYmCBHJ9PRRkzv/pLH9ai9xWzvDMCso/xfwYhXvm7veioe8sYYe1aW1AMEx2uMsvJ94pkYOf1pUrTEpF86vsxHKSYs1DG/1CpPsQWZgP2Z8t7/MrfP+UxeXaDDRbpfmdQipjj6h6xFjkZKiQOf/0cMuSJK23XOem97sGYJU297O8SmLMbN3HS1cUCfv7lrhPv5fefXK8cqTwAo2RChpFtLfwdCD3S7OQEg1836KRLmjGq4MZZrT1rUL8Y9JKSRRX+PaGTd7vFD/1ctDP0TB9hbO/XUWErSmxaPZZCnhXF8jqDr10ct8HJ7+Ef1bkFqEumfXRSbBZJFKeK9CcP4QRpMEKefye/hUXmHZ+HIePqiGa14hp2DEqLLDDbY6aVtskeCwpI0fi0jztQBKaD5P7srmOF1rUNDp6iaiizNvbspxSGzOuBzG2PGHDyBlOkKGkNq1ftUymiFTF3//jU+q5cmKN414x18VDJ2JQ84ysK5SsnDjwXaYceBaxy6rGEbplJ85YnrFgtZsH1J60Se4ki679SuHByWGBBkliCDceN4nbsBouCD7/4qHILxiezTU7xT2LT7ft7lTl8oSLsx5mfOXm4YN/QK/tEFqZv/4iOueME+TMBKq7fuLCT2ND+eI8QFpwAHoabj2DO9HgU8aksEE4JDHzfzoZq//JGV+l4yHIYrlzO1IBsF+naIjUxrH+Zm2fk8QiLeEafhflKYIeS74IBxzkstm/s/S5FBwOGzvzuuIIEod7eKNorOCN4Lj74EwvOtgyobJTHQdNi3n4heVcY8V/s+MyRjJ5RQjXHC+/C0oBdRjVFI1SRDSeNmwWw37Q6rihWCZvqhkYf11Xv+HaTqrgMCw4+3VcQ/XQnB41Jm/qC/3Du3ddoe8FrWOaoZzvmeQ+Kwp5mUNBO4Uj5Qa3VA25A/P7UQ1MsBNtS+YcnStNhvIf5ZI34Uj51vbfiuIA3zI9nmWwrHYSmd5x7/nLkYjNroJVlQSXO0C5MpIuegYFr1pkap6VxILvwvPXFE0i4YPWlaTUSO6MWy/gVt2BqNiiNdRcdDFdApUOfLJZfoPhBF7aNuRtjONgTAmksaRzAqwzEMB3dDzlCOPzI16dNQgjJ9Bm3S/IEYemMvlZ8Tz0WlixRLknKQc+GeJ7PqUNQpXGqoY33EpJ6OT5cXXoLI4HtxSBMj3M8EfQHaiSx5Q177tr8MZx98k7j5NgnIMcmH1KqG9/Zw2GCLTcz+q74mTiNncEBCvHCHDc1aFY6IuLs3IZltitnnnTEox23sIh9syGUhgUyKmiIQ/Ww5sSTRF/ImN9n1R+pg11TXtnv3L/gcNSHQyrdeZh9s9Y97+8QEM+q/eXdImLh7cTsEwefwqxU1vyhNUccxRMXHThqYLZvNEjqxWeoss1sobkwjPMO84cefu6EJYmQHIEH8TBxZX96r7IrkLfJtxQCqVFfc5jgssCG8cwXDOvXjmLT69JXVBn/oPv3gwz/bH0YrDE7IfxoL4z1CV2ocrHE4vOv0l73xUNKxm2OwH1rUHTPOeTE+ePYJqW5LHlJENqUZPccfYIwosJvdFeJEyK7jW0tYwug2V/yQeWOfxJvE2PH7urh4IbJuwcSJByX1fPid+6HxTBH3VWZWQyOv3SalvkUtPoplw+RZWDiLvfPdaUcLPP59cWmix46tcKiGYfTYUmxJWZ2ShcIZ7ztNTh2eH/yV0Wk5wiOnOptdyLvwZ7rz5U0ry9DXMZ/J+89VMSirHM33h7txyrXW1nEWUNs2Hq2GC1JgOKlQLrQ8RZYCVk4pdyWim0bLGphLzmRfCUukwgGOCvo3Xdw+Ad2UqnJaKREYdE267HQTUYs8ayI/cHshf6FYyY1xQ7YlUrF97HIDxpDt6+cK+uBFkWrbTlvTpyN2tp/rVAP5qpf2BxYzMcaeiNvNpoZ7NS/ZUbp8B4mxWXucl+3gL49OtUJdFHQa7DOIXqyBXOvPWfL5n4Dww+nkiKknuCQ+6VBh8690HWxhGdX1AlMIeteV6Bag/LMk7vIqXRYJ58tobY+B3Oyqilv5VShteMNFRO1Tj7lLaGyKmpFrFHafVzyXBO2/kj8Bjt+0F8lYv/Pyh+j/xm7609vYp2lIuupt69geMbiyw36GKgP9oEfEVdwiGfZ7pZPIYwUJRz7WbLTy0l/JF8OtmPqsmvvWM1tzDTNNdK8FIR5Bz21izlNYUQ/50V20W1Md8+stszyw681F5JXjFextdo23Po5K+QMnCKshsRCxxkijZ+iJ0xL2srK7N3xCPecxcnlN9B3dubI8JIV5pX+Lm5bGQAUJtskGp3Chley3/p99kMR01qh/ktBLJ7S+itT7QblTSq0l9RvIqHmaX6Q9wB0MCjukdk1CvOTvHtU57qgun9P4pfw69jGRq99VzAS6hfOM/d8egAkb5oeMwt/mDE79TCnyh3nvnsIFrXcweSwhGKHSRkYcKnQeRbbiTazE5wV/MewWto3xeke5Q7HWTU8sKrGV/vorgOLIxQpZzF2Orbj+HO+fJYcd+z7UnQ04/JzLKtdtDH4JoVd2VUVdN/Nd/47JVXUR1Ug0ujrCBVel6c84Wk75CWI5Z1Fxy8N2kF/l/GP9XZeGLnkMCgodAhiTzGfMnTZBuKRN25Di3FQM6C6Yv6sFUgdKfe43YShwZ3S/7tiMHR4aPeeM/iFpX4J+XmRCbgCCSSDPd7Q8UlwTGP0BHaKTUQk9nZiU0pLU4/XC1h0v/ttEr9iA5fqHUtxP8ittmQ/R50Giw7s+UMXB4As0KDHX6kC4+FRn9hKriDJV0g5e/QwRka+iT0y4YrZxUcZllOGcVD3Dr+BxRy2UT4UZ/73C4mWo48VhtVxKMSZZ920C4uDGnWuOg8iaVvx1pM1N1x+98Qs+9EIzLIgnxP/CxhfdtnM7zgKE/20POd3ei8zc/HZzf0WOBAWd45SrA/St3dHDk9PQsuUVQifYwikd2hrRIk9wHG1TF6yTjy27W2/ZEVyg7TdMiOfnCeBcOVGgQXbFVxqWSZGuM5ApkYdVSsPK6Qcl322P/kyLnNMKnCcPQHzQckpdk/eAbGog9z9334Y3T9XevBUJPRMf38QWDqJGHylVkXkMZAyFWd3mwN0Zk29jDDxwQkDl+6SbWFIVS/uELOuhNA+3VbeFicsPyTY1u60hNhLPnA8IROHAq7GnLkcgG3e7kv9dKcwqftardlLhOGAV0c2J3yh6fPxVYOJGUyN6uJNW3DAIb/AoMtGclDOxKggX/8Cl3uW2X8YPof57RcRDyLP4LCnaNCNrDM4qu9+ufemBwzG+APXZyacDPv5W0VcAmYTKFXhyTjMIUU/s3ArYN8Tl6DhK7hgyu596LwTxJiFNK1dEsNOQeKMSKIa9l8OYohzd8SSRRE1jevxMEgy+BR5yB9GhbW8mut7sY+fKZH4XR0rA/oSvForYDjqKgbEuEFLFj3Hy198WJ51SoNP+CoMF8yfSz/yZsezjtND8G6odudamgppg6gPDolJOcdw0J3tRZg6PaTS5entX2SAQVvfSBqKW5BxQOOlYn8LNkqOXLkWsoRtjz7/reyRASNbte39U9YwsJoRcqZqETP3ZaeedaDDbW1/+kvVjFAibi3pb+4JHftTv11LCcGyXXHknmkTmAtO5ru8JAb1Zw8eL0qZR/Lj7OeGLRHYJLxg2C9tij08j6kmVL9AD7/Xrrldf5E8ST6vOS8Paaw0nyzJTjjsETHScf8+1t2MnDQnPgHSZZfLnrXGOLhbPjXvmxKMaRg8dqLXxIYF3dOVFEPYFZ8Sfj+7GueGxfz2GbyB/OJ+ljRfIk5f5Ru9RM0O37rN5B+2v4e5R/iXtXARlsuPpark5uDcCxFHXSlRJH+YLxf8Vg6TAwaDQa84kUBKXuHwssbZw4cPdsVkY8bnHIssO0bo2SPh+arqKxTJWk8ktN/BVmvfXg9RJiTQ+L30BRUYGLyccEVnZMen46PUl8xxsfeEU+ZDOsh4JLEl0+0Am+w4lCpshHXzY+EKzgnQ875OdVc4B5DIP77+sOyB5Hk1asUFKozuzeAJGvoCAy+pwwwsuHfyN0+/KXIABoVPco0y/Yd1Ly92nnmnDYMXegIHNnY40efg29j2Qhw0/GZT7N6D1S4JDjZx7PjlzqUKncBuJP6yP/ZqZARbnz2zcQv1hyFOwdBnpmo4qiUu9P7bJExl0aSoTStDwaPrHhqnQ6D2PenLWStdXEylWFppeYM91UUne9p5cUEzgPl15ztMvE7xPqTsDqyt0BUyfz6Co6NBv/Z+PI0RywWblXnTWOSt/SxX0gDL1BgVLdS0cf7aKVrXYkqYLpjQOCHoDaOUqiIPtfqgU3HF0bZ4HSqLHtk30CrC2K3YPZEHymG4m+Ir6x1qWLgnevOU0QQSb3qqP+0+CQsCbyWN2F2QRHD7p3u2Ftr2DzfZpv3A3x8XXYPJxrBu+u+CSdQaLoW+ZPv0KhZLq7xSJty6Ed9Yj3nfc4dRqhmdacN8XLF4O9yXzgFdVcI82hXmQG7qMF4Jn4RiIauiZ4KaMJYrqm2sZY2Lf5jmL/YNwqQghTnNhBUu82RzlInH4Gzaod0CLldwUjB6ovesOYzfJeUw0tZC+/Z56kyLUzATbP2BYjZDnurzr/jQ9Yp8Sq3m0ZowSpj//u6ehvYEDs2WTaklCUNhe+qdhldPMWHw/h93PxJ07ybpXTA3hs4Y9SN/Ri/h0KaN6LWIL0h25y17cVIHE3WlYLWSgE2/A5X8497h5Ou4uKKl/5AQdlm/ycgCyL/+Lr/avoPkX59PzmcO4fT7wLw4oiSQZZK+z7fshUVMMaxrVsfhzfVC+r2qUBPc1NfO4wbxctf+GNhpYB5BNVH26jlsEL3OmsY1BkTnUUduYQVo2DP0x4huDWeIjidCHFtgVumuljcvL8zy/Sj1p9fIp6Q+0SK3jx7mE4RMuSMtYOQjHVzXyoH2tPuvYiwlcDwhOC6o6S3mUi2d/XklCKsoFFboLwniJhXbGq+MJU4y/lHSequAxTrjdWWdKtjiftUxV+AhTJM/eBwIzMYmffxHf0UWFyz5Au0WS4FgPfy2zysK+gZ1eRKfHYOJ3/HnRbxskGhU1virMQJmCNbheR/Nsa1u7aZ9hzg28CrSPbhni4tfLOLEbN7h7OqCeQ7cgDG1GeLRnxpALL79mSr3A5Dd1MrjT76HxV3ORR8+6EJrwsmSb8VnYKJT8bio1DGs+2vYp0bJgisV7P2bB7NxQtP2tluuAxKr8zWHBC7BIuM/EZrKTGxOKuSrvFaKhTom1coqT3FVMvfymsYlnDas+ptKLQIdlJHd/qbtOLWwzubByAgrKizW/3mLw6Ji4sjb01z5dKyh2d6dJTg32vk6QOwidObVsKn0foOeuqlL53fPYX9gcaORrjas/Awvzw94jVOP9m79F/cPt53d3l7PqoDNqt1BZlU9mCcw+LLJ2Q/JHdODVXkl8rv36j+2JbAhSUDXY9KTIE9jPao+wXceh2IDvaU2DLEx610wNWs7RAjmUX1/voBzkTcON7h/gI3qh3vvzWfD2At7K2LNAk6cY0haldglTyHVZdj9/B1OJYbIqLjV4Ij7Ypdk7X1Y/uMu95GpEdeqG6KvC5Tj9kGvelsrTRjMDVimuU8NvaayPUXrO3cc/+QpWfEP/iQr2qZXdiIp6+VDfWVRJPBF9nGUnMR6bt+jHYxnoV/HXM/P+y42XGzqMhFbRLKjq9vrsP4dDyTKhl9pgZbtyIAo0yQcNj8gbUXJK7+b6tfhKp0qnI6a372vXgImdXweinUNQGWqVy3ZuBMmdpP4RXMCgVzwkZvbxx17YvQ2DuMFGDpSp3Ty2mXoYN01rNaaCz3XLzNp1O3BBXOmzRPxEbgkIRfVn38PZuvkJr5Ge+KA5lS6tTklTB1w2MMvUA6k08nTLy20YZlJ6Hfo4QBcYpGxp+RaweXi8kc+It74e9Yk1PWaNgxZe57fSrZA17HTMQPXDuJgDu8thdc12LDn9U3KRQeY/fn3x4j4UxhTVhzbdOLGGs5fX5hXqrHf6Vxk7M6eTrzTDg0wU8HNj6tBCw7NEPzSVz/DN0R+tybHOjQnwPK00lDndyGc2BZ3HqYIhpbV02F/OpOwTnNydzq9FdTp50dJUw1Aa8ve59defYd51aKfdRl7YWD49EB/QSnkPLQ9qXBBFTezt+eMFpNhaf/5lKeUGtB59tobgzs6OCtiExtjyyFPUTooGWEYmk8zl3atT5AIi5NlhzZt6IHw89NXbbFoqC7+uvLq4H2cPXb2seFzP6xVElFfsPSFIV42qw+nHXBwkkP0l8gPXLwSN/tLLFueIWc53zc7EOfm/+RonLkHi4a2euWCJkiidSFLy6ohsU1YN4hbGdo4RQ8x3/SEEU77tlN8DrBgnEc9N2yJ01krpZmnhrB+kF9YUp4EBCujq3R6nDix5mxBQZMI49h+jOLYJViQNnucwemAoS2h2XQLb6HtmpmzcSAjjDffNEqK9pCnOiOZw8p9AbM2XofWfnsI8zQ9bBcX9uRTW2sQ/lz5nU8ZCGUPP5yECe9GB/N1kXyKpt6kMq44WMp5qEhPYsVGrXukvsVtXHXNEohS/IX9c2zzx9nqYUnwzDWqVnUovF3mwNiegUvUsuTI5XFcrTL5WkoziwtFd/TENl12uKQx2Xt8GOeJpUxdpvGwvqbLXvlbE9MjDoRrKTxF8ooL9aPiThxx2rddHFoIbu4OarZ1opDx+FHfM60+XP7i2bhEjsMp+vJfwyxbOMl+if+5mSaMOmyuSztrYz+dj+XrczyYR156PmY9gpOfRmKTuZOAMM3UdyWTGbKDXameBIriglZZ+QavCRADFFN4Ltrl72Kmt2T4qg8jdXIdx8EdF8WuT8kX52AdqU3a8ZItkqV1ki4fC8d5VpOjyqKSsMD70ukrUwpMD63eYhpqxCy3Z8yOxnZYyvshKWvIHXtpuPZHuwzD1M/Yk+8O+2CXJiVn5hg7NvfHRByV94PaY0k6xtPuUH5Z2my4cQRnl3Ss+c/J4Ig4w979lesw5ixR9M/sNoykfLNUusYD4/orHL38/wFRJ2xleIsVCK8bOk0WumG5rux6FL9D/i45ucVuSTGcrLQ7nSd/F9s/VHcqPtv+Hx0ll90=" +} \ No newline at end of file diff --git a/test/test_transform.py b/test/test_transform.py index 0b2cae58..4b3e620f 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -600,3 +600,40 @@ def test_similarity_init(): assert(isinstance(tform, renderapi.transform.SimilarityModel)) assert(np.abs(tform.M[0, 0]-2) < EPSILON) assert(tform.M[0, 2] == 10) + + +def test_thinplatespline(): + j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + assert (j['dataString'] == t.dataString) + t.aMtx = np.zeros(4) + t.bVec = np.zeros(2) + t2 = renderapi.transform.ThinPlateSplineTransform( + dataString=t.dataString) + assert (t == t2) + # test incorrect lengths + with pytest.raises(renderapi.errors.RenderError): + s = t2.dataString.split(' ') + s[1] = str(int(s[1])+1) + t3 = renderapi.transform.ThinPlateSplineTransform( + dataString=" ".join(s)) + with pytest.raises(renderapi.errors.RenderError): + s = t2.dataString.split(' ') + s[2] = str(int(s[2])-4) + t3 = renderapi.transform.ThinPlateSplineTransform( + dataString=" ".join(s)) + + +def test_encode64(): + # case for Stephan's '@' character + s = '@QAkh+fAbhm6/8AAAAAAAAA==' + x = renderapi.utils.decodeBase64(s) + assert(x.size == 2) + assert(x[0] == 3.14159) + assert(x[1] == -1.0) + # all other + x = np.array([1.0, 3.0, 3.14159, -4.0, 10.0]) + s = renderapi.utils.encodeBase64(x) + y = renderapi.utils.decodeBase64(s) + assert(np.all(x == y)) From 3458c9cc3083914c91eb9e58904508e25fcc0ca5 Mon Sep 17 00:00:00 2001 From: sharmishtaa Date: Mon, 2 Jul 2018 16:21:13 -0700 Subject: [PATCH 634/766] updated pointmatch client to take 2 arguments (#108) * updated pointmatch client to take 2 arguments * added integration test for pointmatching between two stacks --- integration_tests/test_client_integrated.py | 31 +++++++++++++++++++++ renderapi/client.py | 22 ++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 5adc9cfb..30db6043 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -171,6 +171,16 @@ def teststack(render, render_example_tilespec_and_transforms): yield stack renderapi.stack.delete_stack(stack, render=render) +@pytest.fixture(scope="module") +def teststack2(render, render_example_tilespec_and_transforms): + #copy of teststack for the purpose of testing pointmatchClient with two stacks + stack = 'teststack2' + test_import_jsonfiles(render, render_example_tilespec_and_transforms, + stack=stack) + yield stack + renderapi.stack.delete_stack(stack, render=render) + + def test_tile_pair_client(render, teststack, **kwargs): zvalues = np.array(renderapi.stack.get_z_values_for_stack( @@ -298,6 +308,27 @@ def test_point_match_client(teststack, render, tmpdir): assert(len(pms) > 0) +def test_point_match_client_2args(teststack, teststack2, render, tmpdir): + collection = 'test_client_collection' + zvalues = np.array(renderapi.stack.get_z_values_for_stack( + teststack, render=render)) + tilepairjson = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), render=render) + + tile_pairs = [(tp['p']['id'], tp['q']['id']) for tp + in tilepairjson['neighborPairs'][0:1]] # tile pairs are the same because the two stacks have the same information + sift_options = renderapi.client.SiftPointMatchOptions(renderScale=.25) + renderapi.client.pointMatchClient(teststack, + collection, + tile_pairs, stack2 = teststack2, + debugDirectory=tmpdir, + sift_options=sift_options, + render=render) + tp = tilepairjson['neighborPairs'][0] + pms = renderapi.pointmatch.get_matches_involving_tile( + collection, tp['p']['groupId'], tp['p']['id'], render=render) + assert(len(pms) > 0) + def test_call_run_ws_client_renderclient(render, teststack): # class for this test should be something relatively lightweight.... test_class = 'org.janelia.render.client.ValidateTilesClient' diff --git a/renderapi/client.py b/renderapi/client.py index ce0f1dfe..317da0be 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -1040,6 +1040,7 @@ def __init__(self, renderScale=None, fillWithNoise=None, **kwargs): @renderclientaccess def pointMatchClient(stack, collection, tile_pairs, + stack2 = None, sift_options=None, pointMatchRender=None, debugDirectory=None, @@ -1058,6 +1059,8 @@ def pointMatchClient(stack, collection, tile_pairs, ---------- stack : str stack containing the tiles + stack2 : str + second optional stack containing tiles (if stack2 is not none, then tile_pair['p'] comes from stack and tile_pair['q'] comes from stack2) collection : str point match collection to save results into tile_pairs : iterable @@ -1126,9 +1129,26 @@ def pointMatchClient(stack, collection, tile_pairs, project=project, client_script=client_script) + if stack2 is not None: + canvas_url_template2 = get_canvas_url_template( + stack2, + filter, + renderWithoutMask, + normalizeForMatching, + excludeTransformsAfterLast, + excludeFirstTransformAndAllAfter, + excludeAllTransforms, + host=host, + port=port, + owner=owner, + project=project, + client_script=client_script) + else: + canvas_url_template2 = canvas_url_template + for tile1, tile2 in tile_pairs: argvs += [canvas_url_template.format(tile1), - canvas_url_template.format(tile2)] + canvas_url_template2.format(tile2)] call_run_ws_client('org.janelia.render.client.PointMatchClient', memGB=memGB, client_script=client_script, From cea2e5d2b808cdb108f21b65982bbefcfbf4cc48 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Tue, 10 Jul 2018 07:28:58 -0700 Subject: [PATCH 635/766] processpools: add standard library processing pools (#113) * processpools: add stnadard library processing pools * processpools: test parallel functions with all standardlib processpools, fix dummy pool --- integration_tests/test_client_integrated.py | 28 +++++++-- renderapi/client.py | 56 ++++++++--------- .../external/processpools/stdlib_pool.py | 60 +++++++++++++++++++ 3 files changed, 113 insertions(+), 31 deletions(-) create mode 100644 renderapi/external/processpools/stdlib_pool.py diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 30db6043..a2f0ea59 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -10,6 +10,8 @@ client_script_location, tilespec_file, tform_file, test_pool_size) import PIL +from renderapi.external.processpools.stdlib_pool import ( + WithThreadPool, WithDummyMapPool, WithMultiprocessingPool) root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -97,14 +99,14 @@ def test_failed_jsonfiles_validate_client( def test_import_jsonfiles_parallel( render, render_example_tilespec_and_transforms, stack, use_rest, - poolsize=test_pool_size): + poolsize=test_pool_size, **kwargs): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms (tfiles, transformFile) = render_example_json_files( render_example_tilespec_and_transforms) renderapi.client.import_jsonfiles_parallel( stack, tfiles, transformFile=transformFile, - render=render, poolsize=poolsize, use_rest=use_rest) + render=render, poolsize=poolsize, use_rest=use_rest, **kwargs) validate_stack_import(render, stack, tilespecs) renderapi.stack.delete_stack(stack, render=render) @@ -141,12 +143,13 @@ def test_import_jsonfiles_parallel_multiple( def test_import_tilespecs_parallel(render, render_example_tilespec_and_transforms, - stack='test_import_tilespecs_parallel'): + stack='test_import_tilespecs_parallel', + **kwargs): renderapi.stack.create_stack(stack, render=render) (tilespecs, tforms) = render_example_tilespec_and_transforms renderapi.client.import_tilespecs_parallel( stack, tilespecs, sharedTransforms=tforms, - poolsize=test_pool_size, render=render) + poolsize=test_pool_size, render=render, **kwargs) validate_stack_import(render, stack, tilespecs) @@ -329,6 +332,7 @@ def test_point_match_client_2args(teststack, teststack2, render, tmpdir): collection, tp['p']['groupId'], tp['p']['id'], render=render) assert(len(pms) > 0) + def test_call_run_ws_client_renderclient(render, teststack): # class for this test should be something relatively lightweight.... test_class = 'org.janelia.render.client.ValidateTilesClient' @@ -338,3 +342,19 @@ def test_call_run_ws_client_renderclient(render, teststack): render.DEFAULT_PROJECT, teststack) + [zvalues[0]] assert not renderapi.client.call_run_ws_client( test_class, add_args=args, subprocess_mode='call', renderclient=render) + + +@pytest.mark.parametrize("stackbase,poolclass", [ + ("ThreadPooltest", WithThreadPool), + ("DummyMapPoolTest", WithDummyMapPool), + ("MultiprocessingPoolTest", WithMultiprocessingPool)]) +def test_processpools_parallelfuncs( + render, render_example_tilespec_and_transforms, + stackbase, poolclass, poolsize=test_pool_size): + test_import_tilespecs_parallel( + render, render_example_tilespec_and_transforms, + "{}_tilespecs".format(stackbase), mpPool=poolclass) + test_import_jsonfiles_parallel( + render, render_example_tilespec_and_transforms, + "{}_jsonfiles".format(stackbase), False, poolsize=poolsize, + mpPool=poolclass) diff --git a/renderapi/client.py b/renderapi/client.py index 317da0be..ab39120b 100755 --- a/renderapi/client.py +++ b/renderapi/client.py @@ -8,7 +8,7 @@ import logging import subprocess import tempfile -from multiprocessing.pool import Pool +# from multiprocessing.pool import Pool from decorator import decorator from .errors import ClientScriptError from .utils import NullHandler, renderdump_temp, fitargspec @@ -16,6 +16,7 @@ format_preamble, format_baseurl) from .stack import set_stack_state, make_stack_params from .resolvedtiles import put_tilespecs +from renderapi.external.processpools.stdlib_pool import WithMultiprocessingPool # setup logger logger = logging.getLogger(__name__) @@ -78,32 +79,33 @@ def renderclientaccess(f, *args, **kwargs): return f(*args, **kwargs) -class WithPool(Pool): - """Multiprocessing.pool.Pool with functioning __exit__ call - - Parameters - ---------- - *args - variable length argument list matching input - to multiprocessing.pool.Pool - **kwargs - keyword argument input matching multiprocessing.pool.Pool - - Examples - -------- - >>> with WithPool(number_processes) as pool: - >>> pool.map(myfunc, myInput) - """ - - def __init__(self, *args, **kwargs): - super(WithPool, self).__init__(*args, **kwargs) - - def __enter__(self): - return self - - def __exit__(self, *args, **kwargs): - self.close() - self.join() +WithPool = WithMultiprocessingPool +# class WithPool(Pool): +# """Multiprocessing.pool.Pool with functioning __exit__ call +# +# Parameters +# ---------- +# *args +# variable length argument list matching input +# to multiprocessing.pool.Pool +# **kwargs +# keyword argument input matching multiprocessing.pool.Pool +# +# Examples +# -------- +# >>> with WithPool(number_processes) as pool: +# >>> pool.map(myfunc, myInput) +# """ +# +# def __init__(self, *args, **kwargs): +# super(WithPool, self).__init__(*args, **kwargs) +# +# def __enter__(self): +# return self +# +# def __exit__(self, *args, **kwargs): +# self.close() +# self.join() @renderclientaccess diff --git a/renderapi/external/processpools/stdlib_pool.py b/renderapi/external/processpools/stdlib_pool.py new file mode 100644 index 00000000..2fd2d54d --- /dev/null +++ b/renderapi/external/processpools/stdlib_pool.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +""" +WithPool style helper functions using python's standard library +""" +from multiprocessing.pool import Pool, ThreadPool + + +class WithThreadPool(ThreadPool): + def __init__(self, *args, **kwargs): + super(WithThreadPool, self).__init__(*args, **kwargs) + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + self.join() + + +class WithDummyMapPool: + @staticmethod + def map(*args, **kwargs): + return list(map(*args, **kwargs)) + + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + pass + + +class WithMultiprocessingPool(Pool): + """Multiprocessing.pool.Pool with functioning __exit__ call + + Parameters + ---------- + *args + variable length argument list matching input + to multiprocessing.pool.Pool + **kwargs + keyword argument input matching multiprocessing.pool.Pool + + Examples + -------- + >>> with WithMultiprocessingPool(number_processes) as pool: + >>> pool.map(myfunc, myInput) + """ + + def __init__(self, *args, **kwargs): + super(WithMultiprocessingPool, self).__init__(*args, **kwargs) + + def __enter__(self): + return self + + def __exit__(self, *args, **kwargs): + self.close() + self.join() From bb301bfcd7b3c64605e44ba5a1db9c347fb64815 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Wed, 11 Jul 2018 11:35:14 -0700 Subject: [PATCH 636/766] Better image pyramid (#109) * trying to implement a simplier ImagePyramid interface changed image pyramid interface to be more straightforward, and changed tests where necessary bumped version to indicate backward compatability break wip fixed auto sorting * v1 of image open interface fixed s3 bucket read added new requirements removed auto-import of image_open making urlparse python3 compatible fixing python3 imports more another import fix * removing image_open test * fix bug * removing .pytest_cache files * adding to gitignore * fixing .vscode * remove from git * fixed bug in from_dict * fixing bugs in new class * fixed bugs * reworking MIpMapLevels and removing deprecated stuff * added more image pyramid unit tests * added more tests * adding __setItem__, DeprecationWarning, and class method * added keyerror exceptions and tests * fixed test * test fix --- .gitignore | 2 + integration_tests/test_stack_integrated.py | 5 +- renderapi/channel.py | 9 +- renderapi/image_pyramid.py | 195 +++++++++++++-------- renderapi/stack.py | 7 - renderapi/tilespec.py | 59 +++---- test/test_channel.py | 9 +- test/test_imagepyramid.py | 98 +++++++++++ test/test_tilespec.py | 13 +- 9 files changed, 266 insertions(+), 131 deletions(-) create mode 100644 test/test_imagepyramid.py diff --git a/.gitignore b/.gitignore index 290b7718..d54ba60a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ docs/_build/ htmlcov/ .cache .tox +.pytest_cache +.vscode \ No newline at end of file diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index afb5512f..563d1512 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -32,7 +32,7 @@ def render(): @pytest.fixture def simpletilespec(): - mml = renderapi.tilespec.MipMapLevel(0, '/not/a/path.jpg') + mml = renderapi.image_pyramid.MipMapLevel(0, '/not/a/path.jpg') tform = renderapi.transform.AffineModel(labels=['simple']) layout = renderapi.tilespec.Layout(sectionId="section0", scopeId="testscope", @@ -249,9 +249,6 @@ def test_get_section_z(render, teststack): z = render.run(renderapi.stack.get_section_z_value, teststack, "3407.0") assert z == 3407 - z = render.run(renderapi.stack.get_z_value_for_section, - teststack, "3407.0") - assert z == 3407 def test_clone_stack(render, teststack): diff --git a/renderapi/channel.py b/renderapi/channel.py index 48e10b89..5b456269 100644 --- a/renderapi/channel.py +++ b/renderapi/channel.py @@ -1,4 +1,4 @@ -from .image_pyramid import ImagePyramid, MipMapLevel +from .image_pyramid import ImagePyramid class Channel: @@ -37,7 +37,7 @@ def to_dict(self): d['minIntensity'] = self.minIntensity if self.maxIntensity is not None: d['maxIntensity'] = self.maxIntensity - d['mipmapLevels'] = self.ip.to_ordered_dict() + d['mipmapLevels'] = self.ip.to_dict() return d def from_dict(self, d): @@ -51,7 +51,4 @@ def from_dict(self, d): self.name = d['name'] self.minIntensity = d['minIntensity'] self.maxIntensity = d['maxIntensity'] - self.ip = ImagePyramid(mipMapLevels=[ - MipMapLevel( - int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) - for l, v in d['mipmapLevels'].items()]) + self.ip = ImagePyramid.from_dict(d['mipmapLevels']) diff --git a/renderapi/image_pyramid.py b/renderapi/image_pyramid.py index 65ff1ba5..ac2ca8e9 100644 --- a/renderapi/image_pyramid.py +++ b/renderapi/image_pyramid.py @@ -1,22 +1,26 @@ -from collections import OrderedDict +from collections import MutableMapping +from .errors import RenderError +import logging +from .utils import NullHandler +import warnings +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) -class MipMapLevel: - """MipMapLevel class to represent a level of an image pyramid. - Can be put in dictionary formatting using dict(mML) + +class MipMap: + """MipMap class to represent a image and its mask Attributes ---------- - level : int - level of 2x downsampling represented by mipmaplevel imageUrl : str or None uri corresponding to image maskUrl : str or None uri corresponding to mask """ - def __init__(self, level, imageUrl=None, maskUrl=None): - self.level = level + + def __init__(self, imageUrl=None, maskUrl=None): self.imageUrl = imageUrl self.maskUrl = maskUrl @@ -37,90 +41,139 @@ def _formatUrls(self): d.update({'maskUrl': self.maskUrl}) return d + def __setitem__(self, key, value): + if key == 'imageUrl': + self.imageUrl = value + elif key == 'maskUrl': + self.maskUrl = value + else: + raise KeyError('{} not a valid mipmap attribute'.format(key)) + + def __getitem__(self, key): + if key == 'imageUrl': + return self.imageUrl + if key == 'maskUrl': + return self.maskUrl + else: + raise KeyError( + '{} is not a valid attribute of a mipmapLevel'.format(key)) + def __iter__(self): - return iter([(self.level, self._formatUrls())]) + return iter(self._formatUrls().items()) + def __eq__(self, b): + try: + return all([self.imageUrl == b.imageUrl, + self.maskUrl == b.maskUrl]) + except AttributeError as e: + return all([self.imageUrl == b.get('imageUrl'), + self.maskUrl == b.get('maskUrl')]) -class ImagePyramid: - '''Image Pyramid class representing a set of MipMapLevels which correspond - to mipmapped (continuously downsmapled by 2x) representations - of an image at level 0 - Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) + +class MipMapLevel: + """MipMapLevel class to represent a level of an image pyramid. + Can be put in dictionary formatting using dict(mML) Attributes ---------- - mipMapLevels : :obj:`list` of :class:`MipMapLevel` - list of :class:`MipMapLevel` objects defining image pyramid + level : int + level of 2x downsampling represented by mipmaplevel + imageUrl : str or None + uri corresponding to image + maskUrl : str or None + uri corresponding to mask - ''' - def __init__(self, mipMapLevels=[]): - self.mipMapLevels = mipMapLevels + """ + + def __init__(self, level, imageUrl=None, maskUrl=None): + warnings.warn( + "use of mipmaplevels deprecated, use MipMap and ImagePyramid", + DeprecationWarning) + self.level = level + self.mipmap = MipMap(imageUrl, maskUrl) def to_dict(self): - """return dictionary representation of this object""" - return dict(self.__iter__()) + """ + Returns + ------- + dict + json compatible dictionary representaton + """ + return dict(self.mipmap) - def to_ordered_dict(self, key=None): - """returns :class:`OrderedDict` represention of this object, - ordered by mipmapLevel + def __getitem__(self, key): + if key == 'imageUrl': + return self.mipmap.imageUrl + if key == 'maskUrl': + return self.mipmap.maskUrl + else: + raise KeyError( + '{} is not a valid attribute of a mipmapLevel'.format(key)) - Parameters - ---------- - key : func - function to sort ordered dict of - :class:`mipMapLevel` dicts (default is by level) + def __iter__(self): + return iter(self.to_dict().items()) - Returns - ------- - OrderedDict - sorted dictionary of :class:`mipMapLevels` in ImagePyramid + def __eq__(self, b): + return self.mipmap == b.mipmap - """ - return OrderedDict(sorted( - self.__iter__(), key=((lambda x: x[0]) if key - is None else key))) - def append(self, mmL): - """appends a MipMapLevel to this ImagePyramid +class TransformedDict(MutableMapping): + """A dictionary that applies an arbitrary key-altering + function before accessing the keys""" - Parameters - ---------- - mml : :class:`MipMapLevel` - :class:`MipMapLevel` to append - """ - self.mipMapLevels.append(mmL) + def __init__(self, *args, **kwargs): + self.store = dict() + self.update(dict(*args, **kwargs)) # use the free update to set keys - def update(self, mmL): - """updates the ImagePyramid with this MipMapLevel. - will overwrite existing mipMapLevels with same level + def __getitem__(self, key): + return self.store[self.__keytransform__(key)] - Args: - mml (MipMapLevel): mipmap level to update in pyramid - """ - self.mipMapLevels = [ - l for l in self.mipMapLevels if l.level != mmL.level] - self.append(mmL) + def __setitem__(self, key, value): + self.store[self.__keytransform__(key)] = value - def get(self, to_get): - """gets a specific mipmap level in dictionary form + def __delitem__(self, key): + del self.store[self.__keytransform__(key)] - Parameters - ---------- - to_get : int - level to get + def __iter__(self): + return iter(self.store) - Returns - ------- - dict - representation of requested MipMapLevel - """ - return self.to_dict()[to_get] # TODO should this default + def __len__(self): + return len(self.store) + + def __keytransform__(self, key): + return key + + +class ImagePyramid(TransformedDict): + '''Image Pyramid class representing a set of MipMapLevels which correspond + to mipmapped (continuously downsmapled by 2x) representations + of an image at level 0 + Can be put into dictionary formatting using dict(ip) or OrderedDict(ip) + ''' + + def __keytransform__(self, key): + try: + level = int(key) + except ValueError as e: + raise RenderError("{} is not a valid mipmap level".format(key)) + if level < 0: + raise RenderError( + "{} is not a valid mipmap level (less than 0)".format(key)) + return "{}".format(level) + + def to_dict(self): + return {k: v.to_dict() for k, v in self.items()} + + @classmethod + def from_dict(cls, d): + return cls({l: MipMap(v.get('imageUrl', None), + v.get('maskUrl', None)) + for l, v in d.items()}) + + def __iter__(self): + return iter(sorted(self.store)) @property def levels(self): """list of MipMapLevels in this ImagePyramid""" - return [int(i.level) for i in self.mipMapLevels] - - def __iter__(self): - return iter([ - l for sl in [list(mmL) for mmL in self.mipMapLevels] for l in sl]) + return self.store.keys() diff --git a/renderapi/stack.py b/renderapi/stack.py index 9e57ecb3..b58cd836 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -571,13 +571,6 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, return get_json(session, request_url) -def get_z_value_for_section(stack, sectionId, **kwargs): - """DEPRECATED (use :func:`get_section_z_value`) instead""" - logger.warning( - "DEPRECATED, use renderapi.stack.get_section_z_value instead") - return get_section_z_value(stack, sectionId, **kwargs) - - # haven't fully supported this yet # @renderaccess # def put_resolved_tilespecs(stack, json_dict, host=None, port=None, diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index e4196ac1..d6953ffc 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -6,7 +6,7 @@ from .utils import NullHandler, get_json from .stack import get_z_values_for_stack from .transform import TransformList, estimate_dstpts -from .image_pyramid import MipMapLevel, ImagePyramid +from .image_pyramid import MipMap, ImagePyramid from .layout import Layout from .channel import Channel @@ -57,27 +57,19 @@ class TileSpec: a list of filters to apply to this tile (not yet implemented) mipMapLevels :obj:`list` of :obj:`MipMapLevel` :class:`MipMapLevel` objects for this tile + (DEPRECATED, use imagePyramid instead) + imagePyramid :obj:`ImagePyramid` + :class:`ImagePyramid` for this tile json : dict or None dictionary to initialize this object with (if not None overrides and ignores all keyword arguments) - scale3Url : str - uri of a mipmap level 3 image of this tile - (DEPRECATED, use mipMapLevels, but will override) - scale2Url : str - uri of a mipmap level 2 image of this tile - (DEPRECATED, use mipMapLevels, but will override) - scale1Url : str - uri of a mipmap level 1 image of this tile - (DEPRECATED, use mipMapLevels, but will override) - ''' def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, minint=0, maxint=65535, layout=None, tforms=[], - inputfilters=[], scale3Url=None, scale2Url=None, - scale1Url=None, json=None, channels=None, - mipMapLevels=[], **kwargs): + inputfilters=[], json=None, channels=None, + mipMapLevels=None, imagePyramid=None, **kwargs): if json is not None: self.from_dict(json) else: @@ -92,23 +84,24 @@ def __init__(self, tileId=None, z=None, width=None, height=None, self.inputfilters = inputfilters self.layout = Layout(**kwargs) if layout is None else layout - self.ip = ImagePyramid(mipMapLevels=mipMapLevels) + if imagePyramid is not None: + self.ip = imagePyramid + else: + if mipMapLevels is not None: + self.ip = ImagePyramid({m.level: m.mipmap + for m in mipMapLevels}) + else: + self.ip = ImagePyramid() + # legacy scaleXUrl self.maskUrl = maskUrl self.imageUrl = imageUrl - self.scale3Url = scale3Url - self.scale2Url = scale2Url - self.scale1Url = scale1Url + self.channels = channels if imageUrl is not None: - self.ip.update(MipMapLevel( - 0, imageUrl=imageUrl, maskUrl=maskUrl)) - if scale1Url is not None: - self.ip.update(MipMapLevel(1, imageUrl=scale1Url)) - if scale2Url is not None: - self.ip.update(MipMapLevel(2, imageUrl=scale2Url)) - if scale3Url is not None: - self.ip.update(MipMapLevel(3, imageUrl=scale3Url)) + self.ip[0] = MipMap( + imageUrl=imageUrl, + maskUrl=maskUrl) @property def bbox(self): @@ -146,10 +139,11 @@ def bbox_transformed(self, ndiv_inner=0, # recursively add points to the boundary while ndiv_inner > 0: - sz = 2*xy.shape[0]-1 + sz = 2 * xy.shape[0] - 1 newxy = np.zeros((sz, 2)).astype('float') newxy[0::2, :] = xy[:, :] - newxy[1:sz:2, :] = 0.5*(newxy[0:(sz-2):2, :] + newxy[2:sz:2, :]) + newxy[1:sz:2, :] = 0.5 * \ + (newxy[0:(sz - 2):2, :] + newxy[2:sz:2, :]) xy = newxy ndiv_inner -= 1 @@ -176,7 +170,7 @@ def to_dict(self): thedict['maxIntensity'] = self.maxint if self.layout is not None: thedict['layout'] = self.layout.to_dict() - thedict['mipmapLevels'] = self.ip.to_ordered_dict() + thedict['mipmapLevels'] = self.ip.to_dict() thedict['transforms'] = {} thedict['transforms']['type'] = 'list' # thedict['transforms']['specList']=[t.to_dict() for t in self.tforms] @@ -228,10 +222,9 @@ def from_dict(self, d): self.maxY = d.get('maxY', None) self.minY = d.get('minY', None) mmld = d.get('mipmapLevels', {}) - self.ip = ImagePyramid(mipMapLevels=[ - MipMapLevel( - int(l), imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) - for l, v in mmld.items()]) + self.ip = ImagePyramid({l: MipMap( + imageUrl=v.get('imageUrl'), maskUrl=v.get('maskUrl')) + for l, v in mmld.items()}) tfl = TransformList(json=d['transforms']) self.tforms = tfl.tforms diff --git a/test/test_channel.py b/test/test_channel.py index 04fc1c9a..e83d222d 100644 --- a/test/test_channel.py +++ b/test/test_channel.py @@ -15,10 +15,11 @@ def test_channel(): - mml = renderapi.image_pyramid.MipMapLevel(0, - imageUrl='file:///not/a/path', - maskUrl='file:///not/a/mask') - ip = renderapi.image_pyramid.ImagePyramid(mipMapLevels=[mml]) + mml = renderapi.image_pyramid.MipMap( + imageUrl='file:///not/a/path', + maskUrl='file:///not/a/mask') + ip = renderapi.image_pyramid.ImagePyramid() + ip[0] = mml channel = renderapi.channel.Channel(name='DAPI', maxIntensity=255, minIntensity=0, diff --git a/test/test_imagepyramid.py b/test/test_imagepyramid.py new file mode 100644 index 00000000..d9a35892 --- /dev/null +++ b/test/test_imagepyramid.py @@ -0,0 +1,98 @@ +from renderapi import image_pyramid +from renderapi.errors import RenderError +import pytest + +image_filename = "not_a_file.jpg" +alt_image_filename = "not_a_file2.jpg" +mask_filename = "not_a_mask.png" +alt_mask_filename = "not_a_mask2.png" + + +def test_basic_pyramid(): + + ip = image_pyramid.ImagePyramid() + ip[0] = image_pyramid.MipMap(imageUrl=image_filename, + maskUrl=mask_filename) + + assert(ip[0].imageUrl == image_filename) + assert(ip[0].maskUrl == mask_filename) + assert(len(ip.levels) == 1) + with pytest.raises(RenderError): + ip['not_a_level'] + + with pytest.raises(RenderError): + ip[-1] + + assert(ip.to_dict()['0']['imageUrl'] == image_filename) + + ip[1] = image_pyramid.MipMap(imageUrl=image_filename, + maskUrl=mask_filename) + assert(ip[1].imageUrl == image_filename) + + # test __setitem__ interface + ip[1]['imageUrl'] = alt_image_filename + ip[1]['maskUrl'] = alt_mask_filename + assert(ip[1].imageUrl == alt_image_filename) + assert(ip[1].maskUrl == alt_mask_filename) + + with pytest.raises(KeyError): + ip[1]['notvalid'] = 'test' + + assert(len(ip) == 2) + ip.pop(1) + assert(len(ip) == 1) + with pytest.raises(KeyError): + ip[1] + + +def test_pyramid_deserialize(): + d = { + "0": { + "imageUrl": image_filename, + "maskUrl": mask_filename + } + } + ip = image_pyramid.ImagePyramid.from_dict(d) + assert(ip[0].imageUrl == image_filename) + assert(ip[0].maskUrl == mask_filename) + + mm2 = image_pyramid.MipMap(imageUrl=image_filename, + maskUrl=mask_filename) + ip2 = image_pyramid.ImagePyramid({0: mm2}) + assert(ip == ip2) + + assert(ip[0] == d["0"]) + mm3 = image_pyramid.MipMap(imageUrl=image_filename) + ip3 = image_pyramid.ImagePyramid({0: mm3}) + assert(ip != ip3) + + assert(mm3['imageUrl'] == image_filename) + assert(mm3['maskUrl'] is None) + + with pytest.raises(KeyError): + mm3['not_a_key'] + + +def test_mipmaplevel_deprecated(): + mml = image_pyramid.MipMapLevel(0, + imageUrl=image_filename, + maskUrl=mask_filename) + assert(mml['imageUrl'] == image_filename) + assert(mml['maskUrl'] == mask_filename) + with pytest.raises(KeyError): + mml['not_a_key'] + + assert(mml == mml) + + mml2 = image_pyramid.MipMapLevel(0, + imageUrl=image_filename) + assert(mml != mml2) + assert(len([k for k, v in mml]) == 2) + + +def test_transformed_mapping(): + d = image_pyramid.TransformedDict() + d[1] = 'a' + d[2] = 'b' + mylist = [k for k in d] + assert(len(mylist) == 2) diff --git a/test/test_tilespec.py b/test/test_tilespec.py index 1e5fa696..9602c29f 100644 --- a/test/test_tilespec.py +++ b/test/test_tilespec.py @@ -2,6 +2,7 @@ from operator import eq import renderapi import rendersettings +from renderapi.image_pyramid import ImagePyramid def test_load_tilespecs_json(): @@ -33,17 +34,17 @@ def test_load_tilespecs_args(): for ts in ts_json] tilespecs = [renderapi.tilespec.TileSpec( - tileId=ts['tileId'], z=ts['z'], width=ts['width'], - height=ts['height'], mipMapLevels=[ - renderapi.tilespec.MipMapLevel( - l, d.get('imageUrl'), d.get('maskUrl')) - for l, d in ts['mipmapLevels'].items()], + tileId=ts['tileId'], + z=ts['z'], + width=ts['width'], + height=ts['height'], + imagePyramid=ImagePyramid.from_dict(ts['mipmapLevels']), layout=renderapi.tilespec.Layout( force_pixelsize=False, **ts['layout']), minint=ts['minIntensity'], maxint=ts['maxIntensity'], tforms=renderapi.transform.TransformList( json=ts['transforms']).tforms) - for ts in ts_json_expected] + for ts in ts_json_expected] assert(all(map( lambda x: eq(*x), zip([ts.to_dict() for ts in tilespecs], ts_json_expected)))) From 2e45370a20b17013904e4d88c093da4985783e9e Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 11:46:36 -0700 Subject: [PATCH 637/766] moving ThinPlateSpline into a sub folder --- renderapi/transform.py | 80 +---------------------- renderapi/transforms/ThinPlateSpline.py | 87 +++++++++++++++++++++++++ renderapi/transforms/__init__.py | 0 3 files changed, 88 insertions(+), 79 deletions(-) create mode 100644 renderapi/transforms/ThinPlateSpline.py create mode 100644 renderapi/transforms/__init__.py diff --git a/renderapi/transform.py b/renderapi/transform.py index 611e82fb..0b184ab2 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -12,6 +12,7 @@ from .errors import ConversionError, EstimationError, RenderError from .utils import NullHandler, encodeBase64, decodeBase64 +from .transforms.ThinPlateSpline import ThinPlateSplineTransform logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -1626,85 +1627,6 @@ def dataString(self): shapestring, betastring, meanstring, varstring, dimstring) -class ThinPlateSplineTransform(Transform): - """ - render-python class that can hold a dataString for - mpicbg.trakem2.transform.ThinPlateSplineTransform class. - Parameters - ---------- - dataString: str or None - data string of transformation - labels : list of str - list of labels to give this transform - json: dict or None - json compatible dictionary representation of the transformation - Returns - ------- - :class:`ThinPlateSplineTransform` - a transform instance - """ - - className = 'mpicbg.trakem2.transform.ThinPlateSplineTransform' - - def __init__(self, dataString=None, json=None, transformId=None, - labels=None): - if json is not None: - self.from_dict(json) - else: - if dataString is not None: - self._process_dataString(dataString) - self.labels = labels - self.transformId = transformId - self.className = ( - 'mpicbg.trakem2.transform.ThinPlateSplineTransform') - - def _process_dataString(self, dataString): - fields = dataString.split(" ") - - self.ndims = int(fields[1]) - self.nLm = int(fields[2]) - - if fields[3] != "null": - try: - values = decodeBase64(fields[3]) - self.aMtx = values[0:self.ndims*self.ndims].reshape( - self.ndims, self.ndims) - self.bVec = values[self.ndims*self.ndims:] - except ValueError: - raise RenderError( - "inconsistent sizes and array lengths, \ - in ThinPlateSplineTransform dataString") - else: - self.aMtx = None - self.bVec = None - - try: - values = decodeBase64(fields[4]) - self.srcPts = values[0:self.ndims*self.nLm].reshape( - self.ndims, self.nLm) - self.dMtxDat = values[self.ndims*self.nLm:].reshape( - self.ndims, self.nLm) - except ValueError: - raise RenderError( - "inconsistent sizes and array lengths, \ - in ThinPlateSplineTransform dataString") - - @property - def dataString(self): - header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) - - if self.aMtx is not None: - blk1 = np.concatenate((self.aMtx.flatten(), self.bVec)) - b64_1 = encodeBase64(blk1) - else: - b64_1 = "null" - - blk2 = np.concatenate((self.srcPts.flatten(), self.dMtxDat.flatten())) - b64_2 = encodeBase64(blk2) - - return '{} {} {}'.format(header, b64_1, b64_2) - - class NonLinearTransform(NonLinearCoordinateTransform): className = 'mpicbg.trakem2.transform.nonLinearTransform' diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py new file mode 100644 index 00000000..3c393922 --- /dev/null +++ b/renderapi/transforms/ThinPlateSpline.py @@ -0,0 +1,87 @@ +import json +import logging +from collections import Iterable +import numpy as np +from .errors import ConversionError, EstimationError, RenderError +from .utils import NullHandler, encodeBase64, decodeBase64 + + +class ThinPlateSplineTransform(Transform): + """ + render-python class that can hold a dataString for + mpicbg.trakem2.transform.ThinPlateSplineTransform class. + Parameters + ---------- + dataString: str or None + data string of transformation + labels : list of str + list of labels to give this transform + json: dict or None + json compatible dictionary representation of the transformation + Returns + ------- + :class:`ThinPlateSplineTransform` + a transform instance + """ + + className = 'mpicbg.trakem2.transform.ThinPlateSplineTransform' + + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + self.labels = labels + self.transformId = transformId + self.className = ( + 'mpicbg.trakem2.transform.ThinPlateSplineTransform') + + def _process_dataString(self, dataString): + fields = dataString.split(" ") + + self.ndims = int(fields[1]) + self.nLm = int(fields[2]) + + if fields[3] != "null": + try: + values = decodeBase64(fields[3]) + self.aMtx = values[0:self.ndims*self.ndims].reshape( + self.ndims, self.ndims) + self.bVec = values[self.ndims*self.ndims:] + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + else: + self.aMtx = None + self.bVec = None + + try: + values = decodeBase64(fields[4]) + self.srcPts = values[0:self.ndims*self.nLm].reshape( + self.ndims, self.nLm) + self.dMtxDat = values[self.ndims*self.nLm:].reshape( + self.ndims, self.nLm) + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + + @property + def dataString(self): + header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) + + if self.aMtx is not None: + blk1 = np.concatenate((self.aMtx.flatten(), self.bVec)) + b64_1 = encodeBase64(blk1) + else: + b64_1 = "null" + + blk2 = np.concatenate((self.srcPts.flatten(), self.dMtxDat.flatten())) + b64_2 = encodeBase64(blk2) + + return '{} {} {}'.format(header, b64_1, b64_2) + + diff --git a/renderapi/transforms/__init__.py b/renderapi/transforms/__init__.py new file mode 100644 index 00000000..e69de29b From 6aaa107628c45b4d247163c57f8f2be8a0c5ca16 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 11:48:48 -0700 Subject: [PATCH 638/766] fixing relative import --- renderapi/transforms/ThinPlateSpline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 3c393922..ad174413 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -2,8 +2,8 @@ import logging from collections import Iterable import numpy as np -from .errors import ConversionError, EstimationError, RenderError -from .utils import NullHandler, encodeBase64, decodeBase64 +from ..errors import ConversionError, EstimationError, RenderError +from ..utils import NullHandler, encodeBase64, decodeBase64 class ThinPlateSplineTransform(Transform): From 54f611d9b31ebcbb34f6250b406ab382f91ec9c4 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 11:50:32 -0700 Subject: [PATCH 639/766] importing Transform class --- renderapi/transforms/ThinPlateSpline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index ad174413..caa0a3bf 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -4,6 +4,7 @@ import numpy as np from ..errors import ConversionError, EstimationError, RenderError from ..utils import NullHandler, encodeBase64, decodeBase64 +from ..transform import Transform class ThinPlateSplineTransform(Transform): From dc171f14193eb4ca9bb918e08c348f6425d7941b Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 12:00:12 -0700 Subject: [PATCH 640/766] moving Transform, to avoid circular import --- renderapi/transform.py | 104 +------------------- renderapi/transforms/ThinPlateSpline.py | 2 +- renderapi/transforms/generalTransforms.py | 114 ++++++++++++++++++++++ 3 files changed, 116 insertions(+), 104 deletions(-) create mode 100644 renderapi/transforms/generalTransforms.py diff --git a/renderapi/transform.py b/renderapi/transform.py index 0b184ab2..3376a7da 100644 --- a/renderapi/transform.py +++ b/renderapi/transform.py @@ -12,6 +12,7 @@ from .errors import ConversionError, EstimationError, RenderError from .utils import NullHandler, encodeBase64, decodeBase64 +from .transforms.generalTransforms import * from .transforms.ThinPlateSpline import ThinPlateSplineTransform logger = logging.getLogger(__name__) @@ -314,109 +315,6 @@ def __iter__(self): return iter([('type', 'ref'), ('refId', self.refId)]) -class Transform(object): - """Base transformation class - - Attributes - ---------- - className : str - mpicbg java classname of this transform - dataString : str - string reprsentation of this transform as speced by - mpicbg java class library - transformId : str, optional - unique Id for this transform (optional) - """ - - def __init__(self, className=None, dataString=None, - transformId=None, labels=None, json=None): - """Initialize Transform - - Parameters - ---------- - className : str - mpicbg java classname of this transform - dataString : str - string reprsentation of this transform as speced - by mpicbg java class library - transformId : str, optional - unique Id for this transform (optional) - labels : list of str - list of labels to give this transform - json : dict - json compatible representation of this transform - (supersedes className, dataString, and transformId if not None) - """ - if json is not None: - self.from_dict(json) - else: - self.className = className - self.dataString = dataString - self.transformId = transformId - self.labels = labels - - def to_dict(self): - """serialization routine - - Returns - ------- - dict - json compatible representation of this transform - """ - d = {} - d['type'] = 'leaf' - d['className'] = self.className - d['dataString'] = self.dataString - if self.transformId is not None: - d['id'] = self.transformId - if self.labels is not None: - d['metaData'] = {'labels': self.labels} - return d - - def from_dict(self, d): - """deserialization routine - - Parameters - ---------- - d : dict - json compatible representation of this transform - """ - self.className = d['className'] - self.transformId = d.get('id', None) - self._process_dataString(d['dataString']) - md = d.get('metaData', None) - if md is not None: - self.labels = md.get('labels', None) - else: - self.labels = None - - def _process_dataString(self, datastring): - """method meant to set state of transform from datastring - generic implementation only saves datastring at self.dataString. - should rewrite for all transform classes that want to - implement tform,fit,etc - - Parameters - ---------- - dataString : str - string which can be used to initialize mpicbg transforms in java - """ - self.dataString = datastring - - def __str__(self): - return 'className:%s\ndataString:%s' % ( - self.className, self.dataString) - - def __repr__(self): - return self.__str__() - - def __eq__(self, other): - return self.__str__() == other.__str__() - - def __hash__(self): - return hash((self.__str__())) - - class AffineModel(Transform): """Linear 2d Transformation mpicbg classname: mpicbg.trakem2.transform.AffineModel2D diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index caa0a3bf..2ddbf8c4 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -4,7 +4,7 @@ import numpy as np from ..errors import ConversionError, EstimationError, RenderError from ..utils import NullHandler, encodeBase64, decodeBase64 -from ..transform import Transform +from .generalTransforms import Transform class ThinPlateSplineTransform(Transform): diff --git a/renderapi/transforms/generalTransforms.py b/renderapi/transforms/generalTransforms.py new file mode 100644 index 00000000..1bfc189a --- /dev/null +++ b/renderapi/transforms/generalTransforms.py @@ -0,0 +1,114 @@ +import json +import logging +from collections import Iterable + +import numpy as np + +from .errors import ConversionError, EstimationError, RenderError +from .utils import NullHandler, encodeBase64, decodeBase64 +from .transforms.ThinPlateSpline import ThinPlateSplineTransform + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + +class Transform(object): + """Base transformation class + + Attributes + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced by + mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + """ + + def __init__(self, className=None, dataString=None, + transformId=None, labels=None, json=None): + """Initialize Transform + + Parameters + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced + by mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + labels : list of str + list of labels to give this transform + json : dict + json compatible representation of this transform + (supersedes className, dataString, and transformId if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.className = className + self.dataString = dataString + self.transformId = transformId + self.labels = labels + + def to_dict(self): + """serialization routine + + Returns + ------- + dict + json compatible representation of this transform + """ + d = {} + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = self.dataString + if self.transformId is not None: + d['id'] = self.transformId + if self.labels is not None: + d['metaData'] = {'labels': self.labels} + return d + + def from_dict(self, d): + """deserialization routine + + Parameters + ---------- + d : dict + json compatible representation of this transform + """ + self.className = d['className'] + self.transformId = d.get('id', None) + self._process_dataString(d['dataString']) + md = d.get('metaData', None) + if md is not None: + self.labels = md.get('labels', None) + else: + self.labels = None + + def _process_dataString(self, datastring): + """method meant to set state of transform from datastring + generic implementation only saves datastring at self.dataString. + should rewrite for all transform classes that want to + implement tform,fit,etc + + Parameters + ---------- + dataString : str + string which can be used to initialize mpicbg transforms in java + """ + self.dataString = datastring + + def __str__(self): + return 'className:%s\ndataString:%s' % ( + self.className, self.dataString) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return self.__str__() == other.__str__() + + def __hash__(self): + return hash((self.__str__())) From 45d8c43e210332c2d3c1fc8fa7a2870b10bc6542 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 12:01:27 -0700 Subject: [PATCH 641/766] fixing imports --- renderapi/transforms/generalTransforms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/renderapi/transforms/generalTransforms.py b/renderapi/transforms/generalTransforms.py index 1bfc189a..7debeef4 100644 --- a/renderapi/transforms/generalTransforms.py +++ b/renderapi/transforms/generalTransforms.py @@ -4,9 +4,8 @@ import numpy as np -from .errors import ConversionError, EstimationError, RenderError -from .utils import NullHandler, encodeBase64, decodeBase64 -from .transforms.ThinPlateSpline import ThinPlateSplineTransform +from ..errors import ConversionError, EstimationError, RenderError +from ..utils import NullHandler, encodeBase64, decodeBase64 logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) From fbdd0fe660778ed8e4a7c314f4e1de3bc68732d4 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 6 Aug 2018 16:31:13 -0400 Subject: [PATCH 642/766] fixed deprecated getargspec --- renderapi/utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 8915d25a..25a34e97 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -12,6 +12,10 @@ import numpy import requests +try: + from inspect import getfullargspec +except ImportErorr: + from inspect import getargspec as getfullargspec from .errors import RenderError @@ -349,12 +353,13 @@ def fitargspec(f, oldargs, oldkwargs): kwargs with values filled in according to f spec """ try: - args, varargs, keywords, defaults = inspect.getargspec(f) - num_expected_args = len(args) - len(defaults) + arginfo = getfullargspec(f) + # args, varargs, keywords, defaults = inspect.getargspec(f) + num_expected_args = len(arginfo.args) - len(arginfo.defaults) new_args = tuple(oldargs[:num_expected_args]) new_kwargs = copy.copy(oldkwargs) for i, arg in enumerate(oldargs[num_expected_args:]): - new_kwargs.update({args[i + num_expected_args]: arg}) + new_kwargs.update({arginfo.args[i + num_expected_args]: arg}) return new_args, new_kwargs except Exception as e: logger.error('Cannot fit argspec for {}'.format(f)) From 1fea996007581606a7020ad0b6c72182a5c48927 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 13:36:23 -0700 Subject: [PATCH 643/766] adding skeleton of tform method --- renderapi/transforms/ThinPlateSpline.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 2ddbf8c4..a625114a 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -70,6 +70,28 @@ def _process_dataString(self, dataString): "inconsistent sizes and array lengths, \ in ThinPlateSplineTransform dataString") + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + result = [] + for pt in points: + result.append(self.apply(pt)) + + return np.array(result) + + def apply(self, pt): + return pt + @property def dataString(self): header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) From 822f1a6e63d172604ed8630f86115527118b8a71 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 6 Aug 2018 16:38:46 -0400 Subject: [PATCH 644/766] fixed typo --- renderapi/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 25a34e97..46d390c8 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -14,7 +14,7 @@ import requests try: from inspect import getfullargspec -except ImportErorr: +except ImportError: from inspect import getargspec as getfullargspec from .errors import RenderError From b168ffd780d3f0399eb401131d4f05202adb80b9 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 6 Aug 2018 16:39:05 -0400 Subject: [PATCH 645/766] remove unused inspect --- renderapi/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/renderapi/utils.py b/renderapi/utils.py index 46d390c8..c46b2fb1 100644 --- a/renderapi/utils.py +++ b/renderapi/utils.py @@ -4,7 +4,6 @@ ''' import tempfile import logging -import inspect import copy import json import base64 From 840c57f4267e75003b5c8295651412b5320abe3d Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 14:10:48 -0700 Subject: [PATCH 646/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 43 ++++++++++++++++++++----- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index a625114a..e9803d36 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -1,9 +1,6 @@ -import json -import logging -from collections import Iterable import numpy as np -from ..errors import ConversionError, EstimationError, RenderError -from ..utils import NullHandler, encodeBase64, decodeBase64 +from ..errors import RenderError +from ..utils import encodeBase64, decodeBase64 from .generalTransforms import Transform @@ -90,7 +87,39 @@ def tform(self, points): return np.array(result) def apply(self, pt): - return pt + if not hasattr(self, 'dMtxDat'): + result = pt + return result + + result = self.computeDeformationContribution(pt) + + return result + + def computeDeformationContribution(self, pt): + result = np.zeros(self.ndims).astype(float) + tmpDisplacement = np.zeros(self.ndims).astype(float) + di = 0 + for lnd in range(self.nLm): + tmpDisplacement = self.srcPtDisplacement(lnd, pt) + nrm = self.r2Logr( + np.linalg.norm( + tmpDisplacement)) + for d in range(self.ndims): + result[d] += nrm * self.dMtxDat[d, di] + di += 1 + return result + + def srcPtDisplacement(self, lnd, pt): + result = np.zeros_like(pt) + for d in range(self.ndims): + result[d] = self.srcPts[d][lnd] - pt[d] + return result + + def r2Logr(self, r): + nrm = 0.0 + if r > 1e-8: + nrm = r*r*np.log(r) + return nrm @property def dataString(self): @@ -106,5 +135,3 @@ def dataString(self): b64_2 = encodeBase64(blk2) return '{} {} {}'.format(header, b64_1, b64_2) - - From 8801652ac08b77131d3b7fd215aa307e32ea2d0e Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 14:18:02 -0700 Subject: [PATCH 647/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index e9803d36..74f52246 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -93,6 +93,17 @@ def apply(self, pt): result = self.computeDeformationContribution(pt) + if self.aMtx is not None: + for i in range(self.ndims): + for j in range(self.ndims): + result[i] += self.aMtx[i, j] * pt[j] + else: + result[i] += pt[j] + + if self.bVec is not None: + for i in range(self.ndims): + result[i] += self.bVec[i] + pt[i] + return result def computeDeformationContribution(self, pt): From 132e49d10de032937f46046e13078eef8309ca77 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 14:19:52 -0700 Subject: [PATCH 648/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 74f52246..9ebbf64d 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -98,7 +98,8 @@ def apply(self, pt): for j in range(self.ndims): result[i] += self.aMtx[i, j] * pt[j] else: - result[i] += pt[j] + for i in range(self.ndims): + result[i] += pt[i] if self.bVec is not None: for i in range(self.ndims): From 78ffcb79398abd0b35ce00303f4f50a70dd7ba84 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 14:53:31 -0700 Subject: [PATCH 649/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 9ebbf64d..da8e58a4 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -124,13 +124,13 @@ def computeDeformationContribution(self, pt): def srcPtDisplacement(self, lnd, pt): result = np.zeros_like(pt) for d in range(self.ndims): - result[d] = self.srcPts[d][lnd] - pt[d] + result[d] = self.srcPts[d, lnd] - pt[d] return result def r2Logr(self, r): nrm = 0.0 if r > 1e-8: - nrm = r*r*np.log(r) + nrm = r * r * np.log(r) return nrm @property From 67af6ba0555956daa98ef513af1e356b84d8dd16 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 15:03:12 -0700 Subject: [PATCH 650/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index da8e58a4..dc5d9b94 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -117,7 +117,7 @@ def computeDeformationContribution(self, pt): np.linalg.norm( tmpDisplacement)) for d in range(self.ndims): - result[d] += nrm * self.dMtxDat[d, di] + result[d] += nrm * self.dMtxDat[di, d] di += 1 return result From e231fa188b459f37e3bffaa7adb09356caf51553 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 15:06:53 -0700 Subject: [PATCH 651/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index dc5d9b94..da8e58a4 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -117,7 +117,7 @@ def computeDeformationContribution(self, pt): np.linalg.norm( tmpDisplacement)) for d in range(self.ndims): - result[d] += nrm * self.dMtxDat[di, d] + result[d] += nrm * self.dMtxDat[d, di] di += 1 return result From 9e1b72f232bf0a563f394aea3c9354da7e659a46 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 15:16:48 -0700 Subject: [PATCH 652/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index da8e58a4..8a5484c9 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -122,10 +122,10 @@ def computeDeformationContribution(self, pt): return result def srcPtDisplacement(self, lnd, pt): - result = np.zeros_like(pt) - for d in range(self.ndims): - result[d] = self.srcPts[d, lnd] - pt[d] - return result + #result = np.zeros_like(pt) + #for d in range(self.ndims): + # result[d] = self.srcPts[d, lnd] - pt[d] + return self.srcPts - np.reshape(pt,(self.ndims,1)) def r2Logr(self, r): nrm = 0.0 From 4722ef3856de5976b07c78b2c722d45d824b43f9 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 15:19:14 -0700 Subject: [PATCH 653/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 8a5484c9..116683ed 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -125,7 +125,7 @@ def srcPtDisplacement(self, lnd, pt): #result = np.zeros_like(pt) #for d in range(self.ndims): # result[d] = self.srcPts[d, lnd] - pt[d] - return self.srcPts - np.reshape(pt,(self.ndims,1)) + return self.srcPts[:, lnd] - pt def r2Logr(self, r): nrm = 0.0 From 4e1820074aec4ed47541ea7b686f8452931f3a1e Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 15:28:18 -0700 Subject: [PATCH 654/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 116683ed..e660ffab 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -112,19 +112,16 @@ def computeDeformationContribution(self, pt): tmpDisplacement = np.zeros(self.ndims).astype(float) di = 0 for lnd in range(self.nLm): - tmpDisplacement = self.srcPtDisplacement(lnd, pt) - nrm = self.r2Logr( - np.linalg.norm( - tmpDisplacement)) + tmpD = np.linalg.norm(self.srcPts[:, lnd] - pt) + nrm = 0.0 + if tmpD > 1e-8: + nrm = tmpD * tmpD * np.log(tmpD) for d in range(self.ndims): result[d] += nrm * self.dMtxDat[d, di] di += 1 return result def srcPtDisplacement(self, lnd, pt): - #result = np.zeros_like(pt) - #for d in range(self.ndims): - # result[d] = self.srcPts[d, lnd] - pt[d] return self.srcPts[:, lnd] - pt def r2Logr(self, r): From 5d13e91fd6698284718f30cc4fb00d57654146a4 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 15:35:45 -0700 Subject: [PATCH 655/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 32 ++++++++----------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index e660ffab..4181540f 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -93,43 +93,31 @@ def apply(self, pt): result = self.computeDeformationContribution(pt) - if self.aMtx is not None: - for i in range(self.ndims): - for j in range(self.ndims): - result[i] += self.aMtx[i, j] * pt[j] - else: - for i in range(self.ndims): - result[i] += pt[i] + for i in range(self.ndims): + result[i] += pt[i] - if self.bVec is not None: - for i in range(self.ndims): - result[i] += self.bVec[i] + pt[i] + #if self.aMtx is not None: + # for i in range(self.ndims): + # for j in range(self.ndims): + # result[i] += self.aMtx[i, j] * pt[j] + #if self.bVec is not None: + # for i in range(self.ndims): + # result[i] += self.bVec[i] + pt[i] return result def computeDeformationContribution(self, pt): result = np.zeros(self.ndims).astype(float) tmpDisplacement = np.zeros(self.ndims).astype(float) - di = 0 for lnd in range(self.nLm): tmpD = np.linalg.norm(self.srcPts[:, lnd] - pt) nrm = 0.0 if tmpD > 1e-8: nrm = tmpD * tmpD * np.log(tmpD) for d in range(self.ndims): - result[d] += nrm * self.dMtxDat[d, di] - di += 1 + result[d] += nrm * self.dMtxDat[d, lnd] return result - def srcPtDisplacement(self, lnd, pt): - return self.srcPts[:, lnd] - pt - - def r2Logr(self, r): - nrm = 0.0 - if r > 1e-8: - nrm = r * r * np.log(r) - return nrm - @property def dataString(self): header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) From 2e4093b6094cc0666379d73670585cf6cd887ca9 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 15:53:48 -0700 Subject: [PATCH 656/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 4181540f..f0d5d0bb 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -108,14 +108,14 @@ def apply(self, pt): def computeDeformationContribution(self, pt): result = np.zeros(self.ndims).astype(float) - tmpDisplacement = np.zeros(self.ndims).astype(float) for lnd in range(self.nLm): tmpD = np.linalg.norm(self.srcPts[:, lnd] - pt) nrm = 0.0 if tmpD > 1e-8: nrm = tmpD * tmpD * np.log(tmpD) + nrm = 1./nrm for d in range(self.ndims): - result[d] += nrm * self.dMtxDat[d, lnd] + result[d] += (nrm * self.dMtxDat[d, lnd]) return result @property From 8d9df78a3eb59d2ba49bc843eedd164798e53981 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:01:25 -0700 Subject: [PATCH 657/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index f0d5d0bb..406384a1 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -113,7 +113,6 @@ def computeDeformationContribution(self, pt): nrm = 0.0 if tmpD > 1e-8: nrm = tmpD * tmpD * np.log(tmpD) - nrm = 1./nrm for d in range(self.ndims): result[d] += (nrm * self.dMtxDat[d, lnd]) return result From 576fdb34f842ca2137b96fd02cc67461ffeaffbb Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:17:08 -0700 Subject: [PATCH 658/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 406384a1..7f76d2b2 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -114,7 +114,7 @@ def computeDeformationContribution(self, pt): if tmpD > 1e-8: nrm = tmpD * tmpD * np.log(tmpD) for d in range(self.ndims): - result[d] += (nrm * self.dMtxDat[d, lnd]) + result[d] += (nrm * self.dMtxDat[d, -lnd]) return result @property From 84c5f1bf5bc5213d71f36bf7c28d8ff209f195c8 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:25:48 -0700 Subject: [PATCH 659/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 7f76d2b2..8996f386 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -59,9 +59,9 @@ def _process_dataString(self, dataString): try: values = decodeBase64(fields[4]) self.srcPts = values[0:self.ndims*self.nLm].reshape( - self.ndims, self.nLm) + self.ndims, self.nLm, order='F') self.dMtxDat = values[self.ndims*self.nLm:].reshape( - self.ndims, self.nLm) + self.ndims, self.nLm, order='F') except ValueError: raise RenderError( "inconsistent sizes and array lengths, \ @@ -114,7 +114,7 @@ def computeDeformationContribution(self, pt): if tmpD > 1e-8: nrm = tmpD * tmpD * np.log(tmpD) for d in range(self.ndims): - result[d] += (nrm * self.dMtxDat[d, -lnd]) + result[d] += (nrm * self.dMtxDat[d, lnd]) return result @property From b48a2f49ccf30bad10532644fa105d8c858f97fb Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:27:18 -0700 Subject: [PATCH 660/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 8996f386..4a4023c9 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -59,7 +59,7 @@ def _process_dataString(self, dataString): try: values = decodeBase64(fields[4]) self.srcPts = values[0:self.ndims*self.nLm].reshape( - self.ndims, self.nLm, order='F') + self.ndims, self.nLm, order='C') self.dMtxDat = values[self.ndims*self.nLm:].reshape( self.ndims, self.nLm, order='F') except ValueError: From 9523ce8e9b4b85b8de0e071f7124ccd8743eb566 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:27:55 -0700 Subject: [PATCH 661/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 4a4023c9..9833f2c1 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -61,7 +61,7 @@ def _process_dataString(self, dataString): self.srcPts = values[0:self.ndims*self.nLm].reshape( self.ndims, self.nLm, order='C') self.dMtxDat = values[self.ndims*self.nLm:].reshape( - self.ndims, self.nLm, order='F') + self.ndims, self.nLm, order='C') except ValueError: raise RenderError( "inconsistent sizes and array lengths, \ From 0eb668819ae0d8ccf67b0facc155c1e3f99ed934 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:28:43 -0700 Subject: [PATCH 662/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 9833f2c1..4c66cf02 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -59,7 +59,7 @@ def _process_dataString(self, dataString): try: values = decodeBase64(fields[4]) self.srcPts = values[0:self.ndims*self.nLm].reshape( - self.ndims, self.nLm, order='C') + self.ndims, self.nLm, order='F') self.dMtxDat = values[self.ndims*self.nLm:].reshape( self.ndims, self.nLm, order='C') except ValueError: From ab572d8f274cb1ccea071f2921ca09551eebef76 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:33:17 -0700 Subject: [PATCH 663/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 4c66cf02..5e3a068f 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -96,13 +96,13 @@ def apply(self, pt): for i in range(self.ndims): result[i] += pt[i] - #if self.aMtx is not None: - # for i in range(self.ndims): - # for j in range(self.ndims): - # result[i] += self.aMtx[i, j] * pt[j] - #if self.bVec is not None: - # for i in range(self.ndims): - # result[i] += self.bVec[i] + pt[i] + if self.aMtx is not None: + for i in range(self.ndims): + for j in range(self.ndims): + result[i] += self.aMtx[i, j] * pt[j] + if self.bVec is not None: + for i in range(self.ndims): + result[i] += self.bVec[i] + pt[i] return result From 32de74e9ee8703c1bc0b78c85e8503c43a9cf10b Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:34:20 -0700 Subject: [PATCH 664/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 5e3a068f..a12468bf 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -102,7 +102,7 @@ def apply(self, pt): result[i] += self.aMtx[i, j] * pt[j] if self.bVec is not None: for i in range(self.ndims): - result[i] += self.bVec[i] + pt[i] + result[i] += self.bVec[i] return result From aaedc5550c5081ed52f6f94f652d568dc1a6ddfe Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:38:27 -0700 Subject: [PATCH 665/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index a12468bf..e4a78948 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -108,11 +108,14 @@ def apply(self, pt): def computeDeformationContribution(self, pt): result = np.zeros(self.ndims).astype(float) + displacements = np.linalg.norm( + self.srcPts[:, lnd] - pt, + axis=0) + for lnd in range(self.nLm): - tmpD = np.linalg.norm(self.srcPts[:, lnd] - pt) nrm = 0.0 - if tmpD > 1e-8: - nrm = tmpD * tmpD * np.log(tmpD) + if tmpD[lnd] > 1e-8: + nrm = tmpD[lnd] * tmpD[lnd] * np.log(tmpD[lnd]) for d in range(self.ndims): result[d] += (nrm * self.dMtxDat[d, lnd]) return result From fde512253f111e90a2fafd0e08b222ca32006398 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:40:04 -0700 Subject: [PATCH 666/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index e4a78948..b2525840 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -109,7 +109,8 @@ def apply(self, pt): def computeDeformationContribution(self, pt): result = np.zeros(self.ndims).astype(float) displacements = np.linalg.norm( - self.srcPts[:, lnd] - pt, + self.srcPts[:, lnd] - + pt.reshape(self.ndims, 1), axis=0) for lnd in range(self.nLm): From fe6e7b78ff7d7f25a9638640c009c7adcdf0887f Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:40:50 -0700 Subject: [PATCH 667/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index b2525840..3bbbef46 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -109,7 +109,7 @@ def apply(self, pt): def computeDeformationContribution(self, pt): result = np.zeros(self.ndims).astype(float) displacements = np.linalg.norm( - self.srcPts[:, lnd] - + self.srcPts - pt.reshape(self.ndims, 1), axis=0) From ce12aeae1f1fe0c524c8d8bbc0776dd6f2f42902 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:41:41 -0700 Subject: [PATCH 668/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 3bbbef46..f914b829 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -108,15 +108,15 @@ def apply(self, pt): def computeDeformationContribution(self, pt): result = np.zeros(self.ndims).astype(float) - displacements = np.linalg.norm( + disp = np.linalg.norm( self.srcPts - pt.reshape(self.ndims, 1), axis=0) for lnd in range(self.nLm): nrm = 0.0 - if tmpD[lnd] > 1e-8: - nrm = tmpD[lnd] * tmpD[lnd] * np.log(tmpD[lnd]) + if disp[lnd] > 1e-8: + nrm = disp[lnd] * disp[lnd] * np.log(disp[lnd]) for d in range(self.ndims): result[d] += (nrm * self.dMtxDat[d, lnd]) return result From ba2f64581d685eec380533065507351ad5bbbeae Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:45:43 -0700 Subject: [PATCH 669/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index f914b829..df7a47b2 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -112,13 +112,16 @@ def computeDeformationContribution(self, pt): self.srcPts - pt.reshape(self.ndims, 1), axis=0) + nrm = np.zeros_like(disp) + ind = disp > 1e-8 + nrm[ind] = disp[ind] * disp[ind] * np.log(disp[ind]) for lnd in range(self.nLm): - nrm = 0.0 - if disp[lnd] > 1e-8: - nrm = disp[lnd] * disp[lnd] * np.log(disp[lnd]) + #nrm = 0.0 + #if disp[lnd] > 1e-8: + # nrm = disp[lnd] * disp[lnd] * np.log(disp[lnd]) for d in range(self.ndims): - result[d] += (nrm * self.dMtxDat[d, lnd]) + result[d] += (nrm[lnd] * self.dMtxDat[d, lnd]) return result @property From 0dddf6ab0f2ba173ce9ff3eedb14486a61c9b858 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 16:50:57 -0700 Subject: [PATCH 670/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index df7a47b2..0e524d13 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -116,12 +116,11 @@ def computeDeformationContribution(self, pt): ind = disp > 1e-8 nrm[ind] = disp[ind] * disp[ind] * np.log(disp[ind]) - for lnd in range(self.nLm): - #nrm = 0.0 - #if disp[lnd] > 1e-8: - # nrm = disp[lnd] * disp[lnd] * np.log(disp[lnd]) - for d in range(self.ndims): - result[d] += (nrm[lnd] * self.dMtxDat[d, lnd]) + result = (nrm * self.dMtxDat).sum(1) + + #for lnd in range(self.nLm): + # for d in range(self.ndims): + # result[d] += (nrm[lnd] * self.dMtxDat[d, lnd]) return result @property From 27561dff71fd391adeb2cad6a6939910bfddc575 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 17:06:05 -0700 Subject: [PATCH 671/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index 0e524d13..d463d4e1 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -97,17 +97,18 @@ def apply(self, pt): result[i] += pt[i] if self.aMtx is not None: - for i in range(self.ndims): - for j in range(self.ndims): - result[i] += self.aMtx[i, j] * pt[j] + result += self.aMtx.dot(pt) + #for i in range(self.ndims): + # for j in range(self.ndims): + # result[i] += self.aMtx[i, j] * pt[j] if self.bVec is not None: - for i in range(self.ndims): - result[i] += self.bVec[i] + result += self.bVec + #for i in range(self.ndims): + # result[i] += self.bVec[i] return result def computeDeformationContribution(self, pt): - result = np.zeros(self.ndims).astype(float) disp = np.linalg.norm( self.srcPts - pt.reshape(self.ndims, 1), @@ -115,12 +116,7 @@ def computeDeformationContribution(self, pt): nrm = np.zeros_like(disp) ind = disp > 1e-8 nrm[ind] = disp[ind] * disp[ind] * np.log(disp[ind]) - result = (nrm * self.dMtxDat).sum(1) - - #for lnd in range(self.nLm): - # for d in range(self.ndims): - # result[d] += (nrm[lnd] * self.dMtxDat[d, lnd]) return result @property From f308101dcd0f042b846b4adec2149a7c4bbbc7cb Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 17:08:04 -0700 Subject: [PATCH 672/766] building up tform function call chain --- renderapi/transforms/ThinPlateSpline.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index d463d4e1..a58879c9 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -91,20 +91,11 @@ def apply(self, pt): result = pt return result - result = self.computeDeformationContribution(pt) - - for i in range(self.ndims): - result[i] += pt[i] - + result = pt + self.computeDeformationContribution(pt) if self.aMtx is not None: result += self.aMtx.dot(pt) - #for i in range(self.ndims): - # for j in range(self.ndims): - # result[i] += self.aMtx[i, j] * pt[j] if self.bVec is not None: result += self.bVec - #for i in range(self.ndims): - # result[i] += self.bVec[i] return result From b322a9966c4e8c39fcaf8ec5973b2dbae57fe0a8 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 17:40:09 -0700 Subject: [PATCH 673/766] including 'F' ordering in process_dataString' --- renderapi/transforms/ThinPlateSpline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transforms/ThinPlateSpline.py index a58879c9..567318e3 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transforms/ThinPlateSpline.py @@ -120,7 +120,7 @@ def dataString(self): else: b64_1 = "null" - blk2 = np.concatenate((self.srcPts.flatten(), self.dMtxDat.flatten())) + blk2 = np.concatenate((self.srcPts.flatten(order='F'), self.dMtxDat.flatten(order='C'))) b64_2 = encodeBase64(blk2) return '{} {} {}'.format(header, b64_1, b64_2) From 5bc0b04c95ce279935524a7899d347732c0efd04 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 17:45:57 -0700 Subject: [PATCH 674/766] fixing up test to pass. Was failing on '@' character, which python never generates, but java does --- test/test_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_transform.py b/test/test_transform.py index 4b3e620f..6720a9ae 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -606,7 +606,7 @@ def test_thinplatespline(): j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) t = renderapi.transform.ThinPlateSplineTransform( dataString=j['dataString']) - assert (j['dataString'] == t.dataString) + assert (j['dataString'].split(' ')[-1] == t.dataString.split(' ')[-1]) t.aMtx = np.zeros(4) t.bVec = np.zeros(2) t2 = renderapi.transform.ThinPlateSplineTransform( From 11412c10827a41ef0be033828a062284b7f31b02 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 21:07:33 -0700 Subject: [PATCH 675/766] increasing codecov on transforms --- test/test_transform.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/test_transform.py b/test/test_transform.py index 6720a9ae..e053d6ed 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -23,6 +23,8 @@ def test_TransformList_init(): def test_simple_TransformList_init(): aff = renderapi.transform.AffineModel() tlist = renderapi.transform.TransformList(tforms=[aff]) # noqa: F841 + j = tlist.to_json() + assert(isinstance(j, dict)) def test_fail_TransformList_init(): @@ -88,6 +90,16 @@ def test_affine_rot_90(): print(str(am)) +def test_affine_fail(): + am = renderapi.transform.AffineModel() + # setup a 90 degree clockwise rotation + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float) + points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float) + # catch error + with pytest.raises(renderapi.errors.EstimationError): + am.estimate(points_in, points_out[0:-2, :]) + + def test_affine_random(): am = renderapi.transform.AffineModel(M00=.9, M10=-0.2, @@ -148,6 +160,13 @@ def noscipy_import(name, globals=None, locals=None, derived_pt = renderapi.transform.Polynomial2DTransform( src=srcpts, dst=dstpts) assert(np.allclose(derived_pt.params, default_pt.params)) + assert(not default_pt.is_affine()) + + with pytest.raises(renderapi.errors.EstimationError): + derived_pt.estimate(srcpts, dstpts[0:-2, :]) + + with pytest.raises(renderapi.errors.EstimationError): + derived_pt.estimate(srcpts, dstpts[0:-2, :], order=500) if use_numpy: builtins.__import__ = realimport @@ -380,6 +399,8 @@ def estimate_homography_transform( dst_pts = target_tform.tform(src_pts) tform = transformclass() tform.estimate(src_pts, dst_pts, return_params=False) + M = tform.estimate(src_pts, dst_pts, return_params=True) + assert(M.shape == (2,3)) assert np.allclose(target_tform.M, tform.M) if do_scale: From 55f63c0f7c8498c8c9d8612c1eac2337b6adead0 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 21:14:00 -0700 Subject: [PATCH 676/766] adding test on tform() for TPS --- test/test_transform.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_transform.py b/test/test_transform.py index e053d6ed..f43d51c3 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -645,6 +645,13 @@ def test_thinplatespline(): t3 = renderapi.transform.ThinPlateSplineTransform( dataString=" ".join(s)) + x = np.linspace(0,3840,10) + xt, yt = np.meshgrid(x,x) + src_pts = np.transpose( + np.vstack((xt.flatten(), yt.flatten()))) + dst_pts = t.tform(src_pts) + assert(dst_pts.shape == src_pts.shape) + def test_encode64(): # case for Stephan's '@' character From b3c3b143fafc65f17fc453440c9b8d20f35014cd Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 21:15:09 -0700 Subject: [PATCH 677/766] adding test on tform() for TPS --- test/test_transform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_transform.py b/test/test_transform.py index f43d51c3..58f29839 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -400,7 +400,7 @@ def estimate_homography_transform( tform = transformclass() tform.estimate(src_pts, dst_pts, return_params=False) M = tform.estimate(src_pts, dst_pts, return_params=True) - assert(M.shape == (2,3)) + assert(M.shape == (2, 3)) assert np.allclose(target_tform.M, tform.M) if do_scale: @@ -645,8 +645,8 @@ def test_thinplatespline(): t3 = renderapi.transform.ThinPlateSplineTransform( dataString=" ".join(s)) - x = np.linspace(0,3840,10) - xt, yt = np.meshgrid(x,x) + x = np.linspace(0, 3840, 10) + xt, yt = np.meshgrid(x, x) src_pts = np.transpose( np.vstack((xt.flatten(), yt.flatten()))) dst_pts = t.tform(src_pts) From 62c24f7ad753831f6fa4ffcac888fffb2aaec460 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 21:18:31 -0700 Subject: [PATCH 678/766] fixing up tests --- test/test_transform.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_transform.py b/test/test_transform.py index 58f29839..db01bc5e 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -23,7 +23,9 @@ def test_TransformList_init(): def test_simple_TransformList_init(): aff = renderapi.transform.AffineModel() tlist = renderapi.transform.TransformList(tforms=[aff]) # noqa: F841 - j = tlist.to_json() + js = tlist.to_json() + assert(isinstance(js, str)) + j = json.loads(js) assert(isinstance(j, dict)) @@ -160,7 +162,7 @@ def noscipy_import(name, globals=None, locals=None, derived_pt = renderapi.transform.Polynomial2DTransform( src=srcpts, dst=dstpts) assert(np.allclose(derived_pt.params, default_pt.params)) - assert(not default_pt.is_affine()) + assert(not default_pt.is_affine) with pytest.raises(renderapi.errors.EstimationError): derived_pt.estimate(srcpts, dstpts[0:-2, :]) From 36989a4f27f5f7bbba5bc472ae65baa327bf45f2 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 21:28:40 -0700 Subject: [PATCH 679/766] fixing up tests --- test/test_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_transform.py b/test/test_transform.py index db01bc5e..4e02dbee 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -402,7 +402,7 @@ def estimate_homography_transform( tform = transformclass() tform.estimate(src_pts, dst_pts, return_params=False) M = tform.estimate(src_pts, dst_pts, return_params=True) - assert(M.shape == (2, 3)) + assert(M.shape == (3, 3)) assert np.allclose(target_tform.M, tform.M) if do_scale: From 82e93fd5930966479459b901cb9f57a1b63846e6 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 6 Aug 2018 21:42:15 -0700 Subject: [PATCH 680/766] fixing up tests --- test/test_transform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_transform.py b/test/test_transform.py index 4e02dbee..37cf7b2a 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -651,6 +651,8 @@ def test_thinplatespline(): xt, yt = np.meshgrid(x, x) src_pts = np.transpose( np.vstack((xt.flatten(), yt.flatten()))) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) dst_pts = t.tform(src_pts) assert(dst_pts.shape == src_pts.shape) From 09b99d465e32f290480a8c3cd6982ccb0cdeb4c2 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 08:26:51 -0700 Subject: [PATCH 681/766] reorg transform module --- .../ThinPlateSpline.py | 6 +- renderapi/transform/__init__.py | 2 + renderapi/{ => transform}/transform.py | 107 ++++++++++++++++- renderapi/transforms/__init__.py | 0 renderapi/transforms/generalTransforms.py | 113 ------------------ 5 files changed, 110 insertions(+), 118 deletions(-) rename renderapi/{transforms => transform}/ThinPlateSpline.py (96%) create mode 100644 renderapi/transform/__init__.py rename renderapi/{ => transform}/transform.py (91%) delete mode 100644 renderapi/transforms/__init__.py delete mode 100644 renderapi/transforms/generalTransforms.py diff --git a/renderapi/transforms/ThinPlateSpline.py b/renderapi/transform/ThinPlateSpline.py similarity index 96% rename from renderapi/transforms/ThinPlateSpline.py rename to renderapi/transform/ThinPlateSpline.py index 567318e3..e3a31b26 100644 --- a/renderapi/transforms/ThinPlateSpline.py +++ b/renderapi/transform/ThinPlateSpline.py @@ -1,7 +1,7 @@ import numpy as np from ..errors import RenderError from ..utils import encodeBase64, decodeBase64 -from .generalTransforms import Transform +from .transform import Transform class ThinPlateSplineTransform(Transform): @@ -120,7 +120,9 @@ def dataString(self): else: b64_1 = "null" - blk2 = np.concatenate((self.srcPts.flatten(order='F'), self.dMtxDat.flatten(order='C'))) + blk2 = np.concatenate(( + self.srcPts.flatten(order='F'), + self.dMtxDat.flatten(order='C'))) b64_2 = encodeBase64(blk2) return '{} {} {}'.format(header, b64_1, b64_2) diff --git a/renderapi/transform/__init__.py b/renderapi/transform/__init__.py new file mode 100644 index 00000000..69d16ba6 --- /dev/null +++ b/renderapi/transform/__init__.py @@ -0,0 +1,2 @@ +import * from transform +import * from ThinPlateSpline diff --git a/renderapi/transform.py b/renderapi/transform/transform.py similarity index 91% rename from renderapi/transform.py rename to renderapi/transform/transform.py index 3376a7da..4a34c57b 100644 --- a/renderapi/transform.py +++ b/renderapi/transform/transform.py @@ -11,9 +11,7 @@ import numpy as np from .errors import ConversionError, EstimationError, RenderError -from .utils import NullHandler, encodeBase64, decodeBase64 -from .transforms.generalTransforms import * -from .transforms.ThinPlateSpline import ThinPlateSplineTransform +from .utils import NullHandler logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -28,6 +26,109 @@ from numpy.linalg.linalg import LinAlgError +class Transform(object): + """Base transformation class + + Attributes + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced by + mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + """ + + def __init__(self, className=None, dataString=None, + transformId=None, labels=None, json=None): + """Initialize Transform + + Parameters + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced + by mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + labels : list of str + list of labels to give this transform + json : dict + json compatible representation of this transform + (supersedes className, dataString, and transformId if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.className = className + self.dataString = dataString + self.transformId = transformId + self.labels = labels + + def to_dict(self): + """serialization routine + + Returns + ------- + dict + json compatible representation of this transform + """ + d = {} + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = self.dataString + if self.transformId is not None: + d['id'] = self.transformId + if self.labels is not None: + d['metaData'] = {'labels': self.labels} + return d + + def from_dict(self, d): + """deserialization routine + + Parameters + ---------- + d : dict + json compatible representation of this transform + """ + self.className = d['className'] + self.transformId = d.get('id', None) + self._process_dataString(d['dataString']) + md = d.get('metaData', None) + if md is not None: + self.labels = md.get('labels', None) + else: + self.labels = None + + def _process_dataString(self, datastring): + """method meant to set state of transform from datastring + generic implementation only saves datastring at self.dataString. + should rewrite for all transform classes that want to + implement tform,fit,etc + + Parameters + ---------- + dataString : str + string which can be used to initialize mpicbg transforms in java + """ + self.dataString = datastring + + def __str__(self): + return 'className:%s\ndataString:%s' % ( + self.className, self.dataString) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return self.__str__() == other.__str__() + + def __hash__(self): + return hash((self.__str__())) + + class TransformList: """A list of Transforms diff --git a/renderapi/transforms/__init__.py b/renderapi/transforms/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/renderapi/transforms/generalTransforms.py b/renderapi/transforms/generalTransforms.py deleted file mode 100644 index 7debeef4..00000000 --- a/renderapi/transforms/generalTransforms.py +++ /dev/null @@ -1,113 +0,0 @@ -import json -import logging -from collections import Iterable - -import numpy as np - -from ..errors import ConversionError, EstimationError, RenderError -from ..utils import NullHandler, encodeBase64, decodeBase64 - -logger = logging.getLogger(__name__) -logger.addHandler(NullHandler()) - -class Transform(object): - """Base transformation class - - Attributes - ---------- - className : str - mpicbg java classname of this transform - dataString : str - string reprsentation of this transform as speced by - mpicbg java class library - transformId : str, optional - unique Id for this transform (optional) - """ - - def __init__(self, className=None, dataString=None, - transformId=None, labels=None, json=None): - """Initialize Transform - - Parameters - ---------- - className : str - mpicbg java classname of this transform - dataString : str - string reprsentation of this transform as speced - by mpicbg java class library - transformId : str, optional - unique Id for this transform (optional) - labels : list of str - list of labels to give this transform - json : dict - json compatible representation of this transform - (supersedes className, dataString, and transformId if not None) - """ - if json is not None: - self.from_dict(json) - else: - self.className = className - self.dataString = dataString - self.transformId = transformId - self.labels = labels - - def to_dict(self): - """serialization routine - - Returns - ------- - dict - json compatible representation of this transform - """ - d = {} - d['type'] = 'leaf' - d['className'] = self.className - d['dataString'] = self.dataString - if self.transformId is not None: - d['id'] = self.transformId - if self.labels is not None: - d['metaData'] = {'labels': self.labels} - return d - - def from_dict(self, d): - """deserialization routine - - Parameters - ---------- - d : dict - json compatible representation of this transform - """ - self.className = d['className'] - self.transformId = d.get('id', None) - self._process_dataString(d['dataString']) - md = d.get('metaData', None) - if md is not None: - self.labels = md.get('labels', None) - else: - self.labels = None - - def _process_dataString(self, datastring): - """method meant to set state of transform from datastring - generic implementation only saves datastring at self.dataString. - should rewrite for all transform classes that want to - implement tform,fit,etc - - Parameters - ---------- - dataString : str - string which can be used to initialize mpicbg transforms in java - """ - self.dataString = datastring - - def __str__(self): - return 'className:%s\ndataString:%s' % ( - self.className, self.dataString) - - def __repr__(self): - return self.__str__() - - def __eq__(self, other): - return self.__str__() == other.__str__() - - def __hash__(self): - return hash((self.__str__())) From 7a97cec6b221ba411a4e302f6bf58bbad2452e4d Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 08:35:36 -0700 Subject: [PATCH 682/766] fixing import statement --- renderapi/transform/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform/__init__.py b/renderapi/transform/__init__.py index 69d16ba6..4730f03e 100644 --- a/renderapi/transform/__init__.py +++ b/renderapi/transform/__init__.py @@ -1,2 +1,2 @@ -import * from transform -import * from ThinPlateSpline +from transform import * +from ThinPlateSpline import * From 3ac17becb979bb577f2abd5c25a7fdd78fe26018 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 08:43:03 -0700 Subject: [PATCH 683/766] fixing relative import --- renderapi/transform/transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform/transform.py b/renderapi/transform/transform.py index 4a34c57b..4f7c1ad3 100644 --- a/renderapi/transform/transform.py +++ b/renderapi/transform/transform.py @@ -10,8 +10,8 @@ import numpy as np -from .errors import ConversionError, EstimationError, RenderError -from .utils import NullHandler +from ..errors import ConversionError, EstimationError, RenderError +from ..utils import NullHandler logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) From c5ed31f5641522d8a021917275b11903ae80fe28 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 08:51:33 -0700 Subject: [PATCH 684/766] relative import for python 3? --- renderapi/transform/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform/__init__.py b/renderapi/transform/__init__.py index 4730f03e..38951923 100644 --- a/renderapi/transform/__init__.py +++ b/renderapi/transform/__init__.py @@ -1,2 +1,2 @@ -from transform import * -from ThinPlateSpline import * +from .transform import * +from .ThinPlateSpline import * From 3da5b7ee6a0a5175b35601cfaf2c5dfde62f0776 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 10:18:32 -0700 Subject: [PATCH 685/766] adding another .transform for svd checking --- test/test_transform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_transform.py b/test/test_transform.py index 37cf7b2a..996ec5b4 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -148,8 +148,8 @@ def noscipy_import(name, globals=None, locals=None, cross_py23_reload(renderapi.transform) - assert(renderapi.transform.svd is np.linalg.svd - if use_numpy else renderapi.transform.svd is scipy.linalg.svd) + assert(renderapi.transform.transform.svd is np.linalg.svd + if use_numpy else renderapi.transform.transform.svd is scipy.linalg.svd) datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' @@ -173,7 +173,7 @@ def noscipy_import(name, globals=None, locals=None, if use_numpy: builtins.__import__ = realimport cross_py23_reload(renderapi.transform) - assert(renderapi.transform.svd is scipy.linalg.svd) + assert(renderapi.transform.transform.svd is scipy.linalg.svd) def test_Polynomial_estimation_numpy(): From b5758dd0537984b0b3d900ecdbf178f5a1fd4b29 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 10:19:27 -0700 Subject: [PATCH 686/766] adding another .transform for svd checking --- test/test_transform.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_transform.py b/test/test_transform.py index 996ec5b4..1bbc08f0 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -149,7 +149,8 @@ def noscipy_import(name, globals=None, locals=None, cross_py23_reload(renderapi.transform) assert(renderapi.transform.transform.svd is np.linalg.svd - if use_numpy else renderapi.transform.transform.svd is scipy.linalg.svd) + if use_numpy else + renderapi.transform.transform.svd is scipy.linalg.svd) datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' From 7ea28c5a70de51698d6a5525dd1fdcf0e22268b6 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 10:36:38 -0700 Subject: [PATCH 687/766] fixing with cross_py23 reloads --- test/test_transform.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_transform.py b/test/test_transform.py index 1bbc08f0..8bdf6dd0 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -146,7 +146,7 @@ def noscipy_import(name, globals=None, locals=None, return realimport(name, globals, locals, fromlist, level) builtins.__import__ = noscipy_import - cross_py23_reload(renderapi.transform) + cross_py23_reload(renderapi.transform.transform) assert(renderapi.transform.transform.svd is np.linalg.svd if use_numpy else @@ -173,8 +173,9 @@ def noscipy_import(name, globals=None, locals=None, if use_numpy: builtins.__import__ = realimport - cross_py23_reload(renderapi.transform) + cross_py23_reload(renderapi.transform.transform) assert(renderapi.transform.transform.svd is scipy.linalg.svd) + cross_py23_reload(renderapi.transform) def test_Polynomial_estimation_numpy(): From 9b6f44da5daa8f493209a6ff216420442334ac6d Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 11:59:45 -0700 Subject: [PATCH 688/766] splitting up transform even further --- renderapi/transform/AffineModels.py | 628 +++++++++ renderapi/transform/PolynomialModels.py | 577 +++++++++ renderapi/transform/__init__.py | 3 + renderapi/transform/transform.py | 1567 ----------------------- renderapi/transform/utils.py | 382 ++++++ test/test_transform.py | 10 +- 6 files changed, 1595 insertions(+), 1572 deletions(-) create mode 100644 renderapi/transform/AffineModels.py create mode 100644 renderapi/transform/PolynomialModels.py create mode 100644 renderapi/transform/utils.py diff --git a/renderapi/transform/AffineModels.py b/renderapi/transform/AffineModels.py new file mode 100644 index 00000000..598c1d6d --- /dev/null +++ b/renderapi/transform/AffineModels.py @@ -0,0 +1,628 @@ +from .transform import Transform +import numpy as np +from ..errors import ConversionError, EstimationError + +try: + from scipy.linalg import svd, LinAlgError +except ImportError as e: + logger.info(e) + logger.info('scipy-based linalg may or may not lead ' + 'to better parameter fitting') + from numpy.linalg import svd + from numpy.linalg.linalg import LinAlgError + + +class AffineModel(Transform): + """Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + + className = 'mpicbg.trakem2.transform.AffineModel2D' + + def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, + transformId=None, labels=None, json=None): + """Initialize AffineModel, defaulting to identity + + Parameters + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str + unique transformId for this transform (optional) + labels : list of str + list of labels to give this transform + json : dict + json compatible representation of this transform + (will supersede all other parameters if not None) + + """ + if json is not None: + self.from_dict(json) + else: + self.M00 = M00 + self.M01 = M01 + self.M10 = M10 + self.M11 = M11 + self.B0 = B0 + self.B1 = B1 + self.className = 'mpicbg.trakem2.transform.AffineModel2D' + self.labels = labels + self.load_M() + self.transformId = transformId + + @property + def dataString(self): + """dataString string for this transform""" + return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( + self.M[0, 0], self.M[1, 0], self.M[0, 1], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + + def _process_dataString(self, datastring): + """generate datastring and param attributes from datastring""" + dsList = datastring.split() + self.M00 = float(dsList[0]) + self.M10 = float(dsList[1]) + self.M01 = float(dsList[2]) + self.M11 = float(dsList[3]) + self.B0 = float(dsList[4]) + self.B1 = float(dsList[5]) + self.load_M() + + def load_M(self): + """method to take the attribute of self and fill in self.M""" + self.M = np.identity(3, np.double) + self.M[0, 0] = self.M00 + self.M[0, 1] = self.M01 + self.M[1, 0] = self.M10 + self.M[1, 1] = self.M11 + self.M[0, 2] = self.B0 + self.M[1, 2] = self.B1 + + @staticmethod + def fit(A, B): + """function to fit this transform given the corresponding sets of points A & B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + N = A.shape[0] # total points + + M = np.zeros((2 * N, 6)) + Y = np.zeros((2 * N, 1)) + for i in range(N): + M[2 * i, :] = [A[i, 0], A[i, 1], 0, 0, 1, 0] + M[2 * i + 1, :] = [0, 0, A[i, 0], A[i, 1], 0, 1] + Y[2 * i] = B[i, 0] + Y[2 * i + 1] = B[i, 1] + + (Tvec, residuals, rank, s) = np.linalg.lstsq(M, Y) + return Tvec + + def estimate(self, A, B, return_params=True, **kwargs): + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the parameter matrix + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + Tvec = self.fit(A, B, **kwargs) + self.M00 = Tvec[0, 0] + self.M10 = Tvec[2, 0] + self.M01 = Tvec[1, 0] + self.M11 = Tvec[3, 0] + self.B0 = Tvec[4, 0] + self.B1 = Tvec[5, 0] + self.load_M() + if return_params: + return self.M + + def concatenate(self, model): + """concatenate a model to this model -- ported from trakEM2 below: + :: + + final double a00 = m00 * model.m00 + m01 * model.m10; + final double a01 = m00 * model.m01 + m01 * model.m11; + final double a02 = m00 * model.m02 + m01 * model.m12 + m02; + + final double a10 = m10 * model.m00 + m11 * model.m10; + final double a11 = m10 * model.m01 + m11 * model.m11; + final double a12 = m10 * model.m02 + m11 * model.m12 + m12; + + Parameters + ---------- + model : AffineModel + model to concatenate to this one + + Returns + ------- + AffineModel + model after concatenating model with this model + """ + a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] + a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] + a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + + self.M[0, 2]) + + a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] + a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] + a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + + self.M[1, 2]) + + newmodel = AffineModel(a00, a01, a10, a11, a02, a12) + return newmodel + + def invert(self): + """return an inverted version of this transformation + + Returns + ------- + AffineModel + an inverted version of this transformation + """ + inv_M = np.linalg.inv(self.M) + Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], + inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) + return Ai + + @staticmethod + def convert_to_point_vector(points): + """method to help reshape x,y points to x,y,1 vectors + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx3 array of x,y,1 points used for transformations + """ + Np = points.shape[0] + onevec = np.ones((Np, 1), np.double) + + if points.shape[1] != 2: + raise ConversionError('Points must be of shape (:, 2) ' + '-- got {}'.format(points.shape)) + Nd = 2 + points = np.concatenate((points, onevec), axis=1) + return points, Nd + + @staticmethod + def convert_points_vector_to_array(points, Nd=2): + """method for convertion x,y,K points to x,y vectors + + Parameters + ---------- + points : numpy.array + a Nx3 vector of points after transformation + Nd : int + the number of dimensions to cutoff (should be 2) + + Returns + ------- + numpy.array: a Nx2 array of x,y points + """ + points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T + return points + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(self.M, points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + def inverse_tform(self, points): + """transform a set of points through the inverse of this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after inverse transformation + """ + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(np.linalg.inv(self.M), points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + @property + def scale(self): + """tuple of scale for x, y""" + return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) + for j in range(self.M.shape[1])])[:2] + + @property + def shear(self): + """counter-clockwise shear angle""" + return np.arctan2(-self.M[0, 1], self.M[1, 1]) - self.rotation + + @property + def translation(self): + """tuple of translation in x, y""" + return tuple(self.M[:2, 2]) + + @property + def rotation(self): + """counter-clockwise rotation""" + return np.arctan2(self.M[1, 0], self.M[0, 0]) + + def __str__(self): + return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( + self.M[0, 0], self.M[0, 1], self.M[1, 0], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + + +class TranslationModel(AffineModel): + """Translation fitting and estimation as an :class:`AffineModel` + Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + """ + + className = 'mpicbg.trakem2.transform.TranslationModel2D' + + def __init__(self, *args, **kwargs): + super(TranslationModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected dataString is 'tx ty'""" + tx, ty = map(float, dataString.split(' ')) + self.B0 = tx + self.B1 = ty + self.M00 = 1 + self.M10 = 0 + self.M01 = 0 + self.M11 = 1 + self.load_M() + + @staticmethod + def fit(src, dst): + """function to fit Translation transform given + the corresponding sets of points src & dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + t = dst.mean(axis=0) - src.mean(axis=0) + T = np.eye(3) + T[:2, 2] = t + return T + + def estimate(self, src, dst, return_params=True): + """method for setting this transformation with the best fit + given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + self.M = self.fit(src, dst) + if return_params: + return self.M + + +class RigidModel(AffineModel): + """model for fitting Rigid only transformations + (rotation+translation) + or + (determinate=1, orthonormal eigenvectors) + implemented as an :class:`AffineModel` + + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + className = 'mpicbg.trakem2.transform.RigidModel2D' + + def __init__(self, *args, **kwargs): + super(RigidModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected datastring is 'theta tx ty'""" + theta, tx, ty = map(float, dataString.split(' ')) + self.M00 = np.cos(theta) + self.M01 = -np.sin(theta) + self.M10 = np.sin(theta) + self.M11 = np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() + + @staticmethod + def fit(src, dst, rigid=True, **kwargs): + """function to fit this transform given the corresponding + sets of points src & dst + Umeyama estimation of similarity transformation + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + # TODO shape assertion + num, dim = src.shape + src_cld = src - src.mean(axis=0) + dst_cld = dst - dst.mean(axis=0) + A = np.dot(dst_cld.T, src_cld) / num + d = np.ones((dim, ), dtype=np.double) + if np.linalg.det(A) < 0: + d[dim - 1] = -1 + T = np.eye(dim + 1, dtype=np.double) + + rank = np.linalg.matrix_rank(A) + if rank == 0: + raise EstimationError('zero rank matrix A unacceptable -- ' + 'likely poorly conditioned') + + U, S, V = svd(A) + + if rank == dim - 1: + if np.linalg.det(U) * np.linalg.det(V) > 0: + T[:dim, :dim] = np.dot(U, V) + else: + s = d[dim - 1] + d[dim - 1] = -1 + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V)) + d[dim - 1] = s + else: + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V.T)) + + fit_scale = (1.0 if rigid else + 1.0 / src_cld.var(axis=0).sum() * np.dot(S, d)) + + T[:dim, dim] = dst.mean(axis=0) - fit_scale * np.dot( + T[:dim, :dim], src.mean(axis=0).T) + T[:dim, :dim] *= fit_scale + return T + + def estimate(self, A, B, return_params=True, **kwargs): + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + self.M = self.fit(A, B, **kwargs) + if return_params: + return self.M + + +class SimilarityModel(RigidModel): + """class for fitting Similarity transformations + (translation+rotation+scaling) + or + (orthogonal eigen vectors with equal eigenvalues) + + implemented as an :class:`AffineModel` + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + className = 'mpicbg.trakem2.transform.SimilarityModel2D' + + def __init__(self, *args, **kwargs): + super(SimilarityModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected datastring is 's theta tx ty'""" + s, theta, tx, ty = map(float, dataString.split(' ')) + self.M00 = s * np.cos(theta) + self.M01 = -s * np.sin(theta) + self.M10 = s * np.sin(theta) + self.M11 = s * np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() + + @staticmethod + def fit(src, dst, rigid=False, **kwargs): + """function to fit this transform given the corresponding + sets of points src & dst + Umeyama estimation of similarity transformation + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + return RigidModel.fit(src, dst, rigid=rigid) diff --git a/renderapi/transform/PolynomialModels.py b/renderapi/transform/PolynomialModels.py new file mode 100644 index 00000000..d69477b9 --- /dev/null +++ b/renderapi/transform/PolynomialModels.py @@ -0,0 +1,577 @@ +from .transform import Transform, logger +from .AffineModels import AffineModel +import numpy as np +from ..errors import ConversionError, EstimationError, RenderError + +try: + from scipy.linalg import svd, LinAlgError +except ImportError as e: + logger.info(e) + logger.info('scipy-based linalg may or may not lead ' + 'to better parameter fitting') + from numpy.linalg import svd + from numpy.linalg.linalg import LinAlgError + + +class Polynomial2DTransform(Transform): + """Polynomial2DTransform implemented as in skimage + + Attributes + ---------- + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + + """ + className = 'mpicbg.trakem2.transform.PolynomialTransform2D' + + def __init__(self, dataString=None, src=None, dst=None, order=2, + force_polynomial=True, params=None, identity=False, + labels=None, transformId=None, json=None, **kwargs): + """Initialize Polynomial2DTransform + This provides 5 different ways to initialize the transform which are + mutually exclusive and applied in the order specified here. + 1)json2)dataString,3)identity,4)params,5)(src,dst) + + Parameters + ---------- + json : dict + dictionary representation of the Polynomial2DTransform + generally used by TransformList + dataString : str + dataString representation of transform from mpicpg + identity : bool + whether to make this transform the identity + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + src : numpy.array + Nx2 array of source points to use for fitting (used with dst) + dst : numpy.array + Nx2 array of destination points to use for fitting (used with src) + order : int + degree of polynomial to store + force_polynomial : bool + whether to force this representation to return a Polynomial + regardless of degree (not implemented) + + + """ + if json is not None: + self.from_dict(json) + else: + self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' + if dataString is not None: + self._process_dataString(dataString) + elif identity: + self.params = np.array([[0, 1, 0], [0, 0, 1]]) + elif params is not None: + self.params = params + elif src is not None and dst is not None: + self.estimate(src, dst, order, return_params=False, **kwargs) + + if not force_polynomial and self.is_affine: + raise NotImplementedError('Falling back to Affine model is ' + 'not supported {}') + self.transformId = transformId + self.labels = labels + + @property + def is_affine(self): + """(boolean) TODO allow default to Affine""" + return False + # return self.order + + @property + def order(self): + """(int) order of polynomial""" + no_coeffs = len(self.params.ravel()) + return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) + + @property + def dataString(self): + """dataString of polynomial""" + return Polynomial2DTransform._dataStringfromParams(self.params) + + @staticmethod + def fit(src, dst, order=2): + """function to fit this transform given the corresponding sets + of points src & dst + polynomial fit + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : bool + order of polynomial to fit + + Returns + ------- + numpy.array + a [2,(order+1)*(order+2)/2] array with the best fit parameters + """ + xs = src[:, 0] + ys = src[:, 1] + xd = dst[:, 0] + yd = dst[:, 1] + rows = src.shape[0] + no_coeff = (order + 1) * (order + 2) + + if len(src) != len(dst): + raise EstimationError( + 'source has {} points, but dest has {}!'.format( + len(src), len(dst))) + if no_coeff > len(src): + raise EstimationError( + 'order {} is too large to fit {} points!'.format( + order, len(src))) + + A = np.zeros([rows * 2, no_coeff + 1]) + pidx = 0 + for j in range(order + 1): + for i in range(j + 1): + A[:rows, pidx] = xs ** (j - i) * ys ** i + A[rows:, pidx + no_coeff // 2] = xs ** (j - i) * ys ** i + pidx += 1 + + A[:rows, -1] = xd + A[rows:, -1] = yd + + # right singular vector corresponding to smallest singular value + _, s, V = svd(A) + Vsm = V[np.argmin(s), :] # never trust computers + return (-Vsm[:-1] / Vsm[-1]).reshape((2, no_coeff // 2)) + + def estimate(self, src, dst, order=2, + test_coords=True, max_tries=100, return_params=True, + **kwargs): + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : int + order of polynomial to fit + test_coords : bool + whether to test model after fitting to + make sure it is good (see fitgood) + max_tries : int + how many times to attempt to fit the model (see fitgood) + return_params : bool + whether to return the parameter matrix + **kwargs + dictionary of keyword arguments including those + that can be passed to fitgood + + Returns + ------- + numpy.array + a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix + (or None if return_params=False) + """ + def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): + """check if model produces a 'good' result + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + params : numpy.array + a Kx2 matrix of parameters + atol : float + absolute tolerance as in numpy.allclose for + transformed sample points + rtol : float + relative tolerance as in numpy.allclose for + transformed sample points + + Returns + ------- + bool + whether the goodness condition is met + """ + result = Polynomial2DTransform(params=params).tform(src) + t = np.allclose( + result, dst, + atol=atol, rtol=rtol) + return t + + estimated = False + tries = 0 + while (tries < max_tries and not estimated): + tries += 1 + try: + params = Polynomial2DTransform.fit(src, dst, order=order) + except (LinAlgError, ValueError) as e: + logger.debug('Encountered error {}'.format(e)) + continue + estimated = (fitgood(src, dst, params, **kwargs) if + test_coords else True) + + if tries == max_tries and not estimated: + raise EstimationError('Could not fit Polynomial ' + 'in {} attempts!'.format(tries)) + logger.debug('fit parameters in {} attempts'.format(tries)) + self.params = params + if return_params: + return self.params + + @staticmethod + def _dataStringfromParams(params=None): + """method for producing a dataString from the parameters""" + return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in params.flatten()]).replace('e', 'E') + + def _process_dataString(self, datastring): + """generate datastring and param attributes from datastring""" + dsList = datastring.split(' ') + self.params = Polynomial2DTransform._format_raveled_params(dsList) + + @staticmethod + def _format_raveled_params(raveled_params): + """method to reshape linear parameters into parameter matrix + + Parameters + ---------- + raveled_params : numpy.array + an K long vector of parameters + + Returns + ------- + numpy.array + a (2,K/2) matrix of parameters, with + first row for x and 2nd row for y + """ + halfway = int(len(raveled_params) / 2) + return np.array( + [[float(d) for d in raveled_params[:halfway]], + [float(d) for d in raveled_params[halfway:]]]) + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + dst = np.zeros(points.shape) + x = points[:, 0] + y = points[:, 1] + + o = int((-3 + np.sqrt(9 - 4 * (2 - len(self.params.ravel())))) / 2) + pidx = 0 + for j in range(o + 1): + for i in range(j + 1): + dst[:, 0] += self.params[0, pidx] * x ** (j - i) * y ** i + dst[:, 1] += self.params[1, pidx] * x ** (j - i) * y ** i + pidx += 1 + return dst + + def coefficients(self, order=None): + """determine number of coefficient terms in transform for a given order + + Parameters + ---------- + order : int, optional + order of polynomial, defaults to self.order + + Returns + ------- + int + number of coefficient terms expected in transform + + """ + if order is None: + order = self.order + return (order + 1) * (order + 2) + + def asorder(self, order): + '''return polynomial transform appoximation of this + transformation with a lower order + + Parameters + ---------- + order :int + desired order (must have order> current order) + + Returns + ------- + :class:`Polynomial2DTransform` + transform of lower order + + Raises + ------ + ConversionError + if target order < input order + ''' + if self.order > order: + raise ConversionError( + 'transformation {} is order {} -- conversion to ' + 'order {} not supported'.format( + self.dataString, self.order, order)) + new_params = np.zeros([2, self.coefficients(order) // 2]) + new_params[:self.params.shape[0], :self.params.shape[1]] = self.params + return Polynomial2DTransform(params=new_params) + + @staticmethod + def fromAffine(aff): + """return a polynomial transformation equavalent to a given Affine + + Parameters + ---------- + aff : AffineModel + transform to become equivalent to + + Returns + ------- + Polynomial2DTransform + Order 1 transform equal in effect to aff + + Raises + ------ + ConversionError + if input model is not AffineModel + """ + if not isinstance(aff, AffineModel): + raise ConversionError('attempting to convert a nonaffine model!') + return Polynomial2DTransform(order=1, params=np.array([ + [aff.M[0, 2], aff.M[0, 0], aff.M[0, 1]], + [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) + + +class NonLinearCoordinateTransform(Transform): + """ + render-python class that implements the + mpicbg.trakem2.transform.NonLinearCoordinateTransform class + + Parameters + ---------- + dataString: str or None + data string of transformation + labels : list of str + list of labels to give this transform + json: dict or None + json compatible dictionary representation of the transformation + + Returns + ------- + :class:`NonLinearTransform` + a transform instance + + + """ + + className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' + + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + if labels is not None: + self.labels = labels + self.transformId = transformId + self.className = ( + 'mpicbg.trakem2.transform.NonLinearCoordinateTransform') + + def _process_dataString(self, dataString): + + fields = dataString.split(" ") + + self.dimension = int(fields[0]) + self.length = int(fields[1]) + + # cutoff whitespace if there + fields = fields[0:2 + 4 * self.length + 2] + # last 2 fields are width and height + self.width = int(fields[-2]) + self.height = int(fields[-1]) + + data = np.array(fields[2:-2], dtype='float32') + try: + self.beta = data[0:2 * self.length].reshape(self.length, 2) + except ValueError as e: + raise RenderError( + 'Incorrect number of coefficients in ' + 'NonLinearCoordinateTransform. msg: {}'.format(e)) + if not (self.beta.shape[0] == self.length): + raise RenderError("not correct number of coefficents") + + # normMean and normVar follow + self.normMean = data[self.length * 2:self.length * 3] + self.normVar = data[self.length * 3:self.length * 4] + if not (self.normMean.shape[0] == self.length): + raise RenderError( + "incorrect number of normMean coefficents " + "{} != length {}".format(self.normMean.shape[0], self.length)) + if not (self.normVar.shape[0] == self.length): + raise RenderError( + "incorrect number of normVar coefficents " + "{} != {}".format(self.normVar.shape[0], self.length)) + + def kernelExpand(self, src, normMean=None, normVar=None): + """creates an expanded representation of the x,y + src points in a polynomial form + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a (N x self.length) array of coefficents + """ + x = src[:, 0] + y = src[:, 1] + + expanded = np.zeros([len(x), self.length]) + pidx = 0 + for i in range(1, self.dimension + 1): + for j in range(i, -1, -1): + expanded[:, pidx] = ( + np.power(x, j) * np.power(y, i - j)) + pidx += 1 + + if normMean is None: + normMean = self.normMean + if normVar is None: + normVar = self.normVar + + expanded[:, :-1] = ((expanded[:, :-1] - normMean[:-1]) / + normVar[:-1]) + expanded[:, -1] = 100.0 + return expanded + + def fit(self, A, B): + """function to fit this transform given the corresponding sets of points A & B + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + beta + a self.lengthx2 matrix with polynomial factors + normMean + a self.length vector of expanded means + normVar + a self.length vector of expanded standard deviations + """ + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + normMean = np.zeros(self.length).astype('float') + normVar = np.ones(self.length).astype('float') + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) + normMean = src_exp.mean(0) + normVar = src_exp.std(0) # poorly named variable + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) + + xcoeff, xresiduals, xrank, xs = np.linalg.lstsq(src_exp, B[:, 0]) + ycoeff, yresiduals, yrank, ys = np.linalg.lstsq(src_exp, B[:, 1]) + + beta = np.zeros((self.length, 2)) + beta[:, 0] = xcoeff + beta[:, 1] = ycoeff + + return beta, normMean, normVar + + def estimate(self, A, B, ndim=None, return_params=True, **kwargs): + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the dataString + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + dataString + """ + + beta, normMean, normVar = self.fit(A, B) + self.beta = beta + self.normMean = normMean + self.normVar = normVar + + if return_params: + return self.dataString + + def tform(self, src): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + + # final double[] featureVector = kernelExpand(position); + # return multiply(beta, featureVector); + nsrc = np.array(src, dtype=np.float64) + featureVector = self.kernelExpand(nsrc) + + dst = np.zeros(src.shape) + for i in range(0, featureVector.shape[1]): + dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i, 0]) + dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i, 1]) + return np.array(dst, dtype=src.dtype) + + @property + def dataString(self): + shapestring = '{} {}'.format(self.dimension, self.length) + betastring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.beta.ravel()]).replace('e', 'E') + meanstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normMean]).replace('e', 'E') + varstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normVar]).replace('e', 'E') + dimstring = '{} {}'.format(self.height, self.width) + return '{} {} {} {} {} '.format( + shapestring, betastring, meanstring, varstring, dimstring) + + +class NonLinearTransform(NonLinearCoordinateTransform): + className = 'mpicbg.trakem2.transform.nonLinearTransform' + + +class LensCorrection(NonLinearCoordinateTransform): + """ + a placeholder for the lenscorrection transform, same as NonLinearTransform + for now + """ + className = 'lenscorrection.NonLinearTransform' diff --git a/renderapi/transform/__init__.py b/renderapi/transform/__init__.py index 38951923..d1982071 100644 --- a/renderapi/transform/__init__.py +++ b/renderapi/transform/__init__.py @@ -1,2 +1,5 @@ from .transform import * +from .AffineModels import * from .ThinPlateSpline import * +from .PolynomialModels import * +from .utils import * diff --git a/renderapi/transform/transform.py b/renderapi/transform/transform.py index 4f7c1ad3..ab3bd5bc 100644 --- a/renderapi/transform/transform.py +++ b/renderapi/transform/transform.py @@ -4,27 +4,13 @@ Currently only implemented to facilitate Affine and Polynomial2D used in Khaled Khairy's EM aligner workflow """ -import json import logging -from collections import Iterable -import numpy as np - -from ..errors import ConversionError, EstimationError, RenderError from ..utils import NullHandler logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) -try: - from scipy.linalg import svd, LinAlgError -except ImportError as e: - logger.info(e) - logger.info('scipy-based linalg may or may not lead ' - 'to better parameter fitting') - from numpy.linalg import svd - from numpy.linalg.linalg import LinAlgError - class Transform(object): """Base transformation class @@ -127,1556 +113,3 @@ def __eq__(self, other): def __hash__(self): return hash((self.__str__())) - - -class TransformList: - """A list of Transforms - - Attributes - ---------- - tforms : :obj:`list` of :class:`Transform` - transforms to apply - transformId : str, optional - uniqueId for this TransformList - """ - - def __init__(self, tforms=None, transformId=None, json=None): - """Initialize TransformList - - Parameters - ---------- - tforms : :obj:`list` of :class:`Transform` - transforms to apply - transformId : str, optional - uniqueId for this TransformList - json : dict, optional - json compatible dictionary to create - :class:`TransformList` via :method:`from_dict` - (will supersede tforms and transformId if not None) - """ - if json is not None: - self.from_dict(json) - else: - if tforms is None: - self.tforms = [] - else: - if not isinstance(tforms, list): - raise RenderError( - 'unexpected type {} for transforms!'.format( - type(tforms))) - self.tforms = tforms - self.transformId = transformId - - def to_dict(self): - """serialization function - - Returns - ------- - dict - json & render compatible representation of this TransformList - """ - d = {} - d['type'] = 'list' - d['specList'] = [tform.to_dict() for tform in self.tforms] - if self.transformId is not None: - d['id'] = self.transformId - return d - - def to_json(self): - """serialization function - - Returns - ------- - str - string representation of the json & render - representation of this TransformList - """ - return json.dumps(self.to_dict()) - - def from_dict(self, d): - """deserialization function - - Parameters - ---------- - d : dict - json compatible dictionary representation of this TransformList - """ - self.tforms = [] - if d is not None: - self.transformId = d.get('id') - for td in d['specList']: - self.tforms.append(load_transform_json(td)) - return self.tforms - - -def load_transform_json(d, default_type='leaf'): - """function to get the proper deserialization function - - Parameters - ---------- - d : dict - json compatible representation of Transform - default_type : str - what kind of transform should we assume this - if it is not specified in 'type' ('leaf','list','ref','interpolated') - - Returns - ------- - renderapi.transform.Transform - deserialized transformation using the most appropriate class - - Raises - ------ - RenderError - if d['type'] isn't one of ('leaf','list','ref','interpolated') - """ - handle_load_tform = {'leaf': load_leaf_json, - 'list': lambda x: TransformList(json=x), - 'ref': lambda x: ReferenceTransform(json=x), - 'interpolated': - lambda x: InterpolatedTransform(json=x)} - try: - return handle_load_tform[d.get('type', default_type)](d) - except KeyError as e: - raise RenderError('Unknown Transform Type {}'.format(e)) - - -def load_leaf_json(d): - """function to get the proper deserialization function for leaf transforms - - Parameters - ---------- - d : dict - json compatible representation of leaf transform to deserialize - - Returns - ------- - renderapi.transform.Transform - deserialized transformation - - Raises - ------ - RenderError - if d['type'] != leaf or is omitted - - """ - handle_load_leaf = { - AffineModel.className: lambda x: AffineModel(json=x), - Polynomial2DTransform.className: - lambda x: Polynomial2DTransform(json=x), - TranslationModel.className: lambda x: TranslationModel(json=x), - RigidModel.className: lambda x: RigidModel(json=x), - SimilarityModel.className: lambda x: SimilarityModel(json=x), - NonLinearTransform.className: lambda x: NonLinearTransform(json=x), - LensCorrection.className: lambda x: LensCorrection(json=x), - NonLinearCoordinateTransform.className: - lambda x: NonLinearCoordinateTransform(json=x)} - - tform_type = d.get('type', 'leaf') - if tform_type != 'leaf': - raise RenderError( - 'Unexpected or unknown Transform Type {}'.format(tform_type)) - tform_class = d['className'] - try: - return handle_load_leaf[tform_class](d) - except KeyError as e: - logger.info('Leaf transform class {} not defined in ' - 'transform module, using generic'.format(e)) - return Transform(json=d) - - -class InterpolatedTransform: - """Transform spec defined by linear interpolation of - two other transform specs - - Attributes - ---------- - a : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` - transform at minimum weight - b : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` - transform at maximum weight - lambda_ : float - value in interval [0.,1.] which defines evaluation of the - linear interpolation between a (at 0) and b (at 1) - """ # noqa: E501 - - def __init__(self, a=None, b=None, lambda_=None, json=None): - """Initialize InterpolatedTransform - - Parameters - ---------- - a : :class:`Transform` or :class:`TransformList` - or :class:`InterpolatedTransform` - transform at minimum weight - b : :class:`Transform` or :class:`TransformList` - or :class:`InterpolatedTransform` - transform at maximum weight - lambda_ : float - value in interval [0.,1.] which defines evaluation of the - linear interpolation between a (at 0) and b (at 1) - json : dict - json compatible representation of this transform to - initialize via :method:`self.from_dict` - (will supersede a, b, and lambda_ if not None) - - """ - if json is not None: - self.from_dict(json) - else: - self.a = a - self.b = b - self.lambda_ = lambda_ - - def to_dict(self): - """serialization routine - - Returns - ------- - dict - json compatible representation - """ - return dict(self) - - def from_dict(self, d): - """deserialization routine - - Parameters - ---------- - d : dict - json compatible representation - """ - self.a = load_transform_json(d['a']) - self.b = load_transform_json(d['b']) - self.lambda_ = d['lambda'] - - def __iter__(self): - return iter([('type', 'interpolated'), - ('a', self.a.to_dict()), - ('b', self.b.to_dict()), - ('lambda', self.lambda_)]) - - -class ReferenceTransform: - """Transform which is simply a reference to a transform stored elsewhere - - Attributes - ---------- - refId : str - transformId of the referenced transform - - """ - - def __init__(self, refId=None, json=None): - """Initialize ReferenceTransform - - Parameters - ---------- - refId : str - transformId of the referenced transform - json : dict - json compatible representation of this transform - (will supersede refId if not None) - - """ - if json is not None: - self.from_dict(json) - else: - self.refId = refId - - def to_dict(self): - """serialization routine - - Returns - ------- - dict - json compatible representation of this transform - """ - d = {} - d['type'] = 'ref' - d['refId'] = self.refId - return d - - def from_dict(self, d): - """deserialization routine - - Parameters - ---------- - d : dict - json compatible representation of this transform - """ - self.refId = d['refId'] - - def __str__(self): - return 'ReferenceTransform(%s)' % self.refId - - def __repr__(self): - return self.__str__() - - def __iter__(self): - return iter([('type', 'ref'), ('refId', self.refId)]) - - -class AffineModel(Transform): - """Linear 2d Transformation - mpicbg classname: mpicbg.trakem2.transform.AffineModel2D - implements this simple math - x'=M00*x + M01*x + B0 - y'=M10*x + M11*y + B1 - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - - """ - - className = 'mpicbg.trakem2.transform.AffineModel2D' - - def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, - transformId=None, labels=None, json=None): - """Initialize AffineModel, defaulting to identity - - Parameters - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str - unique transformId for this transform (optional) - labels : list of str - list of labels to give this transform - json : dict - json compatible representation of this transform - (will supersede all other parameters if not None) - - """ - if json is not None: - self.from_dict(json) - else: - self.M00 = M00 - self.M01 = M01 - self.M10 = M10 - self.M11 = M11 - self.B0 = B0 - self.B1 = B1 - self.className = 'mpicbg.trakem2.transform.AffineModel2D' - self.labels = labels - self.load_M() - self.transformId = transformId - - @property - def dataString(self): - """dataString string for this transform""" - return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( - self.M[0, 0], self.M[1, 0], self.M[0, 1], - self.M[1, 1], self.M[0, 2], self.M[1, 2]) - - def _process_dataString(self, datastring): - """generate datastring and param attributes from datastring""" - dsList = datastring.split() - self.M00 = float(dsList[0]) - self.M10 = float(dsList[1]) - self.M01 = float(dsList[2]) - self.M11 = float(dsList[3]) - self.B0 = float(dsList[4]) - self.B1 = float(dsList[5]) - self.load_M() - - def load_M(self): - """method to take the attribute of self and fill in self.M""" - self.M = np.identity(3, np.double) - self.M[0, 0] = self.M00 - self.M[0, 1] = self.M01 - self.M[1, 0] = self.M10 - self.M[1, 1] = self.M11 - self.M[0, 2] = self.B0 - self.M[1, 2] = self.B1 - - @staticmethod - def fit(A, B): - """function to fit this transform given the corresponding sets of points A & B - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): - raise EstimationError( - 'shape mismatch! A shape: {}, B shape {}'.format( - A.shape, B.shape)) - - N = A.shape[0] # total points - - M = np.zeros((2 * N, 6)) - Y = np.zeros((2 * N, 1)) - for i in range(N): - M[2 * i, :] = [A[i, 0], A[i, 1], 0, 0, 1, 0] - M[2 * i + 1, :] = [0, 0, A[i, 0], A[i, 1], 0, 1] - Y[2 * i] = B[i, 0] - Y[2 * i + 1] = B[i, 1] - - (Tvec, residuals, rank, s) = np.linalg.lstsq(M, Y) - return Tvec - - def estimate(self, A, B, return_params=True, **kwargs): - """method for setting this transformation with the best fit - given the corresponding points A,B - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - return_params : boolean - whether to return the parameter matrix - **kwargs - keyword arguments to pass to self.fit - - Returns - ------- - numpy.array - a 2x3 matrix of parameters for this matrix, - laid out (x,y) x (x,y,offset) - (or None if return_params=False) - """ - Tvec = self.fit(A, B, **kwargs) - self.M00 = Tvec[0, 0] - self.M10 = Tvec[2, 0] - self.M01 = Tvec[1, 0] - self.M11 = Tvec[3, 0] - self.B0 = Tvec[4, 0] - self.B1 = Tvec[5, 0] - self.load_M() - if return_params: - return self.M - - def concatenate(self, model): - """concatenate a model to this model -- ported from trakEM2 below: - :: - - final double a00 = m00 * model.m00 + m01 * model.m10; - final double a01 = m00 * model.m01 + m01 * model.m11; - final double a02 = m00 * model.m02 + m01 * model.m12 + m02; - - final double a10 = m10 * model.m00 + m11 * model.m10; - final double a11 = m10 * model.m01 + m11 * model.m11; - final double a12 = m10 * model.m02 + m11 * model.m12 + m12; - - Parameters - ---------- - model : AffineModel - model to concatenate to this one - - Returns - ------- - AffineModel - model after concatenating model with this model - """ - a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] - a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] - a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + - self.M[0, 2]) - - a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] - a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] - a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + - self.M[1, 2]) - - newmodel = AffineModel(a00, a01, a10, a11, a02, a12) - return newmodel - - def invert(self): - """return an inverted version of this transformation - - Returns - ------- - AffineModel - an inverted version of this transformation - """ - inv_M = np.linalg.inv(self.M) - Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], - inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) - return Ai - - @staticmethod - def convert_to_point_vector(points): - """method to help reshape x,y points to x,y,1 vectors - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx3 array of x,y,1 points used for transformations - """ - Np = points.shape[0] - onevec = np.ones((Np, 1), np.double) - - if points.shape[1] != 2: - raise ConversionError('Points must be of shape (:, 2) ' - '-- got {}'.format(points.shape)) - Nd = 2 - points = np.concatenate((points, onevec), axis=1) - return points, Nd - - @staticmethod - def convert_points_vector_to_array(points, Nd=2): - """method for convertion x,y,K points to x,y vectors - - Parameters - ---------- - points : numpy.array - a Nx3 vector of points after transformation - Nd : int - the number of dimensions to cutoff (should be 2) - - Returns - ------- - numpy.array: a Nx2 array of x,y points - """ - points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T - return points - - def tform(self, points): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ - points, Nd = self.convert_to_point_vector(points) - pt = np.dot(self.M, points.T).T - return self.convert_points_vector_to_array(pt, Nd) - - def inverse_tform(self, points): - """transform a set of points through the inverse of this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after inverse transformation - """ - points, Nd = self.convert_to_point_vector(points) - pt = np.dot(np.linalg.inv(self.M), points.T).T - return self.convert_points_vector_to_array(pt, Nd) - - @property - def scale(self): - """tuple of scale for x, y""" - return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) - for j in range(self.M.shape[1])])[:2] - - @property - def shear(self): - """counter-clockwise shear angle""" - return np.arctan2(-self.M[0, 1], self.M[1, 1]) - self.rotation - - @property - def translation(self): - """tuple of translation in x, y""" - return tuple(self.M[:2, 2]) - - @property - def rotation(self): - """counter-clockwise rotation""" - return np.arctan2(self.M[1, 0], self.M[0, 0]) - - def __str__(self): - return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( - self.M[0, 0], self.M[0, 1], self.M[1, 0], - self.M[1, 1], self.M[0, 2], self.M[1, 2]) - - -class TranslationModel(AffineModel): - """Translation fitting and estimation as an :class:`AffineModel` - Linear 2d Transformation - mpicbg classname: mpicbg.trakem2.transform.AffineModel2D - implements this simple math - x'=M00*x + M01*x + B0 - y'=M10*x + M11*y + B1 - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - """ - - className = 'mpicbg.trakem2.transform.TranslationModel2D' - - def __init__(self, *args, **kwargs): - super(TranslationModel, self).__init__(*args, **kwargs) - - def _process_dataString(self, dataString): - """expected dataString is 'tx ty'""" - tx, ty = map(float, dataString.split(' ')) - self.B0 = tx - self.B1 = ty - self.M00 = 1 - self.M10 = 0 - self.M01 = 0 - self.M11 = 1 - self.load_M() - - @staticmethod - def fit(src, dst): - """function to fit Translation transform given - the corresponding sets of points src & dst - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - t = dst.mean(axis=0) - src.mean(axis=0) - T = np.eye(3) - T[:2, 2] = t - return T - - def estimate(self, src, dst, return_params=True): - """method for setting this transformation with the best fit - given the corresponding points src,dst - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - return_params : bool - whether to return the parameter matrix - - Returns - ------- - numpy.array - a 2x3 matrix of parameters for this matrix, - laid out (x,y) x (x,y,offset) - (or None if return_params=False) - """ - self.M = self.fit(src, dst) - if return_params: - return self.M - - -class RigidModel(AffineModel): - """model for fitting Rigid only transformations - (rotation+translation) - or - (determinate=1, orthonormal eigenvectors) - implemented as an :class:`AffineModel` - - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - - """ - className = 'mpicbg.trakem2.transform.RigidModel2D' - - def __init__(self, *args, **kwargs): - super(RigidModel, self).__init__(*args, **kwargs) - - def _process_dataString(self, dataString): - """expected datastring is 'theta tx ty'""" - theta, tx, ty = map(float, dataString.split(' ')) - self.M00 = np.cos(theta) - self.M01 = -np.sin(theta) - self.M10 = np.sin(theta) - self.M11 = np.sin(theta) - self.B0 = tx - self.B1 = ty - self.load_M() - - @staticmethod - def fit(src, dst, rigid=True, **kwargs): - """function to fit this transform given the corresponding - sets of points src & dst - Umeyama estimation of similarity transformation - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - rigid : bool - whether to constrain this transform to be rigid - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - # TODO shape assertion - num, dim = src.shape - src_cld = src - src.mean(axis=0) - dst_cld = dst - dst.mean(axis=0) - A = np.dot(dst_cld.T, src_cld) / num - d = np.ones((dim, ), dtype=np.double) - if np.linalg.det(A) < 0: - d[dim - 1] = -1 - T = np.eye(dim + 1, dtype=np.double) - - rank = np.linalg.matrix_rank(A) - if rank == 0: - raise EstimationError('zero rank matrix A unacceptable -- ' - 'likely poorly conditioned') - - U, S, V = svd(A) - - if rank == dim - 1: - if np.linalg.det(U) * np.linalg.det(V) > 0: - T[:dim, :dim] = np.dot(U, V) - else: - s = d[dim - 1] - d[dim - 1] = -1 - T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V)) - d[dim - 1] = s - else: - T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V.T)) - - fit_scale = (1.0 if rigid else - 1.0 / src_cld.var(axis=0).sum() * np.dot(S, d)) - - T[:dim, dim] = dst.mean(axis=0) - fit_scale * np.dot( - T[:dim, :dim], src.mean(axis=0).T) - T[:dim, :dim] *= fit_scale - return T - - def estimate(self, A, B, return_params=True, **kwargs): - """method for setting this transformation with the - best fit given the corresponding points src,dst - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - return_params : bool - whether to return the parameter matrix - - Returns - ------- - numpy.array - a 2x3 matrix of parameters for this matrix, - laid out (x,y) x (x,y,offset) - (or None if return_params=False) - """ - self.M = self.fit(A, B, **kwargs) - if return_params: - return self.M - - -class SimilarityModel(RigidModel): - """class for fitting Similarity transformations - (translation+rotation+scaling) - or - (orthogonal eigen vectors with equal eigenvalues) - - implemented as an :class:`AffineModel` - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - - """ - className = 'mpicbg.trakem2.transform.SimilarityModel2D' - - def __init__(self, *args, **kwargs): - super(SimilarityModel, self).__init__(*args, **kwargs) - - def _process_dataString(self, dataString): - """expected datastring is 's theta tx ty'""" - s, theta, tx, ty = map(float, dataString.split(' ')) - self.M00 = s * np.cos(theta) - self.M01 = -s * np.sin(theta) - self.M10 = s * np.sin(theta) - self.M11 = s * np.sin(theta) - self.B0 = tx - self.B1 = ty - self.load_M() - - @staticmethod - def fit(src, dst, rigid=False, **kwargs): - """function to fit this transform given the corresponding - sets of points src & dst - Umeyama estimation of similarity transformation - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - rigid : bool - whether to constrain this transform to be rigid - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - return RigidModel.fit(src, dst, rigid=rigid) - - -class Polynomial2DTransform(Transform): - """Polynomial2DTransform implemented as in skimage - - Attributes - ---------- - params : numpy.array - 2xK matrix of polynomial coefficents up to order K - - """ - className = 'mpicbg.trakem2.transform.PolynomialTransform2D' - - def __init__(self, dataString=None, src=None, dst=None, order=2, - force_polynomial=True, params=None, identity=False, - labels=None, transformId=None, json=None, **kwargs): - """Initialize Polynomial2DTransform - This provides 5 different ways to initialize the transform which are - mutually exclusive and applied in the order specified here. - 1)json2)dataString,3)identity,4)params,5)(src,dst) - - Parameters - ---------- - json : dict - dictionary representation of the Polynomial2DTransform - generally used by TransformList - dataString : str - dataString representation of transform from mpicpg - identity : bool - whether to make this transform the identity - params : numpy.array - 2xK matrix of polynomial coefficents up to order K - src : numpy.array - Nx2 array of source points to use for fitting (used with dst) - dst : numpy.array - Nx2 array of destination points to use for fitting (used with src) - order : int - degree of polynomial to store - force_polynomial : bool - whether to force this representation to return a Polynomial - regardless of degree (not implemented) - - - """ - if json is not None: - self.from_dict(json) - else: - self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' - if dataString is not None: - self._process_dataString(dataString) - elif identity: - self.params = np.array([[0, 1, 0], [0, 0, 1]]) - elif params is not None: - self.params = params - elif src is not None and dst is not None: - self.estimate(src, dst, order, return_params=False, **kwargs) - - if not force_polynomial and self.is_affine: - raise NotImplementedError('Falling back to Affine model is ' - 'not supported {}') - self.transformId = transformId - self.labels = labels - - @property - def is_affine(self): - """(boolean) TODO allow default to Affine""" - return False - # return self.order - - @property - def order(self): - """(int) order of polynomial""" - no_coeffs = len(self.params.ravel()) - return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) - - @property - def dataString(self): - """dataString of polynomial""" - return Polynomial2DTransform._dataStringfromParams(self.params) - - @staticmethod - def fit(src, dst, order=2): - """function to fit this transform given the corresponding sets - of points src & dst - polynomial fit - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - order : bool - order of polynomial to fit - - Returns - ------- - numpy.array - a [2,(order+1)*(order+2)/2] array with the best fit parameters - """ - xs = src[:, 0] - ys = src[:, 1] - xd = dst[:, 0] - yd = dst[:, 1] - rows = src.shape[0] - no_coeff = (order + 1) * (order + 2) - - if len(src) != len(dst): - raise EstimationError( - 'source has {} points, but dest has {}!'.format( - len(src), len(dst))) - if no_coeff > len(src): - raise EstimationError( - 'order {} is too large to fit {} points!'.format( - order, len(src))) - - A = np.zeros([rows * 2, no_coeff + 1]) - pidx = 0 - for j in range(order + 1): - for i in range(j + 1): - A[:rows, pidx] = xs ** (j - i) * ys ** i - A[rows:, pidx + no_coeff // 2] = xs ** (j - i) * ys ** i - pidx += 1 - - A[:rows, -1] = xd - A[rows:, -1] = yd - - # right singular vector corresponding to smallest singular value - _, s, V = svd(A) - Vsm = V[np.argmin(s), :] # never trust computers - return (-Vsm[:-1] / Vsm[-1]).reshape((2, no_coeff // 2)) - - def estimate(self, src, dst, order=2, - test_coords=True, max_tries=100, return_params=True, - **kwargs): - """method for setting this transformation with the - best fit given the corresponding points src,dst - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - order : int - order of polynomial to fit - test_coords : bool - whether to test model after fitting to - make sure it is good (see fitgood) - max_tries : int - how many times to attempt to fit the model (see fitgood) - return_params : bool - whether to return the parameter matrix - **kwargs - dictionary of keyword arguments including those - that can be passed to fitgood - - Returns - ------- - numpy.array - a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix - (or None if return_params=False) - """ - def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): - """check if model produces a 'good' result - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - params : numpy.array - a Kx2 matrix of parameters - atol : float - absolute tolerance as in numpy.allclose for - transformed sample points - rtol : float - relative tolerance as in numpy.allclose for - transformed sample points - - Returns - ------- - bool - whether the goodness condition is met - """ - result = Polynomial2DTransform(params=params).tform(src) - t = np.allclose( - result, dst, - atol=atol, rtol=rtol) - return t - - estimated = False - tries = 0 - while (tries < max_tries and not estimated): - tries += 1 - try: - params = Polynomial2DTransform.fit(src, dst, order=order) - except (LinAlgError, ValueError) as e: - logger.debug('Encountered error {}'.format(e)) - continue - estimated = (fitgood(src, dst, params, **kwargs) if - test_coords else True) - - if tries == max_tries and not estimated: - raise EstimationError('Could not fit Polynomial ' - 'in {} attempts!'.format(tries)) - logger.debug('fit parameters in {} attempts'.format(tries)) - self.params = params - if return_params: - return self.params - - @staticmethod - def _dataStringfromParams(params=None): - """method for producing a dataString from the parameters""" - return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in params.flatten()]).replace('e', 'E') - - def _process_dataString(self, datastring): - """generate datastring and param attributes from datastring""" - dsList = datastring.split(' ') - self.params = Polynomial2DTransform._format_raveled_params(dsList) - - @staticmethod - def _format_raveled_params(raveled_params): - """method to reshape linear parameters into parameter matrix - - Parameters - ---------- - raveled_params : numpy.array - an K long vector of parameters - - Returns - ------- - numpy.array - a (2,K/2) matrix of parameters, with - first row for x and 2nd row for y - """ - halfway = int(len(raveled_params) / 2) - return np.array( - [[float(d) for d in raveled_params[:halfway]], - [float(d) for d in raveled_params[halfway:]]]) - - def tform(self, points): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ - dst = np.zeros(points.shape) - x = points[:, 0] - y = points[:, 1] - - o = int((-3 + np.sqrt(9 - 4 * (2 - len(self.params.ravel())))) / 2) - pidx = 0 - for j in range(o + 1): - for i in range(j + 1): - dst[:, 0] += self.params[0, pidx] * x ** (j - i) * y ** i - dst[:, 1] += self.params[1, pidx] * x ** (j - i) * y ** i - pidx += 1 - return dst - - def coefficients(self, order=None): - """determine number of coefficient terms in transform for a given order - - Parameters - ---------- - order : int, optional - order of polynomial, defaults to self.order - - Returns - ------- - int - number of coefficient terms expected in transform - - """ - if order is None: - order = self.order - return (order + 1) * (order + 2) - - def asorder(self, order): - '''return polynomial transform appoximation of this - transformation with a lower order - - Parameters - ---------- - order :int - desired order (must have order> current order) - - Returns - ------- - :class:`Polynomial2DTransform` - transform of lower order - - Raises - ------ - ConversionError - if target order < input order - ''' - if self.order > order: - raise ConversionError( - 'transformation {} is order {} -- conversion to ' - 'order {} not supported'.format( - self.dataString, self.order, order)) - new_params = np.zeros([2, self.coefficients(order) // 2]) - new_params[:self.params.shape[0], :self.params.shape[1]] = self.params - return Polynomial2DTransform(params=new_params) - - @staticmethod - def fromAffine(aff): - """return a polynomial transformation equavalent to a given Affine - - Parameters - ---------- - aff : AffineModel - transform to become equivalent to - - Returns - ------- - Polynomial2DTransform - Order 1 transform equal in effect to aff - - Raises - ------ - ConversionError - if input model is not AffineModel - """ - if not isinstance(aff, AffineModel): - raise ConversionError('attempting to convert a nonaffine model!') - return Polynomial2DTransform(order=1, params=np.array([ - [aff.M[0, 2], aff.M[0, 0], aff.M[0, 1]], - [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) - - -def estimate_dstpts(transformlist, src=None, reference_tforms=None): - """estimate destination points for list of transforms. Recurses - through lists. - - Parameters - ---------- - transformlist : :obj:list of :obj:Transform - transforms that have a tform method implemented - src : numpy.array - a Nx2 array of source points - - Returns - ------- - numpy.array - Nx2 array of destination points - """ - dstpts = src - for tform in transformlist: - if isinstance(tform, list): - dstpts = estimate_dstpts(tform, dstpts, reference_tforms) - elif isinstance(tform, TransformList): - dstpts = estimate_dstpts(tform.tforms, dstpts, reference_tforms) - elif isinstance(tform, ReferenceTransform): - try: - tform_deref = next((tf for tf in reference_tforms - if tf.transformId == tform.refId)) - except TypeError: - raise RenderError( - "you supplied a set of tranforms that includes a " - "reference transform, but didn't supply a set of " - "reference transforms to enable dereferencing") - except StopIteration: - raise RenderError( - "the list of transforms you provided references " - "transorm {} but that transform could not be found " - "in the list of reference transforms".format(tform.refId)) - dstpts = estimate_dstpts([tform_deref], dstpts, reference_tforms) - else: - dstpts = tform.tform(dstpts) - return dstpts - - -class NonLinearCoordinateTransform(Transform): - """ - render-python class that implements the - mpicbg.trakem2.transform.NonLinearCoordinateTransform class - - Parameters - ---------- - dataString: str or None - data string of transformation - labels : list of str - list of labels to give this transform - json: dict or None - json compatible dictionary representation of the transformation - - Returns - ------- - :class:`NonLinearTransform` - a transform instance - - - """ - - className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' - - def __init__(self, dataString=None, json=None, transformId=None, - labels=None): - if json is not None: - self.from_dict(json) - else: - if dataString is not None: - self._process_dataString(dataString) - if labels is not None: - self.labels = labels - self.transformId = transformId - self.className = ( - 'mpicbg.trakem2.transform.NonLinearCoordinateTransform') - - def _process_dataString(self, dataString): - - fields = dataString.split(" ") - - self.dimension = int(fields[0]) - self.length = int(fields[1]) - - # cutoff whitespace if there - fields = fields[0:2 + 4 * self.length + 2] - # last 2 fields are width and height - self.width = int(fields[-2]) - self.height = int(fields[-1]) - - data = np.array(fields[2:-2], dtype='float32') - try: - self.beta = data[0:2 * self.length].reshape(self.length, 2) - except ValueError as e: - raise RenderError( - 'Incorrect number of coefficients in ' - 'NonLinearCoordinateTransform. msg: {}'.format(e)) - if not (self.beta.shape[0] == self.length): - raise RenderError("not correct number of coefficents") - - # normMean and normVar follow - self.normMean = data[self.length * 2:self.length * 3] - self.normVar = data[self.length * 3:self.length * 4] - if not (self.normMean.shape[0] == self.length): - raise RenderError( - "incorrect number of normMean coefficents " - "{} != length {}".format(self.normMean.shape[0], self.length)) - if not (self.normVar.shape[0] == self.length): - raise RenderError( - "incorrect number of normVar coefficents " - "{} != {}".format(self.normVar.shape[0], self.length)) - - def kernelExpand(self, src, normMean=None, normVar=None): - """creates an expanded representation of the x,y - src points in a polynomial form - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a (N x self.length) array of coefficents - """ - x = src[:, 0] - y = src[:, 1] - - expanded = np.zeros([len(x), self.length]) - pidx = 0 - for i in range(1, self.dimension + 1): - for j in range(i, -1, -1): - expanded[:, pidx] = ( - np.power(x, j) * np.power(y, i - j)) - pidx += 1 - - if normMean is None: - normMean = self.normMean - if normVar is None: - normVar = self.normVar - - expanded[:, :-1] = ((expanded[:, :-1] - normMean[:-1]) / - normVar[:-1]) - expanded[:, -1] = 100.0 - return expanded - - def fit(self, A, B): - """function to fit this transform given the corresponding sets of points A & B - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - - Returns - ------- - beta - a self.lengthx2 matrix with polynomial factors - normMean - a self.length vector of expanded means - normVar - a self.length vector of expanded standard deviations - """ - if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): - raise EstimationError( - 'shape mismatch! A shape: {}, B shape {}'.format( - A.shape, B.shape)) - - normMean = np.zeros(self.length).astype('float') - normVar = np.ones(self.length).astype('float') - src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) - normMean = src_exp.mean(0) - normVar = src_exp.std(0) # poorly named variable - src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) - - xcoeff, xresiduals, xrank, xs = np.linalg.lstsq(src_exp, B[:, 0]) - ycoeff, yresiduals, yrank, ys = np.linalg.lstsq(src_exp, B[:, 1]) - - beta = np.zeros((self.length, 2)) - beta[:, 0] = xcoeff - beta[:, 1] = ycoeff - - return beta, normMean, normVar - - def estimate(self, A, B, ndim=None, return_params=True, **kwargs): - """method for setting this transformation with the best fit - given the corresponding points A,B - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - return_params : boolean - whether to return the dataString - **kwargs - keyword arguments to pass to self.fit - - Returns - ------- - dataString - """ - - beta, normMean, normVar = self.fit(A, B) - self.beta = beta - self.normMean = normMean - self.normVar = normVar - - if return_params: - return self.dataString - - def tform(self, src): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ - - # final double[] featureVector = kernelExpand(position); - # return multiply(beta, featureVector); - nsrc = np.array(src, dtype=np.float64) - featureVector = self.kernelExpand(nsrc) - - dst = np.zeros(src.shape) - for i in range(0, featureVector.shape[1]): - dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i, 0]) - dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i, 1]) - return np.array(dst, dtype=src.dtype) - - @property - def dataString(self): - shapestring = '{} {}'.format(self.dimension, self.length) - betastring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.beta.ravel()]).replace('e', 'E') - meanstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.normMean]).replace('e', 'E') - varstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.normVar]).replace('e', 'E') - dimstring = '{} {}'.format(self.height, self.width) - return '{} {} {} {} {} '.format( - shapestring, betastring, meanstring, varstring, dimstring) - - -class NonLinearTransform(NonLinearCoordinateTransform): - className = 'mpicbg.trakem2.transform.nonLinearTransform' - - -class LensCorrection(NonLinearCoordinateTransform): - """ - a placeholder for the lenscorrection transform, same as NonLinearTransform - for now - """ - className = 'lenscorrection.NonLinearTransform' - - -def estimate_transformsum(transformlist, src=None, order=2): - """pseudo-composition of transforms in list of transforms - using source point transformation and a single estimation. - Will produce an Affine Model if all input transforms are Affine, - otherwise will produce a Polynomial of specified order - - Parameters - ---------- - transformlist : :obj:`list` of :obj:`Transform` - list of transform objects that implement tform - src : numpy.array - Nx2 array of source points for estimation - order : int - order of Polynomial output if transformlist - inputs are non-Affine - Returns - ------- - :class:`AffineModel` or :class:`Polynomial2DTransform` - best estimate of transformlist in a single transform of this order - """ - def flatten(l): - """generator-iterator to flatten deep lists of lists""" - for i in l: - if isinstance(i, Iterable): - try: - notstring = isinstance(i, basestring) - except NameError: - notstring = isinstance(i, str) - if notstring: - for sub in flatten(i): - yield sub - else: - yield i - - dstpts = estimate_dstpts(transformlist, src) - tforms = flatten(transformlist) - if all([(tform.className == AffineModel.className) - for tform in tforms]): - am = AffineModel() - am.estimate(A=src, B=dstpts, return_params=False) - return am - return Polynomial2DTransform(src=src, dst=dstpts, order=order) diff --git a/renderapi/transform/utils.py b/renderapi/transform/utils.py new file mode 100644 index 00000000..dbc4f8dd --- /dev/null +++ b/renderapi/transform/utils.py @@ -0,0 +1,382 @@ +import json +from collections import Iterable +from .transform import Transform, logger +from ..errors import RenderError +from .AffineModels import ( + AffineModel, + TranslationModel, + RigidModel, + SimilarityModel) +from .PolynomialModels import ( + Polynomial2DTransform, + NonLinearTransform, + NonLinearCoordinateTransform, + LensCorrection) +from .ThinPlateSpline import ( + ThinPlateSplineTransform) + + +def load_transform_json(d, default_type='leaf'): + """function to get the proper deserialization function + + Parameters + ---------- + d : dict + json compatible representation of Transform + default_type : str + what kind of transform should we assume this + if it is not specified in 'type' ('leaf','list','ref','interpolated') + + Returns + ------- + renderapi.transform.Transform + deserialized transformation using the most appropriate class + + Raises + ------ + RenderError + if d['type'] isn't one of ('leaf','list','ref','interpolated') + """ + handle_load_tform = {'leaf': load_leaf_json, + 'list': lambda x: TransformList(json=x), + 'ref': lambda x: ReferenceTransform(json=x), + 'interpolated': + lambda x: InterpolatedTransform(json=x)} + try: + return handle_load_tform[d.get('type', default_type)](d) + except KeyError as e: + raise RenderError('Unknown Transform Type {}'.format(e)) + + +def load_leaf_json(d): + """function to get the proper deserialization function for leaf transforms + + Parameters + ---------- + d : dict + json compatible representation of leaf transform to deserialize + + Returns + ------- + renderapi.transform.Transform + deserialized transformation + + Raises + ------ + RenderError + if d['type'] != leaf or is omitted + + """ + handle_load_leaf = { + AffineModel.className: lambda x: AffineModel(json=x), + Polynomial2DTransform.className: + lambda x: Polynomial2DTransform(json=x), + TranslationModel.className: lambda x: TranslationModel(json=x), + RigidModel.className: lambda x: RigidModel(json=x), + SimilarityModel.className: lambda x: SimilarityModel(json=x), + NonLinearTransform.className: lambda x: NonLinearTransform(json=x), + LensCorrection.className: lambda x: LensCorrection(json=x), + ThinPlateSplineTransform.className: + lambda x: ThinPlateSplineTransform(json=x), + NonLinearCoordinateTransform.className: + lambda x: NonLinearCoordinateTransform(json=x)} + + tform_type = d.get('type', 'leaf') + if tform_type != 'leaf': + raise RenderError( + 'Unexpected or unknown Transform Type {}'.format(tform_type)) + tform_class = d['className'] + try: + return handle_load_leaf[tform_class](d) + except KeyError as e: + logger.info('Leaf transform class {} not defined in ' + 'transform module, using generic'.format(e)) + return Transform(json=d) + + +def estimate_transformsum(transformlist, src=None, order=2): + """pseudo-composition of transforms in list of transforms + using source point transformation and a single estimation. + Will produce an Affine Model if all input transforms are Affine, + otherwise will produce a Polynomial of specified order + + Parameters + ---------- + transformlist : :obj:`list` of :obj:`Transform` + list of transform objects that implement tform + src : numpy.array + Nx2 array of source points for estimation + order : int + order of Polynomial output if transformlist + inputs are non-Affine + Returns + ------- + :class:`AffineModel` or :class:`Polynomial2DTransform` + best estimate of transformlist in a single transform of this order + """ + def flatten(l): + """generator-iterator to flatten deep lists of lists""" + for i in l: + if isinstance(i, Iterable): + try: + notstring = isinstance(i, basestring) + except NameError: + notstring = isinstance(i, str) + if notstring: + for sub in flatten(i): + yield sub + else: + yield i + + dstpts = estimate_dstpts(transformlist, src) + tforms = flatten(transformlist) + if all([(tform.className == AffineModel.className) + for tform in tforms]): + am = AffineModel() + am.estimate(A=src, B=dstpts, return_params=False) + return am + return Polynomial2DTransform(src=src, dst=dstpts, order=order) + + +def estimate_dstpts(transformlist, src=None, reference_tforms=None): + """estimate destination points for list of transforms. Recurses + through lists. + + Parameters + ---------- + transformlist : :obj:list of :obj:Transform + transforms that have a tform method implemented + src : numpy.array + a Nx2 array of source points + + Returns + ------- + numpy.array + Nx2 array of destination points + """ + dstpts = src + for tform in transformlist: + if isinstance(tform, list): + dstpts = estimate_dstpts(tform, dstpts, reference_tforms) + elif isinstance(tform, TransformList): + dstpts = estimate_dstpts(tform.tforms, dstpts, reference_tforms) + elif isinstance(tform, ReferenceTransform): + try: + tform_deref = next((tf for tf in reference_tforms + if tf.transformId == tform.refId)) + except TypeError: + raise RenderError( + "you supplied a set of tranforms that includes a " + "reference transform, but didn't supply a set of " + "reference transforms to enable dereferencing") + except StopIteration: + raise RenderError( + "the list of transforms you provided references " + "transorm {} but that transform could not be found " + "in the list of reference transforms".format(tform.refId)) + dstpts = estimate_dstpts([tform_deref], dstpts, reference_tforms) + else: + dstpts = tform.tform(dstpts) + return dstpts + + +class TransformList: + """A list of Transforms + + Attributes + ---------- + tforms : :obj:`list` of :class:`Transform` + transforms to apply + transformId : str, optional + uniqueId for this TransformList + """ + + def __init__(self, tforms=None, transformId=None, json=None): + """Initialize TransformList + + Parameters + ---------- + tforms : :obj:`list` of :class:`Transform` + transforms to apply + transformId : str, optional + uniqueId for this TransformList + json : dict, optional + json compatible dictionary to create + :class:`TransformList` via :method:`from_dict` + (will supersede tforms and transformId if not None) + """ + if json is not None: + self.from_dict(json) + else: + if tforms is None: + self.tforms = [] + else: + if not isinstance(tforms, list): + raise RenderError( + 'unexpected type {} for transforms!'.format( + type(tforms))) + self.tforms = tforms + self.transformId = transformId + + def to_dict(self): + """serialization function + + Returns + ------- + dict + json & render compatible representation of this TransformList + """ + d = {} + d['type'] = 'list' + d['specList'] = [tform.to_dict() for tform in self.tforms] + if self.transformId is not None: + d['id'] = self.transformId + return d + + def to_json(self): + """serialization function + + Returns + ------- + str + string representation of the json & render + representation of this TransformList + """ + return json.dumps(self.to_dict()) + + def from_dict(self, d): + """deserialization function + + Parameters + ---------- + d : dict + json compatible dictionary representation of this TransformList + """ + self.tforms = [] + if d is not None: + self.transformId = d.get('id') + for td in d['specList']: + self.tforms.append(load_transform_json(td)) + return self.tforms + + +class InterpolatedTransform: + """Transform spec defined by linear interpolation of + two other transform specs + + Attributes + ---------- + a : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + """ # noqa: E501 + + def __init__(self, a=None, b=None, lambda_=None, json=None): + """Initialize InterpolatedTransform + + Parameters + ---------- + a : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + json : dict + json compatible representation of this transform to + initialize via :method:`self.from_dict` + (will supersede a, b, and lambda_ if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.a = a + self.b = b + self.lambda_ = lambda_ + + def to_dict(self): + """serialization routine + Returns + ------- + dict + json compatible representation + """ + return dict(self) + + def from_dict(self, d): + """deserialization routine + Parameters + ---------- + d : dict + json compatible representation + """ + self.a = load_transform_json(d['a']) + self.b = load_transform_json(d['b']) + self.lambda_ = d['lambda'] + + def __iter__(self): + return iter([('type', 'interpolated'), + ('a', self.a.to_dict()), + ('b', self.b.to_dict()), + ('lambda', self.lambda_)]) + + +class ReferenceTransform: + """Transform which is simply a reference to a transform stored elsewhere + Attributes + ---------- + refId : str + transformId of the referenced transform + """ + + def __init__(self, refId=None, json=None): + """Initialize ReferenceTransform + Parameters + ---------- + refId : str + transformId of the referenced transform + json : dict + json compatible representation of this transform + (will supersede refId if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.refId = refId + + def to_dict(self): + """serialization routine + Returns + ------- + dict + json compatible representation of this transform + """ + d = {} + d['type'] = 'ref' + d['refId'] = self.refId + return d + + def from_dict(self, d): + """deserialization routine + Parameters + ---------- + d : dict + json compatible representation of this transform + """ + self.refId = d['refId'] + + def __str__(self): + return 'ReferenceTransform(%s)' % self.refId + + def __repr__(self): + return self.__str__() + + def __iter__(self): + return iter([('type', 'ref'), ('refId', self.refId)]) diff --git a/test/test_transform.py b/test/test_transform.py index 8bdf6dd0..9a7477c8 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -146,11 +146,11 @@ def noscipy_import(name, globals=None, locals=None, return realimport(name, globals, locals, fromlist, level) builtins.__import__ = noscipy_import - cross_py23_reload(renderapi.transform.transform) + cross_py23_reload(renderapi.transform.PolynomialModels) - assert(renderapi.transform.transform.svd is np.linalg.svd + assert(renderapi.transform.PolynomialModels.svd is np.linalg.svd if use_numpy else - renderapi.transform.transform.svd is scipy.linalg.svd) + renderapi.transform.PolynomialModels.svd is scipy.linalg.svd) datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' @@ -173,8 +173,8 @@ def noscipy_import(name, globals=None, locals=None, if use_numpy: builtins.__import__ = realimport - cross_py23_reload(renderapi.transform.transform) - assert(renderapi.transform.transform.svd is scipy.linalg.svd) + cross_py23_reload(renderapi.transform.PolynomialModels) + assert(renderapi.transform.PolynomialModels.svd is scipy.linalg.svd) cross_py23_reload(renderapi.transform) From a7a64ee474aca1e0b5d6ef5d778855e0064c5887 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 7 Aug 2018 13:27:36 -0700 Subject: [PATCH 689/766] adding all to transform/__init__ --- renderapi/transform/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/renderapi/transform/__init__.py b/renderapi/transform/__init__.py index d1982071..b43f812d 100644 --- a/renderapi/transform/__init__.py +++ b/renderapi/transform/__init__.py @@ -1,5 +1,11 @@ from .transform import * +from .utils import * from .AffineModels import * -from .ThinPlateSpline import * from .PolynomialModels import * -from .utils import * +from .ThinPlateSpline import * +__all__ = [ + 'transform', + 'utils', + 'AffineModels', + 'PolynomialModels', + 'ThinPlateSpline'] From 2851ac45c10ad93bdfb09a75c0a7d67257717717 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Wed, 8 Aug 2018 09:29:58 -0700 Subject: [PATCH 690/766] moving to Russel's suggested reorg --- renderapi/transform/__init__.py | 10 +- renderapi/transform/leaf/AffineModels.py | 628 +++++++++++++++++++ renderapi/transform/leaf/PolynomialModels.py | 577 +++++++++++++++++ renderapi/transform/leaf/ThinPlateSpline.py | 128 ++++ renderapi/transform/leaf/__init__.py | 5 + renderapi/transform/leaf/transform.py | 115 ++++ renderapi/transform/leaf/utils.py | 60 ++ renderapi/transform/transform.py | 256 ++++++-- renderapi/transform/utils.py | 472 +++----------- 9 files changed, 1794 insertions(+), 457 deletions(-) create mode 100644 renderapi/transform/leaf/AffineModels.py create mode 100644 renderapi/transform/leaf/PolynomialModels.py create mode 100644 renderapi/transform/leaf/ThinPlateSpline.py create mode 100644 renderapi/transform/leaf/__init__.py create mode 100644 renderapi/transform/leaf/transform.py create mode 100644 renderapi/transform/leaf/utils.py diff --git a/renderapi/transform/__init__.py b/renderapi/transform/__init__.py index b43f812d..3a1277b9 100644 --- a/renderapi/transform/__init__.py +++ b/renderapi/transform/__init__.py @@ -1,11 +1,3 @@ from .transform import * +from .leaf import * from .utils import * -from .AffineModels import * -from .PolynomialModels import * -from .ThinPlateSpline import * -__all__ = [ - 'transform', - 'utils', - 'AffineModels', - 'PolynomialModels', - 'ThinPlateSpline'] diff --git a/renderapi/transform/leaf/AffineModels.py b/renderapi/transform/leaf/AffineModels.py new file mode 100644 index 00000000..049828d6 --- /dev/null +++ b/renderapi/transform/leaf/AffineModels.py @@ -0,0 +1,628 @@ +from .transform import Transform +import numpy as np +from renderapi.errors import ConversionError, EstimationError + +try: + from scipy.linalg import svd, LinAlgError +except ImportError as e: + logger.info(e) + logger.info('scipy-based linalg may or may not lead ' + 'to better parameter fitting') + from numpy.linalg import svd + from numpy.linalg.linalg import LinAlgError + + +class AffineModel(Transform): + """Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + + className = 'mpicbg.trakem2.transform.AffineModel2D' + + def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, + transformId=None, labels=None, json=None): + """Initialize AffineModel, defaulting to identity + + Parameters + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str + unique transformId for this transform (optional) + labels : list of str + list of labels to give this transform + json : dict + json compatible representation of this transform + (will supersede all other parameters if not None) + + """ + if json is not None: + self.from_dict(json) + else: + self.M00 = M00 + self.M01 = M01 + self.M10 = M10 + self.M11 = M11 + self.B0 = B0 + self.B1 = B1 + self.className = 'mpicbg.trakem2.transform.AffineModel2D' + self.labels = labels + self.load_M() + self.transformId = transformId + + @property + def dataString(self): + """dataString string for this transform""" + return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( + self.M[0, 0], self.M[1, 0], self.M[0, 1], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + + def _process_dataString(self, datastring): + """generate datastring and param attributes from datastring""" + dsList = datastring.split() + self.M00 = float(dsList[0]) + self.M10 = float(dsList[1]) + self.M01 = float(dsList[2]) + self.M11 = float(dsList[3]) + self.B0 = float(dsList[4]) + self.B1 = float(dsList[5]) + self.load_M() + + def load_M(self): + """method to take the attribute of self and fill in self.M""" + self.M = np.identity(3, np.double) + self.M[0, 0] = self.M00 + self.M[0, 1] = self.M01 + self.M[1, 0] = self.M10 + self.M[1, 1] = self.M11 + self.M[0, 2] = self.B0 + self.M[1, 2] = self.B1 + + @staticmethod + def fit(A, B): + """function to fit this transform given the corresponding sets of points A & B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + N = A.shape[0] # total points + + M = np.zeros((2 * N, 6)) + Y = np.zeros((2 * N, 1)) + for i in range(N): + M[2 * i, :] = [A[i, 0], A[i, 1], 0, 0, 1, 0] + M[2 * i + 1, :] = [0, 0, A[i, 0], A[i, 1], 0, 1] + Y[2 * i] = B[i, 0] + Y[2 * i + 1] = B[i, 1] + + (Tvec, residuals, rank, s) = np.linalg.lstsq(M, Y) + return Tvec + + def estimate(self, A, B, return_params=True, **kwargs): + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the parameter matrix + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + Tvec = self.fit(A, B, **kwargs) + self.M00 = Tvec[0, 0] + self.M10 = Tvec[2, 0] + self.M01 = Tvec[1, 0] + self.M11 = Tvec[3, 0] + self.B0 = Tvec[4, 0] + self.B1 = Tvec[5, 0] + self.load_M() + if return_params: + return self.M + + def concatenate(self, model): + """concatenate a model to this model -- ported from trakEM2 below: + :: + + final double a00 = m00 * model.m00 + m01 * model.m10; + final double a01 = m00 * model.m01 + m01 * model.m11; + final double a02 = m00 * model.m02 + m01 * model.m12 + m02; + + final double a10 = m10 * model.m00 + m11 * model.m10; + final double a11 = m10 * model.m01 + m11 * model.m11; + final double a12 = m10 * model.m02 + m11 * model.m12 + m12; + + Parameters + ---------- + model : AffineModel + model to concatenate to this one + + Returns + ------- + AffineModel + model after concatenating model with this model + """ + a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] + a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] + a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + + self.M[0, 2]) + + a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] + a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] + a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + + self.M[1, 2]) + + newmodel = AffineModel(a00, a01, a10, a11, a02, a12) + return newmodel + + def invert(self): + """return an inverted version of this transformation + + Returns + ------- + AffineModel + an inverted version of this transformation + """ + inv_M = np.linalg.inv(self.M) + Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], + inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) + return Ai + + @staticmethod + def convert_to_point_vector(points): + """method to help reshape x,y points to x,y,1 vectors + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx3 array of x,y,1 points used for transformations + """ + Np = points.shape[0] + onevec = np.ones((Np, 1), np.double) + + if points.shape[1] != 2: + raise ConversionError('Points must be of shape (:, 2) ' + '-- got {}'.format(points.shape)) + Nd = 2 + points = np.concatenate((points, onevec), axis=1) + return points, Nd + + @staticmethod + def convert_points_vector_to_array(points, Nd=2): + """method for convertion x,y,K points to x,y vectors + + Parameters + ---------- + points : numpy.array + a Nx3 vector of points after transformation + Nd : int + the number of dimensions to cutoff (should be 2) + + Returns + ------- + numpy.array: a Nx2 array of x,y points + """ + points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T + return points + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(self.M, points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + def inverse_tform(self, points): + """transform a set of points through the inverse of this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after inverse transformation + """ + points, Nd = self.convert_to_point_vector(points) + pt = np.dot(np.linalg.inv(self.M), points.T).T + return self.convert_points_vector_to_array(pt, Nd) + + @property + def scale(self): + """tuple of scale for x, y""" + return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) + for j in range(self.M.shape[1])])[:2] + + @property + def shear(self): + """counter-clockwise shear angle""" + return np.arctan2(-self.M[0, 1], self.M[1, 1]) - self.rotation + + @property + def translation(self): + """tuple of translation in x, y""" + return tuple(self.M[:2, 2]) + + @property + def rotation(self): + """counter-clockwise rotation""" + return np.arctan2(self.M[1, 0], self.M[0, 0]) + + def __str__(self): + return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( + self.M[0, 0], self.M[0, 1], self.M[1, 0], + self.M[1, 1], self.M[0, 2], self.M[1, 2]) + + +class TranslationModel(AffineModel): + """Translation fitting and estimation as an :class:`AffineModel` + Linear 2d Transformation + mpicbg classname: mpicbg.trakem2.transform.AffineModel2D + implements this simple math + x'=M00*x + M01*x + B0 + y'=M10*x + M11*y + B1 + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + """ + + className = 'mpicbg.trakem2.transform.TranslationModel2D' + + def __init__(self, *args, **kwargs): + super(TranslationModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected dataString is 'tx ty'""" + tx, ty = map(float, dataString.split(' ')) + self.B0 = tx + self.B1 = ty + self.M00 = 1 + self.M10 = 0 + self.M01 = 0 + self.M11 = 1 + self.load_M() + + @staticmethod + def fit(src, dst): + """function to fit Translation transform given + the corresponding sets of points src & dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + t = dst.mean(axis=0) - src.mean(axis=0) + T = np.eye(3) + T[:2, 2] = t + return T + + def estimate(self, src, dst, return_params=True): + """method for setting this transformation with the best fit + given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + self.M = self.fit(src, dst) + if return_params: + return self.M + + +class RigidModel(AffineModel): + """model for fitting Rigid only transformations + (rotation+translation) + or + (determinate=1, orthonormal eigenvectors) + implemented as an :class:`AffineModel` + + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + className = 'mpicbg.trakem2.transform.RigidModel2D' + + def __init__(self, *args, **kwargs): + super(RigidModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected datastring is 'theta tx ty'""" + theta, tx, ty = map(float, dataString.split(' ')) + self.M00 = np.cos(theta) + self.M01 = -np.sin(theta) + self.M10 = np.sin(theta) + self.M11 = np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() + + @staticmethod + def fit(src, dst, rigid=True, **kwargs): + """function to fit this transform given the corresponding + sets of points src & dst + Umeyama estimation of similarity transformation + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + # TODO shape assertion + num, dim = src.shape + src_cld = src - src.mean(axis=0) + dst_cld = dst - dst.mean(axis=0) + A = np.dot(dst_cld.T, src_cld) / num + d = np.ones((dim, ), dtype=np.double) + if np.linalg.det(A) < 0: + d[dim - 1] = -1 + T = np.eye(dim + 1, dtype=np.double) + + rank = np.linalg.matrix_rank(A) + if rank == 0: + raise EstimationError('zero rank matrix A unacceptable -- ' + 'likely poorly conditioned') + + U, S, V = svd(A) + + if rank == dim - 1: + if np.linalg.det(U) * np.linalg.det(V) > 0: + T[:dim, :dim] = np.dot(U, V) + else: + s = d[dim - 1] + d[dim - 1] = -1 + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V)) + d[dim - 1] = s + else: + T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V.T)) + + fit_scale = (1.0 if rigid else + 1.0 / src_cld.var(axis=0).sum() * np.dot(S, d)) + + T[:dim, dim] = dst.mean(axis=0) - fit_scale * np.dot( + T[:dim, :dim], src.mean(axis=0).T) + T[:dim, :dim] *= fit_scale + return T + + def estimate(self, A, B, return_params=True, **kwargs): + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : bool + whether to return the parameter matrix + + Returns + ------- + numpy.array + a 2x3 matrix of parameters for this matrix, + laid out (x,y) x (x,y,offset) + (or None if return_params=False) + """ + self.M = self.fit(A, B, **kwargs) + if return_params: + return self.M + + +class SimilarityModel(RigidModel): + """class for fitting Similarity transformations + (translation+rotation+scaling) + or + (orthogonal eigen vectors with equal eigenvalues) + + implemented as an :class:`AffineModel` + + Attributes + ---------- + M00 : float + x'+=M00*x + M01 : float + x'+=M01*y + M10 : float + y'+=M10*x + M11 : float + y'+=M11*y + B0 : float + x'+=B0 + B1 : float + y'+=B1 + transformId : str, optional + unique transformId for this transform + labels : list of str + list of labels to give this transform + M : numpy.array + 3x3 numpy array representing 2d Affine with homogeneous coordinates + populates with values from M00, M01, M10, M11, B0, B1 with load_M() + + """ + className = 'mpicbg.trakem2.transform.SimilarityModel2D' + + def __init__(self, *args, **kwargs): + super(SimilarityModel, self).__init__(*args, **kwargs) + + def _process_dataString(self, dataString): + """expected datastring is 's theta tx ty'""" + s, theta, tx, ty = map(float, dataString.split(' ')) + self.M00 = s * np.cos(theta) + self.M01 = -s * np.sin(theta) + self.M10 = s * np.sin(theta) + self.M11 = s * np.sin(theta) + self.B0 = tx + self.B1 = ty + self.load_M() + + @staticmethod + def fit(src, dst, rigid=False, **kwargs): + """function to fit this transform given the corresponding + sets of points src & dst + Umeyama estimation of similarity transformation + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + rigid : bool + whether to constrain this transform to be rigid + + Returns + ------- + numpy.array + a 6x1 matrix with the best fit parameters + ordered M00,M01,M10,M11,B0,B1 + """ + return RigidModel.fit(src, dst, rigid=rigid) diff --git a/renderapi/transform/leaf/PolynomialModels.py b/renderapi/transform/leaf/PolynomialModels.py new file mode 100644 index 00000000..7e227d32 --- /dev/null +++ b/renderapi/transform/leaf/PolynomialModels.py @@ -0,0 +1,577 @@ +from .transform import Transform, logger +from .AffineModels import AffineModel +import numpy as np +from renderapi.errors import ConversionError, EstimationError, RenderError + +try: + from scipy.linalg import svd, LinAlgError +except ImportError as e: + logger.info(e) + logger.info('scipy-based linalg may or may not lead ' + 'to better parameter fitting') + from numpy.linalg import svd + from numpy.linalg.linalg import LinAlgError + + +class Polynomial2DTransform(Transform): + """Polynomial2DTransform implemented as in skimage + + Attributes + ---------- + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + + """ + className = 'mpicbg.trakem2.transform.PolynomialTransform2D' + + def __init__(self, dataString=None, src=None, dst=None, order=2, + force_polynomial=True, params=None, identity=False, + labels=None, transformId=None, json=None, **kwargs): + """Initialize Polynomial2DTransform + This provides 5 different ways to initialize the transform which are + mutually exclusive and applied in the order specified here. + 1)json2)dataString,3)identity,4)params,5)(src,dst) + + Parameters + ---------- + json : dict + dictionary representation of the Polynomial2DTransform + generally used by TransformList + dataString : str + dataString representation of transform from mpicpg + identity : bool + whether to make this transform the identity + params : numpy.array + 2xK matrix of polynomial coefficents up to order K + src : numpy.array + Nx2 array of source points to use for fitting (used with dst) + dst : numpy.array + Nx2 array of destination points to use for fitting (used with src) + order : int + degree of polynomial to store + force_polynomial : bool + whether to force this representation to return a Polynomial + regardless of degree (not implemented) + + + """ + if json is not None: + self.from_dict(json) + else: + self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' + if dataString is not None: + self._process_dataString(dataString) + elif identity: + self.params = np.array([[0, 1, 0], [0, 0, 1]]) + elif params is not None: + self.params = params + elif src is not None and dst is not None: + self.estimate(src, dst, order, return_params=False, **kwargs) + + if not force_polynomial and self.is_affine: + raise NotImplementedError('Falling back to Affine model is ' + 'not supported {}') + self.transformId = transformId + self.labels = labels + + @property + def is_affine(self): + """(boolean) TODO allow default to Affine""" + return False + # return self.order + + @property + def order(self): + """(int) order of polynomial""" + no_coeffs = len(self.params.ravel()) + return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) + + @property + def dataString(self): + """dataString of polynomial""" + return Polynomial2DTransform._dataStringfromParams(self.params) + + @staticmethod + def fit(src, dst, order=2): + """function to fit this transform given the corresponding sets + of points src & dst + polynomial fit + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : bool + order of polynomial to fit + + Returns + ------- + numpy.array + a [2,(order+1)*(order+2)/2] array with the best fit parameters + """ + xs = src[:, 0] + ys = src[:, 1] + xd = dst[:, 0] + yd = dst[:, 1] + rows = src.shape[0] + no_coeff = (order + 1) * (order + 2) + + if len(src) != len(dst): + raise EstimationError( + 'source has {} points, but dest has {}!'.format( + len(src), len(dst))) + if no_coeff > len(src): + raise EstimationError( + 'order {} is too large to fit {} points!'.format( + order, len(src))) + + A = np.zeros([rows * 2, no_coeff + 1]) + pidx = 0 + for j in range(order + 1): + for i in range(j + 1): + A[:rows, pidx] = xs ** (j - i) * ys ** i + A[rows:, pidx + no_coeff // 2] = xs ** (j - i) * ys ** i + pidx += 1 + + A[:rows, -1] = xd + A[rows:, -1] = yd + + # right singular vector corresponding to smallest singular value + _, s, V = svd(A) + Vsm = V[np.argmin(s), :] # never trust computers + return (-Vsm[:-1] / Vsm[-1]).reshape((2, no_coeff // 2)) + + def estimate(self, src, dst, order=2, + test_coords=True, max_tries=100, return_params=True, + **kwargs): + """method for setting this transformation with the + best fit given the corresponding points src,dst + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + order : int + order of polynomial to fit + test_coords : bool + whether to test model after fitting to + make sure it is good (see fitgood) + max_tries : int + how many times to attempt to fit the model (see fitgood) + return_params : bool + whether to return the parameter matrix + **kwargs + dictionary of keyword arguments including those + that can be passed to fitgood + + Returns + ------- + numpy.array + a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix + (or None if return_params=False) + """ + def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): + """check if model produces a 'good' result + + Parameters + ---------- + src : numpy.array + a Nx2 matrix of source points + dst : numpy.array + a Nx2 matrix of destination points + params : numpy.array + a Kx2 matrix of parameters + atol : float + absolute tolerance as in numpy.allclose for + transformed sample points + rtol : float + relative tolerance as in numpy.allclose for + transformed sample points + + Returns + ------- + bool + whether the goodness condition is met + """ + result = Polynomial2DTransform(params=params).tform(src) + t = np.allclose( + result, dst, + atol=atol, rtol=rtol) + return t + + estimated = False + tries = 0 + while (tries < max_tries and not estimated): + tries += 1 + try: + params = Polynomial2DTransform.fit(src, dst, order=order) + except (LinAlgError, ValueError) as e: + logger.debug('Encountered error {}'.format(e)) + continue + estimated = (fitgood(src, dst, params, **kwargs) if + test_coords else True) + + if tries == max_tries and not estimated: + raise EstimationError('Could not fit Polynomial ' + 'in {} attempts!'.format(tries)) + logger.debug('fit parameters in {} attempts'.format(tries)) + self.params = params + if return_params: + return self.params + + @staticmethod + def _dataStringfromParams(params=None): + """method for producing a dataString from the parameters""" + return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in params.flatten()]).replace('e', 'E') + + def _process_dataString(self, datastring): + """generate datastring and param attributes from datastring""" + dsList = datastring.split(' ') + self.params = Polynomial2DTransform._format_raveled_params(dsList) + + @staticmethod + def _format_raveled_params(raveled_params): + """method to reshape linear parameters into parameter matrix + + Parameters + ---------- + raveled_params : numpy.array + an K long vector of parameters + + Returns + ------- + numpy.array + a (2,K/2) matrix of parameters, with + first row for x and 2nd row for y + """ + halfway = int(len(raveled_params) / 2) + return np.array( + [[float(d) for d in raveled_params[:halfway]], + [float(d) for d in raveled_params[halfway:]]]) + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + dst = np.zeros(points.shape) + x = points[:, 0] + y = points[:, 1] + + o = int((-3 + np.sqrt(9 - 4 * (2 - len(self.params.ravel())))) / 2) + pidx = 0 + for j in range(o + 1): + for i in range(j + 1): + dst[:, 0] += self.params[0, pidx] * x ** (j - i) * y ** i + dst[:, 1] += self.params[1, pidx] * x ** (j - i) * y ** i + pidx += 1 + return dst + + def coefficients(self, order=None): + """determine number of coefficient terms in transform for a given order + + Parameters + ---------- + order : int, optional + order of polynomial, defaults to self.order + + Returns + ------- + int + number of coefficient terms expected in transform + + """ + if order is None: + order = self.order + return (order + 1) * (order + 2) + + def asorder(self, order): + '''return polynomial transform appoximation of this + transformation with a lower order + + Parameters + ---------- + order :int + desired order (must have order> current order) + + Returns + ------- + :class:`Polynomial2DTransform` + transform of lower order + + Raises + ------ + ConversionError + if target order < input order + ''' + if self.order > order: + raise ConversionError( + 'transformation {} is order {} -- conversion to ' + 'order {} not supported'.format( + self.dataString, self.order, order)) + new_params = np.zeros([2, self.coefficients(order) // 2]) + new_params[:self.params.shape[0], :self.params.shape[1]] = self.params + return Polynomial2DTransform(params=new_params) + + @staticmethod + def fromAffine(aff): + """return a polynomial transformation equavalent to a given Affine + + Parameters + ---------- + aff : AffineModel + transform to become equivalent to + + Returns + ------- + Polynomial2DTransform + Order 1 transform equal in effect to aff + + Raises + ------ + ConversionError + if input model is not AffineModel + """ + if not isinstance(aff, AffineModel): + raise ConversionError('attempting to convert a nonaffine model!') + return Polynomial2DTransform(order=1, params=np.array([ + [aff.M[0, 2], aff.M[0, 0], aff.M[0, 1]], + [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) + + +class NonLinearCoordinateTransform(Transform): + """ + render-python class that implements the + mpicbg.trakem2.transform.NonLinearCoordinateTransform class + + Parameters + ---------- + dataString: str or None + data string of transformation + labels : list of str + list of labels to give this transform + json: dict or None + json compatible dictionary representation of the transformation + + Returns + ------- + :class:`NonLinearTransform` + a transform instance + + + """ + + className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' + + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + if labels is not None: + self.labels = labels + self.transformId = transformId + self.className = ( + 'mpicbg.trakem2.transform.NonLinearCoordinateTransform') + + def _process_dataString(self, dataString): + + fields = dataString.split(" ") + + self.dimension = int(fields[0]) + self.length = int(fields[1]) + + # cutoff whitespace if there + fields = fields[0:2 + 4 * self.length + 2] + # last 2 fields are width and height + self.width = int(fields[-2]) + self.height = int(fields[-1]) + + data = np.array(fields[2:-2], dtype='float32') + try: + self.beta = data[0:2 * self.length].reshape(self.length, 2) + except ValueError as e: + raise RenderError( + 'Incorrect number of coefficients in ' + 'NonLinearCoordinateTransform. msg: {}'.format(e)) + if not (self.beta.shape[0] == self.length): + raise RenderError("not correct number of coefficents") + + # normMean and normVar follow + self.normMean = data[self.length * 2:self.length * 3] + self.normVar = data[self.length * 3:self.length * 4] + if not (self.normMean.shape[0] == self.length): + raise RenderError( + "incorrect number of normMean coefficents " + "{} != length {}".format(self.normMean.shape[0], self.length)) + if not (self.normVar.shape[0] == self.length): + raise RenderError( + "incorrect number of normVar coefficents " + "{} != {}".format(self.normVar.shape[0], self.length)) + + def kernelExpand(self, src, normMean=None, normVar=None): + """creates an expanded representation of the x,y + src points in a polynomial form + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a (N x self.length) array of coefficents + """ + x = src[:, 0] + y = src[:, 1] + + expanded = np.zeros([len(x), self.length]) + pidx = 0 + for i in range(1, self.dimension + 1): + for j in range(i, -1, -1): + expanded[:, pidx] = ( + np.power(x, j) * np.power(y, i - j)) + pidx += 1 + + if normMean is None: + normMean = self.normMean + if normVar is None: + normVar = self.normVar + + expanded[:, :-1] = ((expanded[:, :-1] - normMean[:-1]) / + normVar[:-1]) + expanded[:, -1] = 100.0 + return expanded + + def fit(self, A, B): + """function to fit this transform given the corresponding sets of points A & B + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + beta + a self.lengthx2 matrix with polynomial factors + normMean + a self.length vector of expanded means + normVar + a self.length vector of expanded standard deviations + """ + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + normMean = np.zeros(self.length).astype('float') + normVar = np.ones(self.length).astype('float') + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) + normMean = src_exp.mean(0) + normVar = src_exp.std(0) # poorly named variable + src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) + + xcoeff, xresiduals, xrank, xs = np.linalg.lstsq(src_exp, B[:, 0]) + ycoeff, yresiduals, yrank, ys = np.linalg.lstsq(src_exp, B[:, 1]) + + beta = np.zeros((self.length, 2)) + beta[:, 0] = xcoeff + beta[:, 1] = ycoeff + + return beta, normMean, normVar + + def estimate(self, A, B, ndim=None, return_params=True, **kwargs): + """method for setting this transformation with the best fit + given the corresponding points A,B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + return_params : boolean + whether to return the dataString + **kwargs + keyword arguments to pass to self.fit + + Returns + ------- + dataString + """ + + beta, normMean, normVar = self.fit(A, B) + self.beta = beta + self.normMean = normMean + self.normVar = normVar + + if return_params: + return self.dataString + + def tform(self, src): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + + # final double[] featureVector = kernelExpand(position); + # return multiply(beta, featureVector); + nsrc = np.array(src, dtype=np.float64) + featureVector = self.kernelExpand(nsrc) + + dst = np.zeros(src.shape) + for i in range(0, featureVector.shape[1]): + dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i, 0]) + dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i, 1]) + return np.array(dst, dtype=src.dtype) + + @property + def dataString(self): + shapestring = '{} {}'.format(self.dimension, self.length) + betastring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.beta.ravel()]).replace('e', 'E') + meanstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normMean]).replace('e', 'E') + varstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') + for i in self.normVar]).replace('e', 'E') + dimstring = '{} {}'.format(self.height, self.width) + return '{} {} {} {} {} '.format( + shapestring, betastring, meanstring, varstring, dimstring) + + +class NonLinearTransform(NonLinearCoordinateTransform): + className = 'mpicbg.trakem2.transform.nonLinearTransform' + + +class LensCorrection(NonLinearCoordinateTransform): + """ + a placeholder for the lenscorrection transform, same as NonLinearTransform + for now + """ + className = 'lenscorrection.NonLinearTransform' diff --git a/renderapi/transform/leaf/ThinPlateSpline.py b/renderapi/transform/leaf/ThinPlateSpline.py new file mode 100644 index 00000000..2bb8a4e1 --- /dev/null +++ b/renderapi/transform/leaf/ThinPlateSpline.py @@ -0,0 +1,128 @@ +import numpy as np +from renderapi.errors import RenderError +from renderapi.utils import encodeBase64, decodeBase64 +from .transform import Transform + + +class ThinPlateSplineTransform(Transform): + """ + render-python class that can hold a dataString for + mpicbg.trakem2.transform.ThinPlateSplineTransform class. + Parameters + ---------- + dataString: str or None + data string of transformation + labels : list of str + list of labels to give this transform + json: dict or None + json compatible dictionary representation of the transformation + Returns + ------- + :class:`ThinPlateSplineTransform` + a transform instance + """ + + className = 'mpicbg.trakem2.transform.ThinPlateSplineTransform' + + def __init__(self, dataString=None, json=None, transformId=None, + labels=None): + if json is not None: + self.from_dict(json) + else: + if dataString is not None: + self._process_dataString(dataString) + self.labels = labels + self.transformId = transformId + self.className = ( + 'mpicbg.trakem2.transform.ThinPlateSplineTransform') + + def _process_dataString(self, dataString): + fields = dataString.split(" ") + + self.ndims = int(fields[1]) + self.nLm = int(fields[2]) + + if fields[3] != "null": + try: + values = decodeBase64(fields[3]) + self.aMtx = values[0:self.ndims*self.ndims].reshape( + self.ndims, self.ndims) + self.bVec = values[self.ndims*self.ndims:] + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + else: + self.aMtx = None + self.bVec = None + + try: + values = decodeBase64(fields[4]) + self.srcPts = values[0:self.ndims*self.nLm].reshape( + self.ndims, self.nLm, order='F') + self.dMtxDat = values[self.ndims*self.nLm:].reshape( + self.ndims, self.nLm, order='C') + except ValueError: + raise RenderError( + "inconsistent sizes and array lengths, \ + in ThinPlateSplineTransform dataString") + + def tform(self, points): + """transform a set of points through this transformation + + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + + Returns + ------- + numpy.array + a Nx2 array of x,y points after transformation + """ + result = [] + for pt in points: + result.append(self.apply(pt)) + + return np.array(result) + + def apply(self, pt): + if not hasattr(self, 'dMtxDat'): + result = pt + return result + + result = pt + self.computeDeformationContribution(pt) + if self.aMtx is not None: + result += self.aMtx.dot(pt) + if self.bVec is not None: + result += self.bVec + + return result + + def computeDeformationContribution(self, pt): + disp = np.linalg.norm( + self.srcPts - + pt.reshape(self.ndims, 1), + axis=0) + nrm = np.zeros_like(disp) + ind = disp > 1e-8 + nrm[ind] = disp[ind] * disp[ind] * np.log(disp[ind]) + result = (nrm * self.dMtxDat).sum(1) + return result + + @property + def dataString(self): + header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) + + if self.aMtx is not None: + blk1 = np.concatenate((self.aMtx.flatten(), self.bVec)) + b64_1 = encodeBase64(blk1) + else: + b64_1 = "null" + + blk2 = np.concatenate(( + self.srcPts.flatten(order='F'), + self.dMtxDat.flatten(order='C'))) + b64_2 = encodeBase64(blk2) + + return '{} {} {}'.format(header, b64_1, b64_2) diff --git a/renderapi/transform/leaf/__init__.py b/renderapi/transform/leaf/__init__.py new file mode 100644 index 00000000..4416faf6 --- /dev/null +++ b/renderapi/transform/leaf/__init__.py @@ -0,0 +1,5 @@ +from .transform import * +from .AffineModels import * +from .PolynomialModels import * +from .ThinPlateSpline import * +from .utils import * diff --git a/renderapi/transform/leaf/transform.py b/renderapi/transform/leaf/transform.py new file mode 100644 index 00000000..807bd9ab --- /dev/null +++ b/renderapi/transform/leaf/transform.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +"""handling mpicbg transforms in python + +Currently only implemented to facilitate Affine and Polynomial2D + used in Khaled Khairy's EM aligner workflow +""" +import logging + +from renderapi.utils import NullHandler + +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + + +class Transform(object): + """Base transformation class + + Attributes + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced by + mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + """ + + def __init__(self, className=None, dataString=None, + transformId=None, labels=None, json=None): + """Initialize Transform + + Parameters + ---------- + className : str + mpicbg java classname of this transform + dataString : str + string reprsentation of this transform as speced + by mpicbg java class library + transformId : str, optional + unique Id for this transform (optional) + labels : list of str + list of labels to give this transform + json : dict + json compatible representation of this transform + (supersedes className, dataString, and transformId if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.className = className + self.dataString = dataString + self.transformId = transformId + self.labels = labels + + def to_dict(self): + """serialization routine + + Returns + ------- + dict + json compatible representation of this transform + """ + d = {} + d['type'] = 'leaf' + d['className'] = self.className + d['dataString'] = self.dataString + if self.transformId is not None: + d['id'] = self.transformId + if self.labels is not None: + d['metaData'] = {'labels': self.labels} + return d + + def from_dict(self, d): + """deserialization routine + + Parameters + ---------- + d : dict + json compatible representation of this transform + """ + self.className = d['className'] + self.transformId = d.get('id', None) + self._process_dataString(d['dataString']) + md = d.get('metaData', None) + if md is not None: + self.labels = md.get('labels', None) + else: + self.labels = None + + def _process_dataString(self, datastring): + """method meant to set state of transform from datastring + generic implementation only saves datastring at self.dataString. + should rewrite for all transform classes that want to + implement tform,fit,etc + + Parameters + ---------- + dataString : str + string which can be used to initialize mpicbg transforms in java + """ + self.dataString = datastring + + def __str__(self): + return 'className:%s\ndataString:%s' % ( + self.className, self.dataString) + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return self.__str__() == other.__str__() + + def __hash__(self): + return hash((self.__str__())) diff --git a/renderapi/transform/leaf/utils.py b/renderapi/transform/leaf/utils.py new file mode 100644 index 00000000..34e216ea --- /dev/null +++ b/renderapi/transform/leaf/utils.py @@ -0,0 +1,60 @@ +from .transform import Transform, logger +from renderapi.errors import RenderError +from .AffineModels import ( + AffineModel, + TranslationModel, + RigidModel, + SimilarityModel) +from .PolynomialModels import ( + Polynomial2DTransform, + NonLinearTransform, + NonLinearCoordinateTransform, + LensCorrection) +from .ThinPlateSpline import ( + ThinPlateSplineTransform) + + +def load_leaf_json(d): + """function to get the proper deserialization function for leaf transforms + + Parameters + ---------- + d : dict + json compatible representation of leaf transform to deserialize + + Returns + ------- + renderapi.transform.Transform + deserialized transformation + + Raises + ------ + RenderError + if d['type'] != leaf or is omitted + + """ + handle_load_leaf = { + AffineModel.className: lambda x: AffineModel(json=x), + Polynomial2DTransform.className: + lambda x: Polynomial2DTransform(json=x), + TranslationModel.className: lambda x: TranslationModel(json=x), + RigidModel.className: lambda x: RigidModel(json=x), + SimilarityModel.className: lambda x: SimilarityModel(json=x), + NonLinearTransform.className: lambda x: NonLinearTransform(json=x), + LensCorrection.className: lambda x: LensCorrection(json=x), + ThinPlateSplineTransform.className: + lambda x: ThinPlateSplineTransform(json=x), + NonLinearCoordinateTransform.className: + lambda x: NonLinearCoordinateTransform(json=x)} + + tform_type = d.get('type', 'leaf') + if tform_type != 'leaf': + raise RenderError( + 'Unexpected or unknown Transform Type {}'.format(tform_type)) + tform_class = d['className'] + try: + return handle_load_leaf[tform_class](d) + except KeyError as e: + logger.info('Leaf transform class {} not defined in ' + 'transform module, using generic'.format(e)) + return Transform(json=d) diff --git a/renderapi/transform/transform.py b/renderapi/transform/transform.py index ab3bd5bc..a5ff2e6a 100644 --- a/renderapi/transform/transform.py +++ b/renderapi/transform/transform.py @@ -1,115 +1,239 @@ -#!/usr/bin/env python -"""handling mpicbg transforms in python +import json +from collections import Iterable +from renderapi.errors import RenderError +from renderapi.transform.leaf import ( + load_leaf_json, AffineModel, Polynomial2DTransform) -Currently only implemented to facilitate Affine and Polynomial2D - used in Khaled Khairy's EM aligner workflow -""" -import logging -from ..utils import NullHandler - -logger = logging.getLogger(__name__) -logger.addHandler(NullHandler()) - - -class Transform(object): - """Base transformation class +class TransformList: + """A list of Transforms Attributes ---------- - className : str - mpicbg java classname of this transform - dataString : str - string reprsentation of this transform as speced by - mpicbg java class library + tforms : :obj:`list` of :class:`Transform` + transforms to apply transformId : str, optional - unique Id for this transform (optional) + uniqueId for this TransformList """ - def __init__(self, className=None, dataString=None, - transformId=None, labels=None, json=None): - """Initialize Transform + def __init__(self, tforms=None, transformId=None, json=None): + """Initialize TransformList Parameters ---------- - className : str - mpicbg java classname of this transform - dataString : str - string reprsentation of this transform as speced - by mpicbg java class library + tforms : :obj:`list` of :class:`Transform` + transforms to apply transformId : str, optional - unique Id for this transform (optional) - labels : list of str - list of labels to give this transform - json : dict - json compatible representation of this transform - (supersedes className, dataString, and transformId if not None) + uniqueId for this TransformList + json : dict, optional + json compatible dictionary to create + :class:`TransformList` via :method:`from_dict` + (will supersede tforms and transformId if not None) """ if json is not None: self.from_dict(json) else: - self.className = className - self.dataString = dataString + if tforms is None: + self.tforms = [] + else: + if not isinstance(tforms, list): + raise RenderError( + 'unexpected type {} for transforms!'.format( + type(tforms))) + self.tforms = tforms self.transformId = transformId - self.labels = labels def to_dict(self): - """serialization routine + """serialization function Returns ------- dict - json compatible representation of this transform + json & render compatible representation of this TransformList """ d = {} - d['type'] = 'leaf' - d['className'] = self.className - d['dataString'] = self.dataString + d['type'] = 'list' + d['specList'] = [tform.to_dict() for tform in self.tforms] if self.transformId is not None: d['id'] = self.transformId - if self.labels is not None: - d['metaData'] = {'labels': self.labels} return d + def to_json(self): + """serialization function + + Returns + ------- + str + string representation of the json & render + representation of this TransformList + """ + return json.dumps(self.to_dict()) + def from_dict(self, d): - """deserialization routine + """deserialization function + + Parameters + ---------- + d : dict + json compatible dictionary representation of this TransformList + """ + self.tforms = [] + if d is not None: + self.transformId = d.get('id') + for td in d['specList']: + self.tforms.append(load_transform_json(td)) + return self.tforms + + +class InterpolatedTransform: + """Transform spec defined by linear interpolation of + two other transform specs + + Attributes + ---------- + a : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + """ # noqa: E501 + + def __init__(self, a=None, b=None, lambda_=None, json=None): + """Initialize InterpolatedTransform + + Parameters + ---------- + a : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at minimum weight + b : :class:`Transform` or :class:`TransformList` + or :class:`InterpolatedTransform` + transform at maximum weight + lambda_ : float + value in interval [0.,1.] which defines evaluation of the + linear interpolation between a (at 0) and b (at 1) + json : dict + json compatible representation of this transform to + initialize via :method:`self.from_dict` + (will supersede a, b, and lambda_ if not None) + """ + if json is not None: + self.from_dict(json) + else: + self.a = a + self.b = b + self.lambda_ = lambda_ + + def to_dict(self): + """serialization routine + Returns + ------- + dict + json compatible representation + """ + return dict(self) + def from_dict(self, d): + """deserialization routine Parameters ---------- d : dict + json compatible representation + """ + self.a = load_transform_json(d['a']) + self.b = load_transform_json(d['b']) + self.lambda_ = d['lambda'] + + def __iter__(self): + return iter([('type', 'interpolated'), + ('a', self.a.to_dict()), + ('b', self.b.to_dict()), + ('lambda', self.lambda_)]) + + +class ReferenceTransform: + """Transform which is simply a reference to a transform stored elsewhere + Attributes + ---------- + refId : str + transformId of the referenced transform + """ + + def __init__(self, refId=None, json=None): + """Initialize ReferenceTransform + Parameters + ---------- + refId : str + transformId of the referenced transform + json : dict json compatible representation of this transform + (will supersede refId if not None) """ - self.className = d['className'] - self.transformId = d.get('id', None) - self._process_dataString(d['dataString']) - md = d.get('metaData', None) - if md is not None: - self.labels = md.get('labels', None) + if json is not None: + self.from_dict(json) else: - self.labels = None + self.refId = refId - def _process_dataString(self, datastring): - """method meant to set state of transform from datastring - generic implementation only saves datastring at self.dataString. - should rewrite for all transform classes that want to - implement tform,fit,etc + def to_dict(self): + """serialization routine + Returns + ------- + dict + json compatible representation of this transform + """ + d = {} + d['type'] = 'ref' + d['refId'] = self.refId + return d + def from_dict(self, d): + """deserialization routine Parameters ---------- - dataString : str - string which can be used to initialize mpicbg transforms in java + d : dict + json compatible representation of this transform """ - self.dataString = datastring + self.refId = d['refId'] def __str__(self): - return 'className:%s\ndataString:%s' % ( - self.className, self.dataString) + return 'ReferenceTransform(%s)' % self.refId def __repr__(self): return self.__str__() - def __eq__(self, other): - return self.__str__() == other.__str__() + def __iter__(self): + return iter([('type', 'ref'), ('refId', self.refId)]) - def __hash__(self): - return hash((self.__str__())) + +def load_transform_json(d, default_type='leaf'): + """function to get the proper deserialization function + + Parameters + ---------- + d : dict + json compatible representation of Transform + default_type : str + what kind of transform should we assume this + if it is not specified in 'type' ('leaf','list','ref','interpolated') + + Returns + ------- + renderapi.transform.Transform + deserialized transformation using the most appropriate class + + Raises + ------ + RenderError + if d['type'] isn't one of ('leaf','list','ref','interpolated') + """ + handle_load_tform = {'leaf': load_leaf_json, + 'list': lambda x: TransformList(json=x), + 'ref': lambda x: ReferenceTransform(json=x), + 'interpolated': + lambda x: InterpolatedTransform(json=x)} + try: + return handle_load_tform[d.get('type', default_type)](d) + except KeyError as e: + raise RenderError('Unknown Transform Type {}'.format(e)) diff --git a/renderapi/transform/utils.py b/renderapi/transform/utils.py index dbc4f8dd..789eba7b 100644 --- a/renderapi/transform/utils.py +++ b/renderapi/transform/utils.py @@ -1,382 +1,90 @@ -import json -from collections import Iterable -from .transform import Transform, logger -from ..errors import RenderError -from .AffineModels import ( - AffineModel, - TranslationModel, - RigidModel, - SimilarityModel) -from .PolynomialModels import ( - Polynomial2DTransform, - NonLinearTransform, - NonLinearCoordinateTransform, - LensCorrection) -from .ThinPlateSpline import ( - ThinPlateSplineTransform) - - -def load_transform_json(d, default_type='leaf'): - """function to get the proper deserialization function - - Parameters - ---------- - d : dict - json compatible representation of Transform - default_type : str - what kind of transform should we assume this - if it is not specified in 'type' ('leaf','list','ref','interpolated') - - Returns - ------- - renderapi.transform.Transform - deserialized transformation using the most appropriate class - - Raises - ------ - RenderError - if d['type'] isn't one of ('leaf','list','ref','interpolated') - """ - handle_load_tform = {'leaf': load_leaf_json, - 'list': lambda x: TransformList(json=x), - 'ref': lambda x: ReferenceTransform(json=x), - 'interpolated': - lambda x: InterpolatedTransform(json=x)} - try: - return handle_load_tform[d.get('type', default_type)](d) - except KeyError as e: - raise RenderError('Unknown Transform Type {}'.format(e)) - - -def load_leaf_json(d): - """function to get the proper deserialization function for leaf transforms - - Parameters - ---------- - d : dict - json compatible representation of leaf transform to deserialize - - Returns - ------- - renderapi.transform.Transform - deserialized transformation - - Raises - ------ - RenderError - if d['type'] != leaf or is omitted - - """ - handle_load_leaf = { - AffineModel.className: lambda x: AffineModel(json=x), - Polynomial2DTransform.className: - lambda x: Polynomial2DTransform(json=x), - TranslationModel.className: lambda x: TranslationModel(json=x), - RigidModel.className: lambda x: RigidModel(json=x), - SimilarityModel.className: lambda x: SimilarityModel(json=x), - NonLinearTransform.className: lambda x: NonLinearTransform(json=x), - LensCorrection.className: lambda x: LensCorrection(json=x), - ThinPlateSplineTransform.className: - lambda x: ThinPlateSplineTransform(json=x), - NonLinearCoordinateTransform.className: - lambda x: NonLinearCoordinateTransform(json=x)} - - tform_type = d.get('type', 'leaf') - if tform_type != 'leaf': - raise RenderError( - 'Unexpected or unknown Transform Type {}'.format(tform_type)) - tform_class = d['className'] - try: - return handle_load_leaf[tform_class](d) - except KeyError as e: - logger.info('Leaf transform class {} not defined in ' - 'transform module, using generic'.format(e)) - return Transform(json=d) - - -def estimate_transformsum(transformlist, src=None, order=2): - """pseudo-composition of transforms in list of transforms - using source point transformation and a single estimation. - Will produce an Affine Model if all input transforms are Affine, - otherwise will produce a Polynomial of specified order - - Parameters - ---------- - transformlist : :obj:`list` of :obj:`Transform` - list of transform objects that implement tform - src : numpy.array - Nx2 array of source points for estimation - order : int - order of Polynomial output if transformlist - inputs are non-Affine - Returns - ------- - :class:`AffineModel` or :class:`Polynomial2DTransform` - best estimate of transformlist in a single transform of this order - """ - def flatten(l): - """generator-iterator to flatten deep lists of lists""" - for i in l: - if isinstance(i, Iterable): - try: - notstring = isinstance(i, basestring) - except NameError: - notstring = isinstance(i, str) - if notstring: - for sub in flatten(i): - yield sub - else: - yield i - - dstpts = estimate_dstpts(transformlist, src) - tforms = flatten(transformlist) - if all([(tform.className == AffineModel.className) - for tform in tforms]): - am = AffineModel() - am.estimate(A=src, B=dstpts, return_params=False) - return am - return Polynomial2DTransform(src=src, dst=dstpts, order=order) - - -def estimate_dstpts(transformlist, src=None, reference_tforms=None): - """estimate destination points for list of transforms. Recurses - through lists. - - Parameters - ---------- - transformlist : :obj:list of :obj:Transform - transforms that have a tform method implemented - src : numpy.array - a Nx2 array of source points - - Returns - ------- - numpy.array - Nx2 array of destination points - """ - dstpts = src - for tform in transformlist: - if isinstance(tform, list): - dstpts = estimate_dstpts(tform, dstpts, reference_tforms) - elif isinstance(tform, TransformList): - dstpts = estimate_dstpts(tform.tforms, dstpts, reference_tforms) - elif isinstance(tform, ReferenceTransform): - try: - tform_deref = next((tf for tf in reference_tforms - if tf.transformId == tform.refId)) - except TypeError: - raise RenderError( - "you supplied a set of tranforms that includes a " - "reference transform, but didn't supply a set of " - "reference transforms to enable dereferencing") - except StopIteration: - raise RenderError( - "the list of transforms you provided references " - "transorm {} but that transform could not be found " - "in the list of reference transforms".format(tform.refId)) - dstpts = estimate_dstpts([tform_deref], dstpts, reference_tforms) - else: - dstpts = tform.tform(dstpts) - return dstpts - - -class TransformList: - """A list of Transforms - - Attributes - ---------- - tforms : :obj:`list` of :class:`Transform` - transforms to apply - transformId : str, optional - uniqueId for this TransformList - """ - - def __init__(self, tforms=None, transformId=None, json=None): - """Initialize TransformList - - Parameters - ---------- - tforms : :obj:`list` of :class:`Transform` - transforms to apply - transformId : str, optional - uniqueId for this TransformList - json : dict, optional - json compatible dictionary to create - :class:`TransformList` via :method:`from_dict` - (will supersede tforms and transformId if not None) - """ - if json is not None: - self.from_dict(json) - else: - if tforms is None: - self.tforms = [] - else: - if not isinstance(tforms, list): - raise RenderError( - 'unexpected type {} for transforms!'.format( - type(tforms))) - self.tforms = tforms - self.transformId = transformId - - def to_dict(self): - """serialization function - - Returns - ------- - dict - json & render compatible representation of this TransformList - """ - d = {} - d['type'] = 'list' - d['specList'] = [tform.to_dict() for tform in self.tforms] - if self.transformId is not None: - d['id'] = self.transformId - return d - - def to_json(self): - """serialization function - - Returns - ------- - str - string representation of the json & render - representation of this TransformList - """ - return json.dumps(self.to_dict()) - - def from_dict(self, d): - """deserialization function - - Parameters - ---------- - d : dict - json compatible dictionary representation of this TransformList - """ - self.tforms = [] - if d is not None: - self.transformId = d.get('id') - for td in d['specList']: - self.tforms.append(load_transform_json(td)) - return self.tforms - - -class InterpolatedTransform: - """Transform spec defined by linear interpolation of - two other transform specs - - Attributes - ---------- - a : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` - transform at minimum weight - b : :class:`Transform` or :class:`TransformList` or :class:`InterpolatedTransform` - transform at maximum weight - lambda_ : float - value in interval [0.,1.] which defines evaluation of the - linear interpolation between a (at 0) and b (at 1) - """ # noqa: E501 - - def __init__(self, a=None, b=None, lambda_=None, json=None): - """Initialize InterpolatedTransform - - Parameters - ---------- - a : :class:`Transform` or :class:`TransformList` - or :class:`InterpolatedTransform` - transform at minimum weight - b : :class:`Transform` or :class:`TransformList` - or :class:`InterpolatedTransform` - transform at maximum weight - lambda_ : float - value in interval [0.,1.] which defines evaluation of the - linear interpolation between a (at 0) and b (at 1) - json : dict - json compatible representation of this transform to - initialize via :method:`self.from_dict` - (will supersede a, b, and lambda_ if not None) - """ - if json is not None: - self.from_dict(json) - else: - self.a = a - self.b = b - self.lambda_ = lambda_ - - def to_dict(self): - """serialization routine - Returns - ------- - dict - json compatible representation - """ - return dict(self) - - def from_dict(self, d): - """deserialization routine - Parameters - ---------- - d : dict - json compatible representation - """ - self.a = load_transform_json(d['a']) - self.b = load_transform_json(d['b']) - self.lambda_ = d['lambda'] - - def __iter__(self): - return iter([('type', 'interpolated'), - ('a', self.a.to_dict()), - ('b', self.b.to_dict()), - ('lambda', self.lambda_)]) - - -class ReferenceTransform: - """Transform which is simply a reference to a transform stored elsewhere - Attributes - ---------- - refId : str - transformId of the referenced transform - """ - - def __init__(self, refId=None, json=None): - """Initialize ReferenceTransform - Parameters - ---------- - refId : str - transformId of the referenced transform - json : dict - json compatible representation of this transform - (will supersede refId if not None) - """ - if json is not None: - self.from_dict(json) - else: - self.refId = refId - - def to_dict(self): - """serialization routine - Returns - ------- - dict - json compatible representation of this transform - """ - d = {} - d['type'] = 'ref' - d['refId'] = self.refId - return d - - def from_dict(self, d): - """deserialization routine - Parameters - ---------- - d : dict - json compatible representation of this transform - """ - self.refId = d['refId'] - - def __str__(self): - return 'ReferenceTransform(%s)' % self.refId - - def __repr__(self): - return self.__str__() - - def __iter__(self): - return iter([('type', 'ref'), ('refId', self.refId)]) +from collections import Iterable +from renderapi.errors import RenderError +from .leaf import AffineModel, Polynomial2DTransform +from .transform import TransformList, ReferenceTransform + + +def estimate_dstpts(transformlist, src=None, reference_tforms=None): + """estimate destination points for list of transforms. Recurses + through lists. + + Parameters + ---------- + transformlist : :obj:list of :obj:Transform + transforms that have a tform method implemented + src : numpy.array + a Nx2 array of source points + + Returns + ------- + numpy.array + Nx2 array of destination points + """ + dstpts = src + for tform in transformlist: + if isinstance(tform, list): + dstpts = estimate_dstpts(tform, dstpts, reference_tforms) + elif isinstance(tform, TransformList): + dstpts = estimate_dstpts(tform.tforms, dstpts, reference_tforms) + elif isinstance(tform, ReferenceTransform): + try: + tform_deref = next((tf for tf in reference_tforms + if tf.transformId == tform.refId)) + except TypeError: + raise RenderError( + "you supplied a set of tranforms that includes a " + "reference transform, but didn't supply a set of " + "reference transforms to enable dereferencing") + except StopIteration: + raise RenderError( + "the list of transforms you provided references " + "transorm {} but that transform could not be found " + "in the list of reference transforms".format(tform.refId)) + dstpts = estimate_dstpts([tform_deref], dstpts, reference_tforms) + else: + dstpts = tform.tform(dstpts) + return dstpts + + +def estimate_transformsum(transformlist, src=None, order=2): + """pseudo-composition of transforms in list of transforms + using source point transformation and a single estimation. + Will produce an Affine Model if all input transforms are Affine, + otherwise will produce a Polynomial of specified order + + Parameters + ---------- + transformlist : :obj:`list` of :obj:`Transform` + list of transform objects that implement tform + src : numpy.array + Nx2 array of source points for estimation + order : int + order of Polynomial output if transformlist + inputs are non-Affine + Returns + ------- + :class:`AffineModel` or :class:`Polynomial2DTransform` + best estimate of transformlist in a single transform of this order + """ + def flatten(l): + """generator-iterator to flatten deep lists of lists""" + for i in l: + if isinstance(i, Iterable): + try: + notstring = isinstance(i, basestring) + except NameError: + notstring = isinstance(i, str) + if notstring: + for sub in flatten(i): + yield sub + else: + yield i + + dstpts = estimate_dstpts(transformlist, src) + tforms = flatten(transformlist) + if all([(tform.className == AffineModel.className) + for tform in tforms]): + am = AffineModel() + am.estimate(A=src, B=dstpts, return_params=False) + return am + return Polynomial2DTransform(src=src, dst=dstpts, order=order) From e82acec9b269d39a6f9f73b705416882faf8d251 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Wed, 8 Aug 2018 09:31:13 -0700 Subject: [PATCH 691/766] rm some files --- renderapi/transform/AffineModels.py | 628 ------------------------ renderapi/transform/PolynomialModels.py | 577 ---------------------- renderapi/transform/ThinPlateSpline.py | 128 ----- 3 files changed, 1333 deletions(-) delete mode 100644 renderapi/transform/AffineModels.py delete mode 100644 renderapi/transform/PolynomialModels.py delete mode 100644 renderapi/transform/ThinPlateSpline.py diff --git a/renderapi/transform/AffineModels.py b/renderapi/transform/AffineModels.py deleted file mode 100644 index 598c1d6d..00000000 --- a/renderapi/transform/AffineModels.py +++ /dev/null @@ -1,628 +0,0 @@ -from .transform import Transform -import numpy as np -from ..errors import ConversionError, EstimationError - -try: - from scipy.linalg import svd, LinAlgError -except ImportError as e: - logger.info(e) - logger.info('scipy-based linalg may or may not lead ' - 'to better parameter fitting') - from numpy.linalg import svd - from numpy.linalg.linalg import LinAlgError - - -class AffineModel(Transform): - """Linear 2d Transformation - mpicbg classname: mpicbg.trakem2.transform.AffineModel2D - implements this simple math - x'=M00*x + M01*x + B0 - y'=M10*x + M11*y + B1 - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - - """ - - className = 'mpicbg.trakem2.transform.AffineModel2D' - - def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, - transformId=None, labels=None, json=None): - """Initialize AffineModel, defaulting to identity - - Parameters - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str - unique transformId for this transform (optional) - labels : list of str - list of labels to give this transform - json : dict - json compatible representation of this transform - (will supersede all other parameters if not None) - - """ - if json is not None: - self.from_dict(json) - else: - self.M00 = M00 - self.M01 = M01 - self.M10 = M10 - self.M11 = M11 - self.B0 = B0 - self.B1 = B1 - self.className = 'mpicbg.trakem2.transform.AffineModel2D' - self.labels = labels - self.load_M() - self.transformId = transformId - - @property - def dataString(self): - """dataString string for this transform""" - return "%.10f %.10f %.10f %.10f %.10f %.10f" % ( - self.M[0, 0], self.M[1, 0], self.M[0, 1], - self.M[1, 1], self.M[0, 2], self.M[1, 2]) - - def _process_dataString(self, datastring): - """generate datastring and param attributes from datastring""" - dsList = datastring.split() - self.M00 = float(dsList[0]) - self.M10 = float(dsList[1]) - self.M01 = float(dsList[2]) - self.M11 = float(dsList[3]) - self.B0 = float(dsList[4]) - self.B1 = float(dsList[5]) - self.load_M() - - def load_M(self): - """method to take the attribute of self and fill in self.M""" - self.M = np.identity(3, np.double) - self.M[0, 0] = self.M00 - self.M[0, 1] = self.M01 - self.M[1, 0] = self.M10 - self.M[1, 1] = self.M11 - self.M[0, 2] = self.B0 - self.M[1, 2] = self.B1 - - @staticmethod - def fit(A, B): - """function to fit this transform given the corresponding sets of points A & B - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): - raise EstimationError( - 'shape mismatch! A shape: {}, B shape {}'.format( - A.shape, B.shape)) - - N = A.shape[0] # total points - - M = np.zeros((2 * N, 6)) - Y = np.zeros((2 * N, 1)) - for i in range(N): - M[2 * i, :] = [A[i, 0], A[i, 1], 0, 0, 1, 0] - M[2 * i + 1, :] = [0, 0, A[i, 0], A[i, 1], 0, 1] - Y[2 * i] = B[i, 0] - Y[2 * i + 1] = B[i, 1] - - (Tvec, residuals, rank, s) = np.linalg.lstsq(M, Y) - return Tvec - - def estimate(self, A, B, return_params=True, **kwargs): - """method for setting this transformation with the best fit - given the corresponding points A,B - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - return_params : boolean - whether to return the parameter matrix - **kwargs - keyword arguments to pass to self.fit - - Returns - ------- - numpy.array - a 2x3 matrix of parameters for this matrix, - laid out (x,y) x (x,y,offset) - (or None if return_params=False) - """ - Tvec = self.fit(A, B, **kwargs) - self.M00 = Tvec[0, 0] - self.M10 = Tvec[2, 0] - self.M01 = Tvec[1, 0] - self.M11 = Tvec[3, 0] - self.B0 = Tvec[4, 0] - self.B1 = Tvec[5, 0] - self.load_M() - if return_params: - return self.M - - def concatenate(self, model): - """concatenate a model to this model -- ported from trakEM2 below: - :: - - final double a00 = m00 * model.m00 + m01 * model.m10; - final double a01 = m00 * model.m01 + m01 * model.m11; - final double a02 = m00 * model.m02 + m01 * model.m12 + m02; - - final double a10 = m10 * model.m00 + m11 * model.m10; - final double a11 = m10 * model.m01 + m11 * model.m11; - final double a12 = m10 * model.m02 + m11 * model.m12 + m12; - - Parameters - ---------- - model : AffineModel - model to concatenate to this one - - Returns - ------- - AffineModel - model after concatenating model with this model - """ - a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] - a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] - a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + - self.M[0, 2]) - - a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] - a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] - a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + - self.M[1, 2]) - - newmodel = AffineModel(a00, a01, a10, a11, a02, a12) - return newmodel - - def invert(self): - """return an inverted version of this transformation - - Returns - ------- - AffineModel - an inverted version of this transformation - """ - inv_M = np.linalg.inv(self.M) - Ai = AffineModel(inv_M[0, 0], inv_M[0, 1], inv_M[1, 0], - inv_M[1, 1], inv_M[0, 2], inv_M[1, 2]) - return Ai - - @staticmethod - def convert_to_point_vector(points): - """method to help reshape x,y points to x,y,1 vectors - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx3 array of x,y,1 points used for transformations - """ - Np = points.shape[0] - onevec = np.ones((Np, 1), np.double) - - if points.shape[1] != 2: - raise ConversionError('Points must be of shape (:, 2) ' - '-- got {}'.format(points.shape)) - Nd = 2 - points = np.concatenate((points, onevec), axis=1) - return points, Nd - - @staticmethod - def convert_points_vector_to_array(points, Nd=2): - """method for convertion x,y,K points to x,y vectors - - Parameters - ---------- - points : numpy.array - a Nx3 vector of points after transformation - Nd : int - the number of dimensions to cutoff (should be 2) - - Returns - ------- - numpy.array: a Nx2 array of x,y points - """ - points = points[:, 0:Nd] / np.tile(points[:, 2], (Nd, 1)).T - return points - - def tform(self, points): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ - points, Nd = self.convert_to_point_vector(points) - pt = np.dot(self.M, points.T).T - return self.convert_points_vector_to_array(pt, Nd) - - def inverse_tform(self, points): - """transform a set of points through the inverse of this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after inverse transformation - """ - points, Nd = self.convert_to_point_vector(points) - pt = np.dot(np.linalg.inv(self.M), points.T).T - return self.convert_points_vector_to_array(pt, Nd) - - @property - def scale(self): - """tuple of scale for x, y""" - return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) - for j in range(self.M.shape[1])])[:2] - - @property - def shear(self): - """counter-clockwise shear angle""" - return np.arctan2(-self.M[0, 1], self.M[1, 1]) - self.rotation - - @property - def translation(self): - """tuple of translation in x, y""" - return tuple(self.M[:2, 2]) - - @property - def rotation(self): - """counter-clockwise rotation""" - return np.arctan2(self.M[1, 0], self.M[0, 0]) - - def __str__(self): - return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( - self.M[0, 0], self.M[0, 1], self.M[1, 0], - self.M[1, 1], self.M[0, 2], self.M[1, 2]) - - -class TranslationModel(AffineModel): - """Translation fitting and estimation as an :class:`AffineModel` - Linear 2d Transformation - mpicbg classname: mpicbg.trakem2.transform.AffineModel2D - implements this simple math - x'=M00*x + M01*x + B0 - y'=M10*x + M11*y + B1 - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - """ - - className = 'mpicbg.trakem2.transform.TranslationModel2D' - - def __init__(self, *args, **kwargs): - super(TranslationModel, self).__init__(*args, **kwargs) - - def _process_dataString(self, dataString): - """expected dataString is 'tx ty'""" - tx, ty = map(float, dataString.split(' ')) - self.B0 = tx - self.B1 = ty - self.M00 = 1 - self.M10 = 0 - self.M01 = 0 - self.M11 = 1 - self.load_M() - - @staticmethod - def fit(src, dst): - """function to fit Translation transform given - the corresponding sets of points src & dst - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - t = dst.mean(axis=0) - src.mean(axis=0) - T = np.eye(3) - T[:2, 2] = t - return T - - def estimate(self, src, dst, return_params=True): - """method for setting this transformation with the best fit - given the corresponding points src,dst - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - return_params : bool - whether to return the parameter matrix - - Returns - ------- - numpy.array - a 2x3 matrix of parameters for this matrix, - laid out (x,y) x (x,y,offset) - (or None if return_params=False) - """ - self.M = self.fit(src, dst) - if return_params: - return self.M - - -class RigidModel(AffineModel): - """model for fitting Rigid only transformations - (rotation+translation) - or - (determinate=1, orthonormal eigenvectors) - implemented as an :class:`AffineModel` - - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - - """ - className = 'mpicbg.trakem2.transform.RigidModel2D' - - def __init__(self, *args, **kwargs): - super(RigidModel, self).__init__(*args, **kwargs) - - def _process_dataString(self, dataString): - """expected datastring is 'theta tx ty'""" - theta, tx, ty = map(float, dataString.split(' ')) - self.M00 = np.cos(theta) - self.M01 = -np.sin(theta) - self.M10 = np.sin(theta) - self.M11 = np.sin(theta) - self.B0 = tx - self.B1 = ty - self.load_M() - - @staticmethod - def fit(src, dst, rigid=True, **kwargs): - """function to fit this transform given the corresponding - sets of points src & dst - Umeyama estimation of similarity transformation - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - rigid : bool - whether to constrain this transform to be rigid - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - # TODO shape assertion - num, dim = src.shape - src_cld = src - src.mean(axis=0) - dst_cld = dst - dst.mean(axis=0) - A = np.dot(dst_cld.T, src_cld) / num - d = np.ones((dim, ), dtype=np.double) - if np.linalg.det(A) < 0: - d[dim - 1] = -1 - T = np.eye(dim + 1, dtype=np.double) - - rank = np.linalg.matrix_rank(A) - if rank == 0: - raise EstimationError('zero rank matrix A unacceptable -- ' - 'likely poorly conditioned') - - U, S, V = svd(A) - - if rank == dim - 1: - if np.linalg.det(U) * np.linalg.det(V) > 0: - T[:dim, :dim] = np.dot(U, V) - else: - s = d[dim - 1] - d[dim - 1] = -1 - T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V)) - d[dim - 1] = s - else: - T[:dim, :dim] = np.dot(U, np.dot(np.diag(d), V.T)) - - fit_scale = (1.0 if rigid else - 1.0 / src_cld.var(axis=0).sum() * np.dot(S, d)) - - T[:dim, dim] = dst.mean(axis=0) - fit_scale * np.dot( - T[:dim, :dim], src.mean(axis=0).T) - T[:dim, :dim] *= fit_scale - return T - - def estimate(self, A, B, return_params=True, **kwargs): - """method for setting this transformation with the - best fit given the corresponding points src,dst - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - return_params : bool - whether to return the parameter matrix - - Returns - ------- - numpy.array - a 2x3 matrix of parameters for this matrix, - laid out (x,y) x (x,y,offset) - (or None if return_params=False) - """ - self.M = self.fit(A, B, **kwargs) - if return_params: - return self.M - - -class SimilarityModel(RigidModel): - """class for fitting Similarity transformations - (translation+rotation+scaling) - or - (orthogonal eigen vectors with equal eigenvalues) - - implemented as an :class:`AffineModel` - - Attributes - ---------- - M00 : float - x'+=M00*x - M01 : float - x'+=M01*y - M10 : float - y'+=M10*x - M11 : float - y'+=M11*y - B0 : float - x'+=B0 - B1 : float - y'+=B1 - transformId : str, optional - unique transformId for this transform - labels : list of str - list of labels to give this transform - M : numpy.array - 3x3 numpy array representing 2d Affine with homogeneous coordinates - populates with values from M00, M01, M10, M11, B0, B1 with load_M() - - """ - className = 'mpicbg.trakem2.transform.SimilarityModel2D' - - def __init__(self, *args, **kwargs): - super(SimilarityModel, self).__init__(*args, **kwargs) - - def _process_dataString(self, dataString): - """expected datastring is 's theta tx ty'""" - s, theta, tx, ty = map(float, dataString.split(' ')) - self.M00 = s * np.cos(theta) - self.M01 = -s * np.sin(theta) - self.M10 = s * np.sin(theta) - self.M11 = s * np.sin(theta) - self.B0 = tx - self.B1 = ty - self.load_M() - - @staticmethod - def fit(src, dst, rigid=False, **kwargs): - """function to fit this transform given the corresponding - sets of points src & dst - Umeyama estimation of similarity transformation - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - rigid : bool - whether to constrain this transform to be rigid - - Returns - ------- - numpy.array - a 6x1 matrix with the best fit parameters - ordered M00,M01,M10,M11,B0,B1 - """ - return RigidModel.fit(src, dst, rigid=rigid) diff --git a/renderapi/transform/PolynomialModels.py b/renderapi/transform/PolynomialModels.py deleted file mode 100644 index d69477b9..00000000 --- a/renderapi/transform/PolynomialModels.py +++ /dev/null @@ -1,577 +0,0 @@ -from .transform import Transform, logger -from .AffineModels import AffineModel -import numpy as np -from ..errors import ConversionError, EstimationError, RenderError - -try: - from scipy.linalg import svd, LinAlgError -except ImportError as e: - logger.info(e) - logger.info('scipy-based linalg may or may not lead ' - 'to better parameter fitting') - from numpy.linalg import svd - from numpy.linalg.linalg import LinAlgError - - -class Polynomial2DTransform(Transform): - """Polynomial2DTransform implemented as in skimage - - Attributes - ---------- - params : numpy.array - 2xK matrix of polynomial coefficents up to order K - - """ - className = 'mpicbg.trakem2.transform.PolynomialTransform2D' - - def __init__(self, dataString=None, src=None, dst=None, order=2, - force_polynomial=True, params=None, identity=False, - labels=None, transformId=None, json=None, **kwargs): - """Initialize Polynomial2DTransform - This provides 5 different ways to initialize the transform which are - mutually exclusive and applied in the order specified here. - 1)json2)dataString,3)identity,4)params,5)(src,dst) - - Parameters - ---------- - json : dict - dictionary representation of the Polynomial2DTransform - generally used by TransformList - dataString : str - dataString representation of transform from mpicpg - identity : bool - whether to make this transform the identity - params : numpy.array - 2xK matrix of polynomial coefficents up to order K - src : numpy.array - Nx2 array of source points to use for fitting (used with dst) - dst : numpy.array - Nx2 array of destination points to use for fitting (used with src) - order : int - degree of polynomial to store - force_polynomial : bool - whether to force this representation to return a Polynomial - regardless of degree (not implemented) - - - """ - if json is not None: - self.from_dict(json) - else: - self.className = 'mpicbg.trakem2.transform.PolynomialTransform2D' - if dataString is not None: - self._process_dataString(dataString) - elif identity: - self.params = np.array([[0, 1, 0], [0, 0, 1]]) - elif params is not None: - self.params = params - elif src is not None and dst is not None: - self.estimate(src, dst, order, return_params=False, **kwargs) - - if not force_polynomial and self.is_affine: - raise NotImplementedError('Falling back to Affine model is ' - 'not supported {}') - self.transformId = transformId - self.labels = labels - - @property - def is_affine(self): - """(boolean) TODO allow default to Affine""" - return False - # return self.order - - @property - def order(self): - """(int) order of polynomial""" - no_coeffs = len(self.params.ravel()) - return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) - - @property - def dataString(self): - """dataString of polynomial""" - return Polynomial2DTransform._dataStringfromParams(self.params) - - @staticmethod - def fit(src, dst, order=2): - """function to fit this transform given the corresponding sets - of points src & dst - polynomial fit - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - order : bool - order of polynomial to fit - - Returns - ------- - numpy.array - a [2,(order+1)*(order+2)/2] array with the best fit parameters - """ - xs = src[:, 0] - ys = src[:, 1] - xd = dst[:, 0] - yd = dst[:, 1] - rows = src.shape[0] - no_coeff = (order + 1) * (order + 2) - - if len(src) != len(dst): - raise EstimationError( - 'source has {} points, but dest has {}!'.format( - len(src), len(dst))) - if no_coeff > len(src): - raise EstimationError( - 'order {} is too large to fit {} points!'.format( - order, len(src))) - - A = np.zeros([rows * 2, no_coeff + 1]) - pidx = 0 - for j in range(order + 1): - for i in range(j + 1): - A[:rows, pidx] = xs ** (j - i) * ys ** i - A[rows:, pidx + no_coeff // 2] = xs ** (j - i) * ys ** i - pidx += 1 - - A[:rows, -1] = xd - A[rows:, -1] = yd - - # right singular vector corresponding to smallest singular value - _, s, V = svd(A) - Vsm = V[np.argmin(s), :] # never trust computers - return (-Vsm[:-1] / Vsm[-1]).reshape((2, no_coeff // 2)) - - def estimate(self, src, dst, order=2, - test_coords=True, max_tries=100, return_params=True, - **kwargs): - """method for setting this transformation with the - best fit given the corresponding points src,dst - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - order : int - order of polynomial to fit - test_coords : bool - whether to test model after fitting to - make sure it is good (see fitgood) - max_tries : int - how many times to attempt to fit the model (see fitgood) - return_params : bool - whether to return the parameter matrix - **kwargs - dictionary of keyword arguments including those - that can be passed to fitgood - - Returns - ------- - numpy.array - a (2,(order+1)*(order+2)/2) matrix of parameters for this matrix - (or None if return_params=False) - """ - def fitgood(src, dst, params, atol=1e-3, rtol=0, **kwargs): - """check if model produces a 'good' result - - Parameters - ---------- - src : numpy.array - a Nx2 matrix of source points - dst : numpy.array - a Nx2 matrix of destination points - params : numpy.array - a Kx2 matrix of parameters - atol : float - absolute tolerance as in numpy.allclose for - transformed sample points - rtol : float - relative tolerance as in numpy.allclose for - transformed sample points - - Returns - ------- - bool - whether the goodness condition is met - """ - result = Polynomial2DTransform(params=params).tform(src) - t = np.allclose( - result, dst, - atol=atol, rtol=rtol) - return t - - estimated = False - tries = 0 - while (tries < max_tries and not estimated): - tries += 1 - try: - params = Polynomial2DTransform.fit(src, dst, order=order) - except (LinAlgError, ValueError) as e: - logger.debug('Encountered error {}'.format(e)) - continue - estimated = (fitgood(src, dst, params, **kwargs) if - test_coords else True) - - if tries == max_tries and not estimated: - raise EstimationError('Could not fit Polynomial ' - 'in {} attempts!'.format(tries)) - logger.debug('fit parameters in {} attempts'.format(tries)) - self.params = params - if return_params: - return self.params - - @staticmethod - def _dataStringfromParams(params=None): - """method for producing a dataString from the parameters""" - return ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in params.flatten()]).replace('e', 'E') - - def _process_dataString(self, datastring): - """generate datastring and param attributes from datastring""" - dsList = datastring.split(' ') - self.params = Polynomial2DTransform._format_raveled_params(dsList) - - @staticmethod - def _format_raveled_params(raveled_params): - """method to reshape linear parameters into parameter matrix - - Parameters - ---------- - raveled_params : numpy.array - an K long vector of parameters - - Returns - ------- - numpy.array - a (2,K/2) matrix of parameters, with - first row for x and 2nd row for y - """ - halfway = int(len(raveled_params) / 2) - return np.array( - [[float(d) for d in raveled_params[:halfway]], - [float(d) for d in raveled_params[halfway:]]]) - - def tform(self, points): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ - dst = np.zeros(points.shape) - x = points[:, 0] - y = points[:, 1] - - o = int((-3 + np.sqrt(9 - 4 * (2 - len(self.params.ravel())))) / 2) - pidx = 0 - for j in range(o + 1): - for i in range(j + 1): - dst[:, 0] += self.params[0, pidx] * x ** (j - i) * y ** i - dst[:, 1] += self.params[1, pidx] * x ** (j - i) * y ** i - pidx += 1 - return dst - - def coefficients(self, order=None): - """determine number of coefficient terms in transform for a given order - - Parameters - ---------- - order : int, optional - order of polynomial, defaults to self.order - - Returns - ------- - int - number of coefficient terms expected in transform - - """ - if order is None: - order = self.order - return (order + 1) * (order + 2) - - def asorder(self, order): - '''return polynomial transform appoximation of this - transformation with a lower order - - Parameters - ---------- - order :int - desired order (must have order> current order) - - Returns - ------- - :class:`Polynomial2DTransform` - transform of lower order - - Raises - ------ - ConversionError - if target order < input order - ''' - if self.order > order: - raise ConversionError( - 'transformation {} is order {} -- conversion to ' - 'order {} not supported'.format( - self.dataString, self.order, order)) - new_params = np.zeros([2, self.coefficients(order) // 2]) - new_params[:self.params.shape[0], :self.params.shape[1]] = self.params - return Polynomial2DTransform(params=new_params) - - @staticmethod - def fromAffine(aff): - """return a polynomial transformation equavalent to a given Affine - - Parameters - ---------- - aff : AffineModel - transform to become equivalent to - - Returns - ------- - Polynomial2DTransform - Order 1 transform equal in effect to aff - - Raises - ------ - ConversionError - if input model is not AffineModel - """ - if not isinstance(aff, AffineModel): - raise ConversionError('attempting to convert a nonaffine model!') - return Polynomial2DTransform(order=1, params=np.array([ - [aff.M[0, 2], aff.M[0, 0], aff.M[0, 1]], - [aff.M[1, 2], aff.M[1, 0], aff.M[1, 1]]])) - - -class NonLinearCoordinateTransform(Transform): - """ - render-python class that implements the - mpicbg.trakem2.transform.NonLinearCoordinateTransform class - - Parameters - ---------- - dataString: str or None - data string of transformation - labels : list of str - list of labels to give this transform - json: dict or None - json compatible dictionary representation of the transformation - - Returns - ------- - :class:`NonLinearTransform` - a transform instance - - - """ - - className = 'mpicbg.trakem2.transform.NonLinearCoordinateTransform' - - def __init__(self, dataString=None, json=None, transformId=None, - labels=None): - if json is not None: - self.from_dict(json) - else: - if dataString is not None: - self._process_dataString(dataString) - if labels is not None: - self.labels = labels - self.transformId = transformId - self.className = ( - 'mpicbg.trakem2.transform.NonLinearCoordinateTransform') - - def _process_dataString(self, dataString): - - fields = dataString.split(" ") - - self.dimension = int(fields[0]) - self.length = int(fields[1]) - - # cutoff whitespace if there - fields = fields[0:2 + 4 * self.length + 2] - # last 2 fields are width and height - self.width = int(fields[-2]) - self.height = int(fields[-1]) - - data = np.array(fields[2:-2], dtype='float32') - try: - self.beta = data[0:2 * self.length].reshape(self.length, 2) - except ValueError as e: - raise RenderError( - 'Incorrect number of coefficients in ' - 'NonLinearCoordinateTransform. msg: {}'.format(e)) - if not (self.beta.shape[0] == self.length): - raise RenderError("not correct number of coefficents") - - # normMean and normVar follow - self.normMean = data[self.length * 2:self.length * 3] - self.normVar = data[self.length * 3:self.length * 4] - if not (self.normMean.shape[0] == self.length): - raise RenderError( - "incorrect number of normMean coefficents " - "{} != length {}".format(self.normMean.shape[0], self.length)) - if not (self.normVar.shape[0] == self.length): - raise RenderError( - "incorrect number of normVar coefficents " - "{} != {}".format(self.normVar.shape[0], self.length)) - - def kernelExpand(self, src, normMean=None, normVar=None): - """creates an expanded representation of the x,y - src points in a polynomial form - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a (N x self.length) array of coefficents - """ - x = src[:, 0] - y = src[:, 1] - - expanded = np.zeros([len(x), self.length]) - pidx = 0 - for i in range(1, self.dimension + 1): - for j in range(i, -1, -1): - expanded[:, pidx] = ( - np.power(x, j) * np.power(y, i - j)) - pidx += 1 - - if normMean is None: - normMean = self.normMean - if normVar is None: - normVar = self.normVar - - expanded[:, :-1] = ((expanded[:, :-1] - normMean[:-1]) / - normVar[:-1]) - expanded[:, -1] = 100.0 - return expanded - - def fit(self, A, B): - """function to fit this transform given the corresponding sets of points A & B - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - - Returns - ------- - beta - a self.lengthx2 matrix with polynomial factors - normMean - a self.length vector of expanded means - normVar - a self.length vector of expanded standard deviations - """ - if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): - raise EstimationError( - 'shape mismatch! A shape: {}, B shape {}'.format( - A.shape, B.shape)) - - normMean = np.zeros(self.length).astype('float') - normVar = np.ones(self.length).astype('float') - src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) - normMean = src_exp.mean(0) - normVar = src_exp.std(0) # poorly named variable - src_exp = self.kernelExpand(A, normMean=normMean, normVar=normVar) - - xcoeff, xresiduals, xrank, xs = np.linalg.lstsq(src_exp, B[:, 0]) - ycoeff, yresiduals, yrank, ys = np.linalg.lstsq(src_exp, B[:, 1]) - - beta = np.zeros((self.length, 2)) - beta[:, 0] = xcoeff - beta[:, 1] = ycoeff - - return beta, normMean, normVar - - def estimate(self, A, B, ndim=None, return_params=True, **kwargs): - """method for setting this transformation with the best fit - given the corresponding points A,B - - Parameters - ---------- - A : numpy.array - a Nx2 matrix of source points - B : numpy.array - a Nx2 matrix of destination points - return_params : boolean - whether to return the dataString - **kwargs - keyword arguments to pass to self.fit - - Returns - ------- - dataString - """ - - beta, normMean, normVar = self.fit(A, B) - self.beta = beta - self.normMean = normMean - self.normVar = normVar - - if return_params: - return self.dataString - - def tform(self, src): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ - - # final double[] featureVector = kernelExpand(position); - # return multiply(beta, featureVector); - nsrc = np.array(src, dtype=np.float64) - featureVector = self.kernelExpand(nsrc) - - dst = np.zeros(src.shape) - for i in range(0, featureVector.shape[1]): - dst[:, 0] = dst[:, 0] + (featureVector[:, i] * self.beta[i, 0]) - dst[:, 1] = dst[:, 1] + (featureVector[:, i] * self.beta[i, 1]) - return np.array(dst, dtype=src.dtype) - - @property - def dataString(self): - shapestring = '{} {}'.format(self.dimension, self.length) - betastring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.beta.ravel()]).replace('e', 'E') - meanstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.normMean]).replace('e', 'E') - varstring = ' '.join([str(i).replace('e-0', 'e-').replace('e+0', 'e+') - for i in self.normVar]).replace('e', 'E') - dimstring = '{} {}'.format(self.height, self.width) - return '{} {} {} {} {} '.format( - shapestring, betastring, meanstring, varstring, dimstring) - - -class NonLinearTransform(NonLinearCoordinateTransform): - className = 'mpicbg.trakem2.transform.nonLinearTransform' - - -class LensCorrection(NonLinearCoordinateTransform): - """ - a placeholder for the lenscorrection transform, same as NonLinearTransform - for now - """ - className = 'lenscorrection.NonLinearTransform' diff --git a/renderapi/transform/ThinPlateSpline.py b/renderapi/transform/ThinPlateSpline.py deleted file mode 100644 index e3a31b26..00000000 --- a/renderapi/transform/ThinPlateSpline.py +++ /dev/null @@ -1,128 +0,0 @@ -import numpy as np -from ..errors import RenderError -from ..utils import encodeBase64, decodeBase64 -from .transform import Transform - - -class ThinPlateSplineTransform(Transform): - """ - render-python class that can hold a dataString for - mpicbg.trakem2.transform.ThinPlateSplineTransform class. - Parameters - ---------- - dataString: str or None - data string of transformation - labels : list of str - list of labels to give this transform - json: dict or None - json compatible dictionary representation of the transformation - Returns - ------- - :class:`ThinPlateSplineTransform` - a transform instance - """ - - className = 'mpicbg.trakem2.transform.ThinPlateSplineTransform' - - def __init__(self, dataString=None, json=None, transformId=None, - labels=None): - if json is not None: - self.from_dict(json) - else: - if dataString is not None: - self._process_dataString(dataString) - self.labels = labels - self.transformId = transformId - self.className = ( - 'mpicbg.trakem2.transform.ThinPlateSplineTransform') - - def _process_dataString(self, dataString): - fields = dataString.split(" ") - - self.ndims = int(fields[1]) - self.nLm = int(fields[2]) - - if fields[3] != "null": - try: - values = decodeBase64(fields[3]) - self.aMtx = values[0:self.ndims*self.ndims].reshape( - self.ndims, self.ndims) - self.bVec = values[self.ndims*self.ndims:] - except ValueError: - raise RenderError( - "inconsistent sizes and array lengths, \ - in ThinPlateSplineTransform dataString") - else: - self.aMtx = None - self.bVec = None - - try: - values = decodeBase64(fields[4]) - self.srcPts = values[0:self.ndims*self.nLm].reshape( - self.ndims, self.nLm, order='F') - self.dMtxDat = values[self.ndims*self.nLm:].reshape( - self.ndims, self.nLm, order='C') - except ValueError: - raise RenderError( - "inconsistent sizes and array lengths, \ - in ThinPlateSplineTransform dataString") - - def tform(self, points): - """transform a set of points through this transformation - - Parameters - ---------- - points : numpy.array - a Nx2 array of x,y points - - Returns - ------- - numpy.array - a Nx2 array of x,y points after transformation - """ - result = [] - for pt in points: - result.append(self.apply(pt)) - - return np.array(result) - - def apply(self, pt): - if not hasattr(self, 'dMtxDat'): - result = pt - return result - - result = pt + self.computeDeformationContribution(pt) - if self.aMtx is not None: - result += self.aMtx.dot(pt) - if self.bVec is not None: - result += self.bVec - - return result - - def computeDeformationContribution(self, pt): - disp = np.linalg.norm( - self.srcPts - - pt.reshape(self.ndims, 1), - axis=0) - nrm = np.zeros_like(disp) - ind = disp > 1e-8 - nrm[ind] = disp[ind] * disp[ind] * np.log(disp[ind]) - result = (nrm * self.dMtxDat).sum(1) - return result - - @property - def dataString(self): - header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) - - if self.aMtx is not None: - blk1 = np.concatenate((self.aMtx.flatten(), self.bVec)) - b64_1 = encodeBase64(blk1) - else: - b64_1 = "null" - - blk2 = np.concatenate(( - self.srcPts.flatten(order='F'), - self.dMtxDat.flatten(order='C'))) - b64_2 = encodeBase64(blk2) - - return '{} {} {}'.format(header, b64_1, b64_2) From fc0ef53c444a6314a3ea2705c7a70b5272fd3b95 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Wed, 8 Aug 2018 10:56:31 -0700 Subject: [PATCH 692/766] getting logger into AffineModels --- renderapi/transform/leaf/AffineModels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/transform/leaf/AffineModels.py b/renderapi/transform/leaf/AffineModels.py index 049828d6..4998ce2e 100644 --- a/renderapi/transform/leaf/AffineModels.py +++ b/renderapi/transform/leaf/AffineModels.py @@ -1,4 +1,4 @@ -from .transform import Transform +from .transform import Transform, logger import numpy as np from renderapi.errors import ConversionError, EstimationError From 98314f8ac76c5c59122dbe43bc91cdc49fb78e0d Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 9 Aug 2018 10:53:50 -0700 Subject: [PATCH 693/766] adding inverse_tform function with numpy-based gradient descent --- renderapi/transform/leaf/ThinPlateSpline.py | 50 ++++++++++++++++++++- test/test_transform.py | 19 ++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/renderapi/transform/leaf/ThinPlateSpline.py b/renderapi/transform/leaf/ThinPlateSpline.py index 2bb8a4e1..bb8b627d 100644 --- a/renderapi/transform/leaf/ThinPlateSpline.py +++ b/renderapi/transform/leaf/ThinPlateSpline.py @@ -1,5 +1,5 @@ import numpy as np -from renderapi.errors import RenderError +from renderapi.errors import RenderError, EstimationError from renderapi.utils import encodeBase64, decodeBase64 from .transform import Transform @@ -110,6 +110,54 @@ def computeDeformationContribution(self, pt): result = (nrm * self.dMtxDat).sum(1) return result + def gradient_descent( + self, + pt, + gamma=0.01, + precision=0.0001, + max_iters=1000): + # based on https://en.wikipedia.org/wiki/Gradient_descent#Python + cur_pt = np.copy(pt) + prev_pt = np.copy(pt) + step_size = 1 + iters = 0 + while (step_size > precision) & (iters < max_iters): + prev_pt[:] = cur_pt[:] + cur_pt -= gamma*(self.apply(prev_pt) - pt) + step_size = np.linalg.norm(cur_pt - prev_pt) + iters += 1 + if iters == max_iters: + raise EstimationError( + 'gradient descent for inversion of ThinPlateSpline ' + 'reached maximum iterations: %d' % max_iters) + return cur_pt + + def inverse_tform( + self, + points, + gamma=0.01, + precision=0.0001, + max_iters=1000): + """transform a set of points through the inverse of this transformation + Parameters + ---------- + points : numpy.array + a Nx2 array of x,y points + Returns + ------- + numpy.array + a Nx2 array of x,y points after inverse transformation + """ + newpts = [] + for p in points: + npt = self.gradient_descent( + p, + gamma=gamma, + precision=precision, + max_iters=max_iters) + newpts.append(npt) + return np.array(newpts) + @property def dataString(self): header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) diff --git a/test/test_transform.py b/test/test_transform.py index 9a7477c8..5a3aff17 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -627,6 +627,25 @@ def test_similarity_init(): assert(tform.M[0, 2] == 10) +def test_thinplatespline_inverse(): + j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + assert (j['dataString'].split(' ')[-1] == t.dataString.split(' ')[-1]) + x = np.linspace(0, 3840, 10) + xt, yt = np.meshgrid(x, x) + src_pts = np.transpose( + np.vstack((xt.flatten(), yt.flatten()))) + src_inv_est = t.inverse_tform(t.tform(src_pts)) + assert (src_inv_est.shape == src_pts.shape) + for i in range(src_pts.shape[0]): + assert(np.linalg.norm(src_pts[i, :] - src_inv_est[i, :]) < 0.1) + with pytest.raises(renderapi.errors.EstimationError): + src_inv_est = t.inverse_tform( + t.tform(src_pts), + max_iters=5) + + def test_thinplatespline(): j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) t = renderapi.transform.ThinPlateSplineTransform( From 3cba834e33b60ec77d13cb8249f46014424418fd Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 9 Aug 2018 11:08:41 -0700 Subject: [PATCH 694/766] adding inverse_tform comments and also retriggering failed py2.7 build (apt-get failure) --- renderapi/transform/leaf/ThinPlateSpline.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/renderapi/transform/leaf/ThinPlateSpline.py b/renderapi/transform/leaf/ThinPlateSpline.py index bb8b627d..16c51e8a 100644 --- a/renderapi/transform/leaf/ThinPlateSpline.py +++ b/renderapi/transform/leaf/ThinPlateSpline.py @@ -116,7 +116,22 @@ def gradient_descent( gamma=0.01, precision=0.0001, max_iters=1000): - # based on https://en.wikipedia.org/wiki/Gradient_descent#Python + """based on https://en.wikipedia.org/wiki/Gradient_descent#Python + Parameters + ---------- + pt : numpy array + [x,y] point for estimating inverse + gamma : float + step size is gamma fraction of current gradient + precision : float + criteria for stopping for differences between steps + max_iters : int + limit for iterations, error if reached + Returns + ------- + cur_pt : numpy array + [x,y] point, estimated inverse of pt + """ cur_pt = np.copy(pt) prev_pt = np.copy(pt) step_size = 1 From 7c71c735e551ecd9bcd1a4a9a56f4a19cffb5e40 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 9 Aug 2018 11:19:11 -0700 Subject: [PATCH 695/766] trivial comment change to retrigger Travis apt-get failures --- renderapi/transform/leaf/ThinPlateSpline.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/renderapi/transform/leaf/ThinPlateSpline.py b/renderapi/transform/leaf/ThinPlateSpline.py index 16c51e8a..aacd1f63 100644 --- a/renderapi/transform/leaf/ThinPlateSpline.py +++ b/renderapi/transform/leaf/ThinPlateSpline.py @@ -158,6 +158,12 @@ def inverse_tform( ---------- points : numpy.array a Nx2 array of x,y points + gamma : float + step size is gamma fraction of current gradient + precision : float + criteria for stopping for differences between steps + max_iters : int + limit for iterations, error if reached Returns ------- numpy.array From a90f52f731e31abdd9dce7f4853e33f2e0a2f0e5 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 9 Aug 2018 12:15:09 -0700 Subject: [PATCH 696/766] speeding up gradient_descent with higher default gamma value --- renderapi/transform/leaf/ThinPlateSpline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform/leaf/ThinPlateSpline.py b/renderapi/transform/leaf/ThinPlateSpline.py index aacd1f63..7632914a 100644 --- a/renderapi/transform/leaf/ThinPlateSpline.py +++ b/renderapi/transform/leaf/ThinPlateSpline.py @@ -113,7 +113,7 @@ def computeDeformationContribution(self, pt): def gradient_descent( self, pt, - gamma=0.01, + gamma=1.0, precision=0.0001, max_iters=1000): """based on https://en.wikipedia.org/wiki/Gradient_descent#Python @@ -150,7 +150,7 @@ def gradient_descent( def inverse_tform( self, points, - gamma=0.01, + gamma=1.0, precision=0.0001, max_iters=1000): """transform a set of points through the inverse of this transformation From daa33be8cf75690c9a4d46be145920f59a8378c3 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 9 Aug 2018 14:47:03 -0700 Subject: [PATCH 697/766] renaming transform/leaf* to undo CamelCase --- renderapi/transform/leaf/__init__.py | 6 +++--- .../leaf/{AffineModels.py => affine_models.py} | 0 .../leaf/{PolynomialModels.py => polynomial_models.py} | 2 +- .../leaf/{ThinPlateSpline.py => thin_plate_spline.py} | 0 renderapi/transform/leaf/utils.py | 6 +++--- test/test_transform.py | 10 +++++----- 6 files changed, 12 insertions(+), 12 deletions(-) rename renderapi/transform/leaf/{AffineModels.py => affine_models.py} (100%) rename renderapi/transform/leaf/{PolynomialModels.py => polynomial_models.py} (99%) rename renderapi/transform/leaf/{ThinPlateSpline.py => thin_plate_spline.py} (100%) diff --git a/renderapi/transform/leaf/__init__.py b/renderapi/transform/leaf/__init__.py index 4416faf6..731d2eb1 100644 --- a/renderapi/transform/leaf/__init__.py +++ b/renderapi/transform/leaf/__init__.py @@ -1,5 +1,5 @@ from .transform import * -from .AffineModels import * -from .PolynomialModels import * -from .ThinPlateSpline import * +from .affine_models import * +from .polynomial_models import * +from .thin_plate_spline import * from .utils import * diff --git a/renderapi/transform/leaf/AffineModels.py b/renderapi/transform/leaf/affine_models.py similarity index 100% rename from renderapi/transform/leaf/AffineModels.py rename to renderapi/transform/leaf/affine_models.py diff --git a/renderapi/transform/leaf/PolynomialModels.py b/renderapi/transform/leaf/polynomial_models.py similarity index 99% rename from renderapi/transform/leaf/PolynomialModels.py rename to renderapi/transform/leaf/polynomial_models.py index 7e227d32..76a34290 100644 --- a/renderapi/transform/leaf/PolynomialModels.py +++ b/renderapi/transform/leaf/polynomial_models.py @@ -1,5 +1,5 @@ from .transform import Transform, logger -from .AffineModels import AffineModel +from .affine_models import AffineModel import numpy as np from renderapi.errors import ConversionError, EstimationError, RenderError diff --git a/renderapi/transform/leaf/ThinPlateSpline.py b/renderapi/transform/leaf/thin_plate_spline.py similarity index 100% rename from renderapi/transform/leaf/ThinPlateSpline.py rename to renderapi/transform/leaf/thin_plate_spline.py diff --git a/renderapi/transform/leaf/utils.py b/renderapi/transform/leaf/utils.py index 34e216ea..3c353782 100644 --- a/renderapi/transform/leaf/utils.py +++ b/renderapi/transform/leaf/utils.py @@ -1,16 +1,16 @@ from .transform import Transform, logger from renderapi.errors import RenderError -from .AffineModels import ( +from .affine_models import ( AffineModel, TranslationModel, RigidModel, SimilarityModel) -from .PolynomialModels import ( +from .polynomial_models import ( Polynomial2DTransform, NonLinearTransform, NonLinearCoordinateTransform, LensCorrection) -from .ThinPlateSpline import ( +from .thin_plate_spline import ( ThinPlateSplineTransform) diff --git a/test/test_transform.py b/test/test_transform.py index 5a3aff17..32c8df77 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -146,11 +146,11 @@ def noscipy_import(name, globals=None, locals=None, return realimport(name, globals, locals, fromlist, level) builtins.__import__ = noscipy_import - cross_py23_reload(renderapi.transform.PolynomialModels) + cross_py23_reload(renderapi.transform.polynomial_models) - assert(renderapi.transform.PolynomialModels.svd is np.linalg.svd + assert(renderapi.transform.polynomial_models.svd is np.linalg.svd if use_numpy else - renderapi.transform.PolynomialModels.svd is scipy.linalg.svd) + renderapi.transform.polynomial_models.svd is scipy.linalg.svd) datastring = ('67572.7356991 0.972637082773 -0.0266434803369 ' '-3.08962731867E-06 3.52672451824E-06 1.36924119761E-07 ' @@ -173,8 +173,8 @@ def noscipy_import(name, globals=None, locals=None, if use_numpy: builtins.__import__ = realimport - cross_py23_reload(renderapi.transform.PolynomialModels) - assert(renderapi.transform.PolynomialModels.svd is scipy.linalg.svd) + cross_py23_reload(renderapi.transform.polynomial_models) + assert(renderapi.transform.polynomial_models.svd is scipy.linalg.svd) cross_py23_reload(renderapi.transform) From 2a839b5c89d378471ae9def1c1b4d753cb37202d Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 9 Aug 2018 15:53:08 -0700 Subject: [PATCH 698/766] adding __all__ all over the place --- renderapi/transform/leaf/affine_models.py | 3 +++ renderapi/transform/leaf/polynomial_models.py | 3 +++ renderapi/transform/leaf/thin_plate_spline.py | 1 + renderapi/transform/leaf/transform.py | 1 + renderapi/transform/leaf/utils.py | 1 + renderapi/transform/transform.py | 9 ++++++--- renderapi/transform/utils.py | 3 +++ 7 files changed, 18 insertions(+), 3 deletions(-) diff --git a/renderapi/transform/leaf/affine_models.py b/renderapi/transform/leaf/affine_models.py index 4998ce2e..1ebf7be5 100644 --- a/renderapi/transform/leaf/affine_models.py +++ b/renderapi/transform/leaf/affine_models.py @@ -10,6 +10,9 @@ 'to better parameter fitting') from numpy.linalg import svd from numpy.linalg.linalg import LinAlgError +__all__ = [ + 'AffineModel', 'TranslationModel', + 'SimilarityModel', 'RigidModel'] class AffineModel(Transform): diff --git a/renderapi/transform/leaf/polynomial_models.py b/renderapi/transform/leaf/polynomial_models.py index 76a34290..b5799e59 100644 --- a/renderapi/transform/leaf/polynomial_models.py +++ b/renderapi/transform/leaf/polynomial_models.py @@ -11,6 +11,9 @@ 'to better parameter fitting') from numpy.linalg import svd from numpy.linalg.linalg import LinAlgError +__all__ = [ + 'Polynomial2DTransform', 'NonLinearCoordinateTransform', + 'NonLinearTransform', 'LensCorrection'] class Polynomial2DTransform(Transform): diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py index 7632914a..7ac6eb1f 100644 --- a/renderapi/transform/leaf/thin_plate_spline.py +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -2,6 +2,7 @@ from renderapi.errors import RenderError, EstimationError from renderapi.utils import encodeBase64, decodeBase64 from .transform import Transform +__all__ = ['ThinPlateSplineTransform'] class ThinPlateSplineTransform(Transform): diff --git a/renderapi/transform/leaf/transform.py b/renderapi/transform/leaf/transform.py index 807bd9ab..ef7f6b34 100644 --- a/renderapi/transform/leaf/transform.py +++ b/renderapi/transform/leaf/transform.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) +__all__ = ['Transform'] class Transform(object): diff --git a/renderapi/transform/leaf/utils.py b/renderapi/transform/leaf/utils.py index 3c353782..c46116a1 100644 --- a/renderapi/transform/leaf/utils.py +++ b/renderapi/transform/leaf/utils.py @@ -12,6 +12,7 @@ LensCorrection) from .thin_plate_spline import ( ThinPlateSplineTransform) +__all__ = ['load_leaf_json'] def load_leaf_json(d): diff --git a/renderapi/transform/transform.py b/renderapi/transform/transform.py index a5ff2e6a..f0e14690 100644 --- a/renderapi/transform/transform.py +++ b/renderapi/transform/transform.py @@ -1,8 +1,11 @@ import json -from collections import Iterable from renderapi.errors import RenderError -from renderapi.transform.leaf import ( - load_leaf_json, AffineModel, Polynomial2DTransform) +from renderapi.transform.leaf import load_leaf_json +__all__ = [ + 'TransformList', + 'ReferenceTransform', + 'InterpolatedTransform', + 'load_transform_json'] class TransformList: diff --git a/renderapi/transform/utils.py b/renderapi/transform/utils.py index 789eba7b..20fabfca 100644 --- a/renderapi/transform/utils.py +++ b/renderapi/transform/utils.py @@ -2,6 +2,9 @@ from renderapi.errors import RenderError from .leaf import AffineModel, Polynomial2DTransform from .transform import TransformList, ReferenceTransform +__all__ = [ + 'estimate_dstpts', + 'estimate_transformsum'] def estimate_dstpts(transformlist, src=None, reference_tforms=None): From ebc67c064a2af8a7cb766ac419f1d3250f5961f0 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 9 Aug 2018 16:02:30 -0700 Subject: [PATCH 699/766] trivial change to retrigger failed Travis apt-get --- renderapi/transform/utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/renderapi/transform/utils.py b/renderapi/transform/utils.py index 20fabfca..15b0ce5c 100644 --- a/renderapi/transform/utils.py +++ b/renderapi/transform/utils.py @@ -2,9 +2,8 @@ from renderapi.errors import RenderError from .leaf import AffineModel, Polynomial2DTransform from .transform import TransformList, ReferenceTransform -__all__ = [ - 'estimate_dstpts', - 'estimate_transformsum'] +__all__ = ['estimate_dstpts', + 'estimate_transformsum'] def estimate_dstpts(transformlist, src=None, reference_tforms=None): From 0cd836c8b6b13a4dcf468b9d032d4bff813fff14 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 23 Aug 2018 14:13:20 -0400 Subject: [PATCH 700/766] Refactor client (#119) * client: start refactor * client: refactor client to multiple files * client: add __all__ to files, noqas to init * client: remove call_run_ws in some higher level functions * client: add missing project inputs, PEP8 * client: add WithPool to all for compatibility * client: modify all for test compatilibity --- renderapi/client/__init__.py | 4 + renderapi/client/client.py | 363 ++++++++++++ .../{client.py => client/client_calls.py} | 534 +----------------- renderapi/client/params.py | 73 +++ renderapi/client/utils.py | 65 +++ 5 files changed, 526 insertions(+), 513 deletions(-) create mode 100644 renderapi/client/__init__.py create mode 100644 renderapi/client/client.py rename renderapi/{client.py => client/client_calls.py} (57%) mode change 100755 => 100644 create mode 100644 renderapi/client/params.py create mode 100644 renderapi/client/utils.py diff --git a/renderapi/client/__init__.py b/renderapi/client/__init__.py new file mode 100644 index 00000000..ffa7ec61 --- /dev/null +++ b/renderapi/client/__init__.py @@ -0,0 +1,4 @@ +from .client import * # noqa: F403, F401 +from .client_calls import * # noqa: F403, F401 +from .utils import * # noqa: F403, F401 +from .params import * # noqa: F403, F401 diff --git a/renderapi/client/client.py b/renderapi/client/client.py new file mode 100644 index 00000000..78b6c40d --- /dev/null +++ b/renderapi/client/client.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python +''' +render functions relying on render-ws client scripts +''' +import os +from functools import partial +import logging + +from renderapi.utils import NullHandler, renderdump_temp +from renderapi.render import renderaccess +from renderapi.stack import set_stack_state, make_stack_params +from renderapi.resolvedtiles import put_tilespecs +from renderapi.external.processpools.stdlib_pool import WithMultiprocessingPool + +from .utils import renderclientaccess +from .client_calls import importJsonClient, call_run_ws_client + +# setup logger +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) + +WithPool = WithMultiprocessingPool + + +@renderclientaccess +def import_single_json_file(stack, jsonfile, transformFile=None, + subprocess_mode=None, client_script=None, + memGB=None, host=None, port=None, + owner=None, project=None, render=None, **kwargs): + """calls client script to import given jsonfile + + Parameters + ---------- + stack : str + stack to import into + jsonfile : str + path to jsonfile to import + transformFile : str + path to a file that contains shared + transform references if necessary + render : renderapi.render.RenderClient + render connect object + """ + importJsonClient(stack, [jsonfile], transformFile, + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, client_script=client_script, + memGB=memGB, **kwargs) + + +@renderclientaccess +def import_jsonfiles_and_transforms_parallel_by_z( + stack, jsonfiles, transformfiles, poolsize=20, mpPool=WithPool, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, render=None, **kwargs): + """imports json files and transform files in parallel + + Parameters + ---------- + stack : str + the stack to import within + jsonfiles : :obj:`list` of :obj:`str` + "list of tilespec" json paths to import + transformfiles : :obj:`list` of :obj:`str` + "list of transform files" paths which matches + in a 1-1 way with jsonfiles, so referenced transforms + are shared only within a single element of these matched lists. + Useful cases where there is as single z transforms shared + by all tiles within a single z, but not across z's + poolsize : int, optional + number of processes for multiprocessing pool + close_stack : bool, optional + whether to mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + **kwargs + arbitrary keyword arguments + + """ + set_stack_state(stack, 'LOADING', host, port, owner, project) + partial_import = partial(import_single_json_file, stack, render=render, + client_scripts=client_scripts, host=host, + port=port, owner=owner, project=project) + with mpPool(poolsize) as pool: + pool.map(partial_import, jsonfiles, transformfiles) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderclientaccess +def import_jsonfiles_parallel( + stack, jsonfiles, poolsize=20, transformFile=None, mpPool=WithPool, + client_scripts=None, host=None, port=None, owner=None, + project=None, close_stack=True, render=None, **kwargs): + """import jsons using client script in parallel + + Parameters + ---------- + stack : str + the stack to upload into + jsonfiles : :obj:`list` of :obj:`str` + list of jsonfile paths to upload + poolsize : int + number of upload processes spawned by multiprocessing pool + transformFile : str + a single json file path containing transforms referenced + in the jsonfiles + close_stack : bool + whether to mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + **kwargs + arbitrary keyword arguments + + """ + set_stack_state(stack, 'LOADING', host, port, owner, project) + + partial_import = partial(import_single_json_file, stack, render=render, + transformFile=transformFile, + client_scripts=client_scripts, + host=host, port=port, owner=owner, + project=project, **kwargs) + with mpPool(poolsize) as pool: + pool.map(partial_import, jsonfiles) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderaccess +def import_jsonfiles(stack, jsonfiles, transformFile=None, + subprocess_mode=None, client_script=None, memGB=None, + host=None, port=None, owner=None, project=None, + close_stack=True, render=None, **kwargs): + """import jsons using client script serially + + Parameters + ---------- + jsonfiles : :obj:`list` of :obj:`str` + iterator of filenames to be uploaded + transformFile : str + path to a jsonfile that contains shared + transform references (if necessary) + close_stack : bool + mark render stack as COMPLETE after successful import + render : renderapi.render.Render + render connect object + + """ + + set_stack_state(stack, 'LOADING', host, port, owner, project) + importJsonClient(stack, jsonfiles, transformFile, + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, client_script=client_script, + memGB=memGB, **kwargs) + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderclientaccess +def import_jsonfiles_validate_client(stack, jsonfiles, + transformFile=None, client_script=None, + host=None, port=None, owner=None, + project=None, close_stack=True, mem=6, + validator=None, subprocess_mode=None, + memGB=None, + render=None, **kwargs): + """Uses java client for parallelization and validation + + Parameters + ---------- + stack: str + stack to which jsonfiles should be uploaded + jsonfiles: :obj:`list` of :obj:`str` + tilespecs in json files + transformFile: str, optional + json file listing transformspecs with ids which are referenced + in tilespecs contained in jsonfiles + + """ + transform_params = (['--transformFile', transformFile] + if transformFile is not None else []) + if validator is None: + validator_params = [ + '--validatorClass', + 'org.janelia.alignment.spec.validator.TemTileSpecValidator', + '--validatorData', + 'minCoordinate:-500,maxCoordinate:100000,' + 'minSize:500,maxSize:10000'] + else: + raise NotImplementedError('No custom validation handling!') + + stack_params = make_stack_params(host, port, owner, project, stack) + set_stack_state(stack, 'LOADING', host, port, owner, project) + + call_run_ws_client('org.janelia.render.client.ImportJsonClient', + stack_params + + validator_params + + transform_params + + jsonfiles, client_script=client_script, + memGB=memGB, subprocess_mode=subprocess_mode, + **kwargs) + + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +@renderclientaccess +def import_tilespecs(stack, tilespecs, sharedTransforms=None, + use_rest=False, deriveData=True, + subprocess_mode=None, host=None, port=None, + owner=None, project=None, client_script=None, + memGB=None, render=None, **kwargs): + """method to import tilesepcs directly from + :class:`renderapi.tilespec.TileSpec` objects + + Parameters + ---------- + stack : str + stack to which tilespecs will be added + tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` + list of tilespecs to import + sharedTransforms : :obj:`list` of :class:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional + list of shared referenced transforms to be ingested + use_rest: bool + whether to import the tilespecs using the post method directly with deriveData=True + deriveData: bool + if doing use_rest, will determine whether to have the server calculate bounds (default=True) + render : renderapi.render.Render + render connect object + + """ # noqa: E501 + if use_rest: + put_tilespecs(stack, + deriveData=deriveData, + tilespecs=tilespecs, + shared_transforms=sharedTransforms, + host=host, port=port, owner=owner, + project=project, **kwargs) + else: + tsjson = renderdump_temp(tilespecs) + + if sharedTransforms is not None: + trjson = renderdump_temp(sharedTransforms) + + importJsonClient(stack, tileFiles=[tsjson], transformFile=( + trjson if sharedTransforms is not None else None), + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, + client_script=client_script, memGB=memGB, **kwargs) + + os.remove(tsjson) + if sharedTransforms is not None: + os.remove(trjson) + + +@renderclientaccess +def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, + subprocess_mode=None, poolsize=20, + mpPool=WithPool, + close_stack=True, host=None, port=None, + owner=None, project=None, + client_script=None, memGB=None, render=None, + **kwargs): + """method to import tilesepcs directly from + :class:`renderapi.tilespec.TileSpec` objects using + pathos.multiprocessing to parallelize + + Parameters + ---------- + stack : str + stack to which tilespecs will be added + tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` + list of tilespecs to import + sharedTransforms : obj:`list` of :obj:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional + list of shared referenced transforms to be ingested + poolsize : int + degree of parallelism to use + subprocess_mode : str + subprocess mode used when calling client side java + close_stack : bool + mark render stack as COMPLETE after successful import + render : :class:renderapi.render.Render + render connect object + kwargs: dict .. all other kwargs to pass on to renderapi.client.import_tilespecs + """ # noqa: E501 + set_stack_state(stack, 'LOADING', host, port, owner, project) + partial_import = partial( + import_tilespecs, stack, sharedTransforms=sharedTransforms, + subprocess_mode=subprocess_mode, host=host, port=port, + owner=owner, project=project, client_script=client_script, + memGB=memGB, **kwargs) + + # TODO this is a weird way to do splits.... is that okay? + tilespec_groups = [g for g in + (tilespecs[i::poolsize] for i in range(poolsize)) if g] + with mpPool(poolsize) as pool: + pool.map(partial_import, tilespec_groups) + if close_stack: + set_stack_state(stack, 'COMPLETE', host, port, owner, project) + + +# TODO handle fromJson and toJson persistence in these calls +@renderclientaccess +def local_to_world_array(stack, points, tileId, subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + """placeholder function for coordinateClient localtoworld + + Parameters + ---------- + stack : str + stack to which world coordinates are mapped + points : dict + local points to map to world + tileId : str + tileId to which points correspond + subprocess_mode : str + subprocess mode used when calling + clientside java client + Returns + ------- + list + points in world coordinates corresponding to local points + """ + raise NotImplementedError('Whoops') + + +@renderclientaccess +def world_to_local_array(stack, points, subprocess_mode=None, + host=None, port=None, owner=None, project=None, + client_script=None, memGB=None, + render=None, **kwargs): + """placeholder function for coordinateClient worldtolocal + + Parameters + ---------- + stack : str + stack to which world coordinates are mapped + points : dict + local points to map to world + subprocess_mode : str + subprocess mode used when calling client side java + render : :class:`renderapi.render.Render` + render connect object + + Returns + ------- + :obj:`list` of :obj:`list` + dictionaries defining local coordinates + and tileIds corresponding to world point + """ + raise NotImplementedError('Whoops.') + + +__all__ = [ + "import_single_json_file", + "import_jsonfiles_and_transforms_parallel_by_z", + "import_jsonfiles_parallel", "import_jsonfiles", + "import_jsonfiles_validate_client", "import_tilespecs", + "import_tilespecs_parallel", "local_to_world_array", + "world_to_local_array", "WithPool"] diff --git a/renderapi/client.py b/renderapi/client/client_calls.py old mode 100755 new mode 100644 similarity index 57% rename from renderapi/client.py rename to renderapi/client/client_calls.py index ab39120b..d029a175 --- a/renderapi/client.py +++ b/renderapi/client/client_calls.py @@ -1,455 +1,22 @@ -#!/usr/bin/env python -''' -render functions relying on render-ws client scripts -''' -import os -import json -from functools import partial import logging +import json +import os import subprocess import tempfile -# from multiprocessing.pool import Pool -from decorator import decorator -from .errors import ClientScriptError -from .utils import NullHandler, renderdump_temp, fitargspec -from .render import (RenderClient, renderaccess, Render, - format_preamble, format_baseurl) -from .stack import set_stack_state, make_stack_params -from .resolvedtiles import put_tilespecs -from renderapi.external.processpools.stdlib_pool import WithMultiprocessingPool - -# setup logger -logger = logging.getLogger(__name__) -logger.addHandler(NullHandler()) - - -@decorator -def renderclientaccess(f, *args, **kwargs): - """Decorator allowing functions asking for host, port, owner, project, - client_script to default to a connection defined by :class:`RenderClient` - object using its :func:`RenderClient.make_kwargs` method. - Will also attempt to derive a :class:`RenderClient` from an input - :class:`Render` object and fail if client scripts cannot be reached. - - Parameters - ---------- - f : func - function to decorate - Returns - ------- - obj - output of decorated function - """ - args, kwargs = fitargspec(f, args, kwargs) - render = kwargs.get('render') - if render is not None: - if not isinstance(render, RenderClient): - if isinstance(render, Render): - render = RenderClient(**render.make_kwargs(**kwargs)) - else: - raise ValueError( - 'invalid RenderClient object type {} specified!'.format( - type(render))) - return f(*args, **render.make_kwargs(**kwargs)) - else: - try: - client_script = kwargs.get('client_script') - cs_valid = os.path.isfile(client_script) - except TypeError: - try: - client_scripts = kwargs.get('client_scripts') - if os.path.isdir(client_scripts): - client_script = os.path.join(client_scripts, - 'run_ws_client.sh') - cs_valid = os.path.isfile(client_script) - else: - raise ClientScriptError( - 'invalid client_scripts directory {}'.format( - client_scripts)) - except TypeError: - raise ClientScriptError( - 'No client script information specified: ' - 'client_scripts={} client_script={}'.format( - kwargs.get('client_scripts'), - kwargs.get('client_script'))) - if not cs_valid: - # TODO should also check for executability - raise ClientScriptError( - 'invalid client script: {} not a file'.format(client_script)) - return f(*args, **kwargs) - - -WithPool = WithMultiprocessingPool -# class WithPool(Pool): -# """Multiprocessing.pool.Pool with functioning __exit__ call -# -# Parameters -# ---------- -# *args -# variable length argument list matching input -# to multiprocessing.pool.Pool -# **kwargs -# keyword argument input matching multiprocessing.pool.Pool -# -# Examples -# -------- -# >>> with WithPool(number_processes) as pool: -# >>> pool.map(myfunc, myInput) -# """ -# -# def __init__(self, *args, **kwargs): -# super(WithPool, self).__init__(*args, **kwargs) -# -# def __enter__(self): -# return self -# -# def __exit__(self, *args, **kwargs): -# self.close() -# self.join() - - -@renderclientaccess -def import_single_json_file(stack, jsonfile, transformFile=None, - subprocess_mode=None, client_script=None, - memGB=None, host=None, port=None, - owner=None, project=None, render=None, **kwargs): - """calls client script to import given jsonfile - - Parameters - ---------- - stack : str - stack to import into - jsonfile : str - path to jsonfile to import - transformFile : str - path to a file that contains shared - transform references if necessary - render : renderapi.render.RenderClient - render connect object - """ - if transformFile is None: - transform_params = [] - else: - transform_params = ['--transformFile', transformFile] - stack_params = make_stack_params( - host, port, owner, project, stack) - call_run_ws_client('org.janelia.render.client.ImportJsonClient', - stack_params + transform_params + [jsonfile], - client_script=client_script, memGB=memGB, - subprocess_mode=subprocess_mode, **kwargs) - - -@renderclientaccess -def import_jsonfiles_and_transforms_parallel_by_z( - stack, jsonfiles, transformfiles, poolsize=20, mpPool=WithPool, - client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, render=None, **kwargs): - """imports json files and transform files in parallel - - Parameters - ---------- - stack : str - the stack to import within - jsonfiles : :obj:`list` of :obj:`str` - "list of tilespec" json paths to import - transformfiles : :obj:`list` of :obj:`str` - "list of transform files" paths which matches - in a 1-1 way with jsonfiles, so referenced transforms - are shared only within a single element of these matched lists. - Useful cases where there is as single z transforms shared - by all tiles within a single z, but not across z's - poolsize : int, optional - number of processes for multiprocessing pool - close_stack : bool, optional - whether to mark render stack as COMPLETE after successful import - render : renderapi.render.Render - render connect object - **kwargs - arbitrary keyword arguments - - """ - set_stack_state(stack, 'LOADING', host, port, owner, project) - partial_import = partial(import_single_json_file, stack, render=render, - client_scripts=client_scripts, host=host, - port=port, owner=owner, project=project) - with mpPool(poolsize) as pool: - pool.map(partial_import, jsonfiles, transformfiles) - - if close_stack: - set_stack_state(stack, 'COMPLETE', host, port, owner, project) - - -@renderclientaccess -def import_jsonfiles_parallel( - stack, jsonfiles, poolsize=20, transformFile=None, mpPool=WithPool, - client_scripts=None, host=None, port=None, owner=None, - project=None, close_stack=True, render=None, **kwargs): - """import jsons using client script in parallel - - Parameters - ---------- - stack : str - the stack to upload into - jsonfiles : :obj:`list` of :obj:`str` - list of jsonfile paths to upload - poolsize : int - number of upload processes spawned by multiprocessing pool - transformFile : str - a single json file path containing transforms referenced - in the jsonfiles - close_stack : bool - whether to mark render stack as COMPLETE after successful import - render : renderapi.render.Render - render connect object - **kwargs - arbitrary keyword arguments - - """ - set_stack_state(stack, 'LOADING', host, port, owner, project) - - partial_import = partial(import_single_json_file, stack, render=render, - transformFile=transformFile, - client_scripts=client_scripts, - host=host, port=port, owner=owner, - project=project, **kwargs) - with mpPool(poolsize) as pool: - pool.map(partial_import, jsonfiles) - - if close_stack: - set_stack_state(stack, 'COMPLETE', host, port, owner, project) - - -@renderaccess -def import_jsonfiles(stack, jsonfiles, transformFile=None, - subprocess_mode=None, client_script=None, memGB=None, - host=None, port=None, owner=None, project=None, - close_stack=True, render=None, **kwargs): - """import jsons using client script serially - - Parameters - ---------- - jsonfiles : :obj:`list` of :obj:`str` - iterator of filenames to be uploaded - transformFile : str - path to a jsonfile that contains shared - transform references (if necessary) - close_stack : bool - mark render stack as COMPLETE after successful import - render : renderapi.render.Render - render connect object - - """ - - set_stack_state(stack, 'LOADING', host, port, owner, project) - if transformFile is None: - transform_params = [] - else: - transform_params = ['--transformFile', transformFile] - stack_params = make_stack_params( - host, port, owner, project, stack) - call_run_ws_client('org.janelia.render.client.ImportJsonClient', - stack_params + transform_params + jsonfiles, - client_script=client_script, memGB=memGB, - subprocess_mode=subprocess_mode, **kwargs) - if close_stack: - set_stack_state(stack, 'COMPLETE', host, port, owner, project) - - -@renderclientaccess -def import_jsonfiles_validate_client(stack, jsonfiles, - transformFile=None, client_script=None, - host=None, port=None, owner=None, - project=None, close_stack=True, mem=6, - validator=None, subprocess_mode=None, - memGB=None, - render=None, **kwargs): - """Uses java client for parallelization and validation - - Parameters - ---------- - stack: str - stack to which jsonfiles should be uploaded - jsonfiles: :obj:`list` of :obj:`str` - tilespecs in json files - transformFile: str, optional - json file listing transformspecs with ids which are referenced - in tilespecs contained in jsonfiles - - """ - transform_params = (['--transformFile', transformFile] - if transformFile is not None else []) - if validator is None: - validator_params = [ - '--validatorClass', - 'org.janelia.alignment.spec.validator.TemTileSpecValidator', - '--validatorData', - 'minCoordinate:-500,maxCoordinate:100000,' - 'minSize:500,maxSize:10000'] - else: - raise NotImplementedError('No custom validation handling!') - - stack_params = make_stack_params(host, port, owner, project, stack) - set_stack_state(stack, 'LOADING', host, port, owner, project) - - call_run_ws_client('org.janelia.render.client.ImportJsonClient', - stack_params + - validator_params + - transform_params + - jsonfiles, client_script=client_script, - memGB=memGB, subprocess_mode=subprocess_mode, - **kwargs) - - if close_stack: - set_stack_state(stack, 'COMPLETE', host, port, owner, project) - - -@renderclientaccess -def import_tilespecs(stack, tilespecs, sharedTransforms=None, - use_rest=False, deriveData=True, - subprocess_mode=None, host=None, port=None, - owner=None, project=None, client_script=None, - memGB=None, render=None, **kwargs): - """method to import tilesepcs directly from - :class:`renderapi.tilespec.TileSpec` objects - - Parameters - ---------- - stack : str - stack to which tilespecs will be added - tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` - list of tilespecs to import - sharedTransforms : :obj:`list` of :class:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional - list of shared referenced transforms to be ingested - use_rest: bool - whether to import the tilespecs using the post method directly with deriveData=True - deriveData: bool - if doing use_rest, will determine whether to have the server calculate bounds (default=True) - render : renderapi.render.Render - render connect object - - """ # noqa: E501 - if use_rest: - put_tilespecs(stack, - deriveData=deriveData, - tilespecs=tilespecs, - shared_transforms=sharedTransforms, - host=host, port=port, owner=owner, - project=project, **kwargs) - else: - tsjson = renderdump_temp(tilespecs) - - if sharedTransforms is not None: - trjson = renderdump_temp(sharedTransforms) - - importJsonClient(stack, tileFiles=[tsjson], transformFile=( - trjson if sharedTransforms is not None else None), - subprocess_mode=subprocess_mode, host=host, port=port, - owner=owner, project=project, - client_script=client_script, memGB=memGB, **kwargs) - - os.remove(tsjson) - if sharedTransforms is not None: - os.remove(trjson) - - -@renderclientaccess -def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, - subprocess_mode=None, poolsize=20, - mpPool=WithPool, - close_stack=True, host=None, port=None, - owner=None, project=None, - client_script=None, memGB=None, render=None, - **kwargs): - """method to import tilesepcs directly from - :class:`renderapi.tilespec.TileSpec` objects using - pathos.multiprocessing to parallelize - - Parameters - ---------- - stack : str - stack to which tilespecs will be added - tilespecs : :obj:`list` of :class:`renderapi.tilespec.TileSpec` - list of tilespecs to import - sharedTransforms : obj:`list` of :obj:`renderapi.transform.Transform` or :class:`renderapi.transform.TransformList` or :class:`renderapi.transform.InterpolatedTransform`, optional - list of shared referenced transforms to be ingested - poolsize : int - degree of parallelism to use - subprocess_mode : str - subprocess mode used when calling client side java - close_stack : bool - mark render stack as COMPLETE after successful import - render : :class:renderapi.render.Render - render connect object - kwargs: dict .. all other kwargs to pass on to renderapi.client.import_tilespecs - """ # noqa: E501 - set_stack_state(stack, 'LOADING', host, port, owner, project) - partial_import = partial( - import_tilespecs, stack, sharedTransforms=sharedTransforms, - subprocess_mode=subprocess_mode, host=host, port=port, - owner=owner, project=project, client_script=client_script, - memGB=memGB, **kwargs) - - # TODO this is a weird way to do splits.... is that okay? - tilespec_groups = [g for g in - (tilespecs[i::poolsize] for i in range(poolsize)) if g] - with mpPool(poolsize) as pool: - pool.map(partial_import, tilespec_groups) - if close_stack: - set_stack_state(stack, 'COMPLETE', host, port, owner, project) - -# TODO handle fromJson and toJson persistence in these calls -@renderclientaccess -def local_to_world_array(stack, points, tileId, subprocess_mode=None, - host=None, port=None, owner=None, project=None, - client_script=None, memGB=None, - render=None, **kwargs): - """placeholder function for coordinateClient localtoworld +from renderapi.utils import NullHandler +from renderapi.errors import ClientScriptError +from renderapi.render import (format_baseurl, format_preamble, + RenderClient, Render) +from renderapi.stack import make_stack_params, set_stack_state - Parameters - ---------- - stack : str - stack to which world coordinates are mapped - points : dict - local points to map to world - tileId : str - tileId to which points correspond - subprocess_mode : str - subprocess mode used when calling - clientside java client - Returns - ------- - list - points in world coordinates corresponding to local points - """ - raise NotImplementedError('Whoops') +from .utils import renderclientaccess +from .params import SiftPointMatchOptions -@renderclientaccess -def world_to_local_array(stack, points, subprocess_mode=None, - host=None, port=None, owner=None, project=None, - client_script=None, memGB=None, - render=None, **kwargs): - """placeholder function for coordinateClient worldtolocal - - Parameters - ---------- - stack : str - stack to which world coordinates are mapped - points : dict - local points to map to world - subprocess_mode : str - subprocess mode used when calling client side java - render : :class:`renderapi.render.Render` - render connect object - - Returns - ------- - :obj:`list` of :obj:`list` - dictionaries defining local coordinates - and tileIds corresponding to world point - """ - raise NotImplementedError('Whoops.') +# setup logger +logger = logging.getLogger(__name__) +logger.addHandler(NullHandler()) def run_subprocess_mode(args, subprocess_mode=None, **kwargs): @@ -973,76 +540,9 @@ def get_canvas_url_template( return canvas_url_template -class ArgumentParameters(object): - def __init__(self, *args, **kwargs): - pass - - @staticmethod - def sanitize_cmd(cmd): - def jbool_str(c): - return str(c) if type(c) is not bool else "true" if c else "false" - if any([i is None for i in cmd]): - raise ClientScriptError( - 'missing argument in command "{}"'.format(map(str, cmd))) - return map(jbool_str, cmd) - - @staticmethod - def get_cmd_opt(v, flag=None): - return [] if v is None else [v] if flag is None else [flag, v] - - @staticmethod - def get_flag_cmd(v, flag=None): - # for arity 0 - return [flag] if v else [] - - def to_java_args(self): - args = [] - for key, value in self.__dict__.items(): - if (value is not None) and not (key == 'kwargs'): - args += self.get_cmd_opt(value, "--{}".format(key)) - return self.sanitize_cmd(args) - - -class FeatureExtractionParameters(ArgumentParameters): - def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, - SIFTminScale=None, SIFTsteps=None, **kwargs): - super(FeatureExtractionParameters, self).__init__(**kwargs) - self.SIFTfdSize = SIFTfdSize - self.SIFTmaxScale = SIFTmaxScale - self.SIFTminScale = SIFTminScale - self.SIFTsteps = SIFTsteps - - -class MatchDerivationParameters(ArgumentParameters): - def __init__(self, matchIterations=None, - matchMaxEpsilon=None, matchMaxNumInliers=None, - matchMaxTrust=None, matchMinInlierRatio=None, - matchMinNumInliers=None, - matchModelType=None, matchRod=None, **kwargs): - super(MatchDerivationParameters, self).__init__(**kwargs) - self.matchIterations = matchIterations - self.matchMaxEpsilon = matchMaxEpsilon - self.matchMaxNumInliers = matchMaxNumInliers - self.matchMaxTrust = matchMaxTrust - self.matchMinInlierRatio = matchMinInlierRatio - self.matchMinNumInliers = matchMinNumInliers - self.matchMinNumInliers = matchMinNumInliers - self.matchModelType = matchModelType - self.matchRod = matchRod - - -class SiftPointMatchOptions(MatchDerivationParameters, - FeatureExtractionParameters): - def __init__(self, renderScale=None, fillWithNoise=None, **kwargs): - # TODO add missing parameters - super(SiftPointMatchOptions, self).__init__(**kwargs) - self.renderScale = renderScale - self.fillWithNoise = fillWithNoise - - @renderclientaccess def pointMatchClient(stack, collection, tile_pairs, - stack2 = None, + stack2=None, sift_options=None, pointMatchRender=None, debugDirectory=None, @@ -1156,3 +656,11 @@ def pointMatchClient(stack, collection, tile_pairs, memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) + + +__all__ = [ + "call_run_ws_client", "run_subprocess_mode", + "importJsonClient", "tilePairClient", + "importTransformChangesClient", "coordinateClient", + "renderSectionClient", "transformSectionClient", + "get_canvas_url_template", "pointMatchClient"] diff --git a/renderapi/client/params.py b/renderapi/client/params.py new file mode 100644 index 00000000..ce2d6b64 --- /dev/null +++ b/renderapi/client/params.py @@ -0,0 +1,73 @@ +from renderapi.errors import ClientScriptError + + +class ArgumentParameters(object): + def __init__(self, *args, **kwargs): + pass + + @staticmethod + def sanitize_cmd(cmd): + def jbool_str(c): + return str(c) if type(c) is not bool else "true" if c else "false" + if any([i is None for i in cmd]): + raise ClientScriptError( + 'missing argument in command "{}"'.format(map(str, cmd))) + return map(jbool_str, cmd) + + @staticmethod + def get_cmd_opt(v, flag=None): + return [] if v is None else [v] if flag is None else [flag, v] + + @staticmethod + def get_flag_cmd(v, flag=None): + # for arity 0 + return [flag] if v else [] + + def to_java_args(self): + args = [] + for key, value in self.__dict__.items(): + if (value is not None) and not (key == 'kwargs'): + args += self.get_cmd_opt(value, "--{}".format(key)) + return self.sanitize_cmd(args) + + +class FeatureExtractionParameters(ArgumentParameters): + def __init__(self, SIFTfdSize=None, SIFTmaxScale=None, + SIFTminScale=None, SIFTsteps=None, **kwargs): + super(FeatureExtractionParameters, self).__init__(**kwargs) + self.SIFTfdSize = SIFTfdSize + self.SIFTmaxScale = SIFTmaxScale + self.SIFTminScale = SIFTminScale + self.SIFTsteps = SIFTsteps + + +class MatchDerivationParameters(ArgumentParameters): + def __init__(self, matchIterations=None, + matchMaxEpsilon=None, matchMaxNumInliers=None, + matchMaxTrust=None, matchMinInlierRatio=None, + matchMinNumInliers=None, + matchModelType=None, matchRod=None, **kwargs): + super(MatchDerivationParameters, self).__init__(**kwargs) + self.matchIterations = matchIterations + self.matchMaxEpsilon = matchMaxEpsilon + self.matchMaxNumInliers = matchMaxNumInliers + self.matchMaxTrust = matchMaxTrust + self.matchMinInlierRatio = matchMinInlierRatio + self.matchMinNumInliers = matchMinNumInliers + self.matchMinNumInliers = matchMinNumInliers + self.matchModelType = matchModelType + self.matchRod = matchRod + + +class SiftPointMatchOptions(MatchDerivationParameters, + FeatureExtractionParameters): + def __init__(self, renderScale=None, fillWithNoise=None, **kwargs): + # TODO add missing parameters + super(SiftPointMatchOptions, self).__init__(**kwargs) + self.renderScale = renderScale + self.fillWithNoise = fillWithNoise + + +__all__ = [ + "ArgumentParameters", "FeatureExtractionParameters", + "MatchDerivationParameters", "SiftPointMatchOptions"] diff --git a/renderapi/client/utils.py b/renderapi/client/utils.py new file mode 100644 index 00000000..e2985a10 --- /dev/null +++ b/renderapi/client/utils.py @@ -0,0 +1,65 @@ +from decorator import decorator +import os + +from renderapi.errors import ClientScriptError +from renderapi.utils import fitargspec +from renderapi.render import RenderClient, Render + + +@decorator +def renderclientaccess(f, *args, **kwargs): + """Decorator allowing functions asking for host, port, owner, project, + client_script to default to a connection defined by :class:`RenderClient` + object using its :func:`RenderClient.make_kwargs` method. + Will also attempt to derive a :class:`RenderClient` from an input + :class:`Render` object and fail if client scripts cannot be reached. + + Parameters + ---------- + f : func + function to decorate + Returns + ------- + obj + output of decorated function + """ + args, kwargs = fitargspec(f, args, kwargs) + render = kwargs.get('render') + if render is not None: + if not isinstance(render, RenderClient): + if isinstance(render, Render): + render = RenderClient(**render.make_kwargs(**kwargs)) + else: + raise ValueError( + 'invalid RenderClient object type {} specified!'.format( + type(render))) + return f(*args, **render.make_kwargs(**kwargs)) + else: + try: + client_script = kwargs.get('client_script') + cs_valid = os.path.isfile(client_script) + except TypeError: + try: + client_scripts = kwargs.get('client_scripts') + if os.path.isdir(client_scripts): + client_script = os.path.join(client_scripts, + 'run_ws_client.sh') + cs_valid = os.path.isfile(client_script) + else: + raise ClientScriptError( + 'invalid client_scripts directory {}'.format( + client_scripts)) + except TypeError: + raise ClientScriptError( + 'No client script information specified: ' + 'client_scripts={} client_script={}'.format( + kwargs.get('client_scripts'), + kwargs.get('client_script'))) + if not cs_valid: + # TODO should also check for executability + raise ClientScriptError( + 'invalid client script: {} not a file'.format(client_script)) + return f(*args, **kwargs) + + +__all__ = ["renderclientaccess", "ClientScriptError"] From c75868314f8a7ced807d0dc5213d22714180e93d Mon Sep 17 00:00:00 2001 From: RussTorres Date: Tue, 28 Aug 2018 10:02:17 -0700 Subject: [PATCH 701/766] client: add max_tilespecs_per_group option to import_ts_parallel --- renderapi/client/client.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/renderapi/client/client.py b/renderapi/client/client.py index 78b6c40d..081535ee 100644 --- a/renderapi/client/client.py +++ b/renderapi/client/client.py @@ -258,7 +258,8 @@ def import_tilespecs(stack, tilespecs, sharedTransforms=None, def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, subprocess_mode=None, poolsize=20, mpPool=WithPool, - close_stack=True, host=None, port=None, + close_stack=True, max_tilespecs_per_group=None, + host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): @@ -280,10 +281,16 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, subprocess mode used when calling client side java close_stack : bool mark render stack as COMPLETE after successful import + max_tilespecs_per_group: int + maximum tilespecs per import process, default to len(tilespecs)/poolsize render : :class:renderapi.render.Render render connect object kwargs: dict .. all other kwargs to pass on to renderapi.client.import_tilespecs """ # noqa: E501 + tslists = ( + max((len(tilespecs) // max_tilespecs_per_group) + 1, poolsize) if + max_tilespecs_per_group is not None else poolsize) + set_stack_state(stack, 'LOADING', host, port, owner, project) partial_import = partial( import_tilespecs, stack, sharedTransforms=sharedTransforms, @@ -293,7 +300,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, # TODO this is a weird way to do splits.... is that okay? tilespec_groups = [g for g in - (tilespecs[i::poolsize] for i in range(poolsize)) if g] + (tilespecs[i::tslists] for i in range(tslists)) if g] with mpPool(poolsize) as pool: pool.map(partial_import, tilespec_groups) if close_stack: From 27db1863872bfcfd74d068c48f3d7505ba5fd93a Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Wed, 12 Sep 2018 10:40:31 -0700 Subject: [PATCH 702/766] making affine properties correct for Similarity (#122) * making affiner properties correct for Similarity * trivial change to retrigger 2.7 apt-get build failure * doing away with shear tuple, adding option to force shear as all x (default) or y for affine * one more sub-test for sin(theta)=0 --- renderapi/transform/leaf/affine_models.py | 61 +++++++--- test/test_transform.py | 135 +++++++++++++++++++++- 2 files changed, 175 insertions(+), 21 deletions(-) diff --git a/renderapi/transform/leaf/affine_models.py b/renderapi/transform/leaf/affine_models.py index 1ebf7be5..d4caf3c7 100644 --- a/renderapi/transform/leaf/affine_models.py +++ b/renderapi/transform/leaf/affine_models.py @@ -49,7 +49,7 @@ class AffineModel(Transform): className = 'mpicbg.trakem2.transform.AffineModel2D' def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, - transformId=None, labels=None, json=None): + transformId=None, labels=None, json=None, force_shear='x'): """Initialize AffineModel, defaulting to identity Parameters @@ -88,6 +88,7 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, self.labels = labels self.load_M() self.transformId = transformId + self.force_shear = force_shear @property def dataString(self): @@ -207,17 +208,10 @@ def concatenate(self, model): AffineModel model after concatenating model with this model """ - a00 = self.M[0, 0] * model.M[0, 0] + self.M[0, 1] * model.M[1, 0] - a01 = self.M[0, 0] * model.M[0, 1] + self.M[0, 1] * model.M[1, 1] - a02 = (self.M[0, 0] * model.M[0, 2] + self.M[0, 1] * model.M[1, 2] + - self.M[0, 2]) - - a10 = self.M[1, 0] * model.M[0, 0] + self.M[1, 1] * model.M[1, 0] - a11 = self.M[1, 0] * model.M[0, 1] + self.M[1, 1] * model.M[1, 1] - a12 = (self.M[1, 0] * model.M[0, 2] + self.M[1, 1] * model.M[1, 2] + - self.M[1, 2]) - - newmodel = AffineModel(a00, a01, a10, a11, a02, a12) + A = self.M.dot(model.M) + newmodel = AffineModel( + A[0, 0], A[0, 1], A[1, 0], + A[1, 1], A[0, 2], A[1, 2]) return newmodel def invert(self): @@ -309,16 +303,48 @@ def inverse_tform(self, points): pt = np.dot(np.linalg.inv(self.M), points.T).T return self.convert_points_vector_to_array(pt, Nd) + def calc_properties(self): + if self.force_shear == 'x': + sy = np.sqrt(self.M[1, 0] ** 2 + self.M[1, 1] ** 2) + theta = np.arctan2(self.M[1, 0], self.M[1, 1]) + rc = np.cos(theta) + rs = np.sin(theta) + sx = rc * self.M[0, 0] - rs * self.M[0, 1] + if rs != 0: + cx = (self.M[0, 0] - sx*rc) / (sx * rs) + else: + cx = (self.M[0, 1] - sx*rs) / (sx * rc) + cy = 0.0 + else: + # shear in y direction + sx = np.sqrt(self.M[0, 0] ** 2 + self.M[0, 1] ** 2) + theta = np.arctan2(-self.M[0, 1], self.M[0, 0]) + rc = np.cos(theta) + rs = np.sin(theta) + sy = rs * self.M[1, 0] + rc * self.M[1, 1] + if rs != 0: + cy = (self.M[1, 1] - sy * rc) / (-sy * rs) + else: + cy = (self.M[1, 0] - sy * rs) / (sy * rc) + cx = 0.0 + # room for other cases, for example cx = cy + + return sx, sy, cx, cy, theta + @property def scale(self): """tuple of scale for x, y""" - return tuple([np.sqrt(sum([i ** 2 for i in self.M[:, j]])) - for j in range(self.M.shape[1])])[:2] + sx, sy, cx, cy, theta = self.calc_properties() + return (sx, sy) @property def shear(self): - """counter-clockwise shear angle""" - return np.arctan2(-self.M[0, 1], self.M[1, 1]) - self.rotation + """shear""" + sx, sy, cx, cy, theta = self.calc_properties() + if self.force_shear == 'x': + return cx + else: + return cy @property def translation(self): @@ -328,7 +354,8 @@ def translation(self): @property def rotation(self): """counter-clockwise rotation""" - return np.arctan2(self.M[1, 0], self.M[0, 0]) + sx, sy, cx, cy, theta = self.calc_properties() + return theta def __str__(self): return "M=[[%f,%f],[%f,%f]] B=[%f,%f]" % ( diff --git a/test/test_transform.py b/test/test_transform.py index 32c8df77..52453aca 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -59,8 +59,8 @@ def test_fail_bad_tform(): tform = renderapi.transform.load_transform_json(d) # noqa: F841 -def test_affine_rot_90(): - am = renderapi.transform.AffineModel() +def test_similarity_rot_90(): + am = renderapi.transform.SimilarityModel() # setup a 90 degree clockwise rotation points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float) points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float) @@ -71,7 +71,7 @@ def test_affine_rot_90(): assert(np.abs(am.rotation + np.pi / 2) < .000001) assert(np.abs(am.translation[0]) < .000001) assert(np.abs(am.translation[1]) < .000001) - assert(np.abs(am.shear) < .000001) + assert(np.all(np.abs(am.shear) < .000001)) points = np.array([[20, 30], [1, 2], [10, -5], [-4, 3], [5.6, 2.3]]) new_points = am.tform(points) @@ -88,7 +88,7 @@ def test_affine_rot_90(): assert(np.abs(identity.rotation) < .000001) assert(np.abs(identity.translation[0]) < .000001) assert(np.abs(identity.translation[1]) < .000001) - assert(np.abs(identity.shear) < .000001) + assert(np.all(np.abs(identity.shear) < .000001)) print(str(am)) @@ -627,6 +627,133 @@ def test_similarity_init(): assert(tform.M[0, 2] == 10) +def test_affine_properties(): + # scale + sx = 1.5 + sy = 1.1 + # shear + cx = 0.0 + cy = 0.0 + # angle + theta = 0.0 + pt = np.array( + [1.234, 4.567, 1]) + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + rnd_tf.force_shear = 'y' + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + theta = 0.1234 + pt = np.array( + [1.234, 4.567, 1]) + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + rnd_tf.force_shear = 'y' + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + cx = 0.2 + cy = 0.0 + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + rnd_tf.force_shear = 'y' + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + + cx = 0.0 + cy = 0.1 + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + + rnd_tf.force_shear = 'y' + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + assert (np.abs(rnd_tf.shear - cy) < 1e-8) + + cx = 0.3 + cy = 0.1 + M, testpt, rnd_tf = affine_property_function(sx, sy, cx, cy, theta, pt) + tformpt = rnd_tf.tform(np.array([pt[0:2]])) + assert np.all(np.abs(M - rnd_tf.M) < 1e-8) + assert np.all(np.abs(testpt[0:2] - tformpt) < 1e-8) + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.shear - cx) < 1e-8) + + rnd_tf.force_shear = 'y' + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.scale[0] - sx) < 1e-8) & \ + (np.abs(rnd_tf.scale[1] - sy) < 1e-8) + with pytest.raises(AssertionError): + assert np.abs(rnd_tf.rotation - theta) < 1e-8 + with pytest.raises(AssertionError): + assert (np.abs(rnd_tf.shear - cy) < 1e-8) + + +def affine_property_function(sx, sy, cx, cy, theta, pt): + scale = np.array([ + [sx, 0, 0], + [0, sy, 0], + [0, 0, 1]]) + shear = np.array([ + [1, cx, 0], + [cy, 1, 0], + [0, 0, 1]]) + rotation = np.array([ + [np.cos(theta), -np.sin(theta), 0], + [np.sin(theta), np.cos(theta), 0], + [0, 0, 1]]) + M = scale.dot(shear.dot(rotation)) + + testpt = M.dot(pt)[0:2] + + rnd_tf = renderapi.transform.SimilarityModel( + M00=M[0, 0], + M01=M[0, 1], + M10=M[1, 0], + M11=M[1, 1]) + return M, testpt, rnd_tf + + def test_thinplatespline_inverse(): j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) t = renderapi.transform.ThinPlateSplineTransform( From d3900a845d479ca1ad61987e9ae75518d26692d2 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Wed, 12 Sep 2018 10:40:44 -0700 Subject: [PATCH 703/766] client: add ARGBRender functionality and wrapped call (#121) --- integration_tests/test_client_integrated.py | 35 ++++++++++++++++++ renderapi/client/client.py | 41 ++++++++++++++++++++- renderapi/client/client_calls.py | 28 +++++++++++++- 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index a2f0ea59..55bffb63 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -5,6 +5,7 @@ import logging import sys import json +from PIL import Image import numpy as np from test_data import (render_host, render_port, client_script_location, tilespec_file, tform_file, @@ -358,3 +359,37 @@ def test_processpools_parallelfuncs( render, render_example_tilespec_and_transforms, "{}_jsonfiles".format(stackbase), False, poolsize=poolsize, mpPool=poolclass) + + +@pytest.fixture(scope='module') +def tile_tilespec(): + tile_dims = (256, 256) + with tempfile.NamedTemporaryFile(suffix='.tif', mode='w') as imgf: + arr = np.random.randint(0, 256, size=tile_dims, dtype='uint8') + img = Image.fromarray(arr) + img.save(imgf.name) + + ts = renderapi.tilespec.TileSpec( + tileId='myTestTile', + z=1., + sectionId="z1.0", + width=tile_dims[0], + height=tile_dims[1], + minint=0, + maxint=255, + imageUrl="file://{}".format(imgf.name)) + ts.minX = 0 + ts.maxX = tile_dims[0] + ts.minY = 0 + ts.maxY = tile_dims[1] + + yield imgf.name, ts + + +def test_ARGBrenderclient(render, tile_tilespec): + tile, tspec = tile_tilespec + arr = renderapi.client.render_tilespec(tspec, memGB='512M', render=render) + with Image.open(tile) as tileimg: + tilearr = np.array(tileimg) + assert arr.shape[:-1] == (tspec.width, tspec.height) == tilearr.shape + assert np.all(arr[:, :, 0] == tilearr) diff --git a/renderapi/client/client.py b/renderapi/client/client.py index 081535ee..f9c1a80a 100644 --- a/renderapi/client/client.py +++ b/renderapi/client/client.py @@ -5,6 +5,10 @@ import os from functools import partial import logging +import tempfile + +import numpy +from PIL import Image from renderapi.utils import NullHandler, renderdump_temp from renderapi.render import renderaccess @@ -13,7 +17,7 @@ from renderapi.external.processpools.stdlib_pool import WithMultiprocessingPool from .utils import renderclientaccess -from .client_calls import importJsonClient, call_run_ws_client +from .client_calls import importJsonClient, call_run_ws_client, renderClient # setup logger logger = logging.getLogger(__name__) @@ -361,10 +365,43 @@ def world_to_local_array(stack, points, subprocess_mode=None, raise NotImplementedError('Whoops.') +def _defaultval(v, default=None): + return default if v is None else v + + +@renderclientaccess +def materialize_tilespec_image( + tilespec, out_fn=None, height=None, width=None, + x=None, y=None, res=32, + subprocess_mode=None, + client_script=None, memGB=None, + render=None, **kwargs): + tspecfile = renderdump_temp([tilespec]) + + x = _defaultval(x, tilespec.minX) + y = _defaultval(y, tilespec.minY) + width = _defaultval(width, int(float((tilespec.maxX - tilespec.minX)))) + height = _defaultval(height, int(float((tilespec.maxY - tilespec.minY)))) + renderClient(tile_spec_url=tspecfile, out_fn=out_fn, + height=height, width=width, x=x, y=y, res=res, + subprocess_mode=subprocess_mode, + client_script=client_script, memGB=memGB, **kwargs) + + os.remove(tspecfile) + + +def render_tilespec(*args, **kwargs): + with tempfile.NamedTemporaryFile(suffix='.tif') as f: + materialize_tilespec_image(*args, out_fn=f.name, **kwargs) + arr = numpy.array(Image.open(f.name)) + return arr + + __all__ = [ "import_single_json_file", "import_jsonfiles_and_transforms_parallel_by_z", "import_jsonfiles_parallel", "import_jsonfiles", "import_jsonfiles_validate_client", "import_tilespecs", "import_tilespecs_parallel", "local_to_world_array", - "world_to_local_array", "WithPool"] + "world_to_local_array", "WithPool", + "render_tilespec", "materialize_tilespec_image"] diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index d029a175..260b1cdb 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -11,7 +11,7 @@ from renderapi.stack import make_stack_params, set_stack_state from .utils import renderclientaccess -from .params import SiftPointMatchOptions +from .params import ArgumentParameters, SiftPointMatchOptions # setup logger @@ -82,7 +82,7 @@ def call_run_ws_client(className, add_args=[], renderclient=None, logger.warning('call_run_ws_client requires memory specification -- ' 'defaulting to 1G') memGB = '1G' - args = map(str, [client_script, memGB, className] + add_args) + args = list(map(str, [client_script, memGB, className] + add_args)) try: ret_val = run_subprocess_mode(args, **kwargs) except subprocess.CalledProcessError: @@ -658,6 +658,30 @@ def pointMatchClient(stack, collection, tile_pairs, **kwargs) +@renderclientaccess +def renderClient(tile_spec_url=None, height=None, width=None, in_fn=None, + out_fn=None, x=None, y=None, res=None, + subprocess_mode=None, client_script=None, + memGB=None, render=None, **kwargs): + """call render + """ + get_cmd_opt = ArgumentParameters.get_cmd_opt + + argvs = (get_cmd_opt(tile_spec_url, '--tile_spec_url') + + get_cmd_opt(height, '--height') + + get_cmd_opt(width, '--width') + + get_cmd_opt(in_fn, '--in') + + get_cmd_opt(out_fn, '--out') + + get_cmd_opt(x, '--x') + + get_cmd_opt(y, '--y') + + get_cmd_opt(res, '--res')) + + call_run_ws_client('org.janelia.alignment.Render', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + __all__ = [ "call_run_ws_client", "run_subprocess_mode", "importJsonClient", "tilePairClient", From 4c41492989207eb9585cf9a58a934341671cb3ea Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Wed, 12 Sep 2018 10:42:18 -0700 Subject: [PATCH 704/766] Add fit estimate to tps (#123) * clunky, but working copy of javafor computing dMtxDat * working with tests, still clunky * added stand-alone test, not dependent on pre-computed json from java client --- renderapi/transform/leaf/thin_plate_spline.py | 95 ++++++++++++++++++ test/rendersettings.py | 12 ++- test/test_files/thin_plate_spline_affine.json | 5 + test/test_transform.py | 99 +++++++++++++++++++ 4 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 test/test_files/thin_plate_spline_affine.json diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py index 7ac6eb1f..d16fe63b 100644 --- a/renderapi/transform/leaf/thin_plate_spline.py +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -180,6 +180,101 @@ def inverse_tform( newpts.append(npt) return np.array(newpts) + @staticmethod + def fit(A, B, computeAffine=True): + """function to fit this transform given the corresponding sets of points A & B + + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + + Returns + ------- + dMatrix : numpy.array + ndims x nLm + aMatrix : numpy.array + ndims x ndims, affine matrix + bVector : numpy.array + ndims x 1, translation vector + """ + + if not all([A.shape[0] == B.shape[0], A.shape[1] == B.shape[1] == 2]): + raise EstimationError( + 'shape mismatch! A shape: {}, B shape {}'.format( + A.shape, B.shape)) + + # build displacements + ndims = B.shape[1] + nLm = B.shape[0] + y = (B - A).flatten() + + # compute K + # tempting to matricize this, but, nLm x nLm can get big + # settle for vectorize + kMatrix = np.zeros((ndims * nLm, ndims * nLm)) + for i in range(nLm): + r = np.linalg.norm(A[i, :] - A, axis=1) + nrm = np.zeros_like(r) + ind = np.argwhere(r > 1e-8) + nrm[ind] = r[ind] * r[ind] * np.log(r[ind]) + kMatrix[i * ndims, 0::2] = nrm + kMatrix[(i * ndims + 1)::2, 1::2] = nrm + + # compute L + lMatrix = kMatrix + if computeAffine: + pMatrix = np.tile(np.eye(ndims), (nLm, ndims + 1)) + for d in range(ndims): + pMatrix[0::2, d*ndims] = A[:, d] + pMatrix[1::2, d*ndims + 1] = A[:, d] + lMatrix = np.zeros( + (ndims * (nLm + ndims + 1), ndims * (nLm + ndims + 1))) + lMatrix[ + 0: pMatrix.shape[0], + kMatrix.shape[1]: kMatrix.shape[1] + pMatrix.shape[1]] = \ + pMatrix + pMatrix = np.transpose(pMatrix) + lMatrix[ + kMatrix.shape[0]: kMatrix.shape[0] + pMatrix.shape[0], + 0: pMatrix.shape[1]] = pMatrix + lMatrix[0: ndims * nLm, 0: ndims * nLm] = kMatrix + y = np.append(y, np.zeros(ndims * (ndims + 1))) + + wMatrix = np.linalg.solve(lMatrix, y) + + dMatrix = np.reshape(wMatrix[0: ndims * nLm], (ndims, nLm), order='F') + aMatrix = None + bVector = None + if computeAffine: + aMatrix = np.reshape( + wMatrix[ndims * nLm: ndims * nLm + ndims * ndims], + (ndims, ndims), + order='F') + bVector = wMatrix[ndims * nLm + ndims * ndims:] + + return dMatrix, aMatrix, bVector + + def estimate(self, A, B, computeAffine=True): + """method for setting this transformation with the best fit + given the corresponding points A,B + Parameters + ---------- + A : numpy.array + a Nx2 matrix of source points + B : numpy.array + a Nx2 matrix of destination points + computeAffine: boolean + whether to include an affine computation + """ + + self.dMtxDat, self.aMtx, self.bVec = self.fit( + A, B, computeAffine=computeAffine) + (self.nLm, self.ndims) = B.shape + self.srcPts = np.transpose(A) + @property def dataString(self): header = 'ThinPlateSplineR2LogR {} {}'.format(self.ndims, self.nLm) diff --git a/test/rendersettings.py b/test/rendersettings.py index 613104af..b825aa58 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -9,7 +9,7 @@ 'owner': 'renderowner', 'project': 'renderproject', 'client_scripts': '/path/to/client_scripts' - } + } DEFAULT_RENDER_CLIENT = dict(DEFAULT_RENDER, **{ 'client_script': '/path/to/client_scripts/run_ws_client.sh', @@ -21,7 +21,7 @@ 'RENDER_OWNER': DEFAULT_RENDER['owner'], 'RENDER_PROJECT': DEFAULT_RENDER['project'], 'RENDER_CLIENT_SCRIPTS': DEFAULT_RENDER['client_scripts'] - } + } DEFAULT_RENDER_CLIENT_ENVIRONMENT_VARIABLES = dict( DEFAULT_RENDER_ENVIRONMENT_VARIABLES, **{ @@ -33,8 +33,12 @@ TEST_TILESPECS_FILE = os.path.join(TEST_FILES_DIR, 'tilespecs.json') -TEST_THINPLATESPLINE_FILE = os.path.join(TEST_FILES_DIR, - 'thin_plate_spline.json') +TEST_THINPLATESPLINE_FILE = os.path.join( + TEST_FILES_DIR, + 'thin_plate_spline.json') +TEST_THINPLATESPLINEAFFINE_FILE = os.path.join( + TEST_FILES_DIR, + 'thin_plate_spline_affine.json') INTERPOLATED_TRANSFORM_TILESPEC = os.path.join( TEST_FILES_DIR, 'tilespec_interpolated.json') diff --git a/test/test_files/thin_plate_spline_affine.json b/test/test_files/thin_plate_spline_affine.json new file mode 100644 index 00000000..3a6a6fcf --- /dev/null +++ b/test/test_files/thin_plate_spline_affine.json @@ -0,0 +1,5 @@ +{ + "type" : "leaf", + "className" : "mpicbg.trakem2.transform.ThinPlateSplineTransform", + "dataString" : "ThinPlateSplineR2LogR 2 100 @P4R64SLPrGq8kudjBEERFDykx/MHaYpjP2iTdKla/W/AWp5BXNVtaMBXQhhumRdE eJx9VH1UjHkULps6ks+S1IqOWkQpW9k4Xu3aPUeFxIxEUdmtlTgVq0+bmSaqpVCRYWoxanvLUFrK8qYimzmkqCWl8bErsaeRb0527/x+ve85c+3Z+ec59/bc5zy/e583AwP9n1ciy5YaOUwV6kyWZbf+HSXUuYgvh7/XDvER6p+hLkxaJdRKxC8Wi8UhsmNCzSK+yuA/f7yvj3DAn1DnIv6AP6Ee8CfUSsQf8CfULOKr/n9PH/nCmIv4vD++5v3xtRLxeX98zSK+Sp+P7/bRnrAvjLw/vub98bUS8Xl/fM0ivkqfL/jDd8J7wT74GiPvj6+ViM/742sW8VX6fMEfzg2+E94L9sHXGJWIz/vjaxbxVfp8wR/OMc4NvhPeC/ahRDweeX98zSK+Sp8v+MPfFc4xzg2+E94L9sHXGFnEV+nzBX/4O8ffFc4xzg2+E94L9sEiHo8qfb7gD/3fwd85/q5wjnFu8J3wXrAPoUbInDRbmS9yDOZKG/peG/YXcNK6ZwFmtm7c7rqQ9L7wOdyu+mh3qXk3YKjMdZsX9IMdzarKOGm9lXGo7C3MPWp0C6oGnRUvG236mZxV0fcmKGO5/Nm/jOc8zJmcs/3q8zfdmO2SpwVlRWrmpwW3mGcWpoCaMvtQF+i/+GrFowfA6321cpotzMktPH3DQWed9rvTrcz+COtI81RLTpFaVd8U/RsTnXvpfYN/IRd7LPvJ1e9TuIhp2Qb3ZH2A6+6rk5yhL9k7UbsdeDvT14+8DXMlnhXLI0HH4sozwwgm762ka8LcGk7+IO2a7/FgJlnatqSjOJTzLhiUvTU2kfGLE7VHdY4D7J4bJcrhvBVZ1W3nKoGnGO1zRg5zW47WvksGneQJ85vvMHkFL5r67vlx8vphnTuHHGYSwpTDPRPVXKBoQ3fTLAfGbprGasySU4yds/Pwyph50HcbX/bSGniRd2c4tXDyug8nJy8TgU6vw9rRIVSvPZjqbe4nel+0hxO93xM8iJ5vxWCil6UOJXolylSi5zKpiOr5Xqd6Trb0vYP30/cGBJL3dpYGkPfuan1D3rtBWkbeu/7QbvLejsqZ5L2+K03Ie+tS/iDv/drImN6jfAS9x2Jnco+L5iJyD7V9MrnHg/mL6D2sbOk9ao3pPdKayT3Kn04n99Aeiqd5SUqhedl4muSFW9NJ8lL6IZPmZfQkmpfKQJqXGzdIXgJbltG8SCJJXsJnLaN5dmqlee7R0Dwfy6B5XhBE8/xlB82zVwPNs+YHkueQC440z5vH0jybx+j0TFQ/toD+xsv3vRLBt1VVS10x7GPbJCNtOez5uU37egVF732kP9jahPIaUumciyHVkY4B/V7toahR4FvZZf3wLOyj8ta56puwZ+mI8MO6u5vaqFoZijtCaL8mk/LudNO5G3KqE+8A73utmHG3CPbxbsPjmHjYc3XFUI9CuF/j9fx5NZCL/QVrehIJrm4bR/oHZuwhPNP0q3RunTXVOVWm28dn5Seu6fY8e+2sBN39ikdG+EIusl2VDrm6vIm0cwwJ9sRYkn7FRcobnuNP5yyXU52E97BniWLq3G/gfur3l+3EkIv6vVODNjN+W2SyKxWvGLspigsLJxYSdP02lvRbaw8Q3vSwE3TO3onqaP4CXPJppuM26D/tXfSwCXju/ilB8YxffFWOYXczY+caHZ61NIGg7LAd6ZsF5BGepNKNzC3es4noZHyw0uVi5yKfJ7q8FR3J6NLluK3LyVj3fYwtHTZZ992V7ziYSlCS5kn658WTCe+uhQedc59OdfwzIEejLjDiX2Gvz0XO8XGw5/wWhwYx7L1kb9ZjJ7hD3GKTkmaK6W9If7f/Lco7uJDOxZlSnejVcOeepgN/7oO7K5Z2yM5ADo7bX3LfBLlIDCsqGAo5GRRrVD2e4tI+0i8OHUp4FzX/0LlNNlSn6xPIYeD2jXVvIJdRQ45OuQQ5tUye+XkN5HbrneC8PMix9nZSXhjFgym0/0pDeK79OWTuyHMXqtNo8S+ZhThG" +} \ No newline at end of file diff --git a/test/test_transform.py b/test/test_transform.py index 52453aca..49a4255b 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -7,6 +7,7 @@ import pytest EPSILON = 0.0000000001 +EPSILON2 = 0.000000001 def cross_py23_reload(module): @@ -804,6 +805,104 @@ def test_thinplatespline(): dst_pts = t.tform(src_pts) assert(dst_pts.shape == src_pts.shape) + # check load from json + jt = t.to_dict() + nt = renderapi.transform.ThinPlateSplineTransform( + json=jt) + assert nt == t + + +def test_thinplatespline_apply(): + # tests some copied behavior from trakem2 + j = json.load(open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r')) + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + del t.dMtxDat + pt = np.array([1.234, 45.678]) + npt = t.apply(pt) + assert np.all(pt == npt) + + +def estimate_test(jpath, computeAffine=True): + # test that the estimate method can produce same results + with open(jpath, 'r') as f: + j = json.load(f) + + t = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + # exact points + src1 = np.transpose(t.srcPts) + # some in-between points + x = np.linspace( + src1[:, 0].min(), + src1[:, 0].max(), + int(np.sqrt(t.nLm)) * 3) + y = np.linspace( + src1[:, 1].min(), + src1[:, 1].max(), + int(np.sqrt(t.nLm)) * 3) + xt, yt = np.meshgrid(x, y) + src2 = np.transpose(np.vstack((xt.flatten(), yt.flatten()))) + dst1_a = t.tform(src1) + dst2_a = t.tform(src2) + + # estimate + t.estimate(src1, dst1_a, computeAffine=computeAffine) + dst1_b = t.tform(src1) + dst2_b = t.tform(src2) + delta1 = np.linalg.norm(dst1_b - dst1_a, axis=1) + delta2 = np.linalg.norm(dst2_b - dst2_a, axis=1) + + assert delta1.max() < EPSILON2 + assert delta2.max() < EPSILON2 + + with pytest.raises(renderapi.errors.EstimationError): + t.estimate(src1, dst1_a[1:, :]) + + +def test_thinplatespline_estimate(): + estimate_test( + rendersettings.TEST_THINPLATESPLINE_FILE, + computeAffine=False) + estimate_test( + rendersettings.TEST_THINPLATESPLINEAFFINE_FILE, + computeAffine=True) + thinplate_estimate_nojson(computeAffine=True) + thinplate_estimate_nojson(computeAffine=False) + + +def thinplate_estimate_nojson(computeAffine=True): + # an estimate test that does not depend on pre-computed json + x = np.linspace(0, 1000, 40) + xt, yt = np.meshgrid(x, x) + src = np.transpose(np.vstack((xt.flatten(), yt.flatten()))) + + def poly2d(src): + dst = np.zeros_like(src) + dx = (src[:, 0] - src[:, 0].mean()) / (src[:, 0].ptp()) + dy = (src[:, 1] - src[:, 1].mean()) / (src[:, 1].ptp()) + dst[:, 0] = src[:, 0] + 0.5 * dx * dy + 7.0 * dx * dy * dy + dst[:, 1] = src[:, 1] + 0.7 * dy * dy - 4.0 * dx * dx * dy + a = np.array([[1.1, -0.04], [-0.02, 0.95]]) + b = np.array([3.0, 4.0]) + dst = np.transpose(a.dot(np.transpose(dst))) + b + return dst + + dst = poly2d(src) + t = renderapi.transform.ThinPlateSplineTransform() + t.estimate(src, dst, computeAffine=computeAffine) + + x = np.linspace(0, 1000, 120) + xt, yt = np.meshgrid(x, x) + test_src = np.transpose(np.vstack((xt.flatten(), yt.flatten()))) + p_dst = poly2d(test_src) + t_dst = t.tform(test_src) + + delta = np.linalg.norm(p_dst - t_dst, axis=1) + # can it match the polynomial within a pixel? + # for low-N, it won't + assert delta.max() < 1.0 + def test_encode64(): # case for Stephan's '@' character From 17df3569f01be8ef8c9702b951b01f0a8ec0e9a0 Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Wed, 12 Sep 2018 10:45:04 -0700 Subject: [PATCH 705/766] adding return_all option to affine transform (#124) --- renderapi/transform/leaf/affine_models.py | 4 +++- test/test_transform.py | 25 +++++++++++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/renderapi/transform/leaf/affine_models.py b/renderapi/transform/leaf/affine_models.py index d4caf3c7..a278a88d 100644 --- a/renderapi/transform/leaf/affine_models.py +++ b/renderapi/transform/leaf/affine_models.py @@ -119,7 +119,7 @@ def load_M(self): self.M[1, 2] = self.B1 @staticmethod - def fit(A, B): + def fit(A, B, return_all=False): """function to fit this transform given the corresponding sets of points A & B Parameters @@ -151,6 +151,8 @@ def fit(A, B): Y[2 * i + 1] = B[i, 1] (Tvec, residuals, rank, s) = np.linalg.lstsq(M, Y) + if return_all: + return Tvec, residuals, rank, s return Tvec def estimate(self, A, B, return_params=True, **kwargs): diff --git a/test/test_transform.py b/test/test_transform.py index 49a4255b..95465fc6 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -104,12 +104,18 @@ def test_affine_fail(): def test_affine_random(): - am = renderapi.transform.AffineModel(M00=.9, - M10=-0.2, - M01=0.3, - M11=.85, - B0=245.3, - B1=-234.1) + M00 = .9 + M10 = -0.2 + M01 = 0.3 + M11 = .85 + B0 = 245.3 + B1 = -234.1 + am = renderapi.transform.AffineModel(M00=M00, + M10=M10, + M01=M01, + M11=M11, + B0=B0, + B1=B1) points_in = np.random.rand(10, 2) points_out = am.tform(points_in) @@ -119,6 +125,13 @@ def test_affine_random(): assert(np.sum(np.abs(am.M.ravel() - am_fit.M.ravel())) < (.001 * 6)) + # return_all test + tvec, res, rank, s = am.fit(points_in, points_out, return_all=True) + compare_tvec = np.array([M00, M01, M10, M11, B0, B1]) + tvec = tvec.flatten() + assert(np.linalg.norm(tvec - compare_tvec) < EPSILON) + assert(res < EPSILON) + def test_invert_Affine(): am = renderapi.transform.AffineModel(M00=.9, From 47bfd120cd11036a09e86867893bdd91cf6d0e82 Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Thu, 13 Sep 2018 09:03:46 -0700 Subject: [PATCH 706/766] moved force_shear assignment before load from json in affine (#125) --- renderapi/transform/leaf/affine_models.py | 2 +- test/test_transform.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/renderapi/transform/leaf/affine_models.py b/renderapi/transform/leaf/affine_models.py index a278a88d..36f9a3d1 100644 --- a/renderapi/transform/leaf/affine_models.py +++ b/renderapi/transform/leaf/affine_models.py @@ -75,6 +75,7 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, (will supersede all other parameters if not None) """ + self.force_shear = force_shear if json is not None: self.from_dict(json) else: @@ -88,7 +89,6 @@ def __init__(self, M00=1.0, M01=0.0, M10=0.0, M11=1.0, B0=0.0, B1=0.0, self.labels = labels self.load_M() self.transformId = transformId - self.force_shear = force_shear @property def dataString(self): diff --git a/test/test_transform.py b/test/test_transform.py index 95465fc6..2ea5cb39 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -103,6 +103,28 @@ def test_affine_fail(): am.estimate(points_in, points_out[0:-2, :]) +def test_json_affine(): + M00 = .9 + M10 = -0.2 + M01 = 0.3 + M11 = .85 + B0 = 245.3 + B1 = -234.1 + am = renderapi.transform.AffineModel(M00=M00, + M10=M10, + M01=M01, + M11=M11, + B0=B0, + B1=B1) + + props = np.array(am.calc_properties()) + jam = am.to_dict() + am2 = renderapi.transform.AffineModel(json=jam) + jprops = np.array(am2.calc_properties()) + + assert(np.all(np.abs(props - jprops) < EPSILON)) + + def test_affine_random(): M00 = .9 M10 = -0.2 From 2d248951d203b5b01f05d0d9cd525629a765c00d Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 14 Sep 2018 12:17:07 -0700 Subject: [PATCH 707/766] adding __pycache__ and pyc to docker ignore (#126) --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index a04aefb7..53980b69 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ docker_setup.sh docker_base_setup.sh +**/__pycache__ +**/*.pyc From a363015b88a665ed2111c41b05d0a664472b85a5 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 11 Oct 2018 08:22:45 -0700 Subject: [PATCH 708/766] adding bumpversion --- .bumpversion.cfg | 6 ++++++ renderapi/__init__.py | 1 + 2 files changed, 7 insertions(+) create mode 100644 .bumpversion.cfg diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 00000000..c9cc5cba --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,6 @@ +[bumpversion] +current_version = 2.1.0 +commit = True +tag = True + +[bumpversion:file:renderapi/__init__.py] \ No newline at end of file diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 75a42151..9490f811 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -13,6 +13,7 @@ from .render import connect from .render import Render +__version__ = "2.1.0" __all__ = ['render', 'client', 'tilespec', 'errors', 'stack', 'image', 'pointmatch', 'coordinate', 'connect', 'transform', 'resolvedtiles', 'Render'] From c14d5e0be63adb77ff793a42c98f3256c66d6e57 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 11 Oct 2018 08:23:48 -0700 Subject: [PATCH 709/766] =?UTF-8?q?Bump=20version:=202.1.0=20=E2=86=92=202?= =?UTF-8?q?.1.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 5 +++-- renderapi/__init__.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c9cc5cba..b7e9cfa3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,6 +1,7 @@ [bumpversion] -current_version = 2.1.0 +current_version = 2.1.1 commit = True tag = True -[bumpversion:file:renderapi/__init__.py] \ No newline at end of file +[bumpversion:file:renderapi/__init__.py] + diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 9490f811..de9ab0d1 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -13,7 +13,7 @@ from .render import connect from .render import Render -__version__ = "2.1.0" +__version__ = "2.1.1" __all__ = ['render', 'client', 'tilespec', 'errors', 'stack', 'image', 'pointmatch', 'coordinate', 'connect', 'transform', 'resolvedtiles', 'Render'] From 6029528a98249407472af270d1c71db2f62fdeb7 Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Thu, 29 Nov 2018 11:01:53 -0800 Subject: [PATCH 710/766] adding polynomial2d scale and test (#128) --- renderapi/transform/leaf/polynomial_models.py | 10 ++++++++++ test/test_transform.py | 12 ++++++++++++ 2 files changed, 22 insertions(+) diff --git a/renderapi/transform/leaf/polynomial_models.py b/renderapi/transform/leaf/polynomial_models.py index b5799e59..0fbab48d 100644 --- a/renderapi/transform/leaf/polynomial_models.py +++ b/renderapi/transform/leaf/polynomial_models.py @@ -89,6 +89,16 @@ def order(self): no_coeffs = len(self.params.ravel()) return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) + @property + def scale(self): + """tuple of scale for x, y""" + if self.order > 0: + scale = (self.params[0, 1], self.params[1, 2]) + else: + # translation only has no scale impact + scale = (1.0, 1.0) + return scale + @property def dataString(self): """dataString of polynomial""" diff --git a/test/test_transform.py b/test/test_transform.py index 2ea5cb39..aedad40a 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -167,6 +167,18 @@ def test_invert_Affine(): assert(np.allclose(am.concatenate(Iam).M, np.eye(3))) +def test_polynomial_scale(): + p = np.transpose(np.array([[0, 0]])) + t = renderapi.transform.Polynomial2DTransform(params=p) + assert(t.order==0) + assert(t.scale==(1.0, 1.0)) + + p = np.transpose(np.array([[0, 0], [1.1, 0], [0, 0.9]])) + t = renderapi.transform.Polynomial2DTransform(params=p) + assert(t.order==1) + assert(t.scale==(1.1, 0.9)) + + def test_Polynomial_estimation(use_numpy=False): if use_numpy: try: From 7904f8a75aa3793b78f32e27c5c66e2f372830e2 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Mon, 17 Dec 2018 11:56:15 -0800 Subject: [PATCH 711/766] improving bounds efficency (#130) * improving bounds efficency * typi * adding test for loading stack * adding tilebound call and changing getting sectionId from z * typo * typo --- integration_tests/test_stack_integrated.py | 14 +++++++- renderapi/stack.py | 42 ++++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 563d1512..2538d93a 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -221,6 +221,15 @@ def teststack(request, render, render_example_tilespec_and_transforms): yield stack render.run(renderapi.stack.delete_stack, stack) +@pytest.fixture(scope="module") +def teststack_loading(request, render, render_example_tilespec_and_transforms): + (tilespecs, tforms) = render_example_tilespec_and_transforms + stack = 'test_insert_loading' + r = render.run(renderapi.stack.create_stack, stack, force_resolution=True) + render.run(renderapi.client.import_tilespecs, stack, tilespecs, + sharedTransforms=tforms) + yield stack + render.run(renderapi.stack.delete_stack, stack) def test_stack_bounds(render, teststack): # check the stack bounds @@ -415,12 +424,15 @@ def test_get_tile_specs_from_stack(render, teststack, assert len(ts) == len(tilespecs) -def test_get_sectionId_for_z(render, teststack, +def test_get_sectionId_for_z(render, teststack, teststack_loading, render_example_tilespec_and_transforms): (tilespecs, tforms) = render_example_tilespec_and_transforms sectionId = render.run( renderapi.stack.get_sectionId_for_z, teststack, tilespecs[0].z) assert (sectionId == tilespecs[0].layout.sectionId) + sectionId = render.run( + renderapi.stack.get_sectionId_for_z, teststack_loading, tilespecs[0].z) + assert (sectionId == tilespecs[0].layout.sectionId) def test_get_resolvedtiles_from_z(render, teststack, diff --git a/renderapi/stack.py b/renderapi/stack.py index b58cd836..54089a77 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -649,6 +649,40 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, return get_json(session, request_url) +@renderaccess +def get_tilebounds_for_z(stack, z, host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + """returns the bounds for each tile associated with a particular z value + + :func:`renderapi.render.renderaccess` decorated function + + Parameters + ---------- + stack : str + stack to look within + sectionId : str + sectionId to find z value + render : renderapi.render.Render + render connect object + session : requests.sessions.Session + session object (default start a new one) + + Returns + ------- + list + list of dictionaries with tilebounds + + Raises + ------ + RenderError + + """ + + request_url = format_preamble( + host, port, owner, project, stack) + '/z/{}/tileBounds'.format(z) + return get_json(session, request_url) + @renderaccess def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), @@ -678,10 +712,12 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, RenderError """ - sectionData = get_stack_sectionData( - stack, host, port, owner, project, session) + + + bounds = get_tilebounds_for_z(stack, z, host, port, owner, project, session) + try: - return next(sd['sectionId'] for sd in sectionData if sd['z'] == z) + return bounds[0]['sectionId'] except Exception as e: logger.error(e) raise RenderError('Could not find z value %f in stack %s' % (z, stack)) From 05719f1cc5208155c5b6e9496db44dceefa7afeb Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Mon, 17 Dec 2018 14:32:33 -0800 Subject: [PATCH 712/766] vectorization improvements for TPS transform(), apply(), and inverse() (#129) * vectorization improvements for TPS transform(), apply(), and inverse() * remove scipy.special * slightly faster, avoids divide by zero warning * adding scipy as dependency * removing scipy from test_requirements --- renderapi/transform/leaf/thin_plate_spline.py | 69 ++++++++----------- requirements.txt | 1 + test_requirements.txt | 1 - 3 files changed, 30 insertions(+), 41 deletions(-) diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py index d16fe63b..eae5f291 100644 --- a/renderapi/transform/leaf/thin_plate_spline.py +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -2,6 +2,7 @@ from renderapi.errors import RenderError, EstimationError from renderapi.utils import encodeBase64, decodeBase64 from .transform import Transform +import scipy.spatial __all__ = ['ThinPlateSplineTransform'] @@ -81,47 +82,38 @@ def tform(self, points): numpy.array a Nx2 array of x,y points after transformation """ - result = [] - for pt in points: - result.append(self.apply(pt)) + return self.apply(points) - return np.array(result) - - def apply(self, pt): + def apply(self, points): if not hasattr(self, 'dMtxDat'): - result = pt - return result + return points - result = pt + self.computeDeformationContribution(pt) + result = points + self.computeDeformationContribution(points) if self.aMtx is not None: - result += self.aMtx.dot(pt) + result += self.aMtx.dot(points.transpose()).transpose() if self.bVec is not None: result += self.bVec return result - def computeDeformationContribution(self, pt): - disp = np.linalg.norm( - self.srcPts - - pt.reshape(self.ndims, 1), - axis=0) - nrm = np.zeros_like(disp) - ind = disp > 1e-8 - nrm[ind] = disp[ind] * disp[ind] * np.log(disp[ind]) - result = (nrm * self.dMtxDat).sum(1) - return result + def computeDeformationContribution(self, points): + disp = scipy.spatial.distance.cdist( + points, + self.srcPts.transpose()) + disp = np.power(disp, 2.0) * np.ma.log(disp).filled(0.0) + return disp.dot(self.dMtxDat.transpose()) def gradient_descent( self, - pt, + pts, gamma=1.0, precision=0.0001, max_iters=1000): """based on https://en.wikipedia.org/wiki/Gradient_descent#Python Parameters ---------- - pt : numpy array - [x,y] point for estimating inverse + pts : numpy array + a Nx2 array of x,y points gamma : float step size is gamma fraction of current gradient precision : float @@ -130,23 +122,23 @@ def gradient_descent( limit for iterations, error if reached Returns ------- - cur_pt : numpy array - [x,y] point, estimated inverse of pt + cur_pts : numpy array + a Nx2 array of x,y points, estimated inverse of pt """ - cur_pt = np.copy(pt) - prev_pt = np.copy(pt) + cur_pts = np.copy(pts) + prev_pts = np.copy(pts) step_size = 1 iters = 0 while (step_size > precision) & (iters < max_iters): - prev_pt[:] = cur_pt[:] - cur_pt -= gamma*(self.apply(prev_pt) - pt) - step_size = np.linalg.norm(cur_pt - prev_pt) + prev_pts[:, :] = cur_pts[:, :] + cur_pts -= gamma*(self.apply(prev_pts) - pts) + step_size = np.linalg.norm(cur_pts - prev_pts, axis=1).max() iters += 1 if iters == max_iters: raise EstimationError( 'gradient descent for inversion of ThinPlateSpline ' 'reached maximum iterations: %d' % max_iters) - return cur_pt + return cur_pts def inverse_tform( self, @@ -170,15 +162,12 @@ def inverse_tform( numpy.array a Nx2 array of x,y points after inverse transformation """ - newpts = [] - for p in points: - npt = self.gradient_descent( - p, - gamma=gamma, - precision=precision, - max_iters=max_iters) - newpts.append(npt) - return np.array(newpts) + newpts = self.gradient_descent( + points, + gamma=gamma, + precision=precision, + max_iters=max_iters) + return newpts @staticmethod def fit(A, B, computeAffine=True): diff --git a/requirements.txt b/requirements.txt index 3f22a4bb..84f5af25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ requests +scipy numpy pillow sphinxcontrib-napoleon diff --git a/test_requirements.txt b/test_requirements.txt index d54f0207..2018ec5b 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -7,6 +7,5 @@ pytest-pep8>=1.0.6 pytest-xdist>=1.14 flake8>=3.0.4 pylint>=1.5.4 -scipy ujson jinja2 From d5ccfafd75e5a3694e989e9c97397d315c9b0c52 Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Wed, 19 Dec 2018 14:40:11 -0800 Subject: [PATCH 713/766] Vectorize tps 2.0 (#133) * capitalizing className * 40% time saving from russel with sqeuclidean --- renderapi/transform/leaf/thin_plate_spline.py | 5 +++-- test/test_files/thin_plate_spline.json | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py index eae5f291..47f4083d 100644 --- a/renderapi/transform/leaf/thin_plate_spline.py +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -99,8 +99,9 @@ def apply(self, points): def computeDeformationContribution(self, points): disp = scipy.spatial.distance.cdist( points, - self.srcPts.transpose()) - disp = np.power(disp, 2.0) * np.ma.log(disp).filled(0.0) + self.srcPts.transpose(), + metric='sqeuclidean') + disp *= np.ma.log(np.sqrt(disp)).filled(0.0) return disp.dot(self.dMtxDat.transpose()) def gradient_descent( diff --git a/test/test_files/thin_plate_spline.json b/test/test_files/thin_plate_spline.json index 4718f574..591101be 100644 --- a/test/test_files/thin_plate_spline.json +++ b/test/test_files/thin_plate_spline.json @@ -1,6 +1,6 @@ { "type":"leaf", "id":"", - "classname":"mpicbg.trakem2.transform.ThinPlateSplineTransform", + "className":"mpicbg.trakem2.transform.ThinPlateSplineTransform", "dataString":"ThinPlateSplineR2LogR 2 995 null eJx8u3k4VV//Pk6SWYbmUpIGpaIkVNurJJREMkSJDJGxjMk8z8Mxz2dyznHOUZQ5bcmQZI6ERKYMRSEZwve89ea5nufzu37+sK591t5rr+E13Pe91mZi+v/+gyf//+Xafen/Xf6f32P+NG9OVXGG6KbKUBE2DET8T31AXuMZsuMCRA0UiHYf3wlRSWX0IXcXSOG5/0BX8gcEeP29Lxn+lt7FmB3HPG0hKXZfNz9qA1H4Q7fu9fRDwoN2QqKwEESldNPHe6chyXz5GbUvAYLT9Rb5ZU5D9L1Wh47qHAj4ddS209cWMNjKxstakRB0LlzOUG03xNfL3m38+BMiymWWFxouQMzU0S0nlcXXxhOxVmpwCnrKQpJgu+sMmECY8N/fUyT+lmFbfzxk198GUY/Tl/sIjZB829UPhQvg/vZByeseHvDD/r0vwe5vGTKlsTR2lBPiHuowhVg8hhCTJ/c0lH4CZkg36KHYCPgo8PeTOzMhzoxXSWv5K8RsM9CkCJVDSgjPw0fJwhBc8+/8rP9benl0qpwNngasuOrl6vqnYKf3mKhOL4ekDaXnu3eIQDr6my395hTgLp/M7ts/D+6Izczsj/MQk0o18fx0E2LtdkU6alpAQtuNPSYq2hBf4DFUJGEA8Z58cxyWJRBRy9/mOZcPqWrXb3lz6kCssOkeJDgL4uZE8vE8nyCgao/uq1AniC0Q3uhspQwJnHwGg+aZkHhGdX1NaRqk7hT+Kr0+AxwP7xcrGbCGuO+3RMbPYyCmuLN6vf5TwCApnOJZdEhSsDq2qBD+f+zKm3mrS7eRHSTs+ZW4tVgPYl92LzmWKgK2sldkmT0YEuvXBRy/GQKJu4Tcy5S9wOd2JW2IIANp62cLBz6fhTDuDz5zF0Uh4eYC8cOAB4Q3bp+UMr0LGFGP3UeHH4DrCZ3WtMckSKSe6eHYaA8Bs3/fm5j63/2I/9cu43DS6WfjgyHpSuA4F+cMRLvt/XKlpwXi9M9U+OAkwBc/fv1z7X6I2c8WFhCKBQyO2XdutwkkqFaRI/fjIeZq/kuNY28gzTRc2CpQAJJcKxc6269D1EXXOdNn3BD7vKNkwGoR0vGdW3Zfc4K4fJwo2yVFiAyRUHe1bIeIagWRpzoPIG7sblua1leIPif/YC/tCkRWtpoHUj5AzPjNRsGGUUga5/NNJGlB0M+vopKncyDCZEtX7MbvEJ354FFZjyfE7LyzoMypD8k31Hp2s4tBzNydN4ZuF8DXQF3WvF0PUs5W7xe+PAdxKaJ143qlEKeu6pO62AKJwpxfrx0XgJR9HId633Yy7EDUyL7oDkS/vPxT8foOiBFSkZp41waxOmWB2excEG0rH6kkAZBm0T9gXlsFiRKnNGLsQgFT08G2ES2BhIBfJJ+CQxAVeK8m4hENElnPYs9e4YTguBujbHanIVEpVv1AMDOEO8DZ5vNkSCkmHXg2KQEhkTnCiPEIJJ8oXgrwZazzjyKf7MpaSDuUf2CjwQWIenb28bq2NEgxTKCaTxdA5O/XRW9uXWH4kbXOcoo2YA59dCMp1AHF8L/XOc1AYdsfdWWIzZmW22u8BCkLPvPqMfEQU7nHst99BGLvkKe1yC0Qc5s1gFnaCGJcDh27WDACkQqOi/UziZAkVpqIF2kDv+UGl3TnaxC/sSqPmrANAgveaD16uh1iafjLBy09Iax6f3/n8TCID0xdOMmfCaFTPe+411kDQYIp9ERDMDzeJZ/nlZkNGLnXRzUe9EO0qUOWUlggRFUI3vriYwYZv380qGVzA6bydffL/A7IMB+89H5uGDIShr5tHXsK6UJH5uQnDeAx90/Sh0TGOHzcNnO4eYO917vgD9m/Ia5VrJe44wAkjmbS6zVZIa7e2FOmSgESTPRvyZaWQGwDLnz/9yZIMWUVES6VgfjzXX0PHr2CpDeXZ6pT9ACzLv/51okvgO397/lLXH95aufm4+DdFhFrzvwAkmVO+JuxjoOP2qZ+6ZQ08DVnuMnxKAg/apMjGRoDMb9qKqwprhCkLBJlnmgH7np/ulBmXQjh/WbGKbwJIvad+u17sxRCYWZa5gqOEWe2SFu76oCf+lvL4jAPSBHah+NoMgZ/WT/pP5ujIPhks+FGKXXAHRbTu9woCZmtFLVq6VzA1SgctWnYBAlnM047FglCWi4z96/cGogzcNXY3sAGqbIeB9Pev4D4CaGy0BQqpB4M4C47IgpeDbmHY5eZAN/FtKejZCdgyeTWMo54wGYFEWYTj4ITL9MbudJYSBa/6po0LwBWXk/MbxdaQpq8dV7dfra1eVnNZw7wqNBtXQ6kSk7w+F1lBmd9z8aKWQFIHwymP7I7Bu5Gog/lni5DMsce/H2RPHC9F9oZeyob4ovmcmjHtoLDxfzAD3NOkIC5gfvhJQGP/OuE2+6mQxyHVLOxlC/YmcqONI1gABPou+miHgHS/rBjaOaNkOCR/F0wVnI1jq31K2lu46fUq1kQFHF1NlfnwGreW6sPC6l0r3BFACv9gC59Qx2SCRuwWo7ukPZAbCJOLRcwabx41WofINQ2DfAVSIKn4I7NNxa6IP0+ro6P13KtHdK/8TO9v3Jr7d69EHDVZ0KNeTOkLn753X4WA/HINd/Hl/cBhtvChJGIIGT3/vjzM4qrz621Exm8aZfPlmrA1oT43Yx3gMRlUcddfZGQwWv5lG+fCWDO96TZ8ksAjml0+JUyw0739PVmDJsD/jtzoPz8awgIGUv0qyZChmuCDCNjgC+TwzLT/leQse7FPU/LHZDYx9a464QZpLZtYGlzjodMpmS/La7xgEuoQjDcbICpyGIZtVMFYrrjzhgW97V+4V/9LTFyZ6tOEBMhXIk/aPr4MAQLqyRe5j0PWDVVrdFxYQiRfdP0pCMN0p9gLt8PvA2BZ7qsa9lmIBXd4OI2YgihW/xX4mHqN3ne9x/1INMZb02nVQG2qEp76sgncFywzcGwEAC3rWoouVUcUge/DYw2VULgadGA7KPvIO3uIY7oR3LgczVk+sdHRbDZsS9m3GAcMgxE8GeWQ9f6+6/fwh29hYkmtk2Q7jyktf00BzjY/NJJ/hkEWEStx0iUDcxUNP8xaMBOJ2s61nSuPZ/x7/Npew/yigs8AXeOXU7hC+2ArWt3dPi9D9K+eA+F8JyCjOkbwS1usuB5x0Y8cVAebKR468NLvwP+DYuocVgC3MDib4c/agZc09/2LO7taBG8TQPcF+fQz1XTYCAXcsNH5gbg80P/Wbe19xOxf0t1O39PCaMaIOT8a1//1q/iV7PvmbtP7DgGxLqlNwrCfHDr29hS+ZkMwIphC+wrpyFj7/k+3Wuq4NPgMtSs1wxYDy8JbG4p+FTf69xd+RVw5ywM7T/vBvfNihg+mVuQnu1belXkLdwrK8Wxpmeurvtav3Bb/Y/3T+lB8Bd9RvrRBqwtz6On2wLBZaPD6/ydHyC9Vu/e8CUSuHC1yn5W3gN4ZvmIV7O+YH3giDVTnj6kH0utSeEdhIec1Wn2F8YhQf/ngRGXNvA/KWSWnccCOMMf/SEeDuDInOkoh7wC/DFZ5cSZDeDKVJ748z1jXm2N9cibfcHP/zH3u1e3Ac++8X7MZDR4jSAlP9dJA86206njsST4iXL67BGLBuyDJxs0mXQhoG538QCmFLCe7gsfMmPBauhBlbxsC3h3iqPEsW6I6pfyrV04A/GDQ2ZZPorwcJPG1HBy8Kod/MeuzP+W+myXOEmqDD90tV5Ho43BLWNGKydVV9f5P37jLFD/JaMCDK78WcEX/67rWj3BrLCn//AGsJ0hav++nby6zmv1XpW3JbIvzQDG/IBespcKuFSd2F+o8gSiKqPEfloy8JXH7aLFvgmw7wChW5I3AY/7vrI+9uUzO3PuPAWKZ4DDlSPOcO/AritRe68A8dDfdo3uPlzxe4JqY/FWHmO4q9mB/ZH5AzKj/tYb1mzokNm8A4gdFarY176gUypF8ZAZA+cwl+d26x0Z8ZekVJKVDxajMryNpFSIqJiaOSvZAx6BxxfWj7tBhDaXwFWOdHBM7Uyy7/gG4TJ6zOPfg9fG5cH03+Nc5UXWR5Lvfj8lAiHSWoNPJ3HgeK06z6/dEoI/TK/EQcdmWfVirVkIcN5zavtLgFuKNgHVuvIQ2l8tqv3bYK29EPhb3onRVOcQb4TAitnP1j3Ma/U+f+0akp/lCiyeqAbnl+cbXVyKIPEaJ37HgxR4vPXR8SNmBRCXxGx4Xf04uC+ksDrksUE8HiVSyrXg7p5S1KhxGYgvo05WMx0G5+utt1xIXUC8bdgt/yUB7GSKZcnlj1fnc+29q3676scP4ryRSsNxiCof9Tz7mxkCXD6r+zRqgtvpoOzxF1Jw/yvr4BSvF3j9tUPQbSw3jXKPXrt2sfvmNzubCdaXVb493yUCxmT7KxqnD4PfAd2VdbHrviF+Nv07+P5SCrGm0//PvLv8uHV43QkO8HnosqDwMQ2sHlKbm87XQcBkHVx+OAYOuXwtXvenwb2zUFf/CwV8KhvzzgU3gK+iZFdtPAasnG9/JSuQwDWgX9l2fQrY/E/7Dq++aEp/LgbHet9WywOvwDNo0erhdhEG72VLWDdkAj6Pjb61UvnB3WlqyDurGDyWSxodLs2D/3dkj5zOGPhasOdu4sCDjXHzm8/qDL9ne/tpNmYBgsReLDAIDcTnNXSHGk6CS9sW+W5TBn7aODUwsc4HHKTXL/D7FkD8mTsxy/sHwcdgQaRp0BASDfZ72ruTwLrrZs3bE7yruOU//thBXMEbTnL0G6U/CoEU5HX1ZKQr3NkX7W5pSgWS7AMP5zZRsDMJMbfxMIbMK7yD+4MPwgOFT4VRC6bgVp3w4/u5AHjsFj3mv7sIfBfPy19OpIFT0rcLI8GpEMJ++MXFQW6wG2P9i5ut+fWaN4yDY0CD6AvcEQijqSx8tQoFN5ErNa8aMGv9+ndewUrt5sn+W/pgH/47123wGOg4KfIuvq0CZ5FvBuKfmMG7JT9J9FArhNjamDyTBAi2r34zOvgS/A6OubBcYvDZ6raM6UVrCOUO6GUfyYPAp1Lv0htNINDWrSNTxARCHD8N5onegVD28AMO7HwQFvSm+RS/HiP/etyR+LoM4cuCcedGL4FPVmzhs2AtiFLlsrPYTYHAH/PklJJPgDmlImj0KRb8nrb/uMf6FpKbNr7isB0Dk6U/Dbt090Nq/kAzSd0ILDzCVvJ6qlIg+e1nVrge6BLdt/kkJBr+Hae6yZXUAr8wiIPbol8MO8AqXPdCoVQ+xAZey5L7fRW08Sw6BK31gHn13+sX9W17UlJSMjjMm6zgEQyvJFuhnxmYxo/6Hknigpii/IN41kdwT2Az9XfkBOiJOemTJirAzpkiyuXCAlGPkm7sfXUR/NxavBdT8BC18GaBnicGnpt2X/hyQBiSR4a2lr/MBLv+RmOvddshSS6BtpDyHYy5Hix+fhkFmIcq+cb7SuFBwl0Jp1caENF8Avnycg+Y3/bHohocEPWv36olNl+3Dt0NkfV87TKGyeC613Ql32EwpAhk0g8e9bC8HX+3AFEfMKdmOaLB2oERdh06V+PW2njDs8MHryi3gn3aEMYuxwpicI6VIiyG4EXwe1bo3Azh+a0vncYd4KaQ0T9C0GqcXXt+Vb+5rd4x/d5NEDAH+tTzReXAO879UNO6PvCsTa3RmdoApnZx/qf7qiD4+M4n9Wz+cPvxvYFNXydW4+xae6GkWwrPuk+D2Qevzh0xYhAw0mSl8GsO7p+c2RbKfx4y2oreW6p6AiZS+L7NiQEwMfDHtx9fhPgp840Cr13h9tSlgrSCO5DEty7zq38BmG//XfjGhxNSOMSLzdDHa+9ZtRPtZDr33YEWSLU6B6LPEUgOjqpgX6JDQl306NLQAljciJS2UE6CuAtKE3rx19ae/9dugHJMo75SIRrsFRS6uwQY74nitpWe8IKks4Ki55cokGybJbj7VT6kSJ4N7DoiBUnqB5fYnyxAUtOB/gc3/sGHYtpyGTKQYveQAbNsIOW5ru+iugikSmfArmuvIS36ZN6z+VhIctm636uCA7DlQA1Q/8rg7d9GdkXQgfyvbnZVmIFuN5kBXpCNaGRfBHGnPr4tUtSG9MDAFTydaqL08BolH7Bzr4NCb9yCjEymb/MWKPgFdWYb7lkAbLvphk1bTkOqxBJhy47dDB63V+KmyR7ABkSpNgzSIRHKGO6rC1i5oNtx3aoQvyMnZ/Lxc4hPNT3DLvgOMvquBKs7yUE6qhfPZYuFuKlrpolfeACH4/HfcuMDROVgvkzYlwP+akJ++OaDkHDNQN7lRStkwEbnpyYbIPHd7osSuY+AqHLgtmz8AiTrI2dCttdCfHMcf8+fG4CTaZU6YzcBxP3dlJf7uCH17hsxW1NBILIw0IsaMyQa+TDC8wXIPKukw62tDrFz8ZIPzd8zeK3qNazGRUgqGRi85ooBgo7XieKoHIjvctNDZH8DURQdZFk4DdEFTnc26vEB/rbLzQvRDHxSZivgaeAPBLbJTy2tapA4wLxk8usq4C6FHGw6hIf4JxFE00wnwN2M0OtOPQSJmmd2HL6aC4RdL1b0p9jao1NzRl6QGTvcs5mYDQm+hTvXmysCsUhpS2SEB8QvmT+RbBcEQpcbOUbHEGIiNx/2k+EA8tvoXp+w03D3arr3u0svIFk7p8pkzhOw+rErfkS+845+9EcrWJWlnhesDmDg1a3XT7S8AJxAmDnzg+1A/R9/oqjlf7ekVEPiHa88vSEmyJLjpOydTIJEVnut9YgmhFyl2Evr+gD+RelWhRETSCOIRnYsnoaMyT/ApsMN0TWtEkdgBvCc7GY7vnJDlNbR2ohj4oBvwqSWFFyGjBS2lfiVUbAjaGLpMJAejK7E8QSm0YZ77YzxMwkWNyazQWL/kxt54WeAtO/le+cvxRCTEmf7h98AMoUuruhZmGqL5meDKUBAqlbiKubw4FYjxTgg2VVLHzZwgeT9jqczsjyAvMsl5aPMGYhzGXewKr4HGdeWuZ27AyG5//cDJqFbgO2xTxIPY6y/8tyR138sIONIoTd0A+May6CF+8DF1yHwT7Eo4NXrTAQa+SBrnOsV1gYLaVfxzEoHu4G4h66H76+CuPslJWj2NcDqELKaZ5gh6au0bNsAEfzT/XpvWPACfu/tv/PqWC/5p0UNsCl9XV8nK4Bo28eAQww+LSe+7srJLsAGqXN+WY8B/D0XHoNwhh3n5ml0ogchPUpp372BzUCQMtshVOQCYaJe+rJ9rmv4blVPJTg5ahyrfglRuw6zdkfGAGnbtDID2EGcoUTVmORZyLzLoXR/NhVS+Hzb30dzAz5WYKuqai9ENUancD77ApmBt2TY/Tkh9JflOm6VkrX2V/V78oF1RedFwwF7oKXMCf+LwdsxQRbV1hDJ7V4SFXIASKIba03OvoKoLL71hb8ygez2/DSFex4SXFp4tMU+AsnnvLDciSOQpi/RX598HTLb3VfmAyMiGcAgzED5eefojjEEkspuznqukweyRZVql+sgJPmeYsAAPSAdDkqS3RsCKW+4HnVwTAJ2opZk/dIeMLKChhua2oBUzNpnseUiJMYkymkWDQKluDwqQlcI0rbpmU8XCEPYE++sEzEMfqxqb/qtoh+y4vv6keXt8LCIs2hPfDpQlNkrcipvQvLS+ePvxe8C+YKW5cWdxZBSqdiv+bsRsj5WT0X5/4JknWg3vAAvkAmvFgKmH0Pa4rrSuR0ngXa4+EeF6k7wU86zaVF8DrT15xZ4NGYh/VbG9xF3NsjayRT2lSUHUoLZRL7QdYHi37XCU+OIv8ru87wArOiQ71lDFkilTymGtXAB9vi8LeFPLUTMMheZDZwBHCYVryW2BDHakrM7dB8Atj5tXDtEHaLi5wel1XcDTpGfOlTzG8JfdJ8ac9AFvKiXtOmWLgjdVlw3P8HwywPON67kPoWIuId2h6UCIV3txkq+Dn2qFcZeeAdwrfov0bN8EBje5av86yzgv3wY9b/oAkFn5K5OeyYBdudEE5vgDwgJr31auqMBiBIfKew1B8DXanNKhTwj/ulVPH5Szw7+7J909+uzAIHl1uaWpl3gbbw5bqnTDIg6t/XNWQkQNHlbYcNgPSTuD5JSPf0V8EwhU57oIqTqyrn1/jkOaeXDMZLYMMCTBGNY3I0h/Vkx5ymeQ4Cvyw1qwc1BSmaX4qFD5yDTcSihyRKFyOuVBmoWDD55/kPLPfCGtKhryY3znkDNsLiqPkYDQp0NsfbaRaCQ6yaCil0go12PxTv2FVB7ilZwQerxUS12O35ILXvBX938BcIxBKMkF26g79h+64KwNKTORhWpBHFAVs69s9fOkMEoW3fxVmE0pMcr7HtRPQaRsoz/XG6QJUSA6FeBELSt/mabgRVkFuxLMZs7Cf7h8+sj1ssBsTB7rIM5AkLfH9h6xn8YwkN0JHy1QgDvtc+MV6MFgtzfCAywazLslp+f9Us5RPrv4hK8+QhwzcVn+o4ZADngm0XsNnXAuyhfuV1nArjKJdzwrjxIJbeXEat3Q3JMoZi4ihTg4t5tVq8VAnyAhnx4XjCkiUhwuulVAp5np8v4oXuQciI1mm0eCwTLhBVdPNlOR0mjQQwyZ91YHklehIyY3R6Zs1eBuKlvAbt7AjLUDYf9I39D1h+D2v3XsQxc5b50AmsGWYqnyqvZwxm4Yb3e4DIRKALZHGWpxoA7gK7Xf6oJ5E/eX+PdnkJGSOfQhbJEoP/S1jZ/bQTpR+QmNcpjgS708dBO8h9IJOCOR/H5AeXbtSfdDe6g1dLbwBV9bC3+eKzmp1ltzYDIFvAK4h2zO8ACmWfuGrxwloOgWC2TgrtLQLhW0zDzsBhwvg1nWPp4wHMqNkffbQlwzkb9ChLtkHzOWmpz7gXA+/e/OlPZAjHvW0SiFwAIsq1XOEYigFR42aq8loFLvjumqjhfAKITcoU5QADSvuAu7bS+CJlsfCv7bxlmRcFn3Bn2LvFo882jDL+4Lv1W7sFmSN1/4bH4G3bAB1t03z73FrKu38zd2D4AWNWLQV+Tn6yNZ1XfpPI7CGop6QG+9eeWsdamtfpVfTdrsMLQ0hUPuJKlOx+z6ZAltc9I+9ZRwAv3MhLQIaBVRUranN4MyXXj2/8cDgJqa9iTmrY3kHz6s1HjUs5ae6t6ebaphLRC0DdIFf57TY9Jz8UTuyDFky9NRcMXqL0jxzKPzUGsY/p1XYw5UN19/hG4wPo1uMQLvge68P/ghbESq37dw+AyJN7s9/ohUDfMb/pxnQfcoqT5pdaPQmLlt+uGvwaAsJedr/o3DoieD7OF1gsBdrjUgEtNAGJunNh45Ns4YG3PrcQPD1PmhEETe8gcHkuWWaiFrLc9OGzjNoh7tXfPNXUsxJqmxh5VYeRT11q3rZQYoO6Mt5Nb8ATfP9bZRVKRQLKYtCneKQ9Yu7d/47cwm/FO3SuA244mX532Avp9/0GxkgHwCB8u3ZIqCnHNDa8bLl0GvNDblbhCnfd5bNv3FXDUT1EImRVo2qJD2xqOgt3Qh9aKMzcYcWqDbuVLhn+VHfYZdheHFJL84FuhzZCh9Ej95G4Gj3nK9rCt9yEQhhMsdxxWgyykchqeCkG09dQPqZw+oE590K63Og4JklMbw0yWgKpyYr2YnhUkpnzcK6boADRlJUO95zQIFkxmYyRCyHrJFP8isQa8T6mm+GmZAvWARCou9ANgZzP07+APgn9eubFzehoQAjpW9lUyCsRXeC6WfKzYQG8Z8FvqA2qD/wD28W8OByF28ON4cp/BjAHfVhzgRGbE7ZNHvn1v9WPEC9JsaIILZH3QOho4Gwgx1m9HwuufAyVfNuGOeAHE3A/zeiP3Emhy7MeyDAshyuZd1Gi6LZC79tO2C32GWKVj32QfpQMZd2d73qdvEK2oPZu9n2E/5Vzz1hM1gNGpXlA9ux0o1vyz/XQGTnxfrxlSHgUUz1eHKqwpEF5gU+d9lWE/ki850nntISopXsRbSnnNflf5U5apy5Hh0q0Qyf/+8+6jw0BNNEGH2UohVOfvuQQaTunzrLIbxJx6zql2OgXoipM39V8MQ9ybGcNBoQNAzdp0d6CZBBGRJBVGQgFaaYfSk02PISzBggHbHgF9/5GICEIq+OCPy1//cwDoQfgxr2/zEGVQXLG+bhyoj0VPhfxRhsAAopXT5V6gXSas7PM5/zoreq5uFKiR/IMZR6gQ1002XQzpApqiyleH8TKGH6Su6DHU3Kvsar47wWtpL07rGgpEwVaeqD5GXg/rkGdzq4NUty8CTgscgEusbJtLlYXw4hpFweNBQPANOHNpcQxoNWJHFy2uQsLuNJuQ2S1AG+geeSWSDNFhS/sOzU0z8tXvT65lJRBYpEjoz3AGenMo+USPOsTw+BoUKrpBFBsnTmM3E2TaBNHlAnjBTeKpYe+YOWArBY2kGhCImrW99F1UHwgjd6altvdD9Ga9lMbWMiBWHOy3FrkASbxqfQEPHRk8hif3gvg7wGMUBI48fgq4g/zP2Q6NgE91z4r+QpwcO/qLtRlCNZ6DGjELiDeiw7Z7aK+t56r+SouRM5JZ/gOeJ/XXm18MX6tf1Z+olm+XKPzi4CRAOCaz9ylkq9fNyDCJQLRd8g51njuQ7c+1iRmphFjsv/Hsgs57zlZbiJXgKeG8bgFBvea7lQQGAPfn7YcNkg/BJVD7qidvKxBbePdqSvdATPOQ97dWKuBK9xwLWx8A9GHjDCkeNQi7gtMQWq5a68+qvkufp+8rjDsK/sq4qT2XvkO2EP2N2K808D5C27eJWR7oPz4uJ4g6QFAt695urN7a86v6L533w/Ge8lSwIDPg/KZh8JQxtmIAJCAmNey7HDgA9rLiQbuVBAH34fSXhDtHIVj5VPM5hw9ASCioynHpAOf1XIeuH8MBIdopwbL8B7gnOv2qnfwIBJ+7bWj/B/B/HLqy30x61OUVpJ4KD7o1nDbd1ASSuFQdDfkMD7myRfXFdgHxqo2rg5wJ+LRMFvxMOgWZXfeuxZRygjPpt2RkFStkPg0J7I/IgQfvo58NJs9D5nH3K41clqCZMK9q9PHumm5/r4vV+PtGxvVTGtNyJ0DEImcdZtN9iHiTPdjsFQ03XG65fUx8Dv8q5bCmmK/age7HetJXQ/D+nnQ0scwTguOiMi1yHcFrz5U5X95zkFY11xEl9gQw1Ar+PdynIL10U1643FeIVsGF9TVPAlFMSTZHdAY8THpu1s0z8J9nn4TWCSPwOlow++P8Jcj8nCqPz5YHF1Ois0u3EZC3hwo7jEaA8/y+k07XBIA8SzdnEFVw17fuP2HRC+S+HU6aCg7gRxu98KRZFkh1E1l9pXPgLTp6zo7QDhSZxbH5SA/wRlOVBlKEIZNPavHwXSr4njKtb+MMBpJ3np/GiVlwu7azZZ7HBUgNbU7BtdvBP/BpiuCW20COO7lix8GdfWHSnUFAUkBTZD5wQjAP6w9bi4tA2ojXd0juBp9Y5dGyP11AufDHeXnzbgh8NfDPAQMgmyVfCzzQBBGnTW8UoXNA+kEVWogJgShbgRtDfSJAst1zuNdDD8I9nux7KpkLBI/3bSXPtkHaO+kVnS97vRMvb4EbJOFfuruJ4yCbzQ6HHlGBBGen9rvXWdfsdVXvyuZNW8kHcYPqlws2FEHWwbfY0D8i4KcUIdkugIMshZKNtux1gJNOXhkPnTnztSow8um6TZMqBtvW2lvdH6WdElXy/Hkd8C1bD2yYrwZKRMTKvhZR5/rKuTTatZy+jNQRwFo3ZfiiZpAdLr9je0DJ2v7oanur+5WUpUwX7527gXAxaxkzWQD0PMLJPwGNgB1j6h27VwE00qnJrA+HgFBGvX359iTQ2UXVQ080MvIfU9CsEQnoy7V2OVYXAW/cvMlGlReyUtk+HrTvA+L+bdnmvs5A/rHLkv77IoNXOxTvw6cCxfJTu8rBaEbc+6XnLroNsk9GeAcd/sTInwl6fFsa1vq3uv+a+SVWVtqHgWeiC54d+pQIJF4aySI0HwiU9j8Jcc+BviB7NT/uERAqpV0ldwcA6fe3QltrJyAoq8vPRG0G8obX900bzYFYcC63xoRhLxJPrFqiTwA+Z/IwZ5w/ZB4qiX9+ShUIXCw5lbohjP4nsH1xbAbypr/xhVwSOyyZvReIF61X/JTCnv1U7Go7ZAbUbDPa8wGITY+uhepGANFn3RUtc0Y+zT4SJOLSDJmp91l1dD5ApqJ5lN3CNyC0ZybYTj0BqkGi4adiKqM/YYnnHjH4rN6fgNCMT5AZSVbJrvkD1Fdtj4UlGyGj/k2FuqoK0Ne9W9mHyThVFFFaXQX0nceqdRF/wJoKCrW9716br9X9ZlL5yVyxdbqAm5o6/FPxOpBmfKml7WGQ2bJt/iW2F7DralIHyPlAVTQPz9G5AuSnt9guKd4Bkp7aXSmKB1AKHVueRAgBKaiJU6+uDXAmt39uPfYTqDK/7vZmDAPpUvqKfkK6PqSVYVMNxOdpvxjEF6iNKiYwKcvA9wnyG3uVgeTxckXfpV8Jq6RbdTHmweHuTg8uoN3uEczWHwdiD8mu/ORZyFpsdqisXwDSmR/c14U6gT4ZPdD2kXdNZ6VWjVirnswH8ta+uJEvuUB72dHiJnsaSPpcWvXRE5CVzvsu3pIx77yRMoR8N6AHnFv2OVIGWZ7DeWWR9Ws8YHWeKHkF8skqFZApdN/wWY08ZE1wm9dW5kMW857yl5kCQFjPdSMjlmEX+YgFq/s2IPZxPYobF4RMb4caadOtQGT+lCx4ljE/orsGFLi2AaGfZftXyfeMOFb5l4coJj7WLrIHwtvnxWEeDL/JRhlhYwsQaMqP8L5XAf+1gs2KlRuImcEKW+lVQFTtO/x+ZC8QBO+s7O8TlGJn27e8Bzx28vhBqVnAsQgPWeUyeJqR2hddR27A/4zlGLzFBwS9Z49eZJcDfmtJUWaDPuCrhutONWwDXFePa3R8PBDLLetf9G4G3I70kLx5ZSC4hLi/6y4BnOH57h17RwF/ndVy7Isp4ObYhISUGOv2ru7khzcigP11WbPzIQWIlrkr53ZSF9/YGG2gQSaneV4682dI/7Gl2HN4HAh5xZejlLggA1OhlJmzDMTfbwe3TX1n8DjXz0zNhUD4cIHL4CNjHBmTu/Y3OQBZdaPRu7opwIdr21dOMfKr7esTqr5mgFO/6nIn/BiQ5h5ptf26D3Q1vHxdOsM/DI67UR1DIHuHtrVuzQQQWyNiaULqa+u4mj+zOUdy54ttgcT+P+v8b/ylziiyWqmHAklbgnJlYD1k/KlGXecfAk3k6cEXx78BdaTH5VRKBVCfniMqqyhCukz1PsYCAdEZc7XlJxVSRrYcetcsCoSPIn6PRU0g2WYXnUHUgGAsvbIPlXShR/Tj4m3IFMudqa3fBknceiM0sUEginYK3qZ+gOQi3OON99uBiNFYqPqcCvEcVMXWPWRG3PhtIzRaDwkVimMOOuVALB6wDzmjCwlHeSVqCQeBtFVMWUpVBgjusRX92+WB9N23k2fxB6R7ZZ/zTw0H0i9aOeOGVf35/5yTXv2d+sSX3ElchKwas+un1z+CNM3ZQ+dOjQFB/sWeEA4WSD/64Km7RDbgrzwaTT53DDKyZ/4RJgA3883BcVc/xG4JIhutewSk9fzfy0JTASMwWp/uOwCZz4R3bL7xB+Ke7/JnJCwgp/UUy6RcX9M513hzRjVt0zdhIM37hOmaO0O6kZmX3lFGPO/dtYdsKgVYlQf/+rOvzQdnO0gNfykndAKA5OKtq+khDsmoAN5zpwVkjj3h+DQbC0nss4+8fhkA6d4VHUPZAxCDV/1az6IJZG+1wnfPayDh8p4DlsRYIDsajTzVJwC58KMSp1c4ZB2/EC35jQvofp0FT9y1gfT8Nf/Fd31r/VyNN9g+GxsFqitkr0M93/8qgKwwAep2hXSgbLosz1hooDSx3N3z9C1Qvi8Inh4TA2z5PI9UyghQ3yAlG4RpQAt1WNlnzRK5p8a86T1QAoR97lPGgHr4hbqq8hOg5Xgo3LvdDdQjrmUnAxn8o8PHie9gHNBCflwM+/0O4mj2rxPKMiEzi3haiOAJES53h0b9GbhGhGmF/4fw5fH2WsdCZog1ii3ZAwFy/XZKUUWQaXLtSfKzdPCfsfox18LgG47Jz+9h5yDqYcc7c+UyIC0XSN1IMQasgniw6kU3yIzt0nC9tBVSPF7+kwYgE5H7zVEoC0lYyX8OjADFyFAo46oMYLwXFDaqqDDyyoTLJp5WiIvczmv2jg9IjWIxXL09gKvVn7QVqgLypwrz0M5YoE+YHYg2OQ8Un3eaDxJ2QXbsrmdxI32Qpf4vfsm6v2I/lG221x3Hk4HSOLGiV9PueSkYzBtB2E9p/0S9Nsjc9I4vsdUFInqTjAWmG4Go62WrVOACQcPez5+MxzHshb9FVkgXotqFibfPzwPptEYci+QABNVrD4upZDL860hSq8YfCGw5/jW99zCQAxOCEkbiIbpj2PpYux+QazVtZlkNIb0paRe7gwLQ2sDuxv5JoOGXh+u6GfYjMzfDCARr9vGvbgMUUb0Dg0pbgRK47ceeInGgJkcWiR/5CJRjNyW2KJ8H8u+mosayaiBX/Nn0dTwaKLUne3WT64BstRF1qsmD2BIOoc+ndYGi8AG3++128PsuzRPkwchXlw59xx+/D9TY+x8HTlwEWjjat7eGAiGVVec/DpswcEdHrNv8QXBz8U39KPAYKI882AbJkRBxePocS3w4UK5pFxXEaYLP3adR8S+SgJwo02sqsQH8z2xipDMDoMhlcJ6zI4L3SYI/XDoPpLz9hZZnisCn/FD0tvJEhj3GiOGK5yFMcWPPGRM9IB85cZuRICD0ctiWm2wdQO1oIF7cGgherf1vzWMRyLp4NNr8YQ4E3swu2JljyIg386LUTqk1Xro6b6Eek2cvkGwgi0z+Rydd4y2r8WlVl1y933/dZmDZrwb0kMcr5/BD84veRbBbAaW9DT/78htgdIto14i+QNU7kWJ5+Cf4Vd2x09XJAdrRBrVQDzUIok+yqVE6gdZUYEmMeQuRpXqYXfXxQFXNuLDkowGu3MFc593wQPcsXDnPGulnYaNLOA50RaPfGuvbwM2c55iG6j6gXn6XnJaZCF5VvhcrPBi4mvu2QG13D4TdZ0G/yboCDX1R9WhSCiLHqPYP+AaAphKwSzrBCsLYelZ0K3pRPv8WSjFghqgzYJwGtAn9FZwQ/z/zEy2Tem9LZAQjDjBYj2szRMaSVe5nC0H2sTM129mH1nj86v1evS0YRN8faIt/7Ot1pMEVe6dBUr0LsjSv182gMuCYXNxruKQMtCh+JkMXvjXevPZ87bYzQ4YMv+QqOv/9yw5wFN+5ci4nq++e4EFRSbAbFR4Si9sPlBBpk19X94CTb9ABakQeUIsEGbDm0dr6rfqDTXmqgnimK9Ap2tWlw5oQfSpqw+XUT4z4p2B2vOc1RGOSCmWGmhk8wa7+wjnM2nmK/9UFsrHeVlULswx7OjMd+KcYso8cTD4k1QKODvLvMHKHIVtfLiCWRRfMlHsVS35JARXDHlhrcg6iRef3lc1mQdaAgPosbIOIBYee9d0MPHf/skXq4zqIej5w/kXYSaDc+jNeZvoJMANLG88+HwQKdv3K/nZsRcXhDcZvIItC8D147yrEdH1SWXTdBNlMW1fy2Srvzr7i7eblt2dt/Nl8f8s76VOvO7t8GThXSihn03G48WrhNgufNtDUuPSEu6sBsyz1bD7mKWT5lfnnF7JBwu4tn/pTFIEyuX0ljsUXamw99oORF4wFLVU9T0Gcf5bZ4zu7gDLdp/K6PBcS5ka9P/E8A4qs4/JQWzjEratIM7gsA3QWzQTtt3IQu53r4LsZhv2xFL8PojDu15VuPD8pCPR8W4LWegTil3hvTpakMfz77tlRqx1rOuDq/KcealJ633oIaHZi2zzc2NZ07tX6uKemTJi630AtkMdqKJRBkpzVMnnMisGTZspMuxYhySrXW4/tOdAqTLa4xtX8h/cmBCYE794LyXvbVTkqGoDe+YInTnAOUthPy5XG2jF4DJtcygZvSEut1nGU7AJ6RvwFWUMHSOX8qxtnGx1DmmYOQhr5QB2+6y5kad/1llLeAAmf2joGuP2ANvl9+0swh0QnQ8ejWQFA/RyfppG3BEQeh5VzYFkP2lfO66bcOrlvqa+Wwbv5/vCIy0BKT11V0SctIC9ms0j2UyD16HLADsNWoF44LWOYlA6p48d37p+RAvK1uGfPLwlCOk2t723EDaCcDE9QMy2DdNUPK+coqTd/3L3aMgnJXu9V+z8bAbVdfwUXYMPVChXduIA8ncKtGIQFLERNBYk8giy7/kP2pVFACC5iwMnLQDnHVl9H5oG0p3teZjtuB9pujWJ7zXxI26ey7jNTC8P/jOualkIhg5b/cNOP60AOvbpOaZ0QZOzjn7PoZ+CPP+03Du97BTjkynhFAiOPfpZqsSUsAv7xi8lnskSgDGw525EpAkSPyvKxmjNADjue9kv5OxDdhkZrDHWAElp3rm5UDEiHrrf1PqgCysadUkTVx5Ap/31B3bodKDzmlZ/LX0PmC5l72REKkJXHZtVwxwGI474r58my1LsuZz3wYODA55PizF+B0izDCLQNa/sya/swxdxMjnaM9sfcP1XLbwLyFqMM3I4EoPC5y8bP+wIJm9xvGM7ByLfiShU8EoA3Zj/xh86Ih8v3/+5D9CQYD9C+Ap3VY4s68Tbgnkcf0sMIAe30oZXvNPBdASllPG+Aes3nyRXmAMDHXNV7Jf8baKJ7jp+sPQEEkugpeshpoHGOqN+gH2bw5KG/+zxS3MLcG9cBmXtYW0iBwcNzBZJ3ZiQB8a59a1jeU6D1yIbT9zNwtgvzYd43DHwwarLV1FlzTTdZ4yGWQRxqt/WBfmpoJT9mPnPsZhBooB3n7WYVOwWkE569IRPPgT5zZfFBjhJkmv+gYV7dBaotioR3qjDe/+oGXawMqIEFr3hwWWu8ZW3+PC+unDui7Rq28bj7icE7lre9XTBi+L887y8zFSCmCRPaOc8B/Ztjg1vTqTVe9L+4dm3frGdCPPa5N9B/uh0NO6y4pgtle/Ka908w+CoqKRUqwg70pt/3H5rggP4+LcC09zhQX5zN3NyQzsjbcz779K8BPc25O/A4A7/7Lx9rFDYBsrDdzgCbi4AvIWy7EZIKlF3cMaNHJYBwzWDinJwQZHFblSr/5AJcp8jgI0sGH3GLYtobPgHkB3cn7a/5Almk2Xl0l8Wa7pOtFxAZ4d69pnutjWfi9cr3O9TpV3skp22BPHT5H8GXEU+SNoxLvIOsQ7Gd7/fUA71ulxYTdxyQ91au6MF0s/iV73/IFxu9BHragebTtnJemaIsIhXw4zLQv9pSfnfZAOWJkJaPAAPHHj0jrHkrHTLNroQTXgtA9vGpmEtJF4C8jvfkgbQ5yIZWbr+H5YDlPv3Ojn4G6KHBxNM+V9f6Sfv3e0z6N/kRBwIArV2roMKJe61+NX+s5tG152R2LmzOdwA6Pye7G+ni2n3ZQX/LLKqgbbnuAtA6HQhfm09BVkLRz6v5cYz5aPU107ABGvKwtXXdFcg+87lF8RkGshq0fLRypCFbzTLRWCYPsk87zRvpRQCd+n0ad/Ao0GWH6AGs5gzcXvT+ibkBkLdWljYeqgHM7LkiZ9oJyMLvXfHXUJ29XYGuDDs2CtkaaY6Co/em/CSqEWQzn7VK+NgINvFpOLMt+av9/U+eDL7Wf3iDCTh5/fc4VvPqv/O0dj+N8re8KcC9kXmBkVcTv/grZ70EPcmvNEcTCYi/lqpkYGwMBO1SXr1THyB24tYfn1RG3Gc9lntugQfu/RTI9s5m8Nvsik2mwxiwpVxI5zyPB+OsUml+Kda1fmhcOmHRk0AD7RcO9ZJOKWu4xNC12iBYRBMwbbUxYuf4QC0YQcQkHq/tj5D4ec09FfWAmBHLcS7eBDJCfgR9eF8I+M97qfIsVyBpt/sBiW+SQPbaFrDl+BaIS7o+EpwyA/SPuG8zMkxr7ayut5NKPJnu3QpkY4ErXdhMeHxEpVijqgpIReIfE5+NglMMy+CJ3YYMfsFvIPvgDLj70U8p9VoAOeYdAz44gGGKrN8OOiM/5ekwd0y/B+vKTFG3Qi4G/ln6Jw+tzev/flf4Lx9bu16NE2Y3S3ZuN2TwKUJ0j9Oul2B96Up6o7EXkD3LPBnAFm4ffPj2igwPIx8cidzUdRNua/4DbxyBbHnfMqauGe4PY2vqjeuAFnag3KPkJ2TS/btVnZqBfKLsBntc85pOuabzEk7KfOA0Brprp31byTOg0ruORt1igVued1fiuaejfpvANyuwt145Ry/PnHtpIwiQkXydSVZZB0/kzWjlJLemShmvNp/olVtl6FvOCqv3b7egj4uXJmpI+1Aq16TxLss6ZGBYXiuE7zw6se/43sQccWTAFjE9fv8wErJltjG63hqZbhfAb5guR78UTs+I1rSgbWITZkyFgPwc41Dtq+JGJouG9j68ZIRMXW/R5arlQ7qcAmsxy6/lN/mIpmjNx6IT+7EBPU6GyKjjbMefg23ojMZNWw0cBkFtso85dfMjbbI5L5w7lpCOukd9Ga+V0L5KcaxcwDDyKWGgWOlIOVLF+Sj8m0Q3+nPvYSWxUwPoe2pUkc96STT3ZI5g2sNFpC7B+pt+M3sZ0yJnzzHhXCSJh8v11zZdZIgpFK/G14D2Ti6wtVzEI2MyblWt9Rh05Pf9qTbeCHRIsfjmuvOLaPQxthrTX7JIPz45Z6NSLzLQ/DLv6sQO5FOp6EWN1iX0C63czj29D33fv+S42FyHfI+ntBV+HkUn+Z+pm93qQD/+OsqrJaeLvpQG8cahBfRrflpfzMbfyPMEt6gG4Qyk7VTKRiXndYj3NkPKNh41dK7+zgOMFAv6q+jZlbp7Gmi7GbW22FUUbZdgCpL74oGOa4dJxxfkoU1sGrsFbOzQQdESDkelc8ikQoS+8QcvtJmvRfjalBji+WxB/paUEvI6TC43W/4t0h82Y/z6y2a0d6ZWZlfuE/TT4bCT0UcT0SEf1f1pW6qQFvPWbptjZHTogUTryyZBpH66yeBhmjTSNskxfJqJG3npjPaFE2eR8Mmcqyc3tCEDcNQpiCcWwU4EdeCn3ZHWUe2SLNMy9IOS6Uf/4BG0Yi4r8tJyODLs2agf4YogXQIzkj4R95HvSq/85nK0kTfLCSOJznno190bnAWSCejcSBJr1HAOMilu73lE8yMy2lXVctvxBFqBP7D8kJSK9A86DKd92YyUQQTH7mA2dFBVHRSa/ZDh9NbP+Wki6GuDFwJvs62RVltd97xFA6SCyr2ep5uKzIgdHpZSsUda7xo9Hpa/i9Qqnd0YvM0XacHwjAb7jyBf2lk13l/LR8a17m9MUnogzyRau/n3hh5kKHrfzVMJquiXeD6K275ltGORw/r10T6ksJ0vhCm4DZ3mlAlZ+APIEBp1PVbJAE3aokBXw+Wi9d0LNidVriIjl5TViqIM0Bb7C+9vnShFYi1lkV2Gvmhm46Jra1Qhgv1pOnak2QyZNMjH+LzZjv6gPz+sa1mJzkYKbubvd0V9AlMdngvuRD6/mG3ybRFHJiUdlWae70bG4kN3nHtligxNnmH6WX8H7c18WK+VUoPWJPE73NoviAzHkdSjHG4gy8sLhjHTCWjnxWN64750pDDa15Hp9Sa0+4TvniJfH3m2o3FHsvQ7kDqW577WpdpI8z6vZ+eW96OVdjy40Rg99Edu6PVd46eRlgRvtPI6J9I85R3ru74H6WdJ6kqdE0AnDW3Ks7WK0EmLJz+4Pj5GB10uq2m4yiK9pQZn0vV7EZLCiNOcoiXSfHtzQWLwC7ThF/z65XUb6XM21PN+F4728n4kCPQmoW1Zpy46vjNBuvqDbfDtcfLMG6rurrO7gQ4eKxbbyuqKfHB2s3h+4Q3yyajyScM1Ejqsv2Oc3pSPvrbX2m0dZYF2YB5lVrYeQD8Znv+EsVlGUbDleZ55Hak7xzJdc6AEWdyQ+fjb0DlkULntLe9OE3QIA3biegR0Rt3x8LGJRGRgTHPxKZGG9vqdP5t8mRUtlxy8z935C+3tKz7wZPoW0mkbtIWAbkXfMD2DtnsjyMDhu7vTpBfQkryARo202rJ117J/eaLMZUzmF7Y7sGii0y+55nM7iehI/LPznjqh8izZS0FWNg+QNm0ZgcbaNrTmZUniS+VDCG6rz4LGwwmkTS7iefb0BNL1fKemgm420tcWKC5GPoV+KRvLf80ljVaWW7RkZfKiExW3km59uom2Brzli3DbjYxrN8ZIB9egXZEBVCn2U2hVv0A0q4cGMsRRbsJVa4p0auRdnqTcROsxeo0tDqfRNKenYvpW9sgTr8ST0QZU9Mtyzpafh2zRpo37tqcpv0F/tFsqsHC/QqfCMY0sP/EoXTSw641kE9q8Iaw0MpkRP7PcAvbIJCCTGUctXm6OLGMWDpVMW7RFfk7ZfXwavA/p5ItCYl7rIzOv9puKJG5BZicHn7Xh7qIjR0yZsYl1KGW2bqdvmzUyqoAtzwucRltzNU9v1GpGp0b5Q9VfN6Gzz8YFbS2W5FmJaUbflZWQLpAouVd/EF2spXuNvOREfp7uOlCmE41OiX8XTDEqRweDuLkqDo2jSxZIbHXkM3S67IGXTEYs2npo+f2u49xlHL1aNAv+AWQ64QDt8SQFXfhcqTBex4L8/LrpNKZ8Vp45CFeW8JOCtIvdeMc9PIIm8L+7pallhLayqk69HuxDPxC7XIrnd6Ij0zTaR10fdEThvvqo8hFk5lz81ymbOOST+D6z2HINdCy/htxc9ofRD4mcgCRp9OuL8pCu+m1It3tm7bv6WHmmzdl9lj6zZTxsWBKfcI88y3ez/EkDHLLU3z9+i0VFnl/jSWTa5WF5gTIhg2CBkTI2k4lxyec66Njn8TF/r5fy7NmHzxTG26C9LHyd25Nd0F9boqtLte+gM4O39zv2oejMQ+xxtQvcZWzzScq8ZkXIb1ypYlu3FFo1VGxOUuxG+u2UT+CPxqNfku6dvLXzPjqo3NfSxYoiC23x1DHRm8hHylRzmO5PZMYCynsqC5AvhicUcNt3IS2Cr+IaH+5Ffp5T8HCTnJRnyddi73t1XX59/abt1aVT6KLDoLFmpHoZq3Ji/+mpKKSFZWJHdN1d5Jms/f6F7zbI+9pt68uYSWUsTBtjc0aK5Nmy1hlM3PNEBml50zeNf6FNF1Ed23dh8szN3tMcu2+iP06WOLrLmaDzH+gfYzaWIb+d5KfFXjAhk1GnLTcI/UL6dZJ4g/T2oO89E991qjDicqDjHR1tJbRgs9BXhQRupAlPwlRXT6GfZjIbL7yqRd5et8FdS6tD+4NM5NVJM0hjgqTUtmYdpG/p3BNF7rfITC4mR2eChsy2KSl/3XAJnbtXf3XTYAo6mGQPu1LKkeZg0SK7bCO0zwATvQfLggwmPiPb879Cm/aQTBcGTcvY43a3G3WgyNStI4WB7VnoABfHTOiCN9I//ziicYs9+idkotH1XiHS7p0gmNN+BfmqMeW8pOxQtl5597Ee5zq0U87AeCJcGh0IFLj70NwLfX9aM2fJLBddvuR5Nl78HVol5HO89wMTMleXXLjx5CjSiTFf0FWsk1/ffWww5qwA0r/3yxObSU5kqudbwWTgLNIZxPHq6fPL6Gjw9r2FphNo4xzz6B4vJ/RjatIZu13X0e+IcG6j7T1k4A4Hn6ojHZ122WPMecQA6bS6H7EJyUXa3EKfcP/sRT8fDSglHh9DnocWCOxwOIL0lGWzBBe6IkVH4Bn/gw601/Qc/f23i2iqBrt47QsiMnN6w3Yzn7dof97mjnbWa2jzXSVt0RxZZLg7GtM/ZYWU3Ph5o35+I9LwuaHk4Pa3aMu6P/WzShZILzaas/oMDQky2L7LQH0a7f143jD4+XWkXOzTuJhGB/LJQuXsgmAT8lEH/SqWV4c2jjlbCd2noa0Rb0U+l5xDc7STeLX3N6CT9/bHL17vRTuUkCf+BWeQ9xxzzYeSHyPjDdajTZ7daP98p/mgrifS41/P3Smji2ZdYLXzIuWhr288endjXhNx85iJfxnOjL6f4mvIsGNG27Hvds6dzEFahS/9MvihibxvzcmkvrVAl+p/JyUIPES/CL6v7vP5fw2Xhz8VbBiGSXYhFVIiadkZlfUoKk1RiYRkJEl2ShlZURriE5khM3tHj7333uNwrGMee/v8Ce/vee/7vi6DHY56tcYYwwSFYRKdg9u1MJdpeTBZ+SI0FiCt0NfDMGg2cFLmrTKkBJVLvkZFTHebfDBm8RrHY14ds9a5jIOlI68gNh5eEbj2KnOZYqXQl8jQ9G0oy645KbJkD4000zNdS0kw6Hn27l0PVxzWCY/Kb92FFX5nfy7eOI6kmhfgEHAOUv/w8d4leGAlm46bxvVg+C3QT/JV+AbEhdbUY6uM0P7tjlWA31Mo75NzMqCcQgsGzm7NFTZonEqleM7bDoS3uyap+Ddwxnuv2q9MRVh08XdXDNyH/xGPhYh+yYSqc2Lbh0q0cO3q55QB70mI4ShcajN9Bt2Xs6a1CuqxjjZP5fyQHHR8ebUan/gQ60WfWLx9VwXe2Zxmr224cdqDTUHXMBgm+P/RMpjtxg4Jm/KhV/ehcOSZzZ7Dh3BoD8UapwsjEi/rSh244wLt584XPw2RgbIHvoNtPZkweKry9tNQMex92S5/ogOx5eyavgzXOUh+9N6XINWDfXs32Q58UoU65tG49xWiUH/yyMS20THon/nz31xmJLalWQmJnT0FLS3n1KbWabDw89rFLyaVSByuLRWjU0TCNPduURdfLPdZeNLQmwGVFK2sA4JuGH9dKChJQheaPM4Zni/cwvdnKZyTZn9Awf1vMRZadyG1gL6rui0eW428/vJzVmDXRON5Vg4zTBzN+bwiKY0FwmetqR8ToFN4r8Qrzx9Ql6UxTeZ9idOyj33+ZehA+yGjpPclgzA1QaXAPhANo8xd9RuzbVg55EPUk6jHwSxVId9jM9CiUUqILd2EHqHjZ+d8XYBYcJfD2sMO+sLon4Rf84dyA79eZXYdbLU8eJVxXQnLi4S4rx76D4vGh85eTefCpuDxm/MRnkA4uPtwilE6FMySj7oLseMApXmRUbcozp2i4dUcXcemOQtmatF0mI+0KLk5qwCzmrkfvkTXQVv10e5eBwro5yp4MVnRiH50JjmO+R+wYvxrjUw/4rTaIjk0/jn0drhysm/uhz+OxeeySU3Y3fzm5ONAPxxLspg1LVZBwhElNVLLDJQmHbjRuL2MVUzKvxNHz0DfWrv098FCIBUEShmfPooTz4l+loZKGOaiES9xcg5mhYOm3u5WwivWuj3Mft9x8LVuTlurD0zajFPePfoG/rNSFZfp9oCm9OTERCMmqBqo5GkZJuGQ94EzNx6ZwrQid2L+zRCYsn3rt6r/CxoYk50rpT5Ck4z1oP2DBhxJI93o4PyBaRcFpp/kjcCCdceGy6tdWOB2KVLUyQtq8ltEvOJVseFv8jxXwEUI6X/NqjZyBzcuaZ8Q6MlDokS8ZyJDEQ5YWuR0ATesBi39e733ABYmrsU6v4/EbdmfpPWMJeyrTqhw7lmA5bZHZfBAGJpUzErmYAHybz2uMVsrwjIZ1eIaQSkYNGQ5+NpKFWY043pOnddAwvOzBZWug5Bx4kJSA0MnBi6x30t0NYbGN0Xm382IMPu+XT/nVA42vLL8umLFBYUPtOj7BMnQ21M7Phv+B+Zl7irV3fUD1x+d87dHOnHy2rcEI9oG6Lu860BxWPwO31yKePiSD/K+lybVqtJiz2jE+fi6bVzXcM2s/BEC7fXOG1v2p6CZ0e5IiWkl9Dd2dejV3ccBbOt+VMWD43nDrmIfOqGpv7pOUNMf5xeXPUzUb+C0/a48/40uqNyvLrOWcx5ixP86P//vGU66+f9xYSuECrMj64ESQkD2u2fSdWMdevbW+d3ycMRJlhLnlQuDmP+1W9fmgzK0ZL17/PNiJYxEO1IQzB7g2Glvrk9y+jgq4HJK2s4Ppwz2nlPk1kSyfENIkZogBmRvbTTP58Ewje1Y28QLIOwl8lTbhEG78J5Uj3pFrPYfvG7PFQejGqceGvyqhMUTP79Z/sqF8Q2NfUiwA9+jKVV3DLdhfFe2W5vxDRwS4Y3X//0cRh/oqwu07sXJ8F3IuP85dCSOmXiadkFb2BEhc/NJnHd17ly71QCkdy33KNOu42JNXt5kohH2Pjie5SRxHuroAideKtRDF1V0EW8jG84ez6DiniqRp3h4wZPopZhP+XP6NVuXHXY6aU3/XovBWdfNPI9ORxy5w+QmSt2A4ZYl3yYZOGEwtLRqSmcLh+LmC5rddvhKU7D/QC8HTJ+m65FQkYIt8Z+CCRd8cHOqtXwoWRyGFFveWWdkw6yIm3rnzTWoNMuR3ce9O39X+OPqYRMu7H+4kLOn7BwM0+oVsFX64pKUA4xeiMRCdddXq7L7Yd1us36GKhMH85tqBnT9oYD5cquC9Cb2EYdcVK6fg2XVqKdxH7SRTMtcx6U5B4s0PcYRT5dwkSdifT+TBzSfrq5TNFyEQmFvhmd9b4A0uM+K16sin3o49KDxuhMuj/AJ5zC7QNf+96Nn01ugp+viZQNJJczZ716a/vIEjhU+Za56z4jj3Dnz3P90YcSS+bhLpSEuBrSYzmj6YHmzVvls6V8YZCusOE2rD6Ox4QPntWswvMbnlmSCJOZlMK05n3GGCtfjYhIbTPDhadF/+TOm2FNyO5VCUBY6+mmGRc5LQWFRd+dDws7eqT5i8jnQhL3pjHwSZks4QVupmqM+Dn13BqTUlL5AJ1+Nm51uCHQ+flT/u/8qEBopRw9FiUFD3daFq+f55RkF0pb1Nl/gWPkF59OljDgSx0PYyrDF6j1UFz/fqoAPbdKOrL8DkMzHn97r0IQ9J99oXaTygtmP1rFvqc/j0Kx9v2mLL4TfnFpJPFGM/ZbFAovRFyFpz/0VNaZS8H/w7xud7xIsx+EVDt05JIy4ND/82wetdrIdzflJSErf7nOhv4aEk3YaNplD8M2qMKo5ORVbSr7F656bgdW8Ju9vR25BC2/yBv+4Mow7xPyJdq8GQkXqsYKSTuyMVpuxuJeMBBoJH3HyXSi/BxcaVdSxITRvhcqQC5dOd0Q/i3BEbzuuADZXaajgT0xbUg7C4gLFDePJKpx/w7geZiMPhIAg/lO/+GE5h2dU+lgOdrfJ+dEYysOS9CnLecE4KNnLE6CUIgCNDefSTvsP4NRvJ38OmauY4JVSeeTnJyBa2r6/+OczLMu1mIiX0WKLlZ6I0YYYzCQLXpk59geG5rle9LQFY4PxQ/+H0sq4Xv825MetAmy2arm/7pWP09MWd/iodvYpaFPT5x43zD+/Oe2bdRYWh02oJLpu5NMYM724ceNV/q5/wj9kFcSB+Ot+bjh5FKu4fzE5DlHCMqXURtrVh9guO5nUw1IKixNDX98828TOtNySPpIGlO4S194c44ORDi3SmMBx7KOmjZk5vYHNTFJavl8LcOT7HuP4/npo8lNS+DmhAmMlCmtr2ZpY0zmuryvSAUX6zS8ar7+GUb23agXPnyMx/mOH2CdBaCnSr3qu+QS6FQ+l+UrE4OQvi+z3u4pxUJxl2OufEXbqBfLU9IYgoSA0f6liEeok4t57y9dgt5vT0RXFdsicKczVfpuG7aP3Jh3rvsKYHE+AR9RtDEl2lNp70waJrnNNIYwbSKILT9bq7pLfFdgu/rw1DXNHFt6orvVD+YWOz8W4FxZjXOLzjEohzonisPDQSRwjcam8V9sNBOlYwnfxnf3QJZx/lvMcpyJYwt5tGODwaLX/kals6FSLbtT55wZDA0JXzN5UYlnfH9HLQcXw5+5XDv6mdIgRi1n1V+CGoiuGdCcLTLHRfVQ+S1oKf5yJ3ZTiPQ4LcTduxRyjhQ79FfMJVnvsCr3KPxVcAxXTR7/Y2uTg7L2lCtdDgtjU8NbvzB0VIKWfl2N+fhYnVfea6b5ogYkjbz3jM6awtNv6TviKJlRusBweVKKDwUaPh1wXz8DYkyfl5+/X7Hia23P6BC5sO5qRr2w3j2Fzoa/9it2wYEQ3V8ChG0fUpTejNA3lKd6dOUgIYMqn4TtHmPf/i2Pu2gSf6A+4uM/89H+qejDLP9WsdnoXjv/9+qsl5T8YNPXw6OP/CduxUOwaV4ub8oytNDE0ME/UqXggpQ7Nh//aFn7pwqEoZcrRd8+wtKru7ZPrv7Akzd9768sXSGIzNrV6WwpDF/he6LGtY0n+ETtxkgeOmbP/8B1gxOkA9UWvmnb8nWeWt+zAj3VsiZ8TdvYhOfk6LwttI5LKbuwWLYjEpXqTynMZbTt8aXG6785D2PKiGLBWVIBGb6c/d1kcsK743eBj3XH8Q1J+f42/CEpsZ0t+cUYCaV5Q3eK2KA4wdl1W1zDGjunQqQOlh5BAmLOqm++HdSrK8DkpdihJkaPRDDLOp3/0ok8i7QZsaLUfUPX5CFv+IxfWR4SwNlL2aniHJDQluXdUpnJA96dNymVvehitj3C/WOECUxsagcp6bTjSRbwY8TcbRo7iogQDM7bJC+orx9bi2CO2r3bF2TDRrnznSmAXEFiO7j8g9wua3ML22rzvxEGL5Fwhozs4cs72o/QDIrbu19Ud2c7CycfEe/xakUD+hprPPxTBmP2YcDmfHgze5Xw/lMsFBOqAr3tfa+FCHpHzbfVLnFIdmDuyZoNDHtals47GOKjM8UJb5TouNUn8nI7a8YSWcTlHnlMYQVx4M5fJgpGMZdMJ6Zyw+ZsjeFLYAKaMp+aDP63jKKt7Q+eZqzvvgSBKOTskvQs7X4AO2Ch246vwi8cwr6DzcFK3BR6I9m4P9B2EdoUmq7mkZFx2vLWr+j0FzDzbW/9EdRnWrZfU6jtlgFjHtiEgZI8tFuq8Ed2vcPXjAdYHxuxYu0f072SCD46+G7dlCHWGUdNv6n9HtSHiwLTGJ0k7JDccoVE364a+sazy/NAI0DancS4q28IxgfM3h717YIKdXJNhloPk4DaL6VAtaDY8OaHiSpCnAt1ueseDOFh/z2vm91Esl4v56STYjD29EaNbScVQK0lJbl6bhayQv00nOkax4o7hhTtzr7CZleH4uGMNjImYP7FbeozL/bEN6Xb6WOqet07x3AWHx/bMt701xYZ83eRmhwEYiq3Sr24zAFIKG8VhBcAq9rt/2MIPwtpd3tpQM3Yctz7P5uc7AKOkj4WKFVRYa6cV01zth51Nb+/+/krEYmrFFz+b1nCB8LGJ5+IFqAl88NIR5KG30bek8NRzmHVjPCT42Asnf/fSeRiWYuFlkS3Rw38hBc8Oh2kFw0xHxxJ/iweMu8l7nw+lg8nVL/FHU79jcUvzUKJ5BJLTtJcu3neGUAfeS4zxL6DUMDTp7lF1MGlje9enooOEsILrVNMpONxy8mZnZAcuLZv/N89Xh017j8ktU2fDaNYTTwbewzgjNe95tTcLSXXCMUoy0UB+w1flmuOF351H5WRqHXdy9t7C4E4uvhbqcmmuPInJShRN2w+ssPV7MnM93WHs65Muav1wHIqc1mS9icEwULHAs+aXBJ3+5qaX1S7gRhlJi71MHaY8RGsVNT5DgcRNniOKk1gjUn6tjWoMUnPfWIZO9kKCfs+LF247d730WU/bNgjCx/338fG/hAyNlYjEZAkgHZEyWVHwwqIO1aTNTwE4zqdN2U4kYG543SW1xRBY9am0Gz7ogIVrq98NL/Fi94ZNqcvRCZg7YlfQVqKJCezxsVsrb2C68n6e6r0kUJqRwghtH+y7dKL9CNLALE9Yphm3DbTx7TNhcfwNw26HOD/0fcH57t1EzUPzEHliTqOOJw/ahNLjevlE83f5LAX0UDZin/jp5GuEdGwPo+ExcvWBTvI94c+1NFh+JFS64Yst9nyvaG5buoLxRR8TtkaZcO7PR8l3IUkwq+Qm+SaQBJNCC3pxVxlw4FNT6PaAAmT4OGI5dw8M6o7XyVmcgd+FXrmdKiewTdqJpF/JBGOOR/aNetfB8sEGCWLaVewoS5GzDw6GpT+7lwbPGuG8082Us+8wnzIpyNdD0XmHY94w6LO8QnKeSzb3rp/QJls29niAhM0lVx1j3Tmhb9+Z/aE+etDfQzWe4N8BxJJkmpLvNjgrZ0hIm92L5BQjG46wHJy7S8dZf8gR1/RYc1i1iqFg31zvkRuf4W8Pw+zW4H7svMTDECY9AYVSlQ+EGD1hcM76Iq+9M3jWOR5t//4Xu239f9NVX8QpPDAgvXxR/mDi8n5PCSKMB3/2cadnkN8nJPO40+I1zvLd+Jj5akOeqlrQXjxgIZ9FxY4yUDgIX7ze1C7TOIMNJD/jl6z8OMDFoGRSQIC8AnNz3+yj2N4+Qplssy9/D/WJA3PT+6BZI7lP4ugWLkSyjkYH6OOIghW9dNM/7B08JuKRdg8WDxtsG5P4QE/+svHAkiqWrtImkas/QYfGs7ros2q4zKxWhVGRWDCwFR0poAZTFotBmUYi2NKRpnTUywjWr1Hbd72LxPF0G1/IrcRiU86hmS8jsBF/+v5/Z6WAQKPHHnd4DuKK/kpVpe9wzFPp5m0GVpz1N2CmLq2Afx+DP2mKOOL31R+GvAJvgCyn1Sf8KBLblt9YHZK7jUNnth28Am5g2xXjDostTRh7sBJr9PYqkMtFFN/TKmNrSE0z746Pkpgt33bwDcFYQWhFpMkA9MbnjNNeOA5Ze++3T2fw4bz2eNHKMj+2WrOvPGYKxyojASnp3puQuf7zhI00FwwS5rjv3+eF9u5kdoGpUSw/Xd4nnBaIVVHyOdcidWEgRcJiSSoUJsxzKk90EGDyVqLJlQ+a8hRefumcIpIYY2puJdXWB6tHCdVxt2qgtfJXpd3DdFg029dI9U8IUyvCbwarfkBMfy2Y/UEfO+4cXWeipcKh27P665R3sPsgDXuBuiSuZSgwNQ6EQFZuo8QWFxmXWSYvN9PIY+YzQUZdvUu4Jk7V+Op7J9bX2l1uTEvFqaHuOIJKPPQajfzm8veE5R8X9H76UULVaOD72OwSWFyyY3mYmwIFFwnEJIdz2PZC5v7bcFdoklsrSWyRhDRmAcXbjxihg526lFJAEPqtL4srP9wHVd76Ueu+OxzJHqc/80wCh+uc3gT63sHG2trDhfmUOOQwNpqx5yd0lCpEfNN9DGP6PYf7zfiQnFVCTRkAOHeK0VPaRwlHWGBDJrkB63WFSPn0/EAonbXwCODD5WtcldtDz7Df5IBrqFIqjqZaPk5/NAnpB32lb9WEYEPPVAdViz0QEy7xGzxNgibJlWXRDRNssVcfVSyggTKmdYXHUnLYlHncYGzTHGeYcijzOouxlhAo/CWKH7r7khimHFfRKL5I6cOlVEwKuKVVpXoCF4LFg8fff8DXvvcbO3lHYfab9NEqGiJMvXa9fc/sFI4mdvzHHN2L87UlPU8tq3GyU/LWWxExWCpeuJdfVg4NnLyuURpVaF//4N+lJAkY0B+bWt0liO3Bu6PuUBzBqVRnzUNOdkDQvXpYPuoeTE5S5vz0sMD2P4kyFARmmBXdeiPy4gFO/Cf74+h7X5z1TmXd5jgHZEWiW0j+HPSXuoYrbMdhVj8Xzyphpyd7VGfeBX2HuumAl9y7VHFx4+m11wP+2LCQWna2jEKe4pwK5Y8TY5A4Ys4YzXt2Z1+aBRzOK+CCgJTuncYnSHbR6P0cpQxN84XXU9qeYCQtQYzFtB8Ttl5afWfd2Q+GNzZ0L0nYeIklzOuTEKbxB1He0qWHYtHTf1YekaB/8obSsy17yO3Juvr1BSsWVUd+E8qNwOwXhJGyzKswshIDlAcYsCOmurNN8Ao2BGt57GtehLkf0y/jA29A18L09dxaPiS5+aRt8U1jTyN7WObSAIzBtar02m/Ym2T59Uv7eyzqPMnY4BQF82W/uO9trcHcvUfnR9S/wKyy1oFiP34sUBb2ZNcQgHnvzmbxvFRsMqxQ/a2pKc9ybf7zVa0I7LWfNBy7GorEVqppCe0KIDIwVt0P+Y3E6CgJ95+0MLHQXCwenIfzJczHBC43IZmL7/P2p1VMkPm2+NvTHtuJKZ5+7QiZqXnfq7cJ6Kbd9IiPgQzDd0gUplfXYPBSvrzPsgrOcJ5LvvqGChaeVdt1R9+DVnvG/9xSFOFP+KVHU09dceLESotQ4TSuhWZKk2OP4vDUP2Mni+/Q46Fpv9f8NUyZt8oFFVHCiK3w5yvenUhwWDSIW2XCDF61NL1pgKlk+421TD0k5L835PUOBcJbX/Mr1v2YrPZu8OIAFw4rrXnnOrJht1zjwIyTDS5/D6kqtQeoLTA+TrJ2hT634xHx5fdg1J8+219oDAtulh4/d2gAcppt6JaYuXf4aLGINssKv1n3fE6YuIbDsUHjNLfiYU5dzvxT7BmYDeVW7GZ4CXNvJOSWtwyxOeJ8eHpjIXbPK8+PvPyEOfu+WnERH+NQ53lN6uESHAleqLKLfoXTveQLx+xcIXXWyOHvQTsYP2Tj2xL9H1TK+nB4Tmng+G0WuTnnC7BoeJLzSEcGVuZ2OOkq7vS82FCC6Y5fTj37rUaRNAfRrvyHmFB2h/MTbzZXFMGC5vLaQUUf+LdIty1brYOhKaLJPfx/sErG6ctVdlp5ihedy1Ovb+OEO4lK1ogbp703iyZf6kDvgRvVrom8uLZ5lirlex+szHBfX2V3gg1SmrxlQgEu5CYH+qeHyO+1eRBDXtTG9fJptitVJNwCMzVdgVrYXKa9eLTrMpDt2T6N/HwHXYczQnIia2HgyIu8dHVG6Psm25ybyIyLZ06bXNBgh9Vyf7HV92swdvdZT9Grctz8tovV6u4rrFS/MxcTlYSzf79R7xOJBqKKoXvKU28c/iuX7VNaCEu7fw+FW0dAAuezMzUW9TgVWh7W1tuCQ1JFTvYRXbiod7niLrrAgpYAbWTWO5yq1hynmaLCYgdyAk/ZGk5xSgia3nbErf1iQXNXu3FUmmrGO0sXicc/KSlo0uG0Ce0wTdg6rBwf4wgR2YNh3nQempb78ymrM+KaCzKw+epc5y3+zxjhq621q2Uyn6KGpGNgQIvla9k8S0bHoUGEWOh/QQmWeZmkNpltcHNJ1muD5h1syTNJLvgEylOK/BThv0KEaRF68mUKR3naMdexP6f7cOZLUOKNj4Y4qcxpcM1iD0yY3lGhzX+VT3lk9q8e119cE7XqMs29iqP2XDPCF87g5viVe/JPzuDabp2PURkMMB/1VpMcsx/FP07Y+DzShaXmiAe1VGxIeDZk8dSIBQfkyve6prPDJsXsylTOcxiJGpZqNDTGRg4uvh8X6qCQwT85r8wQFglGfV1zFzDucY126JK8PANzq6wmg6A8BYnFbterx/lMC5EVNA3q+VQ3T4wecDgN7Y8UfNhZjmFdbO/r548cYONjZu2gEAtuP+4t76xMhjXFP92H3tDCwuE6nQ9C8VhEmp4TDgyHxQWzKxM8dDgmOXSNQU0Bp0TuhtfJmEABpfv33doWMHDTRspwUB4wV+9vqFYZzE3rfX6tn4+EFy2zu/KZcHw2/55grjp08D1p6tbWRKKDILtJZSzMhzFNGd3izqcR9VQFeWUY7KY8Rx1oCRPcufN8HbdhddJ8X/1BSuh7b8aRlcqKA+Zud+cOiKG/ClXgq+EveJfOYmCBHJ9PRRkzv/pLH9ai9xWzvDMCso/xfwYhXvm7veioe8sYYe1aW1AMEx2uMsvJ94pkYOf1pUrTEpF86vsxHKSYs1DG/1CpPsQWZgP2Z8t7/MrfP+UxeXaDDRbpfmdQipjj6h6xFjkZKiQOf/0cMuSJK23XOem97sGYJU297O8SmLMbN3HS1cUCfv7lrhPv5fefXK8cqTwAo2RChpFtLfwdCD3S7OQEg1836KRLmjGq4MZZrT1rUL8Y9JKSRRX+PaGTd7vFD/1ctDP0TB9hbO/XUWErSmxaPZZCnhXF8jqDr10ct8HJ7+Ef1bkFqEumfXRSbBZJFKeK9CcP4QRpMEKefye/hUXmHZ+HIePqiGa14hp2DEqLLDDbY6aVtskeCwpI0fi0jztQBKaD5P7srmOF1rUNDp6iaiizNvbspxSGzOuBzG2PGHDyBlOkKGkNq1ftUymiFTF3//jU+q5cmKN414x18VDJ2JQ84ysK5SsnDjwXaYceBaxy6rGEbplJ85YnrFgtZsH1J60Se4ki679SuHByWGBBkliCDceN4nbsBouCD7/4qHILxiezTU7xT2LT7ft7lTl8oSLsx5mfOXm4YN/QK/tEFqZv/4iOueME+TMBKq7fuLCT2ND+eI8QFpwAHoabj2DO9HgU8aksEE4JDHzfzoZq//JGV+l4yHIYrlzO1IBsF+naIjUxrH+Zm2fk8QiLeEafhflKYIeS74IBxzkstm/s/S5FBwOGzvzuuIIEod7eKNorOCN4Lj74EwvOtgyobJTHQdNi3n4heVcY8V/s+MyRjJ5RQjXHC+/C0oBdRjVFI1SRDSeNmwWw37Q6rihWCZvqhkYf11Xv+HaTqrgMCw4+3VcQ/XQnB41Jm/qC/3Du3ddoe8FrWOaoZzvmeQ+Kwp5mUNBO4Uj5Qa3VA25A/P7UQ1MsBNtS+YcnStNhvIf5ZI34Uj51vbfiuIA3zI9nmWwrHYSmd5x7/nLkYjNroJVlQSXO0C5MpIuegYFr1pkap6VxILvwvPXFE0i4YPWlaTUSO6MWy/gVt2BqNiiNdRcdDFdApUOfLJZfoPhBF7aNuRtjONgTAmksaRzAqwzEMB3dDzlCOPzI16dNQgjJ9Bm3S/IEYemMvlZ8Tz0WlixRLknKQc+GeJ7PqUNQpXGqoY33EpJ6OT5cXXoLI4HtxSBMj3M8EfQHaiSx5Q177tr8MZx98k7j5NgnIMcmH1KqG9/Zw2GCLTcz+q74mTiNncEBCvHCHDc1aFY6IuLs3IZltitnnnTEox23sIh9syGUhgUyKmiIQ/Ww5sSTRF/ImN9n1R+pg11TXtnv3L/gcNSHQyrdeZh9s9Y97+8QEM+q/eXdImLh7cTsEwefwqxU1vyhNUccxRMXHThqYLZvNEjqxWeoss1sobkwjPMO84cefu6EJYmQHIEH8TBxZX96r7IrkLfJtxQCqVFfc5jgssCG8cwXDOvXjmLT69JXVBn/oPv3gwz/bH0YrDE7IfxoL4z1CV2ocrHE4vOv0l73xUNKxm2OwH1rUHTPOeTE+ePYJqW5LHlJENqUZPccfYIwosJvdFeJEyK7jW0tYwug2V/yQeWOfxJvE2PH7urh4IbJuwcSJByX1fPid+6HxTBH3VWZWQyOv3SalvkUtPoplw+RZWDiLvfPdaUcLPP59cWmix46tcKiGYfTYUmxJWZ2ShcIZ7ztNTh2eH/yV0Wk5wiOnOptdyLvwZ7rz5U0ry9DXMZ/J+89VMSirHM33h7txyrXW1nEWUNs2Hq2GC1JgOKlQLrQ8RZYCVk4pdyWim0bLGphLzmRfCUukwgGOCvo3Xdw+Ad2UqnJaKREYdE267HQTUYs8ayI/cHshf6FYyY1xQ7YlUrF97HIDxpDt6+cK+uBFkWrbTlvTpyN2tp/rVAP5qpf2BxYzMcaeiNvNpoZ7NS/ZUbp8B4mxWXucl+3gL49OtUJdFHQa7DOIXqyBXOvPWfL5n4Dww+nkiKknuCQ+6VBh8690HWxhGdX1AlMIeteV6Bag/LMk7vIqXRYJ58tobY+B3Oyqilv5VShteMNFRO1Tj7lLaGyKmpFrFHafVzyXBO2/kj8Bjt+0F8lYv/Pyh+j/xm7609vYp2lIuupt69geMbiyw36GKgP9oEfEVdwiGfZ7pZPIYwUJRz7WbLTy0l/JF8OtmPqsmvvWM1tzDTNNdK8FIR5Bz21izlNYUQ/50V20W1Md8+stszyw681F5JXjFextdo23Po5K+QMnCKshsRCxxkijZ+iJ0xL2srK7N3xCPecxcnlN9B3dubI8JIV5pX+Lm5bGQAUJtskGp3Chley3/p99kMR01qh/ktBLJ7S+itT7QblTSq0l9RvIqHmaX6Q9wB0MCjukdk1CvOTvHtU57qgun9P4pfw69jGRq99VzAS6hfOM/d8egAkb5oeMwt/mDE79TCnyh3nvnsIFrXcweSwhGKHSRkYcKnQeRbbiTazE5wV/MewWto3xeke5Q7HWTU8sKrGV/vorgOLIxQpZzF2Orbj+HO+fJYcd+z7UnQ04/JzLKtdtDH4JoVd2VUVdN/Nd/47JVXUR1Ug0ujrCBVel6c84Wk75CWI5Z1Fxy8N2kF/l/GP9XZeGLnkMCgodAhiTzGfMnTZBuKRN25Di3FQM6C6Yv6sFUgdKfe43YShwZ3S/7tiMHR4aPeeM/iFpX4J+XmRCbgCCSSDPd7Q8UlwTGP0BHaKTUQk9nZiU0pLU4/XC1h0v/ttEr9iA5fqHUtxP8ittmQ/R50Giw7s+UMXB4As0KDHX6kC4+FRn9hKriDJV0g5e/QwRka+iT0y4YrZxUcZllOGcVD3Dr+BxRy2UT4UZ/73C4mWo48VhtVxKMSZZ920C4uDGnWuOg8iaVvx1pM1N1x+98Qs+9EIzLIgnxP/CxhfdtnM7zgKE/20POd3ei8zc/HZzf0WOBAWd45SrA/St3dHDk9PQsuUVQifYwikd2hrRIk9wHG1TF6yTjy27W2/ZEVyg7TdMiOfnCeBcOVGgQXbFVxqWSZGuM5ApkYdVSsPK6Qcl322P/kyLnNMKnCcPQHzQckpdk/eAbGog9z9334Y3T9XevBUJPRMf38QWDqJGHylVkXkMZAyFWd3mwN0Zk29jDDxwQkDl+6SbWFIVS/uELOuhNA+3VbeFicsPyTY1u60hNhLPnA8IROHAq7GnLkcgG3e7kv9dKcwqftardlLhOGAV0c2J3yh6fPxVYOJGUyN6uJNW3DAIb/AoMtGclDOxKggX/8Cl3uW2X8YPof57RcRDyLP4LCnaNCNrDM4qu9+ufemBwzG+APXZyacDPv5W0VcAmYTKFXhyTjMIUU/s3ArYN8Tl6DhK7hgyu596LwTxJiFNK1dEsNOQeKMSKIa9l8OYohzd8SSRRE1jevxMEgy+BR5yB9GhbW8mut7sY+fKZH4XR0rA/oSvForYDjqKgbEuEFLFj3Hy198WJ51SoNP+CoMF8yfSz/yZsezjtND8G6odudamgppg6gPDolJOcdw0J3tRZg6PaTS5entX2SAQVvfSBqKW5BxQOOlYn8LNkqOXLkWsoRtjz7/reyRASNbte39U9YwsJoRcqZqETP3ZaeedaDDbW1/+kvVjFAibi3pb+4JHftTv11LCcGyXXHknmkTmAtO5ru8JAb1Zw8eL0qZR/Lj7OeGLRHYJLxg2C9tij08j6kmVL9AD7/Xrrldf5E8ST6vOS8Paaw0nyzJTjjsETHScf8+1t2MnDQnPgHSZZfLnrXGOLhbPjXvmxKMaRg8dqLXxIYF3dOVFEPYFZ8Sfj+7GueGxfz2GbyB/OJ+ljRfIk5f5Ru9RM0O37rN5B+2v4e5R/iXtXARlsuPpark5uDcCxFHXSlRJH+YLxf8Vg6TAwaDQa84kUBKXuHwssbZw4cPdsVkY8bnHIssO0bo2SPh+arqKxTJWk8ktN/BVmvfXg9RJiTQ+L30BRUYGLyccEVnZMen46PUl8xxsfeEU+ZDOsh4JLEl0+0Am+w4lCpshHXzY+EKzgnQ875OdVc4B5DIP77+sOyB5Hk1asUFKozuzeAJGvoCAy+pwwwsuHfyN0+/KXIABoVPco0y/Yd1Ly92nnmnDYMXegIHNnY40efg29j2Qhw0/GZT7N6D1S4JDjZx7PjlzqUKncBuJP6yP/ZqZARbnz2zcQv1hyFOwdBnpmo4qiUu9P7bJExl0aSoTStDwaPrHhqnQ6D2PenLWStdXEylWFppeYM91UUne9p5cUEzgPl15ztMvE7xPqTsDqyt0BUyfz6Co6NBv/Z+PI0RywWblXnTWOSt/SxX0gDL1BgVLdS0cf7aKVrXYkqYLpjQOCHoDaOUqiIPtfqgU3HF0bZ4HSqLHtk30CrC2K3YPZEHymG4m+Ir6x1qWLgnevOU0QQSb3qqP+0+CQsCbyWN2F2QRHD7p3u2Ftr2DzfZpv3A3x8XXYPJxrBu+u+CSdQaLoW+ZPv0KhZLq7xSJty6Ed9Yj3nfc4dRqhmdacN8XLF4O9yXzgFdVcI82hXmQG7qMF4Jn4RiIauiZ4KaMJYrqm2sZY2Lf5jmL/YNwqQghTnNhBUu82RzlInH4Gzaod0CLldwUjB6ovesOYzfJeUw0tZC+/Z56kyLUzATbP2BYjZDnurzr/jQ9Yp8Sq3m0ZowSpj//u6ehvYEDs2WTaklCUNhe+qdhldPMWHw/h93PxJ07ybpXTA3hs4Y9SN/Ri/h0KaN6LWIL0h25y17cVIHE3WlYLWSgE2/A5X8497h5Ou4uKKl/5AQdlm/ycgCyL/+Lr/avoPkX59PzmcO4fT7wLw4oiSQZZK+z7fshUVMMaxrVsfhzfVC+r2qUBPc1NfO4wbxctf+GNhpYB5BNVH26jlsEL3OmsY1BkTnUUduYQVo2DP0x4huDWeIjidCHFtgVumuljcvL8zy/Sj1p9fIp6Q+0SK3jx7mE4RMuSMtYOQjHVzXyoH2tPuvYiwlcDwhOC6o6S3mUi2d/XklCKsoFFboLwniJhXbGq+MJU4y/lHSequAxTrjdWWdKtjiftUxV+AhTJM/eBwIzMYmffxHf0UWFyz5Au0WS4FgPfy2zysK+gZ1eRKfHYOJ3/HnRbxskGhU1virMQJmCNbheR/Nsa1u7aZ9hzg28CrSPbhni4tfLOLEbN7h7OqCeQ7cgDG1GeLRnxpALL79mSr3A5Dd1MrjT76HxV3ORR8+6EJrwsmSb8VnYKJT8bio1DGs+2vYp0bJgisV7P2bB7NxQtP2tluuAxKr8zWHBC7BIuM/EZrKTGxOKuSrvFaKhTom1coqT3FVMvfymsYlnDas+ptKLQIdlJHd/qbtOLWwzubByAgrKizW/3mLw6Ji4sjb01z5dKyh2d6dJTg32vk6QOwidObVsKn0foOeuqlL53fPYX9gcaORrjas/Awvzw94jVOP9m79F/cPt53d3l7PqoDNqt1BZlU9mCcw+LLJ2Q/JHdODVXkl8rv36j+2JbAhSUDXY9KTIE9jPao+wXceh2IDvaU2DLEx610wNWs7RAjmUX1/voBzkTcON7h/gI3qh3vvzWfD2At7K2LNAk6cY0haldglTyHVZdj9/B1OJYbIqLjV4Ij7Ypdk7X1Y/uMu95GpEdeqG6KvC5Tj9kGvelsrTRjMDVimuU8NvaayPUXrO3cc/+QpWfEP/iQr2qZXdiIp6+VDfWVRJPBF9nGUnMR6bt+jHYxnoV/HXM/P+y42XGzqMhFbRLKjq9vrsP4dDyTKhl9pgZbtyIAo0yQcNj8gbUXJK7+b6tfhKp0qnI6a372vXgImdXweinUNQGWqVy3ZuBMmdpP4RXMCgVzwkZvbxx17YvQ2DuMFGDpSp3Ty2mXoYN01rNaaCz3XLzNp1O3BBXOmzRPxEbgkIRfVn38PZuvkJr5Ge+KA5lS6tTklTB1w2MMvUA6k08nTLy20YZlJ6Hfo4QBcYpGxp+RaweXi8kc+It74e9Yk1PWaNgxZe57fSrZA17HTMQPXDuJgDu8thdc12LDn9U3KRQeY/fn3x4j4UxhTVhzbdOLGGs5fX5hXqrHf6Vxk7M6eTrzTDg0wU8HNj6tBCw7NEPzSVz/DN0R+tybHOjQnwPK00lDndyGc2BZ3HqYIhpbV02F/OpOwTnNydzq9FdTp50dJUw1Aa8ve59defYd51aKfdRl7YWD49EB/QSnkPLQ9qXBBFTezt+eMFpNhaf/5lKeUGtB59tobgzs6OCtiExtjyyFPUTooGWEYmk8zl3atT5AIi5NlhzZt6IHw89NXbbFoqC7+uvLq4H2cPXb2seFzP6xVElFfsPSFIV42qw+nHXBwkkP0l8gPXLwSN/tLLFueIWc53zc7EOfm/+RonLkHi4a2euWCJkiidSFLy6ohsU1YN4hbGdo4RQ8x3/SEEU77tlN8DrBgnEc9N2yJ01krpZmnhrB+kF9YUp4EBCujq3R6nDix5mxBQZMI49h+jOLYJViQNnucwemAoS2h2XQLb6HtmpmzcSAjjDffNEqK9pCnOiOZw8p9AbM2XofWfnsI8zQ9bBcX9uRTW2sQ/lz5nU8ZCGUPP5yECe9GB/N1kXyKpt6kMq44WMp5qEhPYsVGrXukvsVtXHXNEohS/IX9c2zzx9nqYUnwzDWqVnUovF3mwNiegUvUsuTI5XFcrTL5WkoziwtFd/TENl12uKQx2Xt8GOeJpUxdpvGwvqbLXvlbE9MjDoRrKTxF8ooL9aPiThxx2rddHFoIbu4OarZ1opDx+FHfM60+XP7i2bhEjsMp+vJfwyxbOMl+if+5mSaMOmyuSztrYz+dj+XrczyYR156PmY9gpOfRmKTuZOAMM3UdyWTGbKDXameBIriglZZ+QavCRADFFN4Ltrl72Kmt2T4qg8jdXIdx8EdF8WuT8kX52AdqU3a8ZItkqV1ki4fC8d5VpOjyqKSsMD70ukrUwpMD63eYhpqxCy3Z8yOxnZYyvshKWvIHXtpuPZHuwzD1M/Yk+8O+2CXJiVn5hg7NvfHRByV94PaY0k6xtPuUH5Z2my4cQRnl3Ss+c/J4Ig4w979lesw5ixR9M/sNoykfLNUusYD4/orHL38/wFRJ2xleIsVCK8bOk0WumG5rux6FL9D/i45ucVuSTGcrLQ7nSd/F9s/VHcqPtv+Hx0ll90=" -} \ No newline at end of file +} From 721ac8845e3067af6c81f3c67f2c0fa24fb1723c Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Sat, 29 Dec 2018 11:39:55 -0800 Subject: [PATCH 714/766] Rendererclient (#134) * render-parameters: initial renderparameters retrieval * client: add rendererClient calls * renderparams: add renderparameter apis * renderer: add renderer client to render renderparameters * client: python 2 compatibility --- integration_tests/test_client_integrated.py | 33 ++++++ integration_tests/test_stack_integrated.py | 49 +++++++++ renderapi/client/client.py | 25 ++++- renderapi/client/client_calls.py | 46 +++++++- renderapi/image.py | 114 +++++++++++++++++++- 5 files changed, 262 insertions(+), 5 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 55bffb63..50761d65 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -361,6 +361,29 @@ def test_processpools_parallelfuncs( mpPool=poolclass) +@pytest.fixture(scope='module') +def example_renderparameters(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + width = (bounds['maxX'] - bounds['minX']) / 2 + height = (bounds['maxY'] - bounds['minY']) / 2 + x = bounds['minX'] + width / 4 + y = bounds['minY'] + width / 4 + renderparams = renderapi.image.get_bb_renderparams( + teststack, z, x, y, width, height, + scale=0.25, render=render) + yield renderparams + + +@pytest.fixture(scope='module') +def renderedimg_renderparams(render, example_renderparameters): + # TODO is there a "render from renderparams" api? + arr = renderapi.image.get_renderparameters_image( + example_renderparameters, render=render) + yield arr, example_renderparameters + + @pytest.fixture(scope='module') def tile_tilespec(): tile_dims = (256, 256) @@ -393,3 +416,13 @@ def test_ARGBrenderclient(render, tile_tilespec): tilearr = np.array(tileimg) assert arr.shape[:-1] == (tspec.width, tspec.height) == tilearr.shape assert np.all(arr[:, :, 0] == tilearr) + + +def testRendererClient(render, renderedimg_renderparams): + renderedarr, renderparams = renderedimg_renderparams + arr = renderapi.client.render_renderparameters(renderparams, render=render) + assert arr.shape[:-1] == ( + int(renderparams['height'] * renderparams['scale']), + int(renderparams['width'] * renderparams['scale']) + ) == renderedarr.shape[:-1] + assert np.all(arr == renderedarr) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 2538d93a..8fba2272 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -323,6 +323,25 @@ def test_bb_image_options(render, teststack): maxIntensity=255) +def test_bb_renderparams(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + bounds = render.run(renderapi.stack.get_bounds_from_z, teststack, z) + width = (bounds['maxX'] - bounds['minX']) / 2 + height = (bounds['maxY'] - bounds['minY']) / 2 + x = bounds['minX'] + width / 4 + y = bounds['minY'] + width / 4 + scale = 0.25 + + rp = renderapi.image.get_bb_renderparams( + teststack, z, x, y, width, height, scale=scale, render=render) + assert int(rp['width'] * rp['scale']) == int(width * scale) + assert int(rp['height'] * rp['scale']) == int(height * scale) + assert int(rp['x']) == int(x) + assert int(rp['y']) == int(y) + assert len(rp['tileSpecs']) + + def test_tile_image(render, teststack, render_example_tilespec_and_transforms, **kwargs): (tilespecs, tforms) = render_example_tilespec_and_transforms @@ -346,6 +365,23 @@ def test_tile_image_options(render, teststack, scale=testscale, filter=True, normalizeForMatching=False) +def test_tile_renderparams(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + + tspecs = renderapi.tilespec.get_tile_specs_from_z( + teststack, z, render=render) + ts = tspecs[0] + + rp = renderapi.image.get_tile_renderparams( + teststack, ts.tileId, render=render) + assert int(rp['x']) == int(ts.minX) + assert int(rp['y']) == int(ts.minY) + assert len(rp['tileSpecs']) == 1 + assert rp['height'] - 1 == int(ts.maxY - ts.minY) + assert rp['width'] - 1 == int(ts.maxX - ts.minX) + + def test_section_image(render, teststack, **kwargs): zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) z = zvalues[0] @@ -367,6 +403,19 @@ def test_section_image_options(render, teststack): maxTileSpecsToRender=50) +def test_section_renderparams(render, teststack): + zvalues = render.run(renderapi.stack.get_z_values_for_stack, teststack) + z = zvalues[0] + tspecs = renderapi.tilespec.get_tile_specs_from_z( + teststack, z, render=render) + + scalefactor = 0.05 + rp = renderapi.image.get_section_renderparams( + teststack, z, scale=scalefactor, render=render) + assert len(rp['tileSpecs']) == len(tspecs) + assert rp['scale'] == scalefactor + + def fail_image_get(render, teststack, render_example_tilespec_and_transforms): with pytest.raises(KeyError): render.run(renderapi.image.get_tile_image_data, teststack, diff --git a/renderapi/client/client.py b/renderapi/client/client.py index f9c1a80a..6d17c626 100644 --- a/renderapi/client/client.py +++ b/renderapi/client/client.py @@ -17,7 +17,7 @@ from renderapi.external.processpools.stdlib_pool import WithMultiprocessingPool from .utils import renderclientaccess -from .client_calls import importJsonClient, call_run_ws_client, renderClient +from .client_calls import importJsonClient, call_run_ws_client, renderClient, rendererClient # setup logger logger = logging.getLogger(__name__) @@ -397,6 +397,26 @@ def render_tilespec(*args, **kwargs): return arr +@renderclientaccess +def materialize_renderparameters_image( + obj, out_fn=None, subprocess_mode=None, client_script=None, memGB=None, + render=None, **kwargs): + tfile = renderdump_temp(obj) + rendererClient(parameters_url=tfile, out_fn=out_fn, + subprocess_mode=subprocess_mode, + client_script=client_script, memGB=memGB, **kwargs) + os.remove(tfile) + + +def render_renderparameters(*args, **kwargs): + # NOTE this is python2 compatible hack for keyword-only args + image_ext = kwargs.get('image_ext', '.png') + with tempfile.NamedTemporaryFile(suffix=image_ext) as f: + materialize_renderparameters_image(*args, out_fn=f.name, **kwargs) + arr = numpy.array(Image.open(f.name)) + return arr + + __all__ = [ "import_single_json_file", "import_jsonfiles_and_transforms_parallel_by_z", @@ -404,4 +424,5 @@ def render_tilespec(*args, **kwargs): "import_jsonfiles_validate_client", "import_tilespecs", "import_tilespecs_parallel", "local_to_world_array", "world_to_local_array", "WithPool", - "render_tilespec", "materialize_tilespec_image"] + "render_tilespec", "materialize_tilespec_image", + "materialize_renderparameters_image", "render_renderparameters"] diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 260b1cdb..3c972a77 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -682,9 +682,53 @@ def renderClient(tile_spec_url=None, height=None, width=None, in_fn=None, **kwargs) +@renderclientaccess +def rendererClient(tile_spec_url=None, meshCellSize=None, minMeshCellSize=None, + in_fn=None, out_fn=None, x=None, y=None, width=None, + height=None, scale=None, area_offset=None, + minIntensity=None, maxIntensity=None, gray=None, + quality=None, threads=None, skip_interpolation=None, + binary_mask=None, exclude_mask=None, parameters_url=None, + do_filter=None, background_color=None, fill_with_noise=None, + channels=None, subprocess_mode=None, client_script=None, + memGB=None, render=None, **kwargs): + get_cmd_opt = ArgumentParameters.get_cmd_opt + + argvs = (get_cmd_opt(tile_spec_url, '--tile_spec_url') + + get_cmd_opt(height, '--height') + + get_cmd_opt(width, '--width') + + get_cmd_opt(in_fn, '--in') + + get_cmd_opt(out_fn, '--out') + + get_cmd_opt(x, '--x') + + get_cmd_opt(y, '--y') + + get_cmd_opt(meshCellSize, '--meshCellSize') + + get_cmd_opt(minMeshCellSize, '--minMeshCellSize') + + get_cmd_opt(scale, '--scale') + + get_cmd_opt(area_offset, '--area_offset') + + get_cmd_opt(minIntensity, '--minIntensity') + + get_cmd_opt(maxIntensity, '--maxIntensity') + + get_cmd_opt(gray, '--gray') + + get_cmd_opt(quality, '--quality') + + get_cmd_opt(threads, '--threads') + + get_cmd_opt(skip_interpolation, '--skip_interpolation') + + get_cmd_opt(binary_mask, '--binary_mask') + + get_cmd_opt(exclude_mask, '--exclude_mask') + + get_cmd_opt(parameters_url, '--parameters_url') + + get_cmd_opt(do_filter, '--do_filter') + + get_cmd_opt(background_color, '--background_color') + + get_cmd_opt(fill_with_noise, '--fill_with_noise') + + get_cmd_opt(channels, '--channels')) + + call_run_ws_client('org.janelia.alignment.Render', + memGB=memGB, client_script=client_script, + subprocess_mode=subprocess_mode, add_args=argvs, + **kwargs) + + __all__ = [ "call_run_ws_client", "run_subprocess_mode", "importJsonClient", "tilePairClient", "importTransformChangesClient", "coordinateClient", "renderSectionClient", "transformSectionClient", - "get_canvas_url_template", "pointMatchClient"] + "get_canvas_url_template", "pointMatchClient", "renderClient", + "rendererClient"] diff --git a/renderapi/image.py b/renderapi/image.py index a5f812c4..c11499b9 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -5,9 +5,9 @@ from PIL import Image import numpy as np import logging -from .render import format_preamble, renderaccess +from .render import format_preamble, format_baseurl, renderaccess from .errors import RenderError -from .utils import NullHandler, jbool +from .utils import NullHandler, jbool, get_json, put_json logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) @@ -25,6 +25,38 @@ None: 'png-image'} # Default to png +def _strip_None_value_dictitems(d, exclude_keys=[]): + return {k: v for k, v in d.items() + if v is not None and k not in exclude_keys} + + +@renderaccess +def get_bb_renderparams(stack, z, x, y, width, height, scale=1.0, + channel=None, minIntensity=None, maxIntensity=None, + binaryMask=None, filter=None, filterListName=None, + convertToGray=None, excludeMask=None, + host=None, port=None, owner=None, + project=None, session=requests.session(), + render=None, **kwargs): + + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/z/%d/box/%d,%d,%d,%d,%f/render-parameters" % ( + z, x, y, width, height, scale) + + qparams = _strip_None_value_dictitems({ + "minIntensity": minIntensity, + "maxIntensity": maxIntensity, + "binaryMask": binaryMask, + "filter": filter, + "filterListName": filterListName, + "convertToGray": convertToGray, + "excludeMask": excludeMask, + "channels": channel}) + + return get_json(session, request_url, params=qparams) + + @renderaccess def get_bb_image(stack, z, x, y, width, height, scale=1.0, channel=None, @@ -109,6 +141,41 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, return RenderError(r.text) +@renderaccess +def get_tile_renderparams( + stack, tileId, channel=None, normalizeForMatching=None, + excludeAllTransforms=None, excludeTransformsAfterLast=None, + excludeFirstTransformAndAllAfter=None, scale=None, + width=None, height=None, minIntensity=None, maxIntensity=None, + filter=None, filterListName=None, excludeMask=None, convertToGray=None, + binaryMask=None, host=None, port=None, owner=None, + project=None, img_format=None, + session=requests.session(), render=None, **kwargs): + request_url = format_preamble( + host, port, owner, project, stack) + \ + "/tile/%s/render-parameters" % ( + tileId) + + qparams = _strip_None_value_dictitems({ + "normalizeForMatching": normalizeForMatching, + "excludeAllTransforms": excludeAllTransforms, + "excludeTransformsAfterLast": excludeTransformsAfterLast, + "excludeFirstTransformAndAllAfter": excludeFirstTransformAndAllAfter, + "scale": scale, + "width": width, + "height": height, + "minIntensity": minIntensity, + "maxIntensity": maxIntensity, + "binaryMask": binaryMask, + "filter": filter, + "filterListName": filterListName, + "convertToGray": convertToGray, + "excludeMask": excludeMask, + "channels": channel}) + + return get_json(session, request_url, params=qparams) + + @renderaccess def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, excludeAllTransforms=False, scale=None, @@ -191,6 +258,32 @@ def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, return RenderError(r.text) +@renderaccess +def get_section_renderparams(stack, z, binaryMask=None, channel=None, + convertToGray=None, excludeMask=None, filter=None, + filterListName=None, minIntensity=None, + maxIntensity=None, scale=None, + host=None, port=None, owner=None, project=None, + session=requests.session(), + render=None, **kwargs): + request_url = format_preamble( + host, port, owner, project, stack) + "/z/{}/render-parameters".format( + z) + + qparams = _strip_None_value_dictitems({ + "scale": scale, + "minIntensity": minIntensity, + "maxIntensity": maxIntensity, + "binaryMask": binaryMask, + "filter": filter, + "filterListName": filterListName, + "convertToGray": convertToGray, + "excludeMask": excludeMask, + "channels": channel}) + + return get_json(session, request_url, params=qparams) + + @renderaccess def get_section_image(stack, z, scale=1.0, channel=None, filter=False, @@ -255,3 +348,20 @@ def get_section_image(stack, z, scale=1.0, channel=None, r = session.get(request_url, params=qparams) return np.asarray(Image.open(io.BytesIO(r.content))) + + +@renderaccess +def get_renderparameters_image(renderparams, img_format=None, + host=None, port=None, owner=None, + session=requests.session(), + render=None, **kwargs): + try: + image_ext = IMAGE_FORMATS[img_format] + except KeyError as e: # pragma: no cover + raise ValueError('{} is not a valid render image format!'.format(e)) + + request_url = format_baseurl(host, port) + '/owner/{owner}/{ext}'.format( + owner=owner, ext=image_ext) + + r = put_json(session, request_url, renderparams) + return np.array(Image.open(io.BytesIO(r.content))) From 43d23878bbdfd9c362c845fd590d944bb6dcb20a Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Fri, 25 Jan 2019 11:40:04 -0800 Subject: [PATCH 715/766] Tps adaptive mesh refinement (#132) * adding adaptive mesh estimate for TPS size reduction * fixing className caps in json * added some documentation header * trivial change to retrigger build * trivial change to retrigger build * separating recursion from setup, raising max_iter exception * Update thin_plate_spline.py trivial change to retrigger build * another trivial change for rebuild * added doc string, default pass self to recursion, custom exception * subclassing custom exception as EstimationError * removing custom class __str__ method * removing test of exception __str__ * reverting to simpler estimation error exception --- renderapi/transform/leaf/thin_plate_spline.py | 130 ++++++++++++++++++ test/test_transform.py | 62 ++++++++- 2 files changed, 185 insertions(+), 7 deletions(-) diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py index 47f4083d..a0714ea6 100644 --- a/renderapi/transform/leaf/thin_plate_spline.py +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -3,9 +3,15 @@ from renderapi.utils import encodeBase64, decodeBase64 from .transform import Transform import scipy.spatial +import logging +import sys __all__ = ['ThinPlateSplineTransform'] +logger = logging.getLogger(__name__) +logger.addHandler(logging.StreamHandler(sys.stdout)) + + class ThinPlateSplineTransform(Transform): """ render-python class that can hold a dataString for @@ -281,3 +287,127 @@ def dataString(self): b64_2 = encodeBase64(blk2) return '{} {} {}'.format(header, b64_1, b64_2) + + @staticmethod + def mesh_refine( + new_src, + old_src, + old_dst, + old_tf=None, + computeAffine=True, + tol=1.0, + max_iter=50, + nworst=10, + niter=0): + """recursive kernel for adaptive_mesh_estimate() + Parameters + ---------- + new_src : numpy.array + Nx2 array of new control source points. Adapts during recursion. + Seeded by adaptive_mesh_estimate. + old_src : numpy.array + Nx2 array of orignal control source points. + old_dst : numpy.array + Nx2 array of orignal control destination points. + old_tf : ThinPlateSplineTransform + transform constructed from old_src and old_dst, passed through + recursion iterations. Created if None. + computeAffine : boolean + whether returned transform will have aMtx + tol : float + in units of pixels, how close should the points match + max_iter: int + some limit on how many recursive attempts + nworst : int + per iteration, the nworst matching srcPts will be added + niter : int + passed through the recursion for stopping criteria + + Returns + ------- + ThinPlateSplineTransform + """ + + if old_tf is None: + old_tf = ThinPlateSplineTransform() + old_tf.estimate(old_src, old_dst, computeAffine=computeAffine) + + new_tf = ThinPlateSplineTransform() + new_tf.estimate( + new_src, + old_tf.tform(new_src), + computeAffine=computeAffine) + new_dst = new_tf.tform(old_src) + + delta = np.linalg.norm(new_dst - old_dst, axis=1) + ind = np.argwhere(delta > tol).flatten() + + if ind.size == 0: + return new_tf + + if niter == max_iter: + raise EstimationError( + "Max number of iterations ({}) reached in" + " ThinPlateSplineTransform.mesh_refine()".format( + max_iter)) + + sortind = np.argsort(delta[ind]) + new_src = np.vstack((new_src, old_src[ind[sortind[0: nworst]]])) + + return ThinPlateSplineTransform.mesh_refine( + new_src, + old_src, + old_dst, + old_tf=old_tf, + computeAffine=computeAffine, + tol=tol, + max_iter=max_iter, + nworst=nworst, + niter=(niter + 1)) + + def adaptive_mesh_estimate( + self, + starting_grid=7, + computeAffine=True, + tol=1.0, + max_iter=50, + nworst=10): + """method for creating a transform with fewer control points + that matches the original transfom within some tolerance. + Parameters + ---------- + starting_grid : int + estimate will start with an n x n grid + computeAffine : boolean + whether returned transform will have aMtx + tol : float + in units of pixels, how close should the points match + max_iter: int + some limit on how many recursive attempts + nworst : int + per iteration, the nworst matching srcPts will be added + + Returns + ------- + ThinPlateSplineTransform + """ + + mn = self.srcPts.min(axis=1) + mx = self.srcPts.max(axis=1) + xt, yt = np.meshgrid( + np.linspace(mn[0], mx[0], starting_grid), + np.linspace(mn[1], mx[1], starting_grid)) + new_src = np.vstack((xt.flatten(), yt.flatten())).transpose() + old_src = self.srcPts.transpose() + old_dst = self.tform(old_src) + + return ThinPlateSplineTransform.mesh_refine( + new_src, + old_src, + old_dst, + old_tf=self, + computeAffine=computeAffine, + tol=tol, + max_iter=max_iter, + nworst=nworst, + niter=0) diff --git a/test/test_transform.py b/test/test_transform.py index aedad40a..58b8332a 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -170,13 +170,13 @@ def test_invert_Affine(): def test_polynomial_scale(): p = np.transpose(np.array([[0, 0]])) t = renderapi.transform.Polynomial2DTransform(params=p) - assert(t.order==0) - assert(t.scale==(1.0, 1.0)) - + assert(t.order == 0) + assert(t.scale == (1.0, 1.0)) + p = np.transpose(np.array([[0, 0], [1.1, 0], [0, 0.9]])) t = renderapi.transform.Polynomial2DTransform(params=p) - assert(t.order==1) - assert(t.scale==(1.1, 0.9)) + assert(t.order == 1) + assert(t.scale == (1.1, 0.9)) def test_Polynomial_estimation(use_numpy=False): @@ -835,12 +835,12 @@ def test_thinplatespline(): with pytest.raises(renderapi.errors.RenderError): s = t2.dataString.split(' ') s[1] = str(int(s[1])+1) - t3 = renderapi.transform.ThinPlateSplineTransform( + renderapi.transform.ThinPlateSplineTransform( dataString=" ".join(s)) with pytest.raises(renderapi.errors.RenderError): s = t2.dataString.split(' ') s[2] = str(int(s[2])-4) - t3 = renderapi.transform.ThinPlateSplineTransform( + renderapi.transform.ThinPlateSplineTransform( dataString=" ".join(s)) x = np.linspace(0, 3840, 10) @@ -963,3 +963,51 @@ def test_encode64(): s = renderapi.utils.encodeBase64(x) y = renderapi.utils.decodeBase64(s) assert(np.all(x == y)) + + +def test_adaptive_estimate(): + with open(rendersettings.TEST_THINPLATESPLINE_FILE, 'r') as f: + j = json.load(f) + + tf = renderapi.transform.ThinPlateSplineTransform( + dataString=j['dataString']) + + tol = 1.0 + ntf = tf.adaptive_mesh_estimate(tol=1.0) + + src = ntf.srcPts.transpose() + dsta = tf.tform(src) + dstb = ntf.tform(src) + assert(np.linalg.norm(dsta - dstb, axis=1).max() <= tol) + + src = tf.srcPts.transpose() + dsta = tf.tform(src) + dstb = ntf.tform(src) + nover = np.argwhere(np.linalg.norm(dsta - dstb, axis=1) >= tol).size + assert(nover == 0) + + with pytest.raises(renderapi.errors.EstimationError): + tf.adaptive_mesh_estimate(max_iter=1) + + # invoke the recursion directly, without passing self + mn = tf.srcPts.min(axis=1) + mx = tf.srcPts.max(axis=1) + xt, yt = np.meshgrid( + np.linspace(mn[0], mx[0], 5), + np.linspace(mn[1], mx[1], 5)) + new_src = np.vstack((xt.flatten(), yt.flatten())).transpose() + old_src = tf.srcPts.transpose() + old_dst = tf.tform(old_src) + ntf = tf.mesh_refine( + new_src, + old_src, + old_dst, + old_tf=None, + computeAffine=True, + tol=1.0, + max_iter=50, + nworst=10, + niter=0) + dsta = tf.tform(src) + dstb = ntf.tform(src) + assert(np.linalg.norm(dsta - dstb, axis=1).max() <= tol) From 21efb514f02d008a0834249c5f801b2e4f4c11ec Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Fri, 15 Mar 2019 16:05:51 -0700 Subject: [PATCH 716/766] switching to testing 3.7dev --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 53b2ebcf..bee3f882 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,9 @@ language: python python: - "2.7" - "3.5" - - "3.5-dev" # 3.5 development branch - "3.6" - "3.6-dev" # 3.6 development branch + - "3.7-dev" services: - docker # command to install dependencies From 71c2d5a5f5268c7f9932aef1dbc3c706211bd825 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 15 Apr 2019 13:49:52 -0700 Subject: [PATCH 717/766] fixing get_sectionId_for_z documentation --- renderapi/stack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 54089a77..74db9e30 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -704,8 +704,8 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, Returns ------- - float - z value of sectionId + str + value of sectionId Raises ------ From 5cfcb36a6b752b2c9ebee9c1df08d4cf54be74e8 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 15 Apr 2019 13:54:06 -0700 Subject: [PATCH 718/766] couple pep8 tweaks --- renderapi/stack.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 74db9e30..43191004 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -651,8 +651,8 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, @renderaccess def get_tilebounds_for_z(stack, z, host=None, port=None, owner=None, - project=None, session=requests.session(), - render=None, **kwargs): + project=None, session=requests.session(), + render=None, **kwargs): """returns the bounds for each tile associated with a particular z value :func:`renderapi.render.renderaccess` decorated function @@ -683,6 +683,7 @@ def get_tilebounds_for_z(stack, z, host=None, port=None, owner=None, host, port, owner, project, stack) + '/z/{}/tileBounds'.format(z) return get_json(session, request_url) + @renderaccess def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, project=None, session=requests.session(), @@ -704,7 +705,7 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, Returns ------- - str + str value of sectionId Raises @@ -712,9 +713,9 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, RenderError """ - - bounds = get_tilebounds_for_z(stack, z, host, port, owner, project, session) + bounds = get_tilebounds_for_z( + stack, z, host, port, owner, project, session) try: return bounds[0]['sectionId'] From b9f7f6a2f5fd7e8e0d28ff6cab6d3c2e49bb2a8c Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Fri, 26 Apr 2019 14:32:53 -0700 Subject: [PATCH 719/766] unify first order properties for affine and polynomial --- renderapi/transform/leaf/affine_models.py | 30 ++-------- renderapi/transform/leaf/common.py | 57 +++++++++++++++++++ renderapi/transform/leaf/polynomial_models.py | 38 +++++++++++-- test/test_transform.py | 26 +++++++++ 4 files changed, 119 insertions(+), 32 deletions(-) create mode 100644 renderapi/transform/leaf/common.py diff --git a/renderapi/transform/leaf/affine_models.py b/renderapi/transform/leaf/affine_models.py index 36f9a3d1..11d67d43 100644 --- a/renderapi/transform/leaf/affine_models.py +++ b/renderapi/transform/leaf/affine_models.py @@ -1,4 +1,5 @@ from .transform import Transform, logger +from .common import calc_first_order_properties import numpy as np from renderapi.errors import ConversionError, EstimationError @@ -306,32 +307,9 @@ def inverse_tform(self, points): return self.convert_points_vector_to_array(pt, Nd) def calc_properties(self): - if self.force_shear == 'x': - sy = np.sqrt(self.M[1, 0] ** 2 + self.M[1, 1] ** 2) - theta = np.arctan2(self.M[1, 0], self.M[1, 1]) - rc = np.cos(theta) - rs = np.sin(theta) - sx = rc * self.M[0, 0] - rs * self.M[0, 1] - if rs != 0: - cx = (self.M[0, 0] - sx*rc) / (sx * rs) - else: - cx = (self.M[0, 1] - sx*rs) / (sx * rc) - cy = 0.0 - else: - # shear in y direction - sx = np.sqrt(self.M[0, 0] ** 2 + self.M[0, 1] ** 2) - theta = np.arctan2(-self.M[0, 1], self.M[0, 0]) - rc = np.cos(theta) - rs = np.sin(theta) - sy = rs * self.M[1, 0] + rc * self.M[1, 1] - if rs != 0: - cy = (self.M[1, 1] - sy * rc) / (-sy * rs) - else: - cy = (self.M[1, 0] - sy * rs) / (sy * rc) - cx = 0.0 - # room for other cases, for example cx = cy - - return sx, sy, cx, cy, theta + return calc_first_order_properties( + self.M[0:2, 0:2], + force_shear=self.force_shear) @property def scale(self): diff --git a/renderapi/transform/leaf/common.py b/renderapi/transform/leaf/common.py new file mode 100644 index 00000000..fa3fe4b8 --- /dev/null +++ b/renderapi/transform/leaf/common.py @@ -0,0 +1,57 @@ +import numpy as np + + +def calc_first_order_properties(M, force_shear='x'): + """calculate scale shear and rotation from a 2x2 matrix + could be M[0:2, 0:2] from an AffineModel or params[:, 1:3] + from a Polynomial2DTransform + + Parameters + ---------- + M : numpy array + 2x2 + force_shear : str + 'x' or 'y' + + Returns + ------- + sx : float + scale in x direction + sy : float + scale in y direction + cx : float + shear in x direction + cy : float + shear in y direction + theta : float + rotation angle in radians + """ + if force_shear == 'x': + sy = np.sqrt(M[1, 0] ** 2 + M[1, 1] ** 2) + theta = np.arctan2(M[1, 0], M[1, 1]) + rc = np.cos(theta) + rs = np.sin(theta) + sx = rc * M[0, 0] - rs * M[0, 1] + if rs != 0: + cx = (M[0, 0] - sx*rc) / (sx * rs) + else: + cx = (M[0, 1] - sx*rs) / (sx * rc) + cy = 0.0 + elif force_shear == 'y': + # shear in y direction + sx = np.sqrt(M[0, 0] ** 2 + M[0, 1] ** 2) + theta = np.arctan2(-M[0, 1], M[0, 0]) + rc = np.cos(theta) + rs = np.sin(theta) + sy = rs * M[1, 0] + rc * M[1, 1] + if rs != 0: + cy = (M[1, 1] - sy * rc) / (-sy * rs) + else: + cy = (M[1, 0] - sy * rs) / (sy * rc) + cx = 0.0 + else: + raise ValueError("%s not a valid option for force_shear." + " should be 'x' or 'y'") + # room for other cases, for example cx = cy + + return sx, sy, cx, cy, theta diff --git a/renderapi/transform/leaf/polynomial_models.py b/renderapi/transform/leaf/polynomial_models.py index 0fbab48d..cf740aeb 100644 --- a/renderapi/transform/leaf/polynomial_models.py +++ b/renderapi/transform/leaf/polynomial_models.py @@ -1,6 +1,7 @@ from .transform import Transform, logger from .affine_models import AffineModel import numpy as np +from .common import calc_first_order_properties from renderapi.errors import ConversionError, EstimationError, RenderError try: @@ -29,7 +30,8 @@ class Polynomial2DTransform(Transform): def __init__(self, dataString=None, src=None, dst=None, order=2, force_polynomial=True, params=None, identity=False, - labels=None, transformId=None, json=None, **kwargs): + labels=None, transformId=None, json=None, + force_shear='x', **kwargs): """Initialize Polynomial2DTransform This provides 5 different ways to initialize the transform which are mutually exclusive and applied in the order specified here. @@ -58,6 +60,7 @@ def __init__(self, dataString=None, src=None, dst=None, order=2, """ + self.force_shear = force_shear if json is not None: self.from_dict(json) else: @@ -89,15 +92,38 @@ def order(self): no_coeffs = len(self.params.ravel()) return int((abs(np.sqrt(4 * no_coeffs + 1)) - 3) / 2) + def calc_properties(self): + if self.order == 0: + return 1.0, 1.0, 0.0, 0.0, 0.0 + return calc_first_order_properties( + self.params[:, 1:3], + force_shear=self.force_shear) + @property def scale(self): """tuple of scale for x, y""" - if self.order > 0: - scale = (self.params[0, 1], self.params[1, 2]) + sx, sy, cx, cy, theta = self.calc_properties() + return (sx, sy) + + @property + def shear(self): + """shear""" + sx, sy, cx, cy, theta = self.calc_properties() + if self.force_shear == 'x': + return cx else: - # translation only has no scale impact - scale = (1.0, 1.0) - return scale + return cy + + @property + def translation(self): + """tuple of translation in x, y""" + return tuple(self.params[:, 0]) + + @property + def rotation(self): + """counter-clockwise rotation""" + sx, sy, cx, cy, theta = self.calc_properties() + return theta @property def dataString(self): diff --git a/test/test_transform.py b/test/test_transform.py index 58b8332a..ef6d6924 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -1011,3 +1011,29 @@ def test_adaptive_estimate(): dsta = tf.tform(src) dstb = ntf.tform(src) assert(np.linalg.norm(dsta - dstb, axis=1).max() <= tol) + + +def test_polynomial_shear(): + # make sure it gives the same answer as affine + M = np.array([ + [1.1, -0.2, 7.0], + [0.3, 0.8, -9.0], + [0.0, 0.0, 1.0]]) + + atf = renderapi.transform.AffineModel() + atf.M = M + + params = np.zeros((2, 3)) + params[:, 1:] = M[0:2, 0:2] + params[:, 0] = M[0:2, 2] + + ptf = renderapi.transform.Polynomial2DTransform(params=params) + + for shear in ['x', 'y']: + atf.force_shear = shear + ptf.force_shear = shear + + assert atf.scale == ptf.scale + assert atf.shear == ptf.shear + assert atf.rotation == ptf.rotation + assert atf.translation == ptf.translation From 0bc91d314f28ad318966426fafe3d19b60288b21 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Fri, 26 Apr 2019 14:37:56 -0700 Subject: [PATCH 720/766] go to java9 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bee3f882..b191a7f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ addons: - libopenblas-base - libopenblas-dev - gfortran - - oracle-java8-set-default + - oracle-java9-set-default - maven install: - pip install codecov From 0be8f53c89326f5b6ae137e324fc02de0422e5c5 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Fri, 10 May 2019 15:54:16 -0700 Subject: [PATCH 721/766] updating incorrectly documented z args --- renderapi/stack.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 43191004..71628d04 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -331,7 +331,7 @@ def delete_section(stack, z, host=None, port=None, owner=None, ---------- stack : str stack to delete section from - z : float + z : int or float or str z value to delete render : renderapi.render.Render render connect object @@ -595,7 +595,7 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, ---------- stack : str stack to get bounds from - z : float + z : int or float or str z value to get bounds for render : renderapi.render.Render render connect object @@ -661,8 +661,8 @@ def get_tilebounds_for_z(stack, z, host=None, port=None, owner=None, ---------- stack : str stack to look within - sectionId : str - sectionId to find z value + z : int or float or str + z value for which to get tile bounds render : renderapi.render.Render render connect object session : requests.sessions.Session @@ -696,8 +696,8 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, ---------- stack : str stack to look within - sectionId : str - sectionId to find z value + z : int or float or str + section z value render : renderapi.render.Render render connect object session : requests.sessions.Session From 02172e06177cf8fe50b4c8498e65dfb7a3608a38 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Fri, 10 May 2019 15:58:55 -0700 Subject: [PATCH 722/766] java 9 to pass tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bee3f882..b191a7f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ addons: - libopenblas-base - libopenblas-dev - gfortran - - oracle-java8-set-default + - oracle-java9-set-default - maven install: - pip install codecov From 09191046758fc58b0c76ff24e3f8f2816d6e6728 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Mon, 13 May 2019 11:22:15 -0700 Subject: [PATCH 723/766] pass render as named kwarg through import_tilespecs_parallel --- renderapi/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client/client.py b/renderapi/client/client.py index 6d17c626..b49209a6 100644 --- a/renderapi/client/client.py +++ b/renderapi/client/client.py @@ -300,7 +300,7 @@ def import_tilespecs_parallel(stack, tilespecs, sharedTransforms=None, import_tilespecs, stack, sharedTransforms=sharedTransforms, subprocess_mode=subprocess_mode, host=host, port=port, owner=owner, project=project, client_script=client_script, - memGB=memGB, **kwargs) + render=render, memGB=memGB, **kwargs) # TODO this is a weird way to do splits.... is that okay? tilespec_groups = [g for g in From 3d65af11af913780274a19920a4c760f4706a3a9 Mon Sep 17 00:00:00 2001 From: Dan Kapner <32312979+djkapner@users.noreply.github.com> Date: Mon, 13 May 2019 11:59:15 -0700 Subject: [PATCH 724/766] Update .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bee3f882..b191a7f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ addons: - libopenblas-base - libopenblas-dev - gfortran - - oracle-java8-set-default + - oracle-java9-set-default - maven install: - pip install codecov From 0225da9012a02a9282bcf638b29176ca45642c80 Mon Sep 17 00:00:00 2001 From: Eric Perlman Date: Sun, 16 Jun 2019 15:24:59 -0700 Subject: [PATCH 725/766] Add channels to pointMatchClient (#144) * Add channels to def pointMatchClient() and get_canvas_url_template(). * Add scale as an option. * Remove scale from pointMatchClient. Can use SiftPointMatchOptions.renderScale instead. * Basic test to test multiple channels with pointmatch client. * Assert for fewer pointmatches, not none, with different channels. * Add test for different channel order. * Tweaked tests & docs. * Remove scale from get_canvas_url_template. * Actually remove scale from get_canvas_url_template. --- integration_tests/test_multichannel.py | 74 ++++++++++++++++++++++++++ renderapi/client/client_calls.py | 21 +++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_multichannel.py b/integration_tests/test_multichannel.py index cbb7ebf2..12d6d109 100644 --- a/integration_tests/test_multichannel.py +++ b/integration_tests/test_multichannel.py @@ -3,6 +3,7 @@ import pytest import logging import sys +import numpy as np root = logging.getLogger() root.setLevel(logging.DEBUG) @@ -33,8 +34,81 @@ def multichannel_test_stack(render): return stack +@pytest.fixture(scope='module') +def multichannel_test_stack_channelflip(render): + stack = 'multichannel_test_channelflip' + tilespecs = [renderapi.tilespec.TileSpec(json=d) + for d in test_2_channels_d] + for tilespec in tilespecs: + tilespec.channels.reverse() + renderapi.stack.create_stack(stack, render=render) + renderapi.client.import_tilespecs(stack, tilespecs, render=render) + renderapi.stack.set_stack_state(stack, 'COMPLETE', render=render) + return stack + + +@pytest.fixture(scope='module') +def test_pm_collection(render): + collection = 'test_multichan_collection' + renderapi.pointmatch.import_matches( + collection, test_matches, render=render) + return collection + + def test_section_image_channels(render, multichannel_test_stack): section_image = renderapi.image.get_section_image(multichannel_test_stack, 1.0, channel='DAPI', render=render) print(section_image.shape) + + +def test_section_image_same_channel_different_channel_order(render, multichannel_test_stack, multichannel_test_stack_channelflip): + section_image = renderapi.image.get_section_image(multichannel_test_stack, + 1.0, channel='DAPI', + render=render) + section_image2 = renderapi.image.get_section_image(multichannel_test_stack_channelflip, + 1.0, channel='DAPI', + render=render) + assert(np.array_equal(section_image, section_image2)) + + +def test_section_image_different_channel_order(render, multichannel_test_stack, multichannel_test_stack_channelflip): + section_image = renderapi.image.get_section_image(multichannel_test_stack, + 1.0, + render=render) + section_image2 = renderapi.image.get_section_image(multichannel_test_stack_channelflip, + 1.0, + render=render) + assert(not np.array_equal(section_image, section_image2)) + + +def test_multichannel_pointmatches(render, multichannel_test_stack): + collection = 'test_multichannel_pointmatch_same_channel' + sift_options = renderapi.client.SiftPointMatchOptions(renderScale=0.5) + tile_pairs = [['100000001003000', '100000001004000']] + renderapi.client.pointMatchClient(multichannel_test_stack, + collection, + tile_pairs, + filter=True, + excludeAllTransforms=True, + stackChannels='DAPI', + sift_options=sift_options, + render=render) + pms1 = renderapi.pointmatch.get_matches_involving_tile( + collection, collection, '100000001003000', render=render) + assert(len(pms1) > 0) + + collection2 = 'test_multichannel_pointmatch_different_channel' + renderapi.client.pointMatchClient(multichannel_test_stack, + collection2, + tile_pairs, + stack2=multichannel_test_stack, + filter=True, + excludeAllTransforms=True, + stackChannels='DAPI', + stack2Channels='TdTomato', + sift_options=sift_options, + render=render) + with pytest.raises(renderapi.errors.RenderError): + pms2 = renderapi.pointmatch.get_matches_involving_tile( + collection2, collection2, '100000001003000', render=render) diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 3c972a77..68adc2f5 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -474,6 +474,7 @@ def get_canvas_url_template( stack, filter=False, renderWithoutMask=False, normalizeForMatching=True, excludeTransformsAfterLast=None, excludeFirstTransformAndAllAfter=None, excludeAllTransforms=False, + channels=None, host=None, port=None, owner=None, project=None, client_script=None, render=None, **kwargs): """function for making a render-parameters url template for point matching @@ -507,6 +508,11 @@ def get_canvas_url_template( excludeAllTransforms: bool alternative to normalizeForMatching which simply removes all transforms from the list. default=False + channels: str + list of channels and weights to render in the format [channel name], [channel name]__[weight] or + channel one name]__[weight]__ ... [channel n name]__[weight]. + e.g., "DAPI" or "DAPI__0.9__TdTomato__0.1" + default = None """ # noqa: E501 request_url = format_preamble(host, port, owner, project, stack) tile_base_url = request_url + "/tile" @@ -534,6 +540,8 @@ def get_canvas_url_template( excludeFirstTransformAndAllAfter) if excludeAllTransforms: url_suffix += '&excludeAllTransforms=true' + if channels: + url_suffix += '&channels={}'.format(channels) canvas_url_template = "%s/{}/%s" % (tile_base_url, url_suffix) @@ -551,6 +559,8 @@ def pointMatchClient(stack, collection, tile_pairs, excludeTransformsAfterLast=None, excludeAllTransforms=None, excludeFirstTransformAndAllAfter=None, + stackChannels=None, + stack2Channels=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, @@ -598,7 +608,13 @@ def pointMatchClient(stack, collection, tile_pairs, transforms that you had applied after it. default= None. excludeAllTransforms: bool alternative to normalizeForMatching which simply removes all transforms from the list. - default=False + default = False + stackChannels: str or None + If specified, option to select which channel is used for the stack. + default = None + stack2Channels: str or None + If specified, option to select which channel is used for stack2, if specified. + default = None """ # noqa: E501 sift_options = (SiftPointMatchOptions(**kwargs) if sift_options is None @@ -625,6 +641,7 @@ def pointMatchClient(stack, collection, tile_pairs, excludeTransformsAfterLast, excludeFirstTransformAndAllAfter, excludeAllTransforms, + channels=stackChannels, host=host, port=port, owner=owner, @@ -640,6 +657,7 @@ def pointMatchClient(stack, collection, tile_pairs, excludeTransformsAfterLast, excludeFirstTransformAndAllAfter, excludeAllTransforms, + channels=stack2Channels, host=host, port=port, owner=owner, @@ -647,7 +665,6 @@ def pointMatchClient(stack, collection, tile_pairs, client_script=client_script) else: canvas_url_template2 = canvas_url_template - for tile1, tile2 in tile_pairs: argvs += [canvas_url_template.format(tile1), canvas_url_template2.format(tile2)] From fe2d4b1b9d3b59f853d052a418af28e8ec49f5df Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 23 Jul 2019 14:56:10 -0700 Subject: [PATCH 726/766] adding method to scale thin plate spline --- renderapi/transform/leaf/thin_plate_spline.py | 88 ++++++++++++++++++- test/rendersettings.py | 3 + test/test_files/thin_plate_from_rough.json | 1 + test/test_transform.py | 42 +++++++++ 4 files changed, 130 insertions(+), 4 deletions(-) create mode 100644 test/test_files/thin_plate_from_rough.json diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py index a0714ea6..c8abc9a7 100644 --- a/renderapi/transform/leaf/thin_plate_spline.py +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -394,10 +394,8 @@ def adaptive_mesh_estimate( mn = self.srcPts.min(axis=1) mx = self.srcPts.max(axis=1) - xt, yt = np.meshgrid( - np.linspace(mn[0], mx[0], starting_grid), - np.linspace(mn[1], mx[1], starting_grid)) - new_src = np.vstack((xt.flatten(), yt.flatten())).transpose() + new_src = self.src_array( + mn[0], mn[1], mx[0], mx[1], starting_grid, starting_grid) old_src = self.srcPts.transpose() old_dst = self.tform(old_src) @@ -411,3 +409,85 @@ def adaptive_mesh_estimate( max_iter=max_iter, nworst=nworst, niter=0) + + @staticmethod + def src_array(xmin, ymin, xmax, ymax, nx, ny): + """create N x 2 array of regularly spaced points + + Parameters + ---------- + xmin : float + minimum of x grid + ymin : float + minimum of y grid + xmax : float + maximum of x grid + ymax : float + maximum of y grid + nx : int + number of points in x axis + ny : int + number of points in y axis + + Returns + ------- + src : :class:`numpy.ndarray` + (nx * ny) x 2 array of coordinated. + + """ + x = np.linspace(xmin, xmax, nx) + y = np.linspace(ymin, ymax, ny) + xt, yt = np.meshgrid(x, y) + src = np.vstack((xt.flatten(), yt.flatten())).transpose() + return src + + def scale_coordinates( + self, + factor, + ngrid=20, + preserve_srcPts=False): + """estimates a new ThinPlateSplineTransform from the current one + in a scaled transform space. + + Parameters + ---------- + factor : float + the factor by which to scale the space + ngrid : int + number of points per axis for the estimation grid + preserve_srcPts : bool + one might want to keep the original scaled srcPts + for example, if pts were made specially for a mask + or a crack or fold + + Returns + ------- + new_tform : :class:`renderapi.transform.ThinPlateSplineTransform` + the new transform in the scaled space + + """ + + new_tform = ThinPlateSplineTransform() + computeAffine = True + if self.aMtx is None: + computeAffine = False + + mn = self.srcPts.min(axis=1) + mx = self.srcPts.max(axis=1) + src = self.src_array(mn[0], mn[1], mx[0], mx[1], ngrid, ngrid) + + if preserve_srcPts: + # do not repeat close points + dist = scipy.spatial.distance.cdist( + src, + self.srcPts.transpose(), + metric='euclidean') + ind = np.invert(np.any(dist < 1e-3, axis=0)) + src = np.vstack((src, self.srcPts.transpose()[ind])) + + new_tform.estimate( + src * factor, + self.tform(src) * factor, + computeAffine=computeAffine) + + return new_tform diff --git a/test/rendersettings.py b/test/rendersettings.py index b825aa58..2152fdae 100644 --- a/test/rendersettings.py +++ b/test/rendersettings.py @@ -36,6 +36,9 @@ TEST_THINPLATESPLINE_FILE = os.path.join( TEST_FILES_DIR, 'thin_plate_spline.json') +TEST_THINPLATEROUGH_FILE = os.path.join( + TEST_FILES_DIR, + 'thin_plate_from_rough.json') TEST_THINPLATESPLINEAFFINE_FILE = os.path.join( TEST_FILES_DIR, 'thin_plate_spline_affine.json') diff --git a/test/test_files/thin_plate_from_rough.json b/test/test_files/thin_plate_from_rough.json new file mode 100644 index 00000000..775c7b11 --- /dev/null +++ b/test/test_files/thin_plate_from_rough.json @@ -0,0 +1 @@ +{"type": "leaf", "className": "mpicbg.trakem2.transform.ThinPlateSplineTransform", "dataString": "ThinPlateSplineR2LogR 2 25 eJwBMADP/78yLRWQifN2vw/ErHm04Cs/IqdJx765j78ab8sN7GLJwCCjMIEXaI2/9qEQnRU3FUt4FyU= eJw9zFtMkmEYAOCJELrmIcvsoJ10Mipnx4VtfWLlRQ2PYFPCCNKVMv65iYeIiEw8QLFCy5b/nNp086KtRCNyvmOuzJjLuiBLRUGaW2ZYU1cbtnXzf9/lc/M4S2+NVnOM7c5LXS9/KrazhMYXIxPTn+aJHyuGOui6R8Tda+cXeywJxH2OV5z1qWVsJ/MJDe0Gu23Kij9i5iNmPmLmwyaf1UgnNy/tIx82/rDxh40/xuSjh2t+py0GyYeNP2z8YeOPMfmeavs8q6uL5MPGHzb+sPHHGPSsQ9JopwIKg2ZvIC8eien7u09ssgJ/JTaHX2RD1Mm43su5bqgKzeqXzFYAa+GLabRgL8oUNLDd3C0op9zMmz9gRjf+LU3T7gSQ58dy5+yfkVKUd+VJ2DdQJ3HLecUPkaQ47Nwc1YQaabbQwZVBibUSkk31KEMdDpPCCTBIHIG3+iY4Pfb6o89GobsCNeV5p4LrliMhH243QtWfo2lD72kweIOlG8pccCY1sXqN6kcmtsfrf84BnWU0qmOrCxrqJgfDHSEgVm3bGaX5jjSChYDaTqH03IEdIvlFkGplqjxxGdypOP5Vc+wUKvC0RW/Um5CupXe9qnkz4rsiWh1NflRb45CMZKSArLjNrA3UoqKrgsyDpgpU6ffFWFPjIXvlXpleIUf14z9ixuQdKH9Pl6dE3gri9F/jPfv/ogvPsqXU4UgoHBxWDuyKQEZWXOdyZAaSPOjU+brPQknDzMwELxRyNaGixBQdKG++uTablfQfS4+iWQ=="} diff --git a/test/test_transform.py b/test/test_transform.py index ef6d6924..58f7268a 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -5,6 +5,8 @@ import rendersettings import importlib import pytest +from scipy.spatial.distance import cdist + EPSILON = 0.0000000001 EPSILON2 = 0.000000001 @@ -1037,3 +1039,43 @@ def test_polynomial_shear(): assert atf.shear == ptf.shear assert atf.rotation == ptf.rotation assert atf.translation == ptf.translation + + +@pytest.mark.parametrize('preserve_srcPts', [True, False]) +@pytest.mark.parametrize('computeAffine', [True, False]) +@pytest.mark.parametrize('factor', [1e-3, 1e3, 1e5]) +def test_scale_thinplate(factor, computeAffine, preserve_srcPts): + # use case is to scale a rough alignment thinplate spline + with open(rendersettings.TEST_THINPLATEROUGH_FILE, 'r') as f: + tform = renderapi.transform.load_transform_json( + json.load(f)) + + if not computeAffine: + tform.aMtx = None + tform.bVec = None + + mn = tform.srcPts.min(axis=0) + mx = tform.srcPts.max(axis=0) + + npts = 1000 + src = np.random.rand(npts, 2) + src[:, 0] = src[:, 0] * (mx[0] - mn[0]) + mn[0] + src[:, 1] = src[:, 1] * (mx[1] - mn[1]) + mn[1] + dst = tform.tform(src) + + scaled_tform = tform.scale_coordinates( + factor, preserve_srcPts=preserve_srcPts) + dst_scaled = scaled_tform.tform(src * factor) + + delta = np.linalg.norm(dst_scaled - dst * factor, axis=1) + scale = dst_scaled.ptp(axis=0).mean() + assert (delta.max() / scale) < 1e-4 + + if preserve_srcPts: + dist = cdist( + tform.srcPts.transpose() * factor, + scaled_tform.srcPts.transpose(), + metric='euclidean') + # check that the original srcPts have a close neighbor still + # in the scaled srcPts + assert np.all(np.any(dist < 1e-3, axis=1)) From dffec92c3abe08dc73b625e305aa28ab50f32939 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 23 Jul 2019 15:04:16 -0700 Subject: [PATCH 727/766] adjusting pytest requirements --- test_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_requirements.txt b/test_requirements.txt index 2018ec5b..74e88cc7 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,7 @@ coverage>=4.1 mock>=2.0.0 pep8>=1.7.0 -pytest>=3.0.5 +pytest>4.6,<5.0 pytest-cov>=2.2.1 pytest-pep8>=1.0.6 pytest-xdist>=1.14 From 945168c8e26dc6e67cb46f2d87cd1e8019827c39 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Tue, 23 Jul 2019 15:43:34 -0700 Subject: [PATCH 728/766] adding test across ngrid parameter --- test/test_transform.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/test_transform.py b/test/test_transform.py index 58f7268a..57c3135e 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -1041,10 +1041,11 @@ def test_polynomial_shear(): assert atf.translation == ptf.translation +@pytest.mark.parametrize('ngrid', [5, 20]) @pytest.mark.parametrize('preserve_srcPts', [True, False]) @pytest.mark.parametrize('computeAffine', [True, False]) @pytest.mark.parametrize('factor', [1e-3, 1e3, 1e5]) -def test_scale_thinplate(factor, computeAffine, preserve_srcPts): +def test_scale_thinplate(factor, computeAffine, preserve_srcPts, ngrid): # use case is to scale a rough alignment thinplate spline with open(rendersettings.TEST_THINPLATEROUGH_FILE, 'r') as f: tform = renderapi.transform.load_transform_json( @@ -1064,12 +1065,15 @@ def test_scale_thinplate(factor, computeAffine, preserve_srcPts): dst = tform.tform(src) scaled_tform = tform.scale_coordinates( - factor, preserve_srcPts=preserve_srcPts) + factor, ngrid=ngrid, preserve_srcPts=preserve_srcPts) dst_scaled = scaled_tform.tform(src * factor) delta = np.linalg.norm(dst_scaled - dst * factor, axis=1) scale = dst_scaled.ptp(axis=0).mean() - assert (delta.max() / scale) < 1e-4 + tol = 1e-4 + if ngrid == 5: + tol = 1e-3 + assert (delta.max() / scale) < tol if preserve_srcPts: dist = cdist( From 3961d073710026dd23275e208e8cd9dc90467b25 Mon Sep 17 00:00:00 2001 From: Dan Kapner Date: Thu, 25 Jul 2019 20:48:45 -0700 Subject: [PATCH 729/766] one-liner for src array --- renderapi/transform/leaf/thin_plate_spline.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/renderapi/transform/leaf/thin_plate_spline.py b/renderapi/transform/leaf/thin_plate_spline.py index c8abc9a7..793fc0f9 100644 --- a/renderapi/transform/leaf/thin_plate_spline.py +++ b/renderapi/transform/leaf/thin_plate_spline.py @@ -435,10 +435,7 @@ def src_array(xmin, ymin, xmax, ymax, nx, ny): (nx * ny) x 2 array of coordinated. """ - x = np.linspace(xmin, xmax, nx) - y = np.linspace(ymin, ymax, ny) - xt, yt = np.meshgrid(x, y) - src = np.vstack((xt.flatten(), yt.flatten())).transpose() + src = np.mgrid[xmin:xmax:nx*1j, ymin:ymax:ny*1j].reshape(2, -1).T return src def scale_coordinates( From 45825438b04f0cc98e7455133dff9b9dd38d7fc6 Mon Sep 17 00:00:00 2001 From: "Eric T. Trautman" Date: Thu, 12 Mar 2020 08:19:51 -0400 Subject: [PATCH 730/766] Add distance z to layout (#147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * switch to openjdk8 and force maven to use it for travis tests * Added distanceZ attribute to layout component of tile spec to capture differing depth of FIBSEM tiles. * add IntelliJ (PyCharm) editor files to .gitignore * Bump version: 2.1.1 → 2.2.0 * Bump version: 2.2.0 → 2.3.0 --- .bumpversion.cfg | 3 +-- .gitignore | 4 +++- .travis.yml | 4 +++- renderapi/__init__.py | 2 +- renderapi/layout.py | 9 ++++++++- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b7e9cfa3..b1814f73 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,6 @@ [bumpversion] -current_version = 2.1.1 +current_version = 2.3.0 commit = True tag = True [bumpversion:file:renderapi/__init__.py] - diff --git a/.gitignore b/.gitignore index d54ba60a..b3e71372 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,6 @@ htmlcov/ .cache .tox .pytest_cache -.vscode \ No newline at end of file +.vscode +.idea/ +*.iml \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index b191a7f5..49a86081 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ addons: - libopenblas-base - libopenblas-dev - gfortran - - oracle-java9-set-default + - openjdk-8-jdk - maven install: - pip install codecov @@ -25,6 +25,8 @@ install: - pip install -r test_requirements.txt before_install: - git clone --depth 1 https://github.com/saalfeldlab/render.git render -b master + - export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/jre" + - mvn -version - mvn package -pl render-ws-java-client -am -DskipTests -f render/pom.xml - export RENDER_CLIENT_JAR=`readlink -m $TRAVIS_BUILD_DIR/render/render-ws-java-client/target/render-ws-java-client-*-standalone.jar` - export RENDER_WS_JAVA_CLIENT_EXAMPLE_DATA=`readlink -m $TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/resources` diff --git a/renderapi/__init__.py b/renderapi/__init__.py index de9ab0d1..9b859720 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -13,7 +13,7 @@ from .render import connect from .render import Render -__version__ = "2.1.1" +__version__ = "2.3.0" __all__ = ['render', 'client', 'tilespec', 'errors', 'stack', 'image', 'pointmatch', 'coordinate', 'connect', 'transform', 'resolvedtiles', 'Render'] diff --git a/renderapi/layout.py b/renderapi/layout.py index 05e5240c..83a1b35a 100644 --- a/renderapi/layout.py +++ b/renderapi/layout.py @@ -21,12 +21,14 @@ class Layout: angle of camera when this was taken pixelsize : float effective size of pixels (in units of choice) + distanceZ : float + distance (in units of choice) from prior layer """ def __init__(self, sectionId=None, scopeId=None, cameraId=None, imageRow=None, imageCol=None, stageX=None, stageY=None, rotation=None, pixelsize=None, - force_pixelsize=True, **kwargs): + force_pixelsize=True, distanceZ=None, **kwargs): """Initialize Layout Parameters @@ -51,6 +53,8 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, effective size of pixels (in units of choice) force_pixelsize : bool whether to default pixelsize to 0.1 + distanceZ : float + distance (in units of choice) from prior layer """ self.sectionId = sectionId @@ -64,6 +68,7 @@ def __init__(self, sectionId=None, scopeId=None, cameraId=None, if force_pixelsize: pixelsize = 0.100 if pixelsize is None else pixelsize self.pixelsize = pixelsize + self.distanceZ = distanceZ def to_dict(self): """return a dictionary representation of this object @@ -83,6 +88,7 @@ def to_dict(self): d['stageY'] = self.stageY d['rotation'] = self.rotation d['pixelsize'] = self.pixelsize + d['distanceZ'] = self.distanceZ d = {k: v for k, v in d.items() if v is not None} return d @@ -104,3 +110,4 @@ def from_dict(self, d): self.stageY = d.get('stageY') self.rotation = d.get('rotation') self.pixelsize = d.get('pixelsize') + self.distanceZ = d.get('distanceZ') From 5fbed0ccec1ea0f39642a1f59381ab12901094a5 Mon Sep 17 00:00:00 2001 From: miket Date: Fri, 3 Apr 2020 12:35:00 -0700 Subject: [PATCH 731/766] Tile image scale Allow a min and max intensity scale in `image.get_tile_image_data()` --- renderapi/image.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/renderapi/image.py b/renderapi/image.py index a5f812c4..4d405155 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -112,6 +112,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, @renderaccess def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, excludeAllTransforms=False, scale=None, + minIntensity=None, maxIntensity=None, filter=None, host=None, port=None, owner=None, project=None, img_format=None, session=requests.session(), render=None, **kwargs): @@ -138,6 +139,10 @@ def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, (or remove till there are max 3 transforms) scale : float force scale of image + minIntensity : int + Minimum pixel value to rescale image + maxIntensity : int + Maximum pixel value to rescale image filter : bool whether to apply server side filtering to image img_format : str @@ -178,6 +183,10 @@ def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, qparams['excludeAllTransforms'] = jbool(excludeAllTransforms) if channel is not None: qparams.update({'channels': channel}) + if minIntensity is not None: + qparams['minIntensity'] = minIntensity + if maxIntensity is not None: + qparams['maxIntensity'] = maxIntensity logger.debug(request_url) r = session.get(request_url, params=qparams) From f2ff3ec3f4b1c924b1a312f377776a50ebd373c3 Mon Sep 17 00:00:00 2001 From: miket Date: Tue, 7 Apr 2020 11:27:41 -0700 Subject: [PATCH 732/766] section intensity Min/Max intensity for the `get_section_image` function --- renderapi/image.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/renderapi/image.py b/renderapi/image.py index 4d405155..3eb8f9cc 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -202,6 +202,7 @@ def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, @renderaccess def get_section_image(stack, z, scale=1.0, channel=None, + minIntensity=None, maxIntensity=None, filter=False, maxTileSpecsToRender=None, img_format=None, host=None, port=None, owner=None, project=None, @@ -223,6 +224,10 @@ def get_section_image(stack, z, scale=1.0, channel=None, channel: str channel name to render, (e.g. 'DAPI') or a weighted average of channels of the format e.g 'DAPI___.8___GFP___.2' + minIntensity : int + Minimum pixel value to rescale image + maxIntensity : int + Maximum pixel value to rescale image filter : bool whether or not to apply server side filtering maxTileSpecsToRender : int @@ -261,6 +266,10 @@ def get_section_image(stack, z, scale=1.0, channel=None, qparams.update({'maxTileSpecsToRender': maxTileSpecsToRender}) if channel is not None: qparams.update({'channels': channel}) + if minIntensity is not None: + qparams['minIntensity'] = minIntensity + if maxIntensity is not None: + qparams['maxIntensity'] = maxIntensity r = session.get(request_url, params=qparams) return np.asarray(Image.open(io.BytesIO(r.content))) From 07b32712a596c9d962b7832616ff9c10bfb9bb64 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 11 Aug 2020 11:37:18 -0700 Subject: [PATCH 733/766] Create LICENSE --- LICENSE | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..8dc056db --- /dev/null +++ b/LICENSE @@ -0,0 +1,33 @@ +Allen Institute Software License – This software license is the 2-clause BSD +license plus a third clause that prohibits redistribution and use for +commercial purposes without further permission. + +Copyright © 2019. Allen Institute. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Redistributions and use for commercial purposes are not permitted without +the Allen Institute’s written permission. For purposes of this license, +commercial purposes are the incorporation of the Allen Institute's software +into anything for which you will charge fees or other compensation or use of +the software to perform a commercial service for a third party. Contact +terms@alleninstitute.org for commercial licensing opportunities. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE From 7208b41f1bc14bc3f027e8553d8fabcbb2cfc572 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Tue, 11 Aug 2020 11:38:18 -0700 Subject: [PATCH 734/766] Create CONTRIBUTING.md --- CONTRIBUTING.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..e55867fc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Allen Institute Contribution Agreement + +This document describes the terms under which you may make “Contributions” — +which may include without limitation, software additions, revisions, bug fixes, configuration changes, +documentation, or any other materials — to any of the projects owned or managed by the Allen Institute. +If you have questions about these terms, please contact us at terms@alleninstitute.org. + +You certify that: + +• Your Contributions are either: + +1. Created in whole or in part by you and you have the right to submit them under the designated license +(described below); or +2. Based upon previous work that, to the best of your knowledge, is covered under an appropriate +open source license and you have the right under that license to submit that work with modifications, +whether created in whole or in part by you, under the designated license; or + +3. Provided directly to you by some other person who certified (1) or (2) and you have not modified them. + +• You are granting your Contributions to the Allen Institute under the terms of the [2-Clause BSD license](https://opensource.org/licenses/BSD-2-Clause) +(the “designated license”). + +• You understand and agree that the Allen Institute projects and your Contributions are public and that +a record of the Contributions (including all metadata and personal information you submit with them) is +maintained indefinitely and may be redistributed consistent with the Allen Institute’s mission and the +2-Clause BSD license. From 7c95f5bb4ccc22fbb65a19e5cf57900d6e3b258f Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Tue, 24 Nov 2020 10:32:23 -0800 Subject: [PATCH 735/766] tilePairClient: add support for more parameters and multi-file support --- integration_tests/test_client_integrated.py | 8 ++- renderapi/client/client_calls.py | 78 +++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 50761d65..473d6e5c 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -185,7 +185,6 @@ def teststack2(render, render_example_tilespec_and_transforms): renderapi.stack.delete_stack(stack, render=render) - def test_tile_pair_client(render, teststack, **kwargs): zvalues = np.array(renderapi.stack.get_z_values_for_stack( teststack, render=render)) @@ -196,6 +195,13 @@ def test_tile_pair_client(render, teststack, **kwargs): assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 + multitpjson, multitpfiles = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, + render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) + + assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) + assert len(multitpfiles) == len(tilepairjson["neighborPairs"]) // 2 + @pytest.mark.parametrize("bounds,raises", [ ({}, True), diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 68adc2f5..c4cb956f 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -1,6 +1,7 @@ import logging import json import os +import re import subprocess import tempfile @@ -135,6 +136,11 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, excludeSameSectionNeighbors=None, excludePairsInMatchCollection=None, minx=None, maxx=None, miny=None, maxy=None, + useRowColPositions=None, existingMatchOwner=None, + minExistingMatchCount=None, onlyIncludeTilesFromStack=None, + onlyIncludeTilesNearTileIdsJson=None, maxPairsPerFile=None, + return_jsondata=True, + return_jsonfiles=False, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, @@ -191,11 +197,29 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, minimum y bound from which tile 'p' is selected maxy : float maximum y bound from wich tile 'p' is selected + useRowColPositions : bool + whether to use raster positions for neighbor analysis rather + than radial cutoff + existingMatchOwner : str + owner for the excludePairsInMatchCollection collection + minExistingMatchCount : int + minimum match threshold to exclude pairs + present in excludePairsInMatchCollection collection + onlyIncludeTilesFromStack : str + only include tiles which exist in this stack + onlyIncludeTilesNearTileIdsJson : str + path to json listing tileIds which should be paired + maxPairsPerFile : int + maximum neighborPairs per file. Generating more pairs than + this number (default 100000) will generate additional json + files with a p[0-9]+ suffix on the root. Returns ------- - :obj:`list` of :obj:`dict` + :obj:`list` of :obj:`dict`, optional list of tilepairs + :obj:`list` of string, optional + list of json files containing tilepairs """ if outjson is None: with tempfile.NamedTemporaryFile( @@ -219,6 +243,18 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, '--excludeSameSectionNeighbors') + get_param(excludePairsInMatchCollection, '--excludePairsInMatchCollection') + + get_param(useRowColPositions, + '--useRowColPositions') + + get_param(existingMatchOwner, + '--existingMatchOwner') + + get_param(minExistingMatchCount, + '--minExistingMatchCount') + + get_param(onlyIncludeTilesFromStack, + "--onlyIncludeTilesFromStack") + + get_param(onlyIncludeTilesNearTileIdsJson, + '--onlyIncludeTilesNearTileIdsJson') + + get_param(maxPairsPerFile, + '--maxPairsPerFile') + ['--toJson', outjson] + get_param(minx, '--minX') + get_param(maxx, '--maxX') + get_param(miny, '--minY') + get_param(maxy, '--maxY')) @@ -228,12 +264,44 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, subprocess_mode=subprocess_mode, add_args=argvs, **kwargs) - with open(outjson, 'r') as f: - jsondata = json.load(f) + # We create the jsonfile, so if it is empty it could be multiple + if os.stat(outjson).st_size == 0: + # outjson we created is empty, so remove it + os.remove(outjson) + + outbn_root, outbn_ext = os.path.splitext(os.path.basename(outjson)) + outbn_pattern = "{root}_p[0-9]+{ext}".format( + root=outbn_root, ext=outbn_ext) + jsonfiles = [ + os.path.join(os.path.dirname(outjson), bn) + for bn in os.listdir(os.path.dirname(outjson)) + if re.match(outbn_pattern, bn)] + else: + jsonfiles = [outjson] + + if return_jsondata: + jsondata_list = [] + for jsonfile in jsonfiles: + with open(jsonfile, 'r') as f: + jsondata_list.append(json.load(f)) + if len({d["renderParametersUrlTemplate"] for d in jsondata_list}) != 1: # pragma: no cover + raise ValueError( + "Found tilepair files with disparate " + "renderParametersUrlTemplate values. " + "Maybe there are additional files " + "matching the outjson pattern?") + pairdata = [i for l in (d["neighborPairs"] for d in jsondata_list) + for i in l] + jsondata = dict(jsondata_list[0], **{"neighborPairs": pairdata}) if delete_json: - os.remove(outjson) - return jsondata + for jsonfile in jsonfiles: + os.remove(jsonfile) + + return ((jsondata, jsonfiles) if return_jsonfiles and return_jsondata + else jsondata if return_jsondata + else jsonfiles if return_jsonfiles + else None) @renderclientaccess From 92733b2bf0a0361a7a510bb1fa3cb720cd91b8f1 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Tue, 24 Nov 2020 11:23:14 -0800 Subject: [PATCH 736/766] tilePairClient: better support for non-null outjson inputs --- integration_tests/test_client_integrated.py | 14 +++++++++++--- renderapi/client/client_calls.py | 7 ++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/integration_tests/test_client_integrated.py b/integration_tests/test_client_integrated.py index 473d6e5c..8e1dc551 100644 --- a/integration_tests/test_client_integrated.py +++ b/integration_tests/test_client_integrated.py @@ -195,12 +195,20 @@ def test_tile_pair_client(render, teststack, **kwargs): assert isinstance(tilepairjson, dict) assert len(tilepairjson['neighborPairs']) > 3 + multitpjson_tmp, multitpfiles_tmp = renderapi.client.tilePairClient( + teststack, np.min(zvalues), np.max(zvalues), outjson=None, + render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) + + with tempfile.NamedTemporaryFile( + suffix=".json", mode='r', delete=False) as f: + outjson_specified = f.name + multitpjson, multitpfiles = renderapi.client.tilePairClient( - teststack, np.min(zvalues), np.max(zvalues), outjson=outjson, + teststack, np.min(zvalues), np.max(zvalues), outjson=outjson_specified, render=render, maxPairsPerFile=2, return_jsonfiles=True, **kwargs) - assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) - assert len(multitpfiles) == len(tilepairjson["neighborPairs"]) // 2 + assert len(multitpjson["neighborPairs"]) == len(tilepairjson["neighborPairs"]) == len(multitpjson_tmp["neighborPairs"]) + assert len(multitpfiles) == len(multitpfiles_tmp) == len(tilepairjson["neighborPairs"]) // 2 @pytest.mark.parametrize("bounds,raises", [ diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index c4cb956f..2baf1235 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -265,9 +265,10 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, add_args=argvs, **kwargs) # We create the jsonfile, so if it is empty it could be multiple - if os.stat(outjson).st_size == 0: - # outjson we created is empty, so remove it - os.remove(outjson) + if not os.path.isfile(outjson) or os.stat(outjson).st_size == 0: + if os.path.isfile(outjson): + # outjson we created is empty, so remove it + os.remove(outjson) outbn_root, outbn_ext = os.path.splitext(os.path.basename(outjson)) outbn_pattern = "{root}_p[0-9]+{ext}".format( From 46657409a280c39f0647fbde23759a5ef513a609 Mon Sep 17 00:00:00 2001 From: "Eric T. Trautman" Date: Thu, 21 Jan 2021 19:42:40 -0500 Subject: [PATCH 737/766] Add labels to tile spec (#152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump version: 2.3.0 → 2.4.0 * Add labels to tile specs. Replace mutable default arguments in constructor. * Add groupId to tile specs. * Create LICENSE * Create CONTRIBUTING.md Co-authored-by: Forrest Collman --- .bumpversion.cfg | 2 +- CONTRIBUTING.md | 26 ++++++++++++++++++++++++++ LICENSE | 33 +++++++++++++++++++++++++++++++++ renderapi/__init__.py | 2 +- renderapi/tilespec.py | 19 +++++++++++++++---- 5 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b1814f73..f3b5dda1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.3.0 +current_version = 2.4.0 commit = True tag = True diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..e55867fc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,26 @@ +# Allen Institute Contribution Agreement + +This document describes the terms under which you may make “Contributions” — +which may include without limitation, software additions, revisions, bug fixes, configuration changes, +documentation, or any other materials — to any of the projects owned or managed by the Allen Institute. +If you have questions about these terms, please contact us at terms@alleninstitute.org. + +You certify that: + +• Your Contributions are either: + +1. Created in whole or in part by you and you have the right to submit them under the designated license +(described below); or +2. Based upon previous work that, to the best of your knowledge, is covered under an appropriate +open source license and you have the right under that license to submit that work with modifications, +whether created in whole or in part by you, under the designated license; or + +3. Provided directly to you by some other person who certified (1) or (2) and you have not modified them. + +• You are granting your Contributions to the Allen Institute under the terms of the [2-Clause BSD license](https://opensource.org/licenses/BSD-2-Clause) +(the “designated license”). + +• You understand and agree that the Allen Institute projects and your Contributions are public and that +a record of the Contributions (including all metadata and personal information you submit with them) is +maintained indefinitely and may be redistributed consistent with the Allen Institute’s mission and the +2-Clause BSD license. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..8dc056db --- /dev/null +++ b/LICENSE @@ -0,0 +1,33 @@ +Allen Institute Software License – This software license is the 2-clause BSD +license plus a third clause that prohibits redistribution and use for +commercial purposes without further permission. + +Copyright © 2019. Allen Institute. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Redistributions and use for commercial purposes are not permitted without +the Allen Institute’s written permission. For purposes of this license, +commercial purposes are the incorporation of the Allen Institute's software +into anything for which you will charge fees or other compensation or use of +the software to perform a commercial service for a third party. Contact +terms@alleninstitute.org for commercial licensing opportunities. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE diff --git a/renderapi/__init__.py b/renderapi/__init__.py index 9b859720..a639e30f 100644 --- a/renderapi/__init__.py +++ b/renderapi/__init__.py @@ -13,7 +13,7 @@ from .render import connect from .render import Render -__version__ = "2.3.0" +__version__ = "2.4.0" __all__ = ['render', 'client', 'tilespec', 'errors', 'stack', 'image', 'pointmatch', 'coordinate', 'connect', 'transform', 'resolvedtiles', 'Render'] diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index d6953ffc..ba06eda9 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -67,8 +67,8 @@ class TileSpec: def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, - minint=0, maxint=65535, layout=None, tforms=[], - inputfilters=[], json=None, channels=None, + minint=0, maxint=65535, layout=None, tforms=None, + labels=None, groupId=None, inputfilters=None, json=None, channels=None, mipMapLevels=None, imagePyramid=None, **kwargs): if json is not None: self.from_dict(json) @@ -80,8 +80,10 @@ def __init__(self, tileId=None, z=None, width=None, height=None, self.layout = layout self.minint = minint self.maxint = maxint - self.tforms = tforms - self.inputfilters = inputfilters + self.tforms = tforms if tforms else [] + self.labels = labels if labels else [] + self.groupId = groupId + self.inputfilters = inputfilters if inputfilters else [] self.layout = Layout(**kwargs) if layout is None else layout if imagePyramid is not None: @@ -188,6 +190,12 @@ def to_dict(self): else: thedict['transforms']['specList'].append(t.to_dict()) + if len(self.labels) > 0: + thedict['labels'] = self.labels + + if self.groupId is not None: + thedict['groupId'] = self.groupId + # TODO filters not implemented ''' if len(self.inputfilters): @@ -234,6 +242,9 @@ def from_dict(self, d): else: self.channels = [Channel(json=ch) for ch in chd] + self.labels = d.get('labels', []) + self.groupId = d.get('groupId', None) + # TODO filters not implemented -- should skip ''' self.inputfilters = [] From 9f208327733b52db03c16b60f39ecdb2f70c95c8 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 4 Feb 2021 09:09:54 -0800 Subject: [PATCH 738/766] travis update for AllenInstitute move (#153) * Create LICENSE * Create CONTRIBUTING.md * travis update for AllenInsitute move * travis: remove 3.5 and 3.6-dev * travis: add docker login * travis: add encrypted vars * Update .travis.yml * Update .travis.yml * Update .travis.yml * README: update badge links * travis: update ci env --- .travis.yml | 9 +++++---- README.rst | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 49a86081..ab33e065 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,7 @@ language: python python: - "2.7" - - "3.5" - "3.6" - - "3.6-dev" # 3.6 development branch - "3.7-dev" services: - docker @@ -35,11 +33,14 @@ before_install: - mkdir -p /tmp/example_1 && cp -R $RENDER_WS_JAVA_CLIENT_EXAMPLE_DATA/example_1 /tmp/. - cp -R $RENDER_APP_EXAMPLE_DATA/* /tmp/. - export RENDER_EXAMPLE_DATA=/tmp + - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin - docker-compose up -d env: global: - RENDER_HOST=localhost RENDER_PORT=8080 RENDER_CLIENT_SCRIPTS=$TRAVIS_BUILD_DIR/render/render-ws-java-client/src/main/scripts - - secure: "gIvBfWzWZizVxIW3eJfMX4V4/wrtn6zVoI9lkDbnXgn42+jIUdZYLEnHdZDtgOYtwUHUiY0UuXX/D9pl5XK+5IrZ+ocRKv8vIL1APJNxJGWJThkk8ifQFn+fuG2MvlA1b3z3903FNMSw3AcG0479YucZkPx7TmexU5DHM/zf/+/KvCV5qDPKjZu4hL3o/njuwNZGpYmT0+pUvugiAhF1c5YBmPBMhaplUYKdjuGMeLOlB/QxmBdkTPWCeVda8ohB3ngGc2yjhtDPAaOo9LKNrdLje6DCr2Zv6e4qo27YNFpXoICN0qUPOaoGOoVJ/HOq8HUZgJMi9KiYH2oqP27RqPOHaGABK7PF5aMg6vvUBcCYP4n93rnbm7g4k/+A/mUeew9lXyrh8+O9d8ul5eNfLdaQ1HGnR093Vx53Sr7Q9lyNU/PUteMGzRYQw2EQGuryGiAKJ5tWD9HIHtqQ7iNZup8ijHCKVdmtLHROTHMvpcagQ33Sl/+U05E50YoDZRzvTNSaWUoNhl01xRIyp35KYy55Ei2X8eMTJlirdzKj1hAeOED98SmKx8ROypxJDzcO/N8AGlPRc9/l+OplKTvpnlD3Nsd/bxErWEmrBDtAkY05rfR48e9xtDyqV4KuAXy0wBJn5eqFEFpNNxzMi78CHsF023UodVsWAzn4HvAojrQ=" + - secure: "AZCuswnHLEvDkfcmijSb4sy1E0vCg3nHn8gk9EvO7z9JeJG4FtHYBW3bLiN4WSjxszzaCK16lku0RY4OL2gCv0C87iHaXkDqAWwN0YIFGOkzDhZJRD5CENTsx1w/JFySqTBNiO82gIAnNP+J19mFm1ArtNhDIYAiUPo+CseG8S0cpbQqQe3tBd+XB/u32PUS1zA1mIg6qDXKupoDCUuWBxCZ01Jdv/YDUH3zxz6K9SQmbzJ4m5B1TCw8E6r/dj6mBSjJSh2XmZlOU0Ha2U9nEkkkr7nbNU1GYQYRt9uiqDH4juvFw6WePP+MVzFJCq0Hqdyb9lk/7jh4BX5ffN2o/vs14FuXZ57wblcHZkkEgD0ViIlGjqgPn0WuTJstkVlpvvwuweVfXXxJrYAZwhf14goKiWneDZMJ6g82Z7r9GVQFwEwl/mu5fdLP/qKZ8i3AAd9SgnGdn5eQ/xD5JVJj9JZFqfLUJs7JQreGYfW2r5URIL3Kd2laRucUD4IPwnzHT+VjgNHFtv07bmNxRGPdwC4pbWOMiyq2BvG3JFw/y8j3bNvLxK6XLhDgHtV13NinZx8uZcocb8yCNyBpez+Pfo3RzTcHKnF5foKoEEZIU5OuyGGgeHUup/vg2qsKGn3Wl1nH5fWSbdR2pPHLSD+mpZdLAA9JjqEfVrhZtoXLUpA=" + - secure: "tnw8YYrVoasmJ+3I3VjENfx6BIBG2qaJ4m0+rUR9dal7cmrGTrcyJ1d41Nm+K1XKRvLvBkEHKBzdEVlg2XaAOVBTam6u+zW/Dc16ZXgz7yK1qXtytwiXBe2P7c9HePXIDA/SCapKXmjF5/Zt0BdpXv4fPyK8kbdIjels9Xt8teDTfSxJ6AfXlfXXPiF6A1Lzgf+z9GRL9GXZrgic6itFdNfgErGThgefQSJKr3wKR/wRyx87QD5juzrZtcwtFzcvvO8X8n6D+oOdc9XfRsE1EgFO8RHla32c0BbcKmRP8vwwtfVUt1tqU3SQZP6+n9EUBiSUR9/B50hCOiX5O2RdtHIMncJT0n4ZQe2PmTmwpcY1bU1NJpUszxPXNZrEHRdGqbXfiQq8+sONAWZcrzbqLsP2cXpt+4FrHk+iepu0RBLpLq/UdcVLozELgkNxM1C9k3OF6OlAC//5jWi4hjeoc2L8LpR09Cxm7PCtBvqbCNO2mPhv+h4dsFzmlyN2JL34lQyfzD1v1Jqz4ifJHYupKKtIdA7UiU+RU40ylDzTKfvlI63MkUwxS2u4BY0Gc5WWy/NoJuF6F39c+peoLgcoHiKnXoIvTyWCNSfieXCnuiHrU7nLnijM9qzSZxM6a43KiSGRZAQInQUXzKX5a7cGHaQIFskd3TK71emnhdewM40=" + - secure: "UucVIVQCg9T2d9NiSdMirj37nlwWEyZi2mybzNNVJ8TMfC05Vz+E/b/1ByxOLpUYmXZTzU22Xx316ueIaIk05L+Q8q4Dy1S/spaKBNcJQ8kDxlcD7y0YBPX+QqyOpXCnB1Kt1oawy92g2qALFDMBoupr7xT8aTvCwDigrJYghMiVM1RVlQQAJBaGkB32qs934uQeknvLqSZV35BTFGHTaqJoWV8OyxNwOZFtf5Z8gdWxrAYHOQg9iW+5CP+zLhybROLrMuI5vhj/yvP2I07Scx+jMKOgGw7snPygHpwR0dNw/NvNdy+XIx3zqUlts5Bb1nqCeaNzNouXd1D5NHoqsw/0VLt/HcbCxUKXuaViJy5KfN/qi9U7lSJ0KC2qazwsQXviFYTrT2q1U2n6zlV/CC9e1sLjQmIF3GAysFD3Fcg88Qz/3aZKl6nkqP/T2Y1t/FOPjBSD5ytlpr8xbs6rpxZMU/k91agD5UeP4/lwbncfsJswPNajdTz5scWjHPACDLIxYTwEry64z808lHcc23c9MIeqrkXMI4nPYnLSPQyhXBOJncEugKernBh053Pu5+hDrg9bsoIjtsGmh+p9GnIjrtPAkc8g4tyU+v0PKMVUkqw+mvkVEgzRlKUzURuRTVy5lTHisu0mo6Xmtgw0qfVkTOa2Gnr9gsfmaTToyXY=" # command to run tests script: - python setup.py test # or py.test for Python versions 3.5 and below @@ -49,4 +50,4 @@ cache: - $HOME/.m2 after_success: - codecov -t 3f12d985-af62-455d-a11d-9669c039640d - - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=fcollman/render-python .travis/merge_script.sh" + - "BRANCHES_TO_MERGE_REGEX='develop' BRANCH_TO_MERGE_INTO=master GITHUB_REPO=AllenInstitute/render-python .travis/merge_script.sh" diff --git a/README.rst b/README.rst index 83ed3ece..02a86add 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,11 @@ .. image:: https://readthedocs.org/projects/render-python/badge/ :target: http://render-python.readthedocs.io/en/latest/ :alt: Documentation Status -.. image:: https://travis-ci.org/fcollman/render-python.svg?branch=master - :target: https://travis-ci.org/fcollman/render-python +.. image:: https://travis-ci.com/AllenInstitute/render-python.svg?branch=master + :target: https://travis-ci.com/AllenInstitute/render-python :alt: Build Status -.. image:: https://codecov.io/gh/fcollman/render-python/branch/master/graph/badge.svg - :target: https://codecov.io/gh/fcollman/render-python +.. image:: https://codecov.io/gh/AllenInstitute/render-python/branch/master/graph/badge.svg + :target: https://codecov.io/gh/AllenInstitute/render-python render-python ############# From 120442fee369929ff42218aae2f255e764a8c2e5 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 11 Feb 2021 15:30:22 -0800 Subject: [PATCH 739/766] resolvedtiles: combine_resolvedtiles function --- renderapi/resolvedtiles.py | 26 ++++++++++++++++++++++++++ test/test_resolvedtiles.py | 15 +++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py index e8f018ce..3f079ea8 100644 --- a/renderapi/resolvedtiles.py +++ b/renderapi/resolvedtiles.py @@ -53,6 +53,32 @@ def from_dict(self, d): """ # noqa: E501 +def combine_resolvedtiles(rts_l): + """combine multiple ResolvedTiles objects into a single ResolvedTiles + + Like render, this has an implicit expectation that transformIds and tileIds + are unique among all input objects + + Parameters + ---------- + rts_l : list[renderapi.resolvedtiles.ResolvedTiles] + list of ResolvedTiles objects to combine + + Returns + ------- + rts : renderapi.resolvedtiles.ResolvedTiles + combined ResolvedTiles object + """ + tforms = list({tform.transformId: tform for l in ( + rts.transforms for rts in rts_l) + for tform in l}.values()) + tspecs = list({ts.tileId: ts for l in ( + rts.tilespecs for rts in rts_l) + for ts in l}.values()) + + return ResolvedTiles(transformList=tforms, tilespecs=tspecs) + + @renderaccess def put_tilespecs(stack, resolved_tiles=None, deriveData=True, tilespecs=None, shared_transforms=None, diff --git a/test/test_resolvedtiles.py b/test/test_resolvedtiles.py index e4da011d..484aa63f 100644 --- a/test/test_resolvedtiles.py +++ b/test/test_resolvedtiles.py @@ -24,6 +24,15 @@ def resolvedtiles_object(referenced_tilespecs_and_transforms): return resolved_tiles +@pytest.fixture(scope="module") +def resolvedtiles_objects(resolvedtiles_object): + # one resolvedtiles object per tile in the resolvedtiles_object + rts_l = [renderapi.resolvedtiles.ResolvedTiles( + tilespecs=[ts], transformList=resolvedtiles_object.transforms) + for ts in resolvedtiles_object.tilespecs] + return rts_l + + def test_resolvedtiles_from_dict(resolvedtiles_object, referenced_tilespecs_and_transforms): tilespecs, transforms = referenced_tilespecs_and_transforms @@ -31,3 +40,9 @@ def test_resolvedtiles_from_dict(resolvedtiles_object, resolved_tiles = renderapi.resolvedtiles.ResolvedTiles(json=d) assert(len(tilespecs) == len(resolved_tiles.tilespecs)) assert(len(transforms) == len(resolved_tiles.transforms)) + + +def test_combine_resolvedtiles(resolvedtiles_object, resolvedtiles_objects): + assert(resolvedtiles_object.to_dict() == + renderapi.resolvedtiles.combine_resolvedtiles( + resolvedtiles_objects).to_dict()) From 20d937eed70791eea4bf92f3e2c44ccc03039c55 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Fri, 12 Feb 2021 14:43:26 -0800 Subject: [PATCH 740/766] pointmatch: add functions for fast and deep copy --- renderapi/pointmatch.py | 45 ++++++++++++++++++ test/test_pointmatch.py | 101 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 test/test_pointmatch.py diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 897907a5..85f9ce5c 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -11,6 +11,51 @@ logger.addHandler(NullHandler()) +def copy_match_explicit(match): + """create independent match dictionary equivalent to the input match + significantly faster than e.g. copy.deepcopy or json serialization + + Parameters + ---------- + match : dict + match dictionary + + Returns + ------- + new_match : dict + match dictionary equivalent to input match + + """ + # explicitly copy match dictionary since it contains lists + new_matchd = { + "p": [i[:] for i in match["matches"]["p"]], + "q": [i[:] for i in match["matches"]["q"]], + "w": match["matches"]["w"][:] + } + + new_match = {k: (match[k] if k != "matches" else new_matchd) + for k in match.keys()} + return new_match + + +def copy_matches_explicit(matches): + """create independent match dictionaries equivalent to the input matches + significantly faster than e.g. copy.deepcopy or json serialization + + Parameters + ---------- + matches : list[dict] + list of match dictionaries to copy + + Returns + ------- + new_matches : list[dict] + list of match dictionaries equivalent to input matches + + """ + return [copy_match_explicit(match) for match in matches] + + @renderaccess def get_matchcollection_owners(host=None, port=None, session=requests.session(), diff --git a/test/test_pointmatch.py b/test/test_pointmatch.py new file mode 100644 index 00000000..11246153 --- /dev/null +++ b/test/test_pointmatch.py @@ -0,0 +1,101 @@ +import copy +import renderapi +import pytest + + +@pytest.fixture(scope="module") +def matches(): + return [{ + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "1", + "qId": "1-1", + "matches": { + "p": [[0, 0], + [100, 100], + [0, 100], + [100, 0]], + "q": [[0, 0], + [100, 100], + [0, 100], + [100, 0]], + "w": [1, 1, 1, 1]}, + "matchCount": 4 + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "2", + "qId": "2-1", + "matches": { + "p": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "q": [ + [0, 0], + [100, 100], + [0, 100], + [100, 0] + ], + "w": [1, 1, 1, 1], + }, + "matchCount": 4 + }, + { + "pGroupId": "0", + "pId": "0-1", + "qGroupId": "0", + "qId": "0-2", + "matches": { + "p": [ + [100, 100], + [100, 0] + ], + "q": [ + [0, 100], + [0, 0] + ], + "w": [1, 1], + }, + "matchCount": 2 + } + ] + + +@pytest.fixture(scope="module") +def match(matches): + return matches[1] + + +def set_tuple_subscript(obj, tuple_subscript, value): + o = obj + for s in tuple_subscript[:-1]: + o = o.__getitem__(s) + return o.__setitem__(tuple_subscript[-1], value) + + +def copymatch_and_compare(input_match, tuple_subscript_to_change, + target_value=None): + orig_match = copy.deepcopy(input_match) + copied_match = renderapi.pointmatch.copy_match_explicit(orig_match) + set_tuple_subscript(orig_match, tuple_subscript_to_change, target_value) + return orig_match == copied_match + + +def test_copy_match(match): + # test for top level keys + for k in (match.keys() - {"matches"}): + assert not copymatch_and_compare(match, (k,)) + # test for nested values in matches + for subtup in (("matches", "p", 0, 0), + ("matches", "q", 0, 0), + ("matches", "w", 0)): + assert not copymatch_and_compare(match, subtup) + + +def test_copy_matches(matches): + assert all([i == j for i, j in zip( + matches, renderapi.pointmatch.copy_matches_explicit(matches))]) From d0947b297cff2c26a9e6ed354c2690b94cb1064e Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Fri, 12 Feb 2021 15:24:42 -0800 Subject: [PATCH 741/766] test_pointmatch: use six --- test/test_pointmatch.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_pointmatch.py b/test/test_pointmatch.py index 11246153..53a0e355 100644 --- a/test/test_pointmatch.py +++ b/test/test_pointmatch.py @@ -1,6 +1,9 @@ import copy -import renderapi + import pytest +import six + +import renderapi @pytest.fixture(scope="module") @@ -87,7 +90,7 @@ def copymatch_and_compare(input_match, tuple_subscript_to_change, def test_copy_match(match): # test for top level keys - for k in (match.keys() - {"matches"}): + for k in (six.viewkeys(match) - {"matches"}): assert not copymatch_and_compare(match, (k,)) # test for nested values in matches for subtup in (("matches", "p", 0, 0), From 38af01fb9541f5c3743fb7c9883a440617c9753f Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Tue, 16 Feb 2021 12:55:56 -0800 Subject: [PATCH 742/766] pointmatch: function to swap p and q in pairs --- renderapi/pointmatch.py | 32 ++++++++++++++++++++++++++++++++ test/test_pointmatch.py | 10 ++++++++++ 2 files changed, 42 insertions(+) diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 85f9ce5c..8be05b6a 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -56,6 +56,38 @@ def copy_matches_explicit(matches): return [copy_match_explicit(match) for match in matches] +def swap_matchpair(match, copy=True): + """ + + Parameters + ---------- + match : dict + match dictionary to swap p->q,q->p + copy : bool + whether to return a copy which, when modified, does not change original + + Returns + ------- + new_match : dict + match dictionary with "p" and "q" swapped + """ + updated_d = { + "pId": match["qId"], + "qId": match["pId"], + "qGroupId": match["pGroupId"], + "pGroupId": match["qGroupId"], + "matches": { + "p": match["matches"]["q"], + "q": match["matches"]["p"], + "w": match["matches"]["w"] # include weight because shallow swap + } + } + + new_match = {k: updated_d.get(k, v) for k, v in match.items()} + + return (copy_match_explicit(new_match) if copy else new_match) + + @renderaccess def get_matchcollection_owners(host=None, port=None, session=requests.session(), diff --git a/test/test_pointmatch.py b/test/test_pointmatch.py index 53a0e355..3239e2a7 100644 --- a/test/test_pointmatch.py +++ b/test/test_pointmatch.py @@ -102,3 +102,13 @@ def test_copy_match(match): def test_copy_matches(matches): assert all([i == j for i, j in zip( matches, renderapi.pointmatch.copy_matches_explicit(matches))]) + + +@pytest.mark.parametrize("do_copy", [True, False]) +def test_swap_matchpair(matches, do_copy): + for match, swapped_match in zip( + matches, (renderapi.pointmatch.swap_matchpair( + match, copy=do_copy) for match in matches)): + assert match != swapped_match + assert match == renderapi.pointmatch.swap_matchpair( + swapped_match, do_copy) From b9b8d0e5d72403fe7248f583f472f0964a7af577 Mon Sep 17 00:00:00 2001 From: Forrest Collman Date: Thu, 10 Jun 2021 09:40:26 -0700 Subject: [PATCH 743/766] Update README.rst --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 83ed3ece..cc9493a0 100644 --- a/README.rst +++ b/README.rst @@ -32,4 +32,8 @@ Documentation ############# http://render-python.readthedocs.io/en/latest/ +Government Sponsorship +###################### +Supported by the Intelligence Advanced Research Projects Activity (IARPA) via Department of Interior / Interior Business Center (DoI/IBC) contract number D16PC00004. The U.S. Government is authorized to reproduce and distribute reprints for Governmental purposes notwithstanding any copyright annotation thereon. Disclaimer: The views and conclusions contained herein are those of the authors and should not be interpreted as necessarily representing the official policies or endorsements, either expressed or implied, of IARPA, DoI/IBC, or the U.S. Government. + .. _render : From 928c0c5f376c6115f236e0f29479e934666fe225 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 24 Jun 2021 10:00:11 -0400 Subject: [PATCH 744/766] fixed minor bug where minute was being used as month for the default create timestamp when creating a stack --- renderapi/stack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 71628d04..7386f395 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -50,7 +50,7 @@ def __init__(self, cycleNumber=None, cycleStepNumber=None, self.stackResolutionZ = stackResolutionZ self.mipmapPathBuilder = mipmapPathBuilder self.materializedBoxRootPath = materializedBoxRootPath - self.createTimestamp = (strftime('%Y-%M-%dT%H:%M:%S.00Z') if + self.createTimestamp = (strftime('%Y-%m-%dT%H:%M:%S.00Z') if createTimestamp is None else createTimestamp) self.versionNotes = versionNotes From 4bca171ce4cb7a070f090764241dba27ae82f353 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Wed, 15 Jun 2022 13:09:32 -0700 Subject: [PATCH 745/766] LICENSE: license as BSD2 --- LICENSE | 46 +++++++++++++++++++--------------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/LICENSE b/LICENSE index 8dc056db..00a05dad 100644 --- a/LICENSE +++ b/LICENSE @@ -1,33 +1,25 @@ -Allen Institute Software License – This software license is the 2-clause BSD -license plus a third clause that prohibits redistribution and use for -commercial purposes without further permission. +BSD 2-Clause License -Copyright © 2019. Allen Institute. All rights reserved. +Copyright (c) 2022, Allen Institute +All rights reserved. -Redistribution and use in source and binary forms, with or without +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. -3. Redistributions and use for commercial purposes are not permitted without -the Allen Institute’s written permission. For purposes of this license, -commercial purposes are the incorporation of the Allen Institute's software -into anything for which you will charge fees or other compensation or use of -the software to perform a commercial service for a third party. Contact -terms@alleninstitute.org for commercial licensing opportunities. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 1576a573fe2ae6441f1edd5e0e138c371d315a52 Mon Sep 17 00:00:00 2001 From: martinschorb <35071867+martinschorb@users.noreply.github.com> Date: Mon, 11 Jul 2022 16:49:32 +0200 Subject: [PATCH 746/766] pass `resolutionUnit` parameter makes the `resolutionUnit` parameter (https://github.com/saalfeldlab/render/commit/0448e3d7e8229b4837988a577411109c4e6b0b92) available. --- renderapi/client/client_calls.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 68adc2f5..f500fb3a 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -346,10 +346,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, maxIntensity=None, minIntensity=None, bounds=None, format=None, channel=None, customOutputFolder=None, customSubFolder=None, padFileNamesWithZeros=None, - doFilter=None, fillWithNoise=None, imageType=None, - subprocess_mode=None, host=None, port=None, owner=None, - project=None, client_script=None, memGB=None, - render=None, **kwargs): + resolutionUnit=None, doFilter=None, fillWithNoise=None, + imageType=None, subprocess_mode=None, host=None, + port=None, owner=None, project=None, + client_script=None, memGB=None, render=None, + **kwargs): """run RenderSectionClient.java Parameters @@ -379,6 +380,9 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, folder to save all images in under outputFolder (overrides default of none) padFileNamesWithZeros: bool whether to pad file names with zeros to make sortable + resolutionUnit: str + if format is tiff and unit is specified (e.g. as 'nm'), include resolution data + in rendered tiff headers. imageType: int 8,16,24 to specify what kind of image type to save doFilter : str @@ -418,9 +422,11 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, get_param(customOutputFolder, '--customOutputFolder') + get_param(imageType, '--imageType') + get_param(channel, '--channels') + - get_param(customSubFolder, '--customSubFolder') + + get_param(customSubFolder, '--customSubFolder') + get_param(padFileNamesWithZeros, '--padFileNamesWithZeros') + + get_param(resolutionUnit, '--resolutionUnit') + bound_param + zs) + call_run_ws_client('org.janelia.render.client.RenderSectionClient', memGB=memGB, client_script=client_script, subprocess_mode=subprocess_mode, add_args=argvs, From 785acd6c35fd4ff86d8caf9b91f22bf510330808 Mon Sep 17 00:00:00 2001 From: thopp-tudelft Date: Tue, 1 Nov 2022 14:48:49 +0100 Subject: [PATCH 747/766] use collections abstract base classes from submodule with python 3.10 the abstract base classes Iterable and MutableMapping are no longer duplicated in the base module, this use has been deprecated since python 3.4 these are now imported from collections.abc instead --- renderapi/image_pyramid.py | 2 +- renderapi/transform/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/image_pyramid.py b/renderapi/image_pyramid.py index ac2ca8e9..e92b3e96 100644 --- a/renderapi/image_pyramid.py +++ b/renderapi/image_pyramid.py @@ -1,4 +1,4 @@ -from collections import MutableMapping +from collections.abc import MutableMapping from .errors import RenderError import logging from .utils import NullHandler diff --git a/renderapi/transform/utils.py b/renderapi/transform/utils.py index 15b0ce5c..9c33b0f1 100644 --- a/renderapi/transform/utils.py +++ b/renderapi/transform/utils.py @@ -1,4 +1,4 @@ -from collections import Iterable +from collections.abc import Iterable from renderapi.errors import RenderError from .leaf import AffineModel, Polynomial2DTransform from .transform import TransformList, ReferenceTransform From 4315f249ee524da817d05ae8301e7282c8a77aa6 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Fri, 14 Jul 2023 10:37:12 -0700 Subject: [PATCH 748/766] actions: add publish on release action --- .github/workflows/release_publish.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/release_publish.yml diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml new file mode 100644 index 00000000..fd83a7b9 --- /dev/null +++ b/.github/workflows/release_publish.yml @@ -0,0 +1,21 @@ +name: release_publish +on: + # workflow_dispatch: + release: + types: [published] +jobs: + pypi_publish: + name: Deploy to pypi + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v2 + + - name: Build + run: python setup.py sdist bdist_wheel + + - name: Publish to Pypi + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From 3cec2a4e7aeea7a2be3de75c60b080bfc87e963f Mon Sep 17 00:00:00 2001 From: Martin Schorb Date: Wed, 6 Dec 2023 09:31:08 +0100 Subject: [PATCH 749/766] pass `convertToGray` parameter pass `resolutionUnit` parameter makes the `resolutionUnit` parameter (https://github.com/saalfeldlab/render/commit/ea46e84ded7230788d08e5820c29f11fc4ae70f7) available. --- renderapi/client/client_calls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 0b709e8c..069ab1fa 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -416,7 +416,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, format=None, channel=None, customOutputFolder=None, customSubFolder=None, padFileNamesWithZeros=None, resolutionUnit=None, doFilter=None, fillWithNoise=None, - imageType=None, subprocess_mode=None, host=None, + convertToGray=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): @@ -452,8 +452,8 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, resolutionUnit: str if format is tiff and unit is specified (e.g. as 'nm'), include resolution data in rendered tiff headers. - imageType: int - 8,16,24 to specify what kind of image type to save + convertToGray: str + string representing java boolean for whether to save output as 8bit uint doFilter : str string representing java boolean for whether to render image with default filter (varies with render version) @@ -489,7 +489,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, get_param(maxIntensity, '--maxIntensity') + get_param(fillWithNoise, '--fillWithNoise') + get_param(customOutputFolder, '--customOutputFolder') + - get_param(imageType, '--imageType') + + get_param(convertToGray, '--convertToGray') + get_param(channel, '--channels') + get_param(customSubFolder, '--customSubFolder') + get_param(padFileNamesWithZeros, '--padFileNamesWithZeros') + From b504ab98d98c1496c0f41c41d4082c354257f2a9 Mon Sep 17 00:00:00 2001 From: Martin Schorb Date: Wed, 6 Dec 2023 09:31:08 +0100 Subject: [PATCH 750/766] pass `convertToGray` parameter makes the `convertToGray` parameter (https://github.com/saalfeldlab/render/commit/ea46e84ded7230788d08e5820c29f11fc4ae70f7) available. --- renderapi/client/client_calls.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 0b709e8c..069ab1fa 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -416,7 +416,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, format=None, channel=None, customOutputFolder=None, customSubFolder=None, padFileNamesWithZeros=None, resolutionUnit=None, doFilter=None, fillWithNoise=None, - imageType=None, subprocess_mode=None, host=None, + convertToGray=None, subprocess_mode=None, host=None, port=None, owner=None, project=None, client_script=None, memGB=None, render=None, **kwargs): @@ -452,8 +452,8 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, resolutionUnit: str if format is tiff and unit is specified (e.g. as 'nm'), include resolution data in rendered tiff headers. - imageType: int - 8,16,24 to specify what kind of image type to save + convertToGray: str + string representing java boolean for whether to save output as 8bit uint doFilter : str string representing java boolean for whether to render image with default filter (varies with render version) @@ -489,7 +489,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, get_param(maxIntensity, '--maxIntensity') + get_param(fillWithNoise, '--fillWithNoise') + get_param(customOutputFolder, '--customOutputFolder') + - get_param(imageType, '--imageType') + + get_param(convertToGray, '--convertToGray') + get_param(channel, '--channels') + get_param(customSubFolder, '--customSubFolder') + get_param(padFileNamesWithZeros, '--padFileNamesWithZeros') + From 331d7086451e9b57fc6b07036c7a75ff72b6d881 Mon Sep 17 00:00:00 2001 From: Martin Schorb Date: Wed, 6 Dec 2023 09:49:39 +0100 Subject: [PATCH 751/766] fix `convertToGray` parameter (bool without value) --- renderapi/client/client_calls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 069ab1fa..abb959e8 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -489,7 +489,7 @@ def renderSectionClient(stack, rootDirectory, zs, scale=None, get_param(maxIntensity, '--maxIntensity') + get_param(fillWithNoise, '--fillWithNoise') + get_param(customOutputFolder, '--customOutputFolder') + - get_param(convertToGray, '--convertToGray') + + (['--convertToGray'] if convertToGray else []) + get_param(channel, '--channels') + get_param(customSubFolder, '--customSubFolder') + get_param(padFileNamesWithZeros, '--padFileNamesWithZeros') + From 7f7808f4c1282f004da6b162f2f1fc444c253cc6 Mon Sep 17 00:00:00 2001 From: Martin Schorb <35071867+martinschorb@users.noreply.github.com> Date: Thu, 7 Dec 2023 13:18:27 +0100 Subject: [PATCH 752/766] Update tilepair client to match asap:develop Make client call compatible with the way Render handles the `useRowColPositions` parameter. ref: https://github.com/AllenInstitute/asap-modules/pull/252 --- renderapi/client/client_calls.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/renderapi/client/client_calls.py b/renderapi/client/client_calls.py index 0b709e8c..1f10de67 100644 --- a/renderapi/client/client_calls.py +++ b/renderapi/client/client_calls.py @@ -243,8 +243,7 @@ def tilePairClient(stack, minz, maxz, outjson=None, delete_json=False, '--excludeSameSectionNeighbors') + get_param(excludePairsInMatchCollection, '--excludePairsInMatchCollection') + - get_param(useRowColPositions, - '--useRowColPositions') + + (['--useRowColPositions'] if useRowColPositions else []) + get_param(existingMatchOwner, '--existingMatchOwner') + get_param(minExistingMatchCount, From 255b02caabbb5b48058878d9aac831f8780d6096 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Wed, 19 Mar 2025 12:21:41 -0700 Subject: [PATCH 753/766] pyproject: setup pyproject with pixi --- .gitignore | 3 +- pyproject.toml | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 42 ---------------------- 3 files changed, 96 insertions(+), 43 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/.gitignore b/.gitignore index b3e71372..c7196dd1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ htmlcov/ .pytest_cache .vscode .idea/ -*.iml \ No newline at end of file +*.iml +.pixi/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..258c31ee --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,94 @@ +[project] +name = "render-python" +description = "python API client to interact with render databases (see https://github.com/saalfeldlab/render)" +readme = "README.rst" +license = {file = "LICENSE"} +authors = [ + {name = "Forrest Collman"}, + {name = "Russel Torres"}, + {name = "Eric Perlman"}, + {name = "Sharmi Seshamani"} +] +maintainers = [ + {name = "Russel Torres"} +] +requires-python = ">3.8,<3.12" +dynamic = ["version"] +dependencies = [ + "requests", + "scipy", + "numpy", + "pillow", + "sphinxcontrib-napoleon", + "decorator", + "six" +] + +[project.urls] +Repository = "https://github.com/AllenInstitute/render-python.git" + +[project.optional-dependencies] +test = [ + "coverage>=4.1", + "mock>=2.0.0", + "pep8>=1.7.0", + # "pytest>4.6,<5.0", + "pytest", + "pytest-cov>=2.2.1", + "pytest-pep8>=1.0.6", + "pytest-xdist>=1.14", + "flake8>=3.0.4", + "pylint>=1.5.4", + "jinja2", + "ujson" +] +pypi-build = [ + "build", + "twine" +] + +[build-system] +requires = ["setuptools>=64", "setuptools_scm>=8"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +py-modules = [ + "renderapi" +] + +[tool.setuptools_scm] +version_file = "renderapi/_version.py" + +# some downgrades from pixi's expectations +[tool.pixi.system-requirements] +linux = "5.0.0" +libc = "2.27" + +[tool.pixi.project] +channels = ["conda-forge"] +platforms = ["linux-64"] + +[tool.pixi.pypi-dependencies] +render-python = { path = ".", editable = true } + +# conda-enabled features +[tool.pixi.feature.pixi-base.dependencies] +numpy = "*" +scipy = "*" + +[tool.pixi.environments] +test = ["test"] +conda = ["pixi-base"] +conda-test = ["pixi-base", "test"] +build = ["pypi-build"] + +[tool.coverage.run] +omit = ["integration_tests/*", "test/*"] + +[tool.pixi.feature.test.tasks] +test = "pytest --cov --cov-report=xml --junitxml=test-reports/test.xml" + +[tool.pixi.feature.pypi-build.tasks] +pypi-build = "python -m build" +pypi-test-upload = "python -m twine upload --repository testpypi dist/*" +pypi-test-deploy = "python -m build && python -m twine upload --repository testpypi dist/*" diff --git a/setup.py b/setup.py deleted file mode 100644 index 5a8e826c..00000000 --- a/setup.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -from setuptools import setup, find_packages -import sys -from setuptools.command.test import test as TestCommand - - -class PyTest(TestCommand): - user_options = [('pytest-args=', 'a', "Arguments to pass to pytest")] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.pytest_args = "" - - def run_tests(self): - # import here, cause outside the eggs aren't loaded - import shlex - import pytest - self.pytest_args += " --cov=renderapi --cov-report html "\ - "--junitxml=test-reports/test.xml" - - errno = pytest.main(shlex.split(self.pytest_args)) - sys.exit(errno) - - -with open('test_requirements.txt', 'r') as f: - test_required = f.read().splitlines() - -with open('requirements.txt', 'r') as f: - required = f.read().splitlines() - -setup(name='render-python', - use_scm_version=True, - description=' a python API to interact via python with render ' - 'databases see https://github.com/saalfeldlab/render', - author='Forrest Collman, Russel Torres, Eric Perlman, Sharmi Seshamani', - author_email='forrest.collman@gmail.com', - url='https://github.com/fcollman/render-python', - packages=find_packages(), - setup_requires=['setuptools_scm'], - install_requires=required, - tests_require=test_required, - cmdclass={'test': PyTest},) From 21276aac4e6a9449c33fb1b6b1d57cd704eb2d67 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Wed, 19 Mar 2025 15:31:37 -0700 Subject: [PATCH 754/766] test_transform: update to numpy.ptp and numpy.float64 for numpy2 --- test/test_transform.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/test_transform.py b/test/test_transform.py index 57c3135e..e977c03f 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -65,8 +65,8 @@ def test_fail_bad_tform(): def test_similarity_rot_90(): am = renderapi.transform.SimilarityModel() # setup a 90 degree clockwise rotation - points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float) - points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float) + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float64) + points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float64) am.estimate(points_in, points_out) assert(np.abs(am.scale[0] - 1.0) < .00001) @@ -98,8 +98,8 @@ def test_similarity_rot_90(): def test_affine_fail(): am = renderapi.transform.AffineModel() # setup a 90 degree clockwise rotation - points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float) - points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float) + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float64) + points_out = np.array([[0, 0], [1, 0], [0, -1], [1, -1]], np.float64) # catch error with pytest.raises(renderapi.errors.EstimationError): am.estimate(points_in, points_out[0:-2, :]) @@ -525,7 +525,7 @@ def test_non_linear_transform(): "9.885988268659214E18 1.1051253229808925E19 1.6002173610912907E19 " "0.0 2048 2048 "), transformId="testing") - ticks = np.arange(0, 2048, 64, np.float) + ticks = np.arange(0, 2048, 64, np.float64) xx, yy = np.meshgrid(ticks, ticks) x = np.ravel(xx).T y = np.ravel(yy).T @@ -626,7 +626,7 @@ def referenced_tilespecs_and_transforms(): def test_estimate_dstpoints_reference(referenced_tilespecs_and_transforms): tilespecs, transforms = referenced_tilespecs_and_transforms - ticks = np.arange(0, 2048, 64, np.float) + ticks = np.arange(0, 2048, 64, np.float64) xx, yy = np.meshgrid(ticks, ticks) x = np.ravel(xx).T y = np.ravel(yy).T @@ -643,7 +643,7 @@ def test_estimate_dstpoints_reference(referenced_tilespecs_and_transforms): def test_fail_convert_points(): - points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float) + points_in = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], np.float64) with pytest.raises(renderapi.errors.ConversionError): renderapi.transform.AffineModel.convert_to_point_vector(points_in.T) @@ -928,8 +928,8 @@ def thinplate_estimate_nojson(computeAffine=True): def poly2d(src): dst = np.zeros_like(src) - dx = (src[:, 0] - src[:, 0].mean()) / (src[:, 0].ptp()) - dy = (src[:, 1] - src[:, 1].mean()) / (src[:, 1].ptp()) + dx = (src[:, 0] - src[:, 0].mean()) / (np.ptp(src[:, 0])) + dy = (src[:, 1] - src[:, 1].mean()) / (np.ptp(src[:, 1])) dst[:, 0] = src[:, 0] + 0.5 * dx * dy + 7.0 * dx * dy * dy dst[:, 1] = src[:, 1] + 0.7 * dy * dy - 4.0 * dx * dx * dy a = np.array([[1.1, -0.04], [-0.02, 0.95]]) @@ -1069,7 +1069,7 @@ def test_scale_thinplate(factor, computeAffine, preserve_srcPts, ngrid): dst_scaled = scaled_tform.tform(src * factor) delta = np.linalg.norm(dst_scaled - dst * factor, axis=1) - scale = dst_scaled.ptp(axis=0).mean() + scale = np.ptp(dst_scaled, axis=0).mean() tol = 1e-4 if ngrid == 5: tol = 1e-3 From 9e342c4d9f13ce93b17ba9d56aab94aaf5f66570 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 20 Mar 2025 00:06:12 -0700 Subject: [PATCH 755/766] test: use imagepyramid in test_imagepyramid --- test/test_imagepyramid.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/test_imagepyramid.py b/test/test_imagepyramid.py index d9a35892..683948e1 100644 --- a/test/test_imagepyramid.py +++ b/test/test_imagepyramid.py @@ -74,18 +74,20 @@ def test_pyramid_deserialize(): def test_mipmaplevel_deprecated(): - mml = image_pyramid.MipMapLevel(0, - imageUrl=image_filename, - maskUrl=mask_filename) + with pytest.deprecated_call(): + mml = image_pyramid.MipMapLevel( + 0, + imageUrl=image_filename, + maskUrl=mask_filename) assert(mml['imageUrl'] == image_filename) assert(mml['maskUrl'] == mask_filename) with pytest.raises(KeyError): mml['not_a_key'] assert(mml == mml) - - mml2 = image_pyramid.MipMapLevel(0, - imageUrl=image_filename) + with pytest.deprecated_call(): + mml2 = image_pyramid.MipMapLevel(0, + imageUrl=image_filename) assert(mml != mml2) assert(len([k for k, v in mml]) == 2) From 50db21226bb54d85fa55a013c99a6047741205ed Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 20 Mar 2025 00:06:38 -0700 Subject: [PATCH 756/766] test: update to imagepyramid --- integration_tests/test_stack_integrated.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_stack_integrated.py b/integration_tests/test_stack_integrated.py index 8fba2272..61345553 100644 --- a/integration_tests/test_stack_integrated.py +++ b/integration_tests/test_stack_integrated.py @@ -32,7 +32,8 @@ def render(): @pytest.fixture def simpletilespec(): - mml = renderapi.image_pyramid.MipMapLevel(0, '/not/a/path.jpg') + mm = renderapi.image_pyramid.MipMap(imageUrl='file://not/a/path.jpg') + ip = renderapi.image_pyramid.ImagePyramid({0: mm}) tform = renderapi.transform.AffineModel(labels=['simple']) layout = renderapi.tilespec.Layout(sectionId="section0", scopeId="testscope", @@ -45,7 +46,7 @@ def simpletilespec(): ts = renderapi.tilespec.TileSpec(tileId="1000", width=2048, height=2048, - mipMapLevels=[mml], + imagePyramid=ip, z=0, tforms=[tform], layout=layout) From 5d7e85bc88990ac214291df63b6195391e7d5b3e Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 20 Mar 2025 00:07:58 -0700 Subject: [PATCH 757/766] transform: update to new numpy.linalg error method --- renderapi/transform/leaf/polynomial_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renderapi/transform/leaf/polynomial_models.py b/renderapi/transform/leaf/polynomial_models.py index cf740aeb..24080a95 100644 --- a/renderapi/transform/leaf/polynomial_models.py +++ b/renderapi/transform/leaf/polynomial_models.py @@ -10,8 +10,8 @@ logger.info(e) logger.info('scipy-based linalg may or may not lead ' 'to better parameter fitting') - from numpy.linalg import svd - from numpy.linalg.linalg import LinAlgError + from numpy.linalg import svd, LinAlgError + __all__ = [ 'Polynomial2DTransform', 'NonLinearCoordinateTransform', 'NonLinearTransform', 'LensCorrection'] From 1e596aa33858f6e68edb193f6450b620d264a8c9 Mon Sep 17 00:00:00 2001 From: thopp-tudelft Date: Fri, 24 Oct 2025 14:22:48 +0200 Subject: [PATCH 758/766] remove session object from being created on import --- renderapi/coordinate.py | 16 ++++++++-------- renderapi/image.py | 14 +++++++------- renderapi/pointmatch.py | 28 ++++++++++++++-------------- renderapi/render.py | 21 ++++++++++++++------- renderapi/resolvedtiles.py | 4 ++-- renderapi/stack.py | 36 ++++++++++++++++++------------------ renderapi/tilespec.py | 14 +++++++------- 7 files changed, 70 insertions(+), 63 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index e022572c..9edf5727 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -20,7 +20,7 @@ @renderaccess def world_to_local_coordinates(stack, z, x, y, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """maps an world x,y,z coordinate in stack to a local coordinate Parameters @@ -64,7 +64,7 @@ def world_to_local_coordinates(stack, z, x, y, host=None, @renderaccess def local_to_world_coordinates(stack, tileId, x, y, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """convert coordinate from local to world with webservice request @@ -110,7 +110,7 @@ def local_to_world_coordinates(stack, tileId, x, y, def world_to_local_coordinates_batch(stack, d, z, host=None, port=None, owner=None, project=None, execute_local=False, - session=requests.session(), + session=None, render=None, **kwargs): """convert coordinate parameters from world to local @@ -174,7 +174,7 @@ def world_to_local_coordinates_batch(stack, d, z, host=None, @renderaccess def local_to_world_coordinates_batch(stack, d, z, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """convert coordinate parameters from local to world @@ -302,7 +302,7 @@ def unpackage_world_to_local_point_match_from_json(json_answer, tileId): # def old_world_to_local_coordinates_array(stack, dataarray, tileId, z=0, # host=None, port=None, # owner=None, project=None, -# session=requests.session(), +# session=None, # render=None, **kwargs): # '''''' @@ -359,7 +359,7 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, owner=None, project=None, client_script=None, doClientSide=False, number_of_threads=20, - session=requests.session(), **kwargs): + session=None, **kwargs): """map world to local coordinates using numpy array Parameters @@ -403,7 +403,7 @@ def world_to_local_coordinates_array(stack, dataarray, tileId, z, # def old_local_to_world_coordinates_array(stack, dataarray, tileId, z=0, # host=None, port=None, # owner=None, project=None, -# session=requests.session(), +# session=None, # render=None, **kwargs): # '''''' # request_url = format_preamble( @@ -439,7 +439,7 @@ def local_to_world_coordinates_array(stack, dataarray, tileId, z, owner=None, project=None, client_script=None, doClientSide=False, number_of_threads=20, - session=requests.session(), **kwargs): + session=None, **kwargs): """map local to world coordinates using numpy array Parameters diff --git a/renderapi/image.py b/renderapi/image.py index df30fd0b..6230f94d 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -36,7 +36,7 @@ def get_bb_renderparams(stack, z, x, y, width, height, scale=1.0, binaryMask=None, filter=None, filterListName=None, convertToGray=None, excludeMask=None, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): request_url = format_preamble( @@ -63,7 +63,7 @@ def get_bb_image(stack, z, x, y, width, height, scale=1.0, minIntensity=None, maxIntensity=None, binaryMask=None, filter=None, maxTileSpecsToRender=None, host=None, port=None, owner=None, project=None, - img_format=None, session=requests.session(), + img_format=None, session=None, render=None, **kwargs): """render image from a bounding box defined in xy and return numpy array: @@ -150,7 +150,7 @@ def get_tile_renderparams( filter=None, filterListName=None, excludeMask=None, convertToGray=None, binaryMask=None, host=None, port=None, owner=None, project=None, img_format=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + \ "/tile/%s/render-parameters" % ( @@ -182,7 +182,7 @@ def get_tile_image_data(stack, tileId, channel=None, normalizeForMatching=True, minIntensity=None, maxIntensity=None, filter=None, host=None, port=None, owner=None, project=None, img_format=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """render image from a tile with all transforms and return numpy array :func:`renderapi.render.renderaccess` decorated function @@ -273,7 +273,7 @@ def get_section_renderparams(stack, z, binaryMask=None, channel=None, filterListName=None, minIntensity=None, maxIntensity=None, scale=None, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): request_url = format_preamble( host, port, owner, project, stack) + "/z/{}/render-parameters".format( @@ -299,7 +299,7 @@ def get_section_image(stack, z, scale=1.0, channel=None, filter=False, maxTileSpecsToRender=None, img_format=None, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """render an section of image @@ -371,7 +371,7 @@ def get_section_image(stack, z, scale=1.0, channel=None, @renderaccess def get_renderparameters_image(renderparams, img_format=None, host=None, port=None, owner=None, - session=requests.session(), + session=None, render=None, **kwargs): try: image_ext = IMAGE_FORMATS[img_format] diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 8be05b6a..9ad5fb9d 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -90,7 +90,7 @@ def swap_matchpair(match, copy=True): @renderaccess def get_matchcollection_owners(host=None, port=None, - session=requests.session(), + session=None, render=None, **kwargs): """get all the matchCollection owners @@ -121,7 +121,7 @@ def get_matchcollection_owners(host=None, port=None, @renderaccess def get_matchcollections(owner=None, host=None, port=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """get all the matchCollections owned by owner :func:`renderapi.render.renderaccess` decorated function @@ -153,7 +153,7 @@ def get_matchcollections(owner=None, host=None, port=None, @renderaccess def get_match_groupIds(matchCollection, owner=None, host=None, - port=None, session=requests.session(), + port=None, session=None, render=None, **kwargs): """get all the groupIds in a matchCollection @@ -190,7 +190,7 @@ def get_match_groupIds(matchCollection, owner=None, host=None, def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, stream=True, owner=None, host=None, - port=None, session=requests.session(), + port=None, session=None, render=None, **kwargs): """get all the matches outside a groupId in a matchCollection returns all matches where pGroupId == groupId and qGroupId != groupId @@ -237,7 +237,7 @@ def get_matches_outside_group(matchCollection, groupId, mergeCollections=None, def get_matches_within_group(matchCollection, groupId, mergeCollections=None, stream=True, owner=None, host=None, port=None, - session=requests.session(), + session=None, render=None, **kwargs): """get all the matches within a groupId in a matchCollection returns all matches where pGroupId == groupId and qGroupId == groupId @@ -286,7 +286,7 @@ def get_matches_from_group_to_group(matchCollection, pgroup, qgroup, mergeCollections=None, stream=True, render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): + session=None, **kwargs): """get all the matches between two specific groups returns all matches where pgroup == pGroupId and qgroup == qGroupId OR pgroup == qGroupId and qgroup == pGroupId @@ -360,7 +360,7 @@ def get_matches_from_tile_to_tile(matchCollection, pgroup, pid, qgroup, qid, mergeCollections=None, render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): + session=None, **kwargs): """get all the matches between two specific tiles returns all matches where pgroup == pGroupId and pid=pId and qgroup == qGroupId and qid == qId @@ -415,7 +415,7 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, stream=True, render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): + session=None, **kwargs): """get all the matches from a specific groups returns all matches where pgroup == pGroupId @@ -461,7 +461,7 @@ def get_matches_with_group(matchCollection, pgroup, mergeCollections=None, def get_match_groupIds_from_only(matchCollection, mergeCollections=None, render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): + session=None, **kwargs): """get all the source pGroupIds in a matchCollection :func:`renderapi.render.renderaccess` decorated function @@ -499,7 +499,7 @@ def get_match_groupIds_from_only(matchCollection, mergeCollections=None, def get_match_groupIds_to_only(matchCollection, mergeCollections=None, render=None, owner=None, host=None, port=None, - session=requests.session(), **kwargs): + session=None, **kwargs): """get all the destination qGroupIds in a matchCollection :func:`renderapi.render.renderaccess` decorated function @@ -538,7 +538,7 @@ def get_match_groupIds_to_only(matchCollection, mergeCollections=None, def get_matches_involving_tile(matchCollection, groupId, id, mergeCollections=None, stream=True, owner=None, host=None, port=None, - session=requests.session(), **kwargs): + session=None, **kwargs): """get all the matches involving a specific tile returns all matches where groupId == pGroupId and id == pId OR groupId == qGroupId and id == qId @@ -586,7 +586,7 @@ def get_matches_involving_tile(matchCollection, groupId, id, @renderaccess def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, render=None, owner=None, host=None, - port=None, session=requests.session(), + port=None, session=None, **kwargs): """delete all the matches between two specific groups deletes all matches where (pgroup == pGroupId and qgroup == qGroupId) @@ -631,7 +631,7 @@ def delete_point_matches_between_groups(matchCollection, pGroupId, qGroupId, @renderaccess def import_matches(matchCollection, data, owner=None, host=None, port=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """import matches into render database :func:`renderapi.render.renderaccess` decorated function @@ -664,7 +664,7 @@ def import_matches(matchCollection, data, owner=None, host=None, port=None, @renderaccess def delete_collection(matchCollection, owner=None, host=None, port=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """delete match collection from render database :func:`renderapi.render.renderaccess` decorated function diff --git a/renderapi/render.py b/renderapi/render.py index 0f466901..137dccd3 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -379,6 +379,9 @@ def renderaccess(f, *args, **kwargs): You can if you wish specify any of the arguments, in which case they will not be filled in by the default values, but you don't have to. + The default value for session is None, which will be replaced with a newly + created requests.Session object. + As such, the documentation omits describing the parameters which are natural to expect will be filled in by the renderaccess decorator. @@ -400,13 +403,17 @@ def renderaccess(f, *args, **kwargs): render = kwargs.get('render') if render is not None: if isinstance(render, Render): - return f(*args, **render.make_kwargs(**kwargs)) + kwargs = render.make_kwargs(**kwargs) else: raise ValueError( 'invalid Render object type {} specified!'.format( type(render))) - else: - return f(*args, **kwargs) + + session = kwargs.get("session") + if session is None: + kwargs["session"] = requests.Session() + + return f(*args, **kwargs) def format_baseurl(host, port): @@ -459,7 +466,7 @@ def format_preamble(host, port, owner, project, stack): @renderaccess -def get_owners(host=None, port=None, session=requests.session(), +def get_owners(host=None, port=None, session=None, render=None, **kwargs): """return list of owners across all Projects and Stacks for a render server @@ -488,7 +495,7 @@ def get_owners(host=None, port=None, session=requests.session(), @renderaccess def get_stack_metadata_by_owner(owner=None, host=None, port=None, - session=requests.session(), + session=None, render=None, **kwargs): """return metadata for all stacks belonging to particular owner on render server @@ -517,7 +524,7 @@ def get_stack_metadata_by_owner(owner=None, host=None, port=None, @renderaccess def get_projects_by_owner(owner=None, host=None, port=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """return list of projects belonging to a single owner for render stack :func:`renderaccess` decorated function @@ -545,7 +552,7 @@ def get_projects_by_owner(owner=None, host=None, port=None, @renderaccess def get_stacks_by_owner_project(owner=None, project=None, host=None, - port=None, session=requests.session(), + port=None, session=None, render=None, **kwargs): """return list of stacks belonging to an owner's project on render server diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py index 3f079ea8..9d842156 100644 --- a/renderapi/resolvedtiles.py +++ b/renderapi/resolvedtiles.py @@ -83,7 +83,7 @@ def combine_resolvedtiles(rts_l): def put_tilespecs(stack, resolved_tiles=None, deriveData=True, tilespecs=None, shared_transforms=None, host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """upload resolved tiles to the server :func:`renderapi.render.renderaccess` decorated function @@ -125,7 +125,7 @@ def put_tilespecs(stack, resolved_tiles=None, deriveData=True, @renderaccess def get_resolved_tiles_from_z(stack, z, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """Get a set of ResolvedTiles from a specific z value. Returns a tuple of tilespecs and referenced transforms. diff --git a/renderapi/stack.py b/renderapi/stack.py index 7386f395..103b847d 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -96,7 +96,7 @@ def from_dict(self, d): @renderaccess def set_stack_metadata(stack, sv, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """sets the stack metadata for a stack @@ -126,7 +126,7 @@ def set_stack_metadata(stack, sv, host=None, port=None, owner=None, @renderaccess def get_full_stack_metadata(stack, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """get stack metadata for stack @@ -193,7 +193,7 @@ def get_stack_metadata(*args, **kwargs): @renderaccess def set_stack_state(stack, state='LOADING', host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """ set state of selected stack. @@ -237,7 +237,7 @@ def set_stack_state(stack, state='LOADING', host=None, port=None, @renderaccess def likelyUniqueId(host=None, port=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """return hex-code nearly-unique id from render server :func:`renderapi.render.renderaccess` decorated function @@ -292,7 +292,7 @@ def make_stack_params(host, port, owner, project, stack): @renderaccess def delete_stack(stack, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """deletes a stack from render server @@ -321,7 +321,7 @@ def delete_stack(stack, host=None, port=None, owner=None, @renderaccess def delete_section(stack, z, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """removes a single z from a stack @@ -352,7 +352,7 @@ def delete_section(stack, z, host=None, port=None, owner=None, @renderaccess def delete_tile(stack, tileId, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """ removes a tile from a stack @@ -388,7 +388,7 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, stackResolutionX=None, stackResolutionY=None, stackResolutionZ=None, force_resolution=True, host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """creates a new stack :func:`renderapi.render.renderaccess` decorated function @@ -448,7 +448,7 @@ def create_stack(stack, cycleNumber=None, cycleStepNumber=None, @renderaccess def rename_stack(stack, to_stack, to_project=None, to_owner=None, host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """ :func:`renderapi.render.renderaccess` decorated function @@ -541,7 +541,7 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, @renderaccess def get_z_values_for_stack(stack, project=None, host=None, port=None, - owner=None, session=requests.session(), + owner=None, session=None, render=None, **kwargs): """get a list of z values for which there are tiles in the stack @@ -575,7 +575,7 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, # @renderaccess # def put_resolved_tilespecs(stack, json_dict, host=None, port=None, # owner=None, project=None, -# session=requests.session(), +# session=None, # render=None, **kwargs): # request_url = format_preamble( # host, port, owner, project, stack) + "/resolvedTiles" @@ -585,7 +585,7 @@ def get_z_values_for_stack(stack, project=None, host=None, port=None, @renderaccess def get_bounds_from_z(stack, z, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """get a bounds dictionary for a specific z @@ -620,7 +620,7 @@ def get_bounds_from_z(stack, z, host=None, port=None, owner=None, @renderaccess def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """get bounds of a whole stack :func:`renderapi.render.renderaccess` decorated function @@ -651,7 +651,7 @@ def get_stack_bounds(stack, host=None, port=None, owner=None, project=None, @renderaccess def get_tilebounds_for_z(stack, z, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """returns the bounds for each tile associated with a particular z value @@ -686,7 +686,7 @@ def get_tilebounds_for_z(stack, z, host=None, port=None, owner=None, @renderaccess def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """returns the sectionId associated with a particular z value @@ -726,7 +726,7 @@ def get_sectionId_for_z(stack, z, host=None, port=None, owner=None, @renderaccess def get_stack_sectionData(stack, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """returns information about the sectionIds of each slice in stack @@ -766,7 +766,7 @@ def get_stack_sectionData(stack, host=None, port=None, owner=None, @renderaccess def get_section_z_value(stack, sectionId, host=None, port=None, - owner=None, project=None, session=requests.session(), + owner=None, project=None, session=None, render=None, **kwargs): """get the z value for a specific sectionId (string) @@ -799,7 +799,7 @@ def get_section_z_value(stack, sectionId, host=None, port=None, @renderaccess def get_stack_tileIds(stack, host=None, port=None, owner=None, project=None, - session=requests.session(), render=None, **kwargs): + session=None, render=None, **kwargs): """get tileIds for a stack :func:`renderapi.render.renderaccess` decorated function diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index ba06eda9..7c86202a 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -259,7 +259,7 @@ def from_dict(self, d): @renderaccess def get_tile_spec_renderparameters(stack, tile, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """renderapi call to get the render parameters of a specific tileId @@ -293,7 +293,7 @@ def get_tile_spec_renderparameters(stack, tile, host=None, port=None, @renderaccess def get_tile_spec(stack, tile, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """renderapi call to get a specific tilespec by tileId note that this will return a tilespec with resolved transform references @@ -328,7 +328,7 @@ def get_tile_spec(stack, tile, host=None, port=None, owner=None, @renderaccess def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """renderapi call to get a specific tilespec by tileId note that this will return a tilespec without resolved transform references @@ -362,7 +362,7 @@ def get_tile_spec_raw(stack, tile, host=None, port=None, owner=None, def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, scale=1.0, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """renderapi call to get all tilespec that exist within a 2d bounding box specified with min and max x,y values @@ -409,7 +409,7 @@ def get_tile_specs_from_minmax_box(stack, z, xmin, xmax, ymin, ymax, @renderaccess def get_tile_specs_from_box(stack, z, x, y, width, height, scale=1.0, host=None, port=None, owner=None, - project=None, session=requests.session(), + project=None, session=None, render=None, **kwargs): """renderapi call to get all tilespec that exist within a 2d bounding box specified with min x,y values and width, height @@ -455,7 +455,7 @@ def get_tile_specs_from_box(stack, z, x, y, width, height, @renderaccess def get_tile_specs_from_z(stack, z, host=None, port=None, - owner=None, project=None, session=requests.session(), + owner=None, project=None, session=None, render=None, **kwargs): """Get all TileSpecs in a specific z values. Returns referenced transforms. @@ -492,7 +492,7 @@ def get_tile_specs_from_z(stack, z, host=None, port=None, @renderaccess def get_tile_specs_from_stack(stack, host=None, port=None, owner=None, project=None, - session=requests.session(), + session=None, render=None, **kwargs): """get flat list of tilespecs for stack using i for sl in l for i in sl From c89959ea13bdc03c8d4e46e5acfd21fc61b077d4 Mon Sep 17 00:00:00 2001 From: thopp-tudelft Date: Fri, 24 Oct 2025 18:03:24 +0200 Subject: [PATCH 759/766] remove now unused imports of requests --- renderapi/coordinate.py | 1 - renderapi/image.py | 1 - renderapi/pointmatch.py | 1 - renderapi/resolvedtiles.py | 1 - renderapi/tilespec.py | 6 +++--- 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/renderapi/coordinate.py b/renderapi/coordinate.py index 9edf5727..beeeced6 100644 --- a/renderapi/coordinate.py +++ b/renderapi/coordinate.py @@ -6,7 +6,6 @@ from .utils import NullHandler, renderdumps, renderdump, get_json from .client import coordinateClient from .errors import RenderError -import requests import json import numpy as np import logging diff --git a/renderapi/image.py b/renderapi/image.py index 6230f94d..76253b32 100644 --- a/renderapi/image.py +++ b/renderapi/image.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import io -import requests from PIL import Image import numpy as np import logging diff --git a/renderapi/pointmatch.py b/renderapi/pointmatch.py index 9ad5fb9d..6e478c28 100644 --- a/renderapi/pointmatch.py +++ b/renderapi/pointmatch.py @@ -2,7 +2,6 @@ ''' Point Match APIs ''' -import requests import logging from .render import format_baseurl, renderaccess from .utils import NullHandler, get_json, put_json, rest_delete diff --git a/renderapi/resolvedtiles.py b/renderapi/resolvedtiles.py index 9d842156..41ef7f43 100644 --- a/renderapi/resolvedtiles.py +++ b/renderapi/resolvedtiles.py @@ -5,7 +5,6 @@ from .render import format_preamble, renderaccess from .errors import RenderError import logging -import requests logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) diff --git a/renderapi/tilespec.py b/renderapi/tilespec.py index 7c86202a..e2fabe43 100644 --- a/renderapi/tilespec.py +++ b/renderapi/tilespec.py @@ -1,6 +1,5 @@ #!/usr/bin/env python import logging -import requests import numpy as np from .render import format_preamble, renderaccess from .utils import NullHandler, get_json @@ -68,8 +67,9 @@ class TileSpec: def __init__(self, tileId=None, z=None, width=None, height=None, imageUrl=None, maskUrl=None, minint=0, maxint=65535, layout=None, tforms=None, - labels=None, groupId=None, inputfilters=None, json=None, channels=None, - mipMapLevels=None, imagePyramid=None, **kwargs): + labels=None, groupId=None, inputfilters=None, json=None, + channels=None, mipMapLevels=None, imagePyramid=None, + **kwargs): if json is not None: self.from_dict(json) else: From 67aaf25bd0cdf0c8002241c0eea32ca09a72ed44 Mon Sep 17 00:00:00 2001 From: thopp-tudelft Date: Fri, 24 Oct 2025 17:39:46 +0200 Subject: [PATCH 760/766] add session object to Render class --- renderapi/render.py | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 137dccd3..7f972936 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -30,16 +30,19 @@ class Render(object): render project to which make_kwargs will default DEFAULT_CLIENT_SCRIPTS : str render client scripts path to which make_kwargs will default + session : requests.sessions.Session + session object to which make_kwargs will default """ def __init__(self, host=None, port=None, owner=None, project=None, - client_scripts=None, **kwargs): + client_scripts=None, session=None, **kwargs): self.DEFAULT_HOST = host self.DEFAULT_PORT = port self.DEFAULT_PROJECT = project self.DEFAULT_OWNER = owner self.DEFAULT_CLIENT_SCRIPTS = client_scripts + self.session = session logger.debug('Render object created with ' 'host={h}, port={p}, project={pr}, ' @@ -52,7 +55,8 @@ def __init__(self, host=None, port=None, owner=None, project=None, def DEFAULT_KWARGS(self): """"kwargs to which the render object falls back. Depends on: self.DEFAULT_HOST, self.DEFAULT_OWNER, self.DEFAULT_PORT, - self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS + self.DEFAULT_PROJECT, self.DEFAULT_CLIENT_SCRIPTS, + self.session Returns ------- @@ -62,7 +66,7 @@ def DEFAULT_KWARGS(self): return self.make_kwargs() def make_kwargs(self, host=None, port=None, owner=None, project=None, - client_scripts=None, **kwargs): + client_scripts=None, session=None, **kwargs): """make kwargs using this render object's defaults and any designated kwargs passed in @@ -78,6 +82,8 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, render webservice project client_scripts : str or None render java client script location + session : requests.sessions.Session + http session to use **kwargs all other keyword arguments passed through @@ -85,7 +91,8 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, ------- dict keyword arguments with missing - host,port,owner,project,client_scripts filled in with defaults + host,port,owner,project,client_scripts,session filled in with + defaults """ processed_kwargs = { 'host': self.DEFAULT_HOST if host is None else host, @@ -93,7 +100,9 @@ def make_kwargs(self, host=None, port=None, owner=None, project=None, 'owner': self.DEFAULT_OWNER if owner is None else owner, 'project': self.DEFAULT_PROJECT if project is None else project, 'client_scripts': (self.DEFAULT_CLIENT_SCRIPTS if client_scripts - is None else client_scripts)} + is None else client_scripts), + 'session': self.session if session is None else session, + } processed_kwargs.update(kwargs) return processed_kwargs @@ -144,6 +153,8 @@ class RenderClient(Render): render project to which make_kwargs will default DEFAULT_CLIENT_SCRIPTS : str render client scripts path to which make_kwargs will default + session : requests.sessions.Session + session object to which make_kwargs will default client_script : str location of wrapper script for java client with input same as Render java client's run_ws_client.sh @@ -220,7 +231,7 @@ def make_kwargs(self, *args, **kwargs): ------- dict keyword arguments with missing - host,port,owner,project,client_scripts,client_script,memGB + host,port,owner,project,client_scripts,session,client_script,memGB filled in with defaults """ # hack to get dictionary defaults to work @@ -235,7 +246,8 @@ def make_kwargs(self, *args, **kwargs): def connect(host=None, port=None, owner=None, project=None, client_scripts=None, client_script=None, memGB=None, - force_http=True, validate_client=True, web_only=False, **kwargs): + force_http=True, validate_client=True, web_only=False, + session=None, **kwargs): """helper function to create a :class:`Render` instance, or :class:`RenderClient` if sufficent parameters are provided. Will default to using environment variables if not specified in call, @@ -277,6 +289,8 @@ def connect(host=None, port=None, owner=None, project=None, web_only : bool whether to check environment variables/prompt user for client_scripts directory if not in arguments + session : requests.sessions.Session + http session to use Returns ------- @@ -361,20 +375,21 @@ def connect(host=None, port=None, owner=None, project=None, host=host, port=port, owner=owner, project=project, client_scripts=client_scripts, - validate_client=validate_client) + validate_client=validate_client, + session=session) except ClientScriptError as e: logger.info(e) logger.warning( 'Could not initiate render Client -- falling back to web') return Render(host=host, port=port, owner=owner, project=project, - client_scripts=client_scripts) + client_scripts=client_scripts, session=session) @decorator def renderaccess(f, *args, **kwargs): - """Decorator allowing functions asking for host, port, owner, project - to default to a connection defined by a :class:`Render` object - using its :func:`RenderClient.make_kwargs` method. + """Decorator allowing functions asking for host, port, owner, project, + and/or session to default to a connection defined by a :class:`Render` + object using its :func:`RenderClient.make_kwargs` method. You can if you wish specify any of the arguments, in which case they will not be filled in by the default values, but you don't have to. @@ -478,7 +493,7 @@ def get_owners(host=None, port=None, session=None, render host (defaults to host from render) port : int render port (default to port from render) - session : requests.Session + session : requests.sessions.Session requests session render : RenderClient RenderClient connection object From 25548b111a2f8704b8a9d3d88f50aa9828accaa2 Mon Sep 17 00:00:00 2001 From: thopp-tudelft Date: Fri, 24 Oct 2025 19:20:16 +0200 Subject: [PATCH 761/766] add option to the Render class to create a new session by default --- renderapi/render.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/renderapi/render.py b/renderapi/render.py index 7f972936..f0e7598d 100755 --- a/renderapi/render.py +++ b/renderapi/render.py @@ -32,17 +32,25 @@ class Render(object): render client scripts path to which make_kwargs will default session : requests.sessions.Session session object to which make_kwargs will default + create_session : bool + create a new session for this instance of render if no session was + provided, allowing the same session to be reused for requests and + increasing performance, defaults to True """ def __init__(self, host=None, port=None, owner=None, project=None, - client_scripts=None, session=None, **kwargs): + client_scripts=None, session=None, create_session=True, + **kwargs): self.DEFAULT_HOST = host self.DEFAULT_PORT = port self.DEFAULT_PROJECT = project self.DEFAULT_OWNER = owner self.DEFAULT_CLIENT_SCRIPTS = client_scripts - self.session = session + if create_session and session is None: + self.session = requests.Session() + else: + self.session = session logger.debug('Render object created with ' 'host={h}, port={p}, project={pr}, ' @@ -155,6 +163,10 @@ class RenderClient(Render): render client scripts path to which make_kwargs will default session : requests.sessions.Session session object to which make_kwargs will default + create_session : bool + create a new session for this instance of render if no session was + provided, allowing the same session to be reused for requests and + increasing performance, defaults to True client_script : str location of wrapper script for java client with input same as Render java client's run_ws_client.sh From 691faebcac89ef27ca77149b431b641ca0dfe58d Mon Sep 17 00:00:00 2001 From: thopp-tudelft Date: Fri, 24 Oct 2025 19:34:46 +0200 Subject: [PATCH 762/766] remove the creation of of new session in clone_stack this is now no longer necessary as it is done by the renderaccess decorator already --- renderapi/stack.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/renderapi/stack.py b/renderapi/stack.py index 103b847d..c7e91ab2 100644 --- a/renderapi/stack.py +++ b/renderapi/stack.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import logging from time import strftime -import requests from .errors import RenderError from .utils import jbool, NullHandler, post_json, put_json, rest_delete from .render import (format_baseurl, format_preamble, @@ -515,7 +514,6 @@ def clone_stack(inputstack, outputstack, skipTransforms=False, toProject=None, requests.session.response server response """ - session = requests.session() if session is None else session sv = StackVersion(**kwargs) newstack_project = project qparams = {} From 7df3b70e980d915a8b36b75660282c43f2620c4d Mon Sep 17 00:00:00 2001 From: thopp-tudelft Date: Thu, 27 Nov 2025 19:44:17 +0100 Subject: [PATCH 763/766] add session to tests closes thopp-tudelft#1 --- test/test_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_client.py b/test/test_client.py index 42d32505..39d9ebb6 100644 --- a/test/test_client.py +++ b/test/test_client.py @@ -2,6 +2,7 @@ import pytest import renderapi import rendersettings +import requests args = { 'host': 'renderhost', @@ -17,6 +18,7 @@ def test_render_client(): def test_default_kwargs(rkwargs=rendersettings.DEFAULT_RENDER, **kwargs): + rkwargs = dict(rkwargs, session=requests.Session()) r = renderapi.connect(**dict(rkwargs, **kwargs)) new_r = renderapi.connect(**dict(r.DEFAULT_KWARGS, **kwargs)) assert(new_r.DEFAULT_KWARGS == r.DEFAULT_KWARGS == rkwargs) @@ -36,7 +38,8 @@ def valstostring(d): old_env = os.environ.copy() os.environ.update(valstostring(renvkwargs)) - env_render = renderapi.connect(**kwargs) + rkwargs = dict(rkwargs, session=requests.Session()) + env_render = renderapi.connect(session=rkwargs["session"], **kwargs) # restore environment os.environ.clear() From f0639e2d8075215114e88739848b408d7e72f1e7 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Wed, 10 Dec 2025 14:02:41 -0800 Subject: [PATCH 764/766] actions: release_publish with pixi task --- .github/workflows/release_publish.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index fd83a7b9..a012f528 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -1,3 +1,4 @@ +# publish to pypi on github release (tagged as vX.Y.Z) name: release_publish on: # workflow_dispatch: @@ -11,8 +12,15 @@ jobs: - name: Check out repo uses: actions/checkout@v2 - - name: Build - run: python setup.py sdist bdist_wheel + - name: Pixi setup + uses: prefix-dev/setup-pixi@v0.8.3 + with: + pixi-version: v0.41.4 + cache: false + run-install: false + + - name: Build with pixi env + run: pixi run pypi-build - name: Publish to Pypi uses: pypa/gh-action-pypi-publish@release/v1 From 0e5af2cf2129e1ef261decc2bac0814cfd6d5a1c Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 11 Dec 2025 08:57:50 -0800 Subject: [PATCH 765/766] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 258c31ee..312c1160 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ requires = ["setuptools>=64", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" [tool.setuptools] -py-modules = [ +packages = [ "renderapi" ] From 668faeac0d43f20964dbe7f6e3482e02b8ca3328 Mon Sep 17 00:00:00 2001 From: Russel Torres Date: Thu, 11 Dec 2025 09:44:37 -0800 Subject: [PATCH 766/766] pyproject: add 3.12 and 3.13 pixi features, make <3.14 --- pyproject.toml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 312c1160..b87d73ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ authors = [ maintainers = [ {name = "Russel Torres"} ] -requires-python = ">3.8,<3.12" +requires-python = ">3.8,<3.14" dynamic = ["version"] dependencies = [ "requests", @@ -76,11 +76,20 @@ render-python = { path = ".", editable = true } numpy = "*" scipy = "*" +[tool.pixi.feature.py312.dependencies] +python = "3.12.*" +[tool.pixi.feature.py313.dependencies] +python = "3.13.*" + + + [tool.pixi.environments] test = ["test"] conda = ["pixi-base"] conda-test = ["pixi-base", "test"] build = ["pypi-build"] +test312 = ["py312", "test"] +test313 = ["py313", "test"] [tool.coverage.run] omit = ["integration_tests/*", "test/*"]