diff --git a/.gitignore b/.gitignore index e44d1b6..240d5e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ #Eclipse .project - +venv +venv_win TestFiles test.txt diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 3368915..565f5d0 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -1,19 +1,16 @@ import Algorithmia import os -from Algorithmia.errors import DataApiError +from Algorithmia.errors import DataApiError, AlgorithmException from Algorithmia.algo_response import AlgoResponse -from Algorithmia.util import md5_for_file, md5_for_str import json, re, requests, six import toml import shutil -from time import time class CLI: def __init__(self): self.client = Algorithmia.client() # algo auth - - def auth(self, apikey, apiaddress, cacert="", profile="default"): + def auth(self, apiaddress, apikey="", cacert="", profile="default", bearer=""): # store api key in local config file and read from it each time a client needs to be created key = self.getconfigfile() @@ -24,20 +21,17 @@ def auth(self, apikey, apiaddress, cacert="", profile="default"): config['profiles'][profile]['api_key'] = apikey config['profiles'][profile]['api_server'] = apiaddress config['profiles'][profile]['ca_cert'] = cacert + config['profiles'][profile]['bearer_token'] = bearer else: - config['profiles'][profile] = {'api_key': apikey, 'api_server': apiaddress, 'ca_cert': cacert} + config['profiles'][profile] = {'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert,'bearer_token':bearer} else: - config['profiles'] = {profile: {'api_key': apikey, 'api_server': apiaddress, 'ca_cert': cacert}} + config['profiles'] = {profile:{'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert,'bearer_token':bearer }} with open(key, "w") as key: - toml.dump(config, key) - client = Algorithmia.client( - api_key=self.getAPIkey(profile), - api_address=self.getAPIaddress(profile), - ca_cert=self.getCert(profile) - ) - self.ls(path=None, client=client) + toml.dump(config,key) + + self.ls(path = None,client = CLI().getClient(profile)) # algo run run the the specified algo def runalgo(self, options, client): @@ -248,28 +242,7 @@ def cat(self, path, client): # algo freeze def freezeAlgo(self, client, manifest_path="model_manifest.json"): - if os.path.exists(manifest_path): - with open(manifest_path, 'r') as f: - manifest_file = json.load(f) - manifest_file['timestamp'] = str(time()) - required_files = manifest_file['required_files'] - optional_files = manifest_file['optional_files'] - for i in range(len(required_files)): - uri = required_files[i]['source_uri'] - local_file = client.file(uri).getFile(as_path=True) - md5_checksum = md5_for_file(local_file) - required_files[i]['md5_checksum'] = md5_checksum - for i in range(len(optional_files)): - uri = required_files[i]['source_uri'] - local_file = client.file(uri).getFile(as_path=True) - md5_checksum = md5_for_file(local_file) - required_files[i]['md5_checksum'] = md5_checksum - lock_md5_checksum = md5_for_str(str(manifest_file)) - manifest_file['lock_checksum'] = lock_md5_checksum - with open('model_manifest.json.freeze', 'w') as f: - json.dump(manifest_file, f) - else: - print("Expected to find a model_manifest.json file, none was discovered in working directory") + client.freeze(manifest_path) # algo cp def cp(self, src, dest, client): @@ -335,12 +308,10 @@ def list_languages(self, client): return table def getBuildLogs(self, user, algo, client): - api_response = client.algo(user + '/' + algo).build_logs() - - if "error" in api_response: - return json.dumps(api_response) + api_response = client.algo(user + '/' + algo).get_builds() return json.dumps(api_response['results'], indent=1) + def getconfigfile(self): if (os.name == "posix"): # if!windows @@ -366,6 +337,7 @@ def getconfigfile(self): file.write("api_key = ''\n") file.write("api_server = ''\n") file.write("ca_cert = ''\n") + file.write("bearer_token = ''\n") key = keyPath + keyFile @@ -383,6 +355,16 @@ def getAPIkey(self, profile): return config_dict['profiles'][profile]['api_key'] else: return None + + def getBearerToken(self,profile): + key = self.getconfigfile() + config_dict = toml.load(key) + if 'profiles' in config_dict and profile in config_dict['profiles'] and \ + config_dict['profiles'][profile]['bearer_token'] != "": + return config_dict['profiles'][profile]['bearer_token'] + else: + return None + def getAPIaddress(self, profile): key = self.getconfigfile() @@ -401,3 +383,14 @@ def getCert(self, profile): return config_dict['profiles'][profile]['ca_cert'] else: return None + + def getClient(self,profile): + apiAddress = self.getAPIaddress(profile) + apiKey = self.getAPIkey(profile) + caCert = self.getCert(profile) + bearer = None + + if apiKey is None: + bearer = self.getBearerToken(profile) + + return Algorithmia.client(api_key=apiKey,api_address=apiAddress,ca_cert=caCert,bearer_token = bearer) diff --git a/Algorithmia/__init__.py b/Algorithmia/__init__.py index 05ed6dc..38e7ed6 100644 --- a/Algorithmia/__init__.py +++ b/Algorithmia/__init__.py @@ -23,8 +23,8 @@ def file(dataUrl): def dir(dataUrl): return getDefaultClient().dir(dataUrl) -def client(api_key=None, api_address=None, ca_cert=None): - return Client(api_key, api_address, ca_cert) +def client(api_key=None, api_address=None, ca_cert=None, bearer_token=None): + return Client(api_key, api_address, ca_cert, bearer_token) def handler(apply_func, load_func=lambda: None): return Handler(apply_func, load_func) diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index 9e67c5c..1b5f7b5 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -6,6 +6,7 @@ import six from Algorithmia.CLI import CLI import argparse +import re #bind input to raw input try: @@ -145,27 +146,26 @@ def main(): APIkey = input("enter API key: ") CACert = input('(optional) enter path to custom CA certificate: ') - if len(APIkey) == 28 and APIkey.startswith("sim"): - if APIaddress == "" or not APIaddress.startswith("https://api."): - APIaddress = "https://api.algorithmia.com" - - CLI().auth(apikey=APIkey, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + if APIaddress == "" or not APIaddress.startswith("https://api."): + print("invalid API address") else: - print("invalid api key") - + if len(APIkey) == 28 and APIkey.startswith("sim"): + CLI().auth(apikey=APIkey, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + else: + jwt = re.compile(r"^([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_\-\+\/=]*)") + Bearer = input("enter JWT token: ") + if jwt.match(Bearer): + CLI().auth(apikey=APIkey, bearer=Bearer, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + else: + print("invalid authentication") + + + if args.cmd == 'help': parser.parse_args(['-h']) #create a client with the appropreate api address and key - client = Algorithmia.client() - if len(CLI().getAPIaddress(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile)) - elif len(CLI().getAPIaddress(args.profile)) > 1 and len(CLI().getCert(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile),CLI().getCert(args.profile)) - elif len(CLI().getAPIaddress(args.profile)) < 1 and len(CLI().getCert(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile),CLI().getCert(args.profile)) - else: - client = Algorithmia.client(CLI().getAPIkey(args.profile)) + client = CLI().getClient(args.profile) if args.cmd == 'run': diff --git a/Algorithmia/algo_response.py b/Algorithmia/algo_response.py index dfce6c1..d5abbc4 100644 --- a/Algorithmia/algo_response.py +++ b/Algorithmia/algo_response.py @@ -1,5 +1,6 @@ import base64 from Algorithmia.errors import raiseAlgoApiError +from Algorithmia.async_response import AsyncResponse import sys @@ -19,8 +20,12 @@ def __repr__(self): @staticmethod def create_algo_response(response): + + # Check if request is async + if 'async_protocol' in response and 'request_id' in response: + return AsyncResponse(response) # Parse response JSON, if it's indeed JSON - if 'error' in response or 'metadata' not in response: + elif 'error' in response or 'metadata' not in response: # Failure raise raiseAlgoApiError(response) else: diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 2fe5820..40be378 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -1,17 +1,18 @@ 'Algorithmia Algorithm API Client (python)' -import base64 import json import re from Algorithmia.async_response import AsyncResponse from Algorithmia.algo_response import AlgoResponse -from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError +from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError, AlgorithmException from enum import Enum from algorithmia_api_client.rest import ApiException -from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, SettingsPublish, \ +from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, \ + SettingsPublish, \ CreateRequestVersionInfo, VersionInfo, VersionInfoPublish -OutputType = Enum('OutputType','default raw void') +OutputType = Enum('OutputType', 'default raw void') + class Algorithm(object): def __init__(self, client, algoRef): @@ -32,113 +33,129 @@ def __init__(self, client, algoRef): raise ValueError('Invalid algorithm URI: ' + algoRef) def set_options(self, timeout=300, stdout=False, output=OutputType.default, **query_parameters): - self.query_parameters = {'timeout':timeout, 'stdout':stdout} + self.query_parameters = {'timeout': timeout, 'stdout': stdout} self.output_type = output self.query_parameters.update(query_parameters) return self + def get_algorithm_id(self): + url = '/v1/algorithms/' + self.username + '/' + self.algoname + print(url) + api_response = self.client.getJsonHelper(url) + if 'id' in api_response: + return api_response['id'] + else: + raise Exception("field 'id' not found in response: ", api_response) + + + def get_secrets(self): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + api_response = self.client.getJsonHelper(url) + return api_response + + + def set_secret(self, short_name, secret_key, secret_value, description=None): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + secret_providers = self.client.get_secret_providers() + provider_id = secret_providers[0]['id'] + + create_parameters = { + "owner_type": "algorithm", + "owner_id": algorithm_id, + "short_name": short_name, + "provider_id": provider_id, + "secret_key": secret_key, + "secret_value": secret_value, + } + if description: + create_parameters['description'] = description + else: + create_parameters['description'] = " " + + print(create_parameters) + api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) + return api_response + + # Create a new algorithm - def create(self, details={}, settings={}, version_info={}): - detailsObj = Details(**details) - settingsObj = SettingsMandatory(**settings) - createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} - create_request = CreateRequest(**create_parameters) - try: - # Create Algorithm - api_response = self.client.manageApi.create_algorithm(self.username, create_request) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def create(self, details, settings, version_info=None, source=None, scmsCredentials=None): + url = "/v1/algorithms/" + self.username + create_parameters = {"name": self.algoname, "details": details, "settings": settings} + if version_info: + create_parameters['version_info'] = version_info + if source: + create_parameters['source'] = source + if scmsCredentials: + create_parameters['scmsCredentials'] = scmsCredentials + + api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) + return api_response # Update the settings in an algorithm - def update(self, details={}, settings={}, version_info={}): - detailsObj = Details(**details) - settingsObj = Settings(**settings) - createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - update_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} - update_request = UpdateRequest(**update_parameters) - try: - # Update Algorithm - api_response = self.client.manageApi.update_algorithm(self.username, self.algoname, update_request) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def update(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + url = "/v1/algorithms/" + self.username + "/" + self.algoname + update_parameters = {"details": details, "settings": settings, + "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + api_response = self.client.putHelper(url, update_parameters) + return api_response # Publish an algorithm - def publish(self, details={}, settings={}, version_info={}): - # detailsObj = Details(**details) - # settingsObj = SettingsPublish(**settings) - # versionRequestObj = VersionInfoPublish(**version_info) - # publish_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": versionRequestObj} - # version_request = VersionRequest(**publish_parameters) # VersionRequest | Publish Version Request - publish_parameters = {"details": details, "settings": settings, "version_info": version_info} - url = "/v1/algorithms/"+self.username+"/"+self.algoname + "/versions" - print(publish_parameters) - api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, **self.query_parameters) + def publish(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions" + publish_parameters = {"details": details, "settings": settings, + "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, retry=True) return api_response - # except ApiException as e: - # error_message = json.loads(e.body) - # raise raiseAlgoApiError(error_message) - def builds(self, limit=56, marker=None): - try: - if marker is not None: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, marker=marker) - else: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def get_builds(self, limit=56, marker=None): + kwargs = {"limit": limit, "marker": marker} + url = "/v1/algorithms/" + self.username + "/" + self.algoname + '/builds' + response = self.client.getJsonHelper(url, **kwargs) + return response def get_build(self, build_id): # Get the build object for a given build_id # The build status can have one of the following value: succeeded, failed, in-progress - try: - api_response = self.client.manageApi.get_algorithm_build_by_id(self.username, self.algoname, build_id) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds/' + build_id + response = self.client.getJsonHelper(url) + return response def get_build_logs(self, build_id): # Get the algorithm build logs for a given build_id - try: - api_response = self.client.manageApi.get_algorithm_build_logs(self.username, self.algoname, build_id) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) - - def build_logs(self): - url = '/v1/algorithms/'+self.username+'/'+self.algoname+'/builds' - response = json.loads(self.client.getHelper(url).content.decode('utf-8')) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds/' + build_id + '/logs' + response = self.client.getJsonHelper(url) return response - def get_scm_status(self): - try: - api_response = self.client.manageApi.get_algorithm_scm_connection_status(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/scm/status' + response = self.client.getJsonHelper(url) + return response # Get info on an algorithm def info(self, algo_hash=None): + # Get Algorithm + if algo_hash: + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions/' + algo_hash + else: + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions' + response = self.client.getJsonHelper(url) + return response + + # Check if an Algorithm exists + def exists(self): try: - # Get Algorithm - if algo_hash: - api_response = self.client.manageApi.get_algorithm_hash_version(self.username, self.algoname, algo_hash) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + _ = self.client.getJsonHelper(url) + return True + except AlgorithmException as e: + if "404" in str(e) or "No such algorithm" in str(e): + return False + elif "403" in str(e): + raise Exception("unable to check exists on algorithms you don't own.") else: - api_response = self.client.manageApi.get_algorithm(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + raise e # Get all versions of the algorithm, with the given filters def versions(self, limit=None, marker=None, published=None, callable=None): @@ -154,24 +171,17 @@ def versions(self, limit=None, marker=None, published=None, callable=None): if callable: c = callable kwargs["callable"] = str(c).lower() if str(c) in bools else c - try: - # Get Algorithm versions - api_response = self.client.manageApi.get_algorithm_versions(self.username, self.algoname, **kwargs) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) - + # Get Algorithm versions + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions' + response = self.client.getJsonHelper(url, **kwargs) + return response # Compile an algorithm def compile(self): - try: - # Compile algorithm - api_response = self.client.manageApi.compile_algorithm(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + # Compile algorithm + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/compile' + response = self.client.postJsonHelper(url, {}, parse_response_as_json=True, retry=True) + return response # Pipe an input into this algorithm def pipe(self, input1): @@ -181,25 +191,26 @@ def pipe(self, input1): elif self.output_type == OutputType.void: return self._postVoidOutput(input1) else: - return AlgoResponse.create_algo_response(self.client.postJsonHelper(self.url, input1, **self.query_parameters)) + return AlgoResponse.create_algo_response( + self.client.postJsonHelper(self.url, input1, **self.query_parameters)) def _postRawOutput(self, input1): - # Don't parse response as json - self.query_parameters['output'] = 'raw' - response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) - # Check HTTP code and throw error as needed - if response.status_code == 400: - # Bad request - raise ApiError(response.text) - elif response.status_code == 500: - raise ApiInternalError(response.text) - else: - return response.text + # Don't parse response as json + self.query_parameters['output'] = 'raw' + response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) + # Check HTTP code and throw error as needed + if response.status_code == 400: + # Bad request + raise ApiError(response.text) + elif response.status_code == 500: + raise ApiInternalError(response.text) + else: + return response.text def _postVoidOutput(self, input1): - self.query_parameters['output'] = 'void' - responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) - if 'error' in responseJson: - raise ApiError(responseJson['error']['message']) - else: - return AsyncResponse(responseJson) + self.query_parameters['output'] = 'void' + responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) + if 'error' in responseJson: + raise ApiError(responseJson['error']['message']) + else: + return AsyncResponse(responseJson) diff --git a/Algorithmia/client.py b/Algorithmia/client.py index ccea0ca..dc26e1a 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -2,16 +2,18 @@ import Algorithmia from Algorithmia.insights import Insights +from Algorithmia.errors import raiseAlgoApiError from Algorithmia.algorithm import Algorithm from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory, AdvancedDataDirectory from algorithmia_api_client import Configuration, DefaultApi, ApiClient - +from Algorithmia.util import md5_for_file, md5_for_str from tempfile import mkstemp import atexit import json, re, requests, six, certifi import tarfile import os +from time import time class Client(object): @@ -21,13 +23,18 @@ class Client(object): apiKey = None apiAddress = None requestSession = None + bearerToken = None - def __init__(self, apiKey=None, apiAddress=None, caCert=None): + def __init__(self, apiKey=None, apiAddress=None, caCert=None, bearerToken=None): # Override apiKey with environment variable config = None self.requestSession = requests.Session() if apiKey is None and 'ALGORITHMIA_API_KEY' in os.environ: apiKey = os.environ['ALGORITHMIA_API_KEY'] + elif bearerToken is None and 'ALGORITHMIA_BEARER_TOKEN' in os.environ: + bearerToken = os.environ['ALGORITHMIA_BEARER_TOKEN'] + + self.bearerToken = bearerToken self.apiKey = apiKey if apiAddress is not None: self.apiAddress = apiAddress @@ -35,6 +42,7 @@ def __init__(self, apiKey=None, apiAddress=None, caCert=None): self.apiAddress = Algorithmia.getApiAddress() if caCert == False: self.requestSession.verify = False + self.requestSession.trust_env = False config = Configuration(use_ssl=False) elif caCert is None and 'REQUESTS_CA_BUNDLE' in os.environ: caCert = os.environ.get('REQUESTS_CA_BUNDLE') @@ -62,6 +70,11 @@ def username(self): username = next(self.dir("").list()).path return username + def scms(self): + url = "/v1/scms" + response = self.getJsonHelper(url) + return response + def file(self, dataUrl, cleanup=False): if dataUrl.startswith('file://'): return LocalDataFile(self, dataUrl) @@ -163,6 +176,11 @@ def get_supported_languages(self): response = self.getHelper(url) return response.json() + def get_secret_providers(self): + url = "/v1/secret-provider" + api_response = self.getJsonHelper(url) + return api_response + def get_organization_errors(self, org_name): """Gets the errors for the organization. @@ -205,18 +223,19 @@ def get_algorithm_errors(self, algorithm_id): """ url = '/v1/algorithms/%s/errors' % algorithm_id - response = self.getHelper(url) - return response.json() + return self.getJsonHelper(url) # Used to send insight data to Algorithm Queue Reader in cluster def report_insights(self, insights): return Insights(insights) # Used internally to post json to the api and parse json response - def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query_parameters): + def postJsonHelper(self, url, input_object, parse_response_as_json=True, retry=False, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken input_json = None if input_object is None: @@ -234,28 +253,61 @@ def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query response = self.requestSession.post(self.apiAddress + url, data=input_json, headers=headers, params=query_parameters) - - if parse_response_as_json and response.status_code == 200: - return response.json() - return response + if 200 <= response.status_code <= 299: + if parse_response_as_json: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + return response + elif retry: + return self.postJsonHelper(url, input_object, parse_response_as_json, False, **query_parameters) + else: + raise raiseAlgoApiError(response) # Used internally to http get a file def getHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) + def getJsonHelper(self, url, **query_parameters): + headers = {} + if self.apiKey is not None: + headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken + response = self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) + if 200 <= response.status_code <= 299: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + if response.content is not None: + response = response.json() + raise raiseAlgoApiError(response) + def getStreamHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters, stream=True) def patchHelper(self, url, params): headers = {'content-type': 'application/json'} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.patch(self.apiAddress + url, headers=headers, data=json.dumps(params)) # Used internally to get http head result @@ -263,6 +315,8 @@ def headHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.head(self.apiAddress + url, headers=headers) # Used internally to http put a file @@ -270,19 +324,29 @@ def putHelper(self, url, data): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken if isJson(data): headers['Content-Type'] = 'application/json' - response = self.requestSession.put(self.apiAddress + url, data=data, headers=headers) if response._content == b'': return response - return response.json() + if 200 <= response.status_code <= 299: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + raise raiseAlgoApiError(response) # Used internally to http delete a file def deleteHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken response = self.requestSession.delete(self.apiAddress + url, headers=headers) if response.reason == "No Content": return response @@ -322,6 +386,31 @@ def exit_handler(self): except OSError as e: print(e) + # Used by CI/CD automation for freezing model manifest files, and by the CLI for manual freezing + def freeze(self, manifest_path, manifest_output_dir="."): + if os.path.exists(manifest_path): + with open(manifest_path, 'r') as f: + manifest_file = json.load(f) + manifest_file['timestamp'] = str(time()) + required_files = manifest_file['required_files'] + optional_files = manifest_file['optional_files'] + for i in range(len(required_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + for i in range(len(optional_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + lock_md5_checksum = md5_for_str(str(manifest_file)) + manifest_file['lock_checksum'] = lock_md5_checksum + with open(manifest_output_dir + '/' + 'model_manifest.json.freeze', 'w') as f: + json.dump(manifest_file, f) + else: + print("Expected to find a model_manifest.json file, none was discovered in working directory") + def isJson(myjson): try: diff --git a/Algorithmia/errors.py b/Algorithmia/errors.py index 9356e24..22ad68e 100644 --- a/Algorithmia/errors.py +++ b/Algorithmia/errors.py @@ -36,7 +36,7 @@ def raiseAlgoApiError(result): if 'message' in result['error']: message = result['error']['message'] else: - message = None + message = result['error'] if 'error_type' in result['error']: err_type = result['error']['error_type'] else: @@ -47,4 +47,4 @@ def raiseAlgoApiError(result): stacktrace = None return AlgorithmException(message=message, stack_trace=stacktrace, error_type=err_type) else: - return Exception(result) + return Exception("Non-Algorithm related Failure: " + str(result)) diff --git a/README.md b/README.md index a7f3214..59a80dc 100644 --- a/README.md +++ b/README.md @@ -72,9 +72,9 @@ If the algorithm output is text, then the `result` field of the response will be ```python algo = client.algo('demo/Hello/0.1.1') response = algo.pipe("HAL 9000") -print response.result # Hello, world! -print response.metadata # Metadata(content_type='text',duration=0.0002127) -print response.metadata.duration # 0.0002127 +print(response.result) # Hello, world! +print(response.metadata) # Metadata(content_type='text',duration=0.0002127) +print(response.metadata.duration) # 0.0002127 ``` ### JSON input/output @@ -119,7 +119,7 @@ This includes support for changing the timeout or indicating that the API should ```python from Algorithmia.algorithm import OutputType response = client.algo('util/echo').set_options(timeout=60, stdout=False) -print response.metadata.stdout +print(response.metadata.stdout) ``` Note: `stdout=True` is only supported if you have access to the algorithm source. @@ -186,15 +186,15 @@ foo = client.dir("data://.my/foo") # List files in "foo" for file in foo.files(): - print file.path + " at URL: " + file.url + " last modified " + file.last_modified + print(file.path + " at URL: " + file.url + " last modified " + file.last_modified) # List directories in "foo" for file in foo.dirs(): - print dir.path + " at URL: " + file.url + print(dir.path + " at URL: " + file.url) # List everything in "foo" for entry in foo.list(): - print entry.path + " at URL: " + entry.url + print(entry.path + " at URL: " + entry.url) ``` ### Manage directory permissions @@ -230,7 +230,7 @@ $ algo auth Configuring authentication for profile: 'default' Enter API Endpoint [https://api.algorithmia.com]: Enter API Key: -(optional) enter path to custom CA certificate: +(optional) enter path to custom CA certificate: Profile is ready to use. Test with 'algo ls' ``` @@ -332,7 +332,7 @@ algo auth --profile second_user Configuring authentication for profile: 'second_user' Enter API Endpoint [https://api.algorithmia.com]: Enter API Key: -(optional) enter path to custom CA certificate: +(optional) enter path to custom CA certificate: ``` Now you may use `algo ls --profile second_user` to list files in your `second_user` account. For more information, see the auth command help with `algo auth --help`. @@ -342,7 +342,7 @@ Now you may use `algo ls --profile second_user` to list files in your `second_us When running commands, the Algorithmia CLI will use the default profile unless otherwise specified with the `--profile ` option. See the following example: ```text -$ algo run kenny/factor -d 17 --profile second_user +$ algo run kenny/factor -d 17 --profile second_user [17] ``` diff --git a/Test/CLI_test.py b/Test/CLI_test.py deleted file mode 100644 index 35ca486..0000000 --- a/Test/CLI_test.py +++ /dev/null @@ -1,234 +0,0 @@ -import sys -# look in ../ BEFORE trying to import Algorithmia. If you append to the -# you will load the version installed on the computer. -sys.path = ['../'] + sys.path - -import unittest -import os -import json -import Algorithmia -from Algorithmia.CLI import CLI -import argparse -import shutil -import toml - -class CLITest(unittest.TestCase): - def setUp(self): - # create a directory to use in testing the cp command - self.client = Algorithmia.client() - CLI().mkdir("data://.my/moredata", self.client) - if(not os.path.exists("./TestFiles/")): - os.mkdir("./TestFiles/") - - def test_ls(self): - parentDir = "data://.my/" - newDir = "test" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(result is not None and "moredata" in result and newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - - def test_mkdir(self): - - parentDir = "data://.my/" - newDir = "test" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - def test_rmdir(self): - parentDir = "data://.my/" - newDir = "testRmdir" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir not in result) - - def test_cat(self): - file = "data://.my/moredata/test.txt" - localfile = "./TestFiles/test.txt" - fileContents = "some text in test file" - - CLI().rm(file, self.client) - testfile = open(localfile, "w") - testfile.write(fileContents) - testfile.close() - - CLI().cp([localfile],file,self.client) - - result = CLI().cat([file],self.client) - self.assertEqual(result, fileContents) - - def test_get_build_logs(self): - user=os.environ.get('ALGO_USER_NAME') - algo="Echo" - - result = json.loads(CLI().getBuildLogs(user,algo,self.client)) - if "error" in result: - print(result) - self.assertTrue("error" not in result) - - -#local to remote - def test_cp_L2R(self): - localfile = "./TestFiles/test.txt" - testfile = open(localfile, "w") - testfile.write("some text") - testfile.close() - - src = [localfile] - dest = "data://.my/moredata/test.txt" - CLI().cp(src,dest,self.client) - - result = CLI().ls("data://.my/moredata/",self.client) - self.assertTrue("test.txt" in result) - -#remote to remote - def test_cp_R2R(self): - - src = ["data://.my/moredata/test.txt"] - dest = "data://.my/moredata/test2.txt" - CLI().cp(src,dest,self.client) - - result = CLI().ls("data://.my/moredata/",self.client) - self.assertTrue("test2.txt" in result) - -#remote to local - def test_cp_R2L(self): - src = ["data://.my/moredata/test.txt"] - dest = "./test.txt" - - CLI().cp(src,dest,self.client) - self.assertTrue(os.path.isfile(dest)) - - def test_run(self): - name = "util/Echo" - inputs = "test" - - parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') - - subparsers = parser.add_subparsers(help = 'sub cmd',dest = 'subparser_name') - parser_run = subparsers.add_parser('run', help = 'algo run [input options] [output options]') - - parser_run.add_argument('algo') - parser_run.add_argument('-d','--data', action = 'store', help = 'detect input type', default = None) - parser_run.add_argument('-t','--text', action = 'store', help = 'treat input as text', default = None) - parser_run.add_argument('-j','--json', action = 'store', help = 'treat input as json data', default = None) - parser_run.add_argument('-b','--binary', action = 'store', help = 'treat input as binary data', default = None) - parser_run.add_argument('-D','--data-file', action = 'store', help = 'specify a path to an input file', default = None) - parser_run.add_argument('-T','--text-file', action = 'store', help = 'specify a path to a text file', default = None) - parser_run.add_argument('-J','--json-file', action = 'store', help = 'specify a path to a json file', default = None) - parser_run.add_argument('-B','--binary-file', action = 'store', help = 'specify a path to a binary file', default = None) - parser_run.add_argument('--timeout', action = 'store',type = int, default = 300, help = 'specify a timeout (seconds)') - parser_run.add_argument('--debug', action = 'store_true', help = 'print the stdout from the algo ') - parser_run.add_argument('--profile', action = 'store', type = str, default = 'default') - parser_run.add_argument('-o', '--output', action = 'store', default = None, type = str) - - args = parser.parse_args(['run',name,'-d',inputs]) - - result = CLI().runalgo(args, self.client) - self.assertEqual(result, inputs) - - def test_auth(self): - #key for test account - key = os.getenv('ALGORITHMIA_API_KEY') - address = 'https://api.algorithmia.com' - profile = 'default' - CLI().auth(key,address,profile=profile) - resultK = CLI().getAPIkey(profile) - resultA = CLI().getAPIaddress(profile) - self.assertEqual(resultK, key) - self.assertEqual(resultA, address) - - def test_auth_cert(self): - - localfile = "./TestFiles/fakecert.pem" - - testfile = open(localfile, "w") - testfile.write("") - testfile.close() - - #key for test account - key = os.getenv('ALGORITHMIA_API_KEY') - address = 'https://api.algorithmia.com' - cacert = localfile - profile = 'test' - - CLI().auth(key,address,cacert,profile) - resultK = CLI().getAPIkey(profile) - resultA = CLI().getAPIaddress(profile) - resultC = CLI().getCert(profile) - self.assertEqual(resultK, key) - self.assertEqual(resultA, address) - self.assertEqual(resultC, cacert) - - def test_get_environment(self): - result = CLI().get_environment_by_language("python2",self.client) - print(result) - if("error" in result): - print(result) - self.assertTrue(result is not None and "display_name" in result) - - def test_list_languages(self): - result = CLI().list_languages(self.client) - if("error" in result[0]): - print(result) - self.assertTrue(result is not None and "anaconda3" in result[1]) - - - def test_rm(self): - localfile = "./TestFiles/testRM.txt" - - testfile = open(localfile, "w") - testfile.write("some text") - testfile.close() - - src = [localfile] - dest = "data://.my/moredata/" - CLI().cp(src,dest,self.client) - - result1 = CLI().ls(dest,self.client) - - CLI().rm("data://.my/moredata/testRM.txt",self.client) - - result2 = CLI().ls(dest,self.client) - - self.assertTrue("testRM.txt" in result1 and "testRM.txt" not in result2) - - def test_get_template(self): - filename = "./temptest" - envid = "36fd467e-fbfe-4ea6-aa66-df3f403b7132" - response = CLI().get_template(envid,filename,self.client) - print(response) - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - def test_api_address_auth(self): - api_key = os.getenv('ALGORITHMIA_TEST_API_KEY') - api_address = "https://api.test.algorithmia.com" - CLI().auth(api_key, api_address) - profile = "default" - - client = Algorithmia.client(CLI().getAPIkey(profile), CLI().getAPIaddress(profile), CLI().getCert(profile)) - result2 = CLI().ls("data://.my", client) - print(result2) - self.assertTrue(result2 != "") - - - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/Test/algo_test.py b/Test/algo_test.py deleted file mode 100644 index b14e99b..0000000 --- a/Test/algo_test.py +++ /dev/null @@ -1,77 +0,0 @@ -import sys -import os -from Algorithmia.errors import AlgorithmException -# look in ../ BEFORE trying to import Algorithmia. If you append to the -# you will load the version installed on the computer. -sys.path = ['../'] + sys.path - -import unittest - -import Algorithmia - -class AlgoTest(unittest.TestCase): - def setUp(self): - self.client = Algorithmia.client() - - def test_call_customCert(self): - open("./test.pem",'w') - c = Algorithmia.client(ca_cert="./test.pem") - result = c.algo('util/Echo').pipe(bytearray('foo','utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo','utf-8'), result.result) - try: - os.remove("./test.pem") - except OSError as e: - print(e) - - def test_call_binary(self): - result = self.client.algo('util/Echo').pipe(bytearray('foo','utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo','utf-8'), result.result) - - def test_text_unicode(self): - telephone = u"\u260E" - - #Unicode input to pipe() - result1 = self.client.algo('util/Echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) - - #Unicode return in .result - result2 = self.client.algo('util/Echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) - - def test_get_build_by_id(self): - result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.build_id is not None) - - def test_get_build_logs(self): - result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.logs is not None) - - def test_get_scm_status(self): - result = self.client.algo("J_bragg/Echo").get_scm_status() - self.assertTrue(result.scm_connection_status is not None) - - def test_exception_ipa_algo(self): - try: - result = self.client.algo('zeryx/raise_exception').pipe("") - except AlgorithmException as e: - self.assertEqual(e.message, "This is an exception") - - # def test_json_unicode(self): - # telephone = [u"\u260E"] - # - # #Unicode input to pipe() - # result1 = self.client.algo('util/Echo').pipe(telephone) - # self.assertEquals('json', result1.metadata.content_type) - # self.assertEquals(telephone, result1.result) - # - # #Unicode return in .result - # result2 = self.client.algo('util/Echo').pipe(result1.result) - # self.assertEquals('json', result2.metadata.content_type) - # self.assertEquals(telephone, result2.result) - -if __name__ == '__main__': - unittest.main() diff --git a/Test/api/__init__.py b/Test/api/__init__.py index 5ca3185..c5ff73e 100644 --- a/Test/api/__init__.py +++ b/Test/api/__init__.py @@ -1,16 +1,3 @@ -import importlib -from fastapi import FastAPI, Response - -app = FastAPI() - -@app.post("/v1/{username}/{algoname}/{version}") -async def throw_error(username, algoname, version): - return Response("Internal Server Error", status_code=500) - - -def create_endpoint(algoname): - module = importlib.import_module(algoname) - @app.get("/invocations") - def invocations(data): - return module.apply(data) +from .app import start_webserver_reg as start_webserver_reg +from .self_signed_app import start_webserver_self_signed as start_webserver_self_signed diff --git a/Test/api/app.py b/Test/api/app.py new file mode 100644 index 0000000..db7efd2 --- /dev/null +++ b/Test/api/app.py @@ -0,0 +1,549 @@ +from fastapi import FastAPI, Request, status +from typing import Optional +from fastapi.responses import Response, JSONResponse +import json +import base64 +from multiprocessing import Process +import uvicorn + +regular_app = FastAPI() + + +def start_webserver_reg(): + def _start_webserver(): + uvicorn.run(regular_app, host="127.0.0.1", port=8080, log_level="debug") + + p = Process(target=_start_webserver) + p.start() + return p + + +@regular_app.post("/v1/algo/{username}/{algoname}") +async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} + content_type = request.headers['Content-Type'] + auth = request.headers.get('Authorization', None) + if auth is None: + return {"error": {"message": "authorization required"}} + request = await request.body() + if output and output == "void": + return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} + elif output and output == "raw": + return Response(request.decode(), status_code=200) + elif algoname == "500": + return Response("Internal Server Error", status_code=500) + elif algoname == "raise_exception": + return {"error": {"message": "This is an exception"}} + else: + if content_type != "application/octet-stream": + request = request.decode('utf-8') + if content_type == "text/plain": + metadata['content_type'] = "text" + elif content_type == "application/json": + request = json.loads(request) + metadata['content_type'] = "json" + else: + metadata['content_type'] = "binary" + request = base64.b64encode(request) + output = {"result": request, "metadata": metadata} + return output + + +@regular_app.post("/v1/algo/{username}/{algoname}/{githash}", status_code=status.HTTP_200_OK) +async def process_hello_world(request: Request, username, algoname, githash): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, + 'content_type': "text"} + request = await request.body() + request = request.decode('utf-8') + return {"result": f"hello {request}", "metadata": metadata} + + +### Algorithm Routes +@regular_app.get('/v1/algorithms/{username}/{algoname}') +async def process_get_algo(username, algoname): + if algoname == "echo" and username == 'quality': + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", + "details": {"summary": "", "label": "echo", "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", + "package_set": "python36", "license": "apl", "royalty_microcredits": 0, + "network_access": "full", "pipeline_enabled": True, "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, + "self_link": "https://api.algorithmia.com/v1/algorithms/quality/echo/versions/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"} + elif algoname == "echo": + return JSONResponse(content={"error": {"id": "1cfb98c5-532e-4cbf-9192-fdd45b86969c", "code": 2001, + "message": "Caller is not authorized to perform the operation"}}, + status_code=403) + else: + return JSONResponse(content={"error": "No such algorithm"}, status_code=404) + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") +async def get_build_id(username, algoname, buildid): + return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", + "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", + "version_info": {"semantic_version": "0.1.1"}} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/scm/status") +async def get_scm_status(username, algoname): + return {"scm_connection_status": "active"} + + +@regular_app.get("/v1/scms") +async def get_scms(): + return {'results': [{'default': True, 'enabled': True, 'id': 'internal', 'name': '', 'provider': 'internal'}, + {'default': False, 'enabled': True, 'id': 'github', 'name': 'https://github.com', + 'provider': 'github', 'scm': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'oauth': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'urls': {'web': 'https://github.com', 'api': 'https://api.github.com', + 'ssh': 'ssh://git@github.com'}}, + {'default': False, 'enabled': True, 'id': 'aadebe70-007f-48ff-ba38-49007c6e0377', + 'name': 'https://gitlab.com', 'provider': 'gitlab', + 'scm': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'oauth': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': True, 'id': '24ad1496-5a1d-43e2-9d96-42fce8e5484f', + 'name': 'IQIVA Public GitLab', 'provider': 'gitlab', + 'scm': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'oauth': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': False, 'id': '83cd96ae-b1f4-4bd9-b9ca-6f7f25c37708', + 'name': 'GitlabTest', 'provider': 'gitlab', + 'scm': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'oauth': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}]} + + +@regular_app.get("/v1/algorithms/{algo_id}/errors") +async def get_algo_errors(algo_id): + return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) + + +@regular_app.post("/v1/algorithms/{username}") +async def create_algorithm(request: Request, username): + payload = await request.json() + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], + "details": {"label": payload["details"]["label"]}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "resource_type": "algorithm"} + + +@regular_app.put('/v1/algorithms/{username}/{algoname}') +async def update_algorithm(request: Request, username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + + +@regular_app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + +fail_cnt = 0 + +@regular_app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + global fail_cnt + if "failonce" == algoname and fail_cnt == 0: + fail_cnt +=1 + return JSONResponse(content="This is an expected failure mode, try again", status_code=400) + elif "failalways" == algoname: + return JSONResponse(status_code=500) + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/versions") +async def versions_of_algorithm(request: Request, username, algoname): + return {"marker": None, "next_link": None, "results": [ + {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "package_set": "python36", + "license": "apl", "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", "started_at": "2022-05-08T22:43:09.050Z", + "finished_at": "2022-05-08T22:43:28.646Z", "version_info": {"semantic_version": "0.1.0"}, + "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"https://api.algorithmia.com/v1/algorithms/{username}/{algoname}/versions" + "/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"}]} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + if algohash == "e85db9bca2fad519f540b445f30d12523e4dec9c": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "language": "python3", + "environment": "cpu", "package_set": "python36", "license": "apl", + "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, "resource_type": "algorithm"} + else: + return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) + + +### Admin Routes +@regular_app.post("/v1/users") +async def create_user(request: Request): + payload = await request.body() + data = json.loads(payload) + username = data['username'] + email = data['email'] + return { + "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", + "username": username, + "email": email, + "fullname": username, + "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" + } + + +@regular_app.get("/v1/users/{user_id}/errors") +async def get_user_errors(user_id): + return [] + + +@regular_app.get("/v1/organization/types") +async def get_org_types(): + return [ + {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, + {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, + {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} + ] + + +@regular_app.post("/v1/organizations") +async def create_org(request: Request): + payload = await request.body() + data = json.loads(payload) + org_name = data["org_name"] + org_email = data["org_email"] + return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": org_email, + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@regular_app.put("/v1/organizations/{orgname}/members/{username}") +async def add_user_to_org(orgname, username): + return Response(status_code=200) + + +@regular_app.get("/v1/organizations/{orgname}/errors") +async def org_errors(orgname): + return [] + + +@regular_app.put("/v1/organizations/{org_name}") +async def edit_org(org_name): + return Response(status_code=204) + + +@regular_app.get("/v1/organizations/{org_name}") +async def get_org_by_name(org_name): + return { + "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg1542@algo.com", + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@regular_app.get("/v1/algorithm-environments/edge/languages") +async def get_supported_langs(): + return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", + "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", + "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, + {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", + "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, + {"name": "python2", "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "python3", "display_name": "Python 3.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "r36", "display_name": "R 3.6.x (Environments)", + "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, + {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", + "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] + + +@regular_app.get("/v1/algorithm-environments/edge/languages/{language}/environments") +async def get_environments_by_lang(language): + return { + "environments": [ + { + "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", + "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", + "display_name": "Python 2.7", + "description": "Generic Python 2.7 installation", + "created_at": "2020-12-21T21:47:53.239", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "CPU" + }, + { + "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", + "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", + "display_name": "Python 2.7 + GPU support", + "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", + "created_at": "2020-08-14T07:22:32.955", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "GPU" + } + ] + } + + +@regular_app.get("/v1/secret-provider") +async def get_service_providers(): + return [ + { + "id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "name": "algorithmia_internal_secret_provider", + "description": "Internal Secret Provider", + "moduleName": "module", + "factoryClassName": "com.algorithmia.plugin.sqlsecretprovider.InternalSecretProviderFactory", + "interfaceVersion": "1.0", + "isEnabled": True, + "isDefault": True, + "created": "2021-03-11T20:42:23Z", + "modified": "2021-03-11T20:42:23Z" + } + ] + + +@regular_app.get("/v1/algorithms/{algorithm_id}/secrets") +async def get_secrets_for_algorithm(algorithm_id): + return { + "secrets": [ + { + "id": "45e97c47-3ae6-46be-87ee-8ab23746706b", + "short_name": "MLOPS_SERVICE_URL", + "description": "", + "secret_key": "MLOPS_SERVICE_URL", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-22T14:36:01Z", + "modified": "2022-07-22T14:36:01Z" + }, + { + "id": "50dca60e-317f-4582-8854-5b83b4d182d0", + "short_name": "deploy_id", + "description": "", + "secret_key": "DEPLOYMENT_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "5a75cdc8-ecc8-4715-8c4b-8038991f1608", + "short_name": "model_path", + "description": "", + "secret_key": "MODEL_PATH", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "80e51ed3-f6db-419d-9349-f59f4bbfdcbb", + "short_name": "model_id", + "description": "", + "secret_key": "MODEL_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:30Z", + "modified": "2022-07-21T19:04:30Z" + }, + { + "id": "8773c654-ea2f-4ac5-9ade-55dfc47fec9d", + "short_name": "datarobot_api_token", + "description": "", + "secret_key": "DATAROBOT_MLOPS_API_TOKEN", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + } + ] + } + + +@regular_app.post("/v1/algorithms/{algorithm_id}/secrets") +async def set_algorithm_secret(algorithm_id): + return { + "id":"959af771-7cd8-4981-91c4-70def15bbcdc", + "short_name":"tst", + "description":"", + "secret_key":"test", + "owner_type":"algorithm", + "owner_id":"fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id":"dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created":"2022-07-22T18:28:42Z", + "modified":"2022-07-22T18:28:42Z" +} \ No newline at end of file diff --git a/Test/api/self_signed_app.py b/Test/api/self_signed_app.py new file mode 100644 index 0000000..693d486 --- /dev/null +++ b/Test/api/self_signed_app.py @@ -0,0 +1,374 @@ +from fastapi import FastAPI, Request +from typing import Optional +from fastapi.responses import Response +import json +import base64 +from multiprocessing import Process +import uvicorn + +self_signed_app = FastAPI() + + +def start_webserver_self_signed(): + def _start_webserver(): + uvicorn.run(self_signed_app, host="127.0.0.1", port=8090, log_level="debug", + ssl_certfile="Test/resources/cert.cert", ssl_keyfile="Test/resources/cert.key") + + p = Process(target=_start_webserver) + p.start() + return p + + +@self_signed_app.post("/v1/algo/{username}/{algoname}") +async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} + content_type = request.headers['Content-Type'] + auth = request.headers.get('Authorization', None) + if auth is None: + return {"error": {"message": "authorization required"}} + request = await request.body() + if output and output == "void": + return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} + elif output and output == "raw": + return Response(request.decode(), status_code=200) + elif algoname == "500": + return Response("Internal Server Error", status_code=500) + elif algoname == "raise_exception": + return {"error": {"message": "This is an exception"}} + else: + if content_type != "application/octet-stream": + request = request.decode('utf-8') + if content_type == "text/plain": + metadata['content_type'] = "text" + elif content_type == "application/json": + request = json.loads(request) + metadata['content_type'] = "json" + else: + metadata['content_type'] = "binary" + request = base64.b64encode(request) + output = {"result": request, "metadata": metadata} + return output + + +@self_signed_app.post("/v1/algo/{username}/{algoname}/{githash}") +async def process_hello_world(request: Request, username, algoname, githash): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, + 'content_type': "text"} + request = await request.body() + request = request.decode('utf-8') + return {"result": f"hello {request}", "metadata": metadata} + + +### Algorithm Routes +@self_signed_app.get('/v1/algorithms/{username}/{algoname}') +async def process_get_algo(request: Request, username, algoname): + if algoname == "echo": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", + "details": {"summary": "", "label": "echo", "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", + "package_set": "python36", "license": "apl", "royalty_microcredits": 0, + "network_access": "full", "pipeline_enabled": True, "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, + "self_link": "https://api.algorithmia.com/v1/algorithms/quality/echo/versions/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"} + else: + return {"error": "No such algorithm"} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") +async def get_build_id(username, algoname, buildid): + return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", + "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", + "version_info": {"semantic_version": "0.1.1"}} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/scm/status") +async def get_scm_status(username, algoname): + return {"scm_connection_status": "active"} + + +@self_signed_app.get("/v1/algorithms/{algo_id}/errors") +async def get_algo_errors(algo_id): + return {"error": {"message": "not found"}} + + +@self_signed_app.post("/v1/algorithms/{username}") +async def create_algorithm(request: Request, username): + payload = await request.json() + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], + "details": {"label": payload["details"]["label"]}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "resource_type": "algorithm"} + + +@self_signed_app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + + +@self_signed_app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/versions") +async def versions_of_algorithm(request: Request, username, algoname): + return {"marker": None, "next_link": None, "results": [ + {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "package_set": "python36", + "license": "apl", "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", "started_at": "2022-05-08T22:43:09.050Z", + "finished_at": "2022-05-08T22:43:28.646Z", "version_info": {"semantic_version": "0.1.0"}, + "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"https://api.algorithmia.com/v1/algorithms/{username}/{algoname}/versions" + "/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"}]} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + if algohash == "e85db9bca2fad519f540b445f30d12523e4dec9c": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "language": "python3", + "environment": "cpu", "package_set": "python36", "license": "apl", + "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, "resource_type": "algorithm"} + else: + return {"error": {"message": "not found"}} + + +### Admin Routes +@self_signed_app.post("/v1/users") +async def create_user(request: Request): + payload = await request.body() + data = json.loads(payload) + username = data['username'] + email = data['email'] + return { + "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", + "username": username, + "email": email, + "fullname": username, + "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" + } + + +@self_signed_app.get("/v1/users/{user_id}/errors") +async def get_user_errors(user_id): + return [] + + +@self_signed_app.get("/v1/organization/types") +async def get_org_types(): + return [ + {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, + {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, + {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} + ] + + +@self_signed_app.post("/v1/organizations") +async def create_org(request: Request): + payload = await request.body() + data = json.loads(payload) + org_name = data["org_name"] + org_email = data["org_email"] + return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": org_email, + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@self_signed_app.put("/v1/organizations/{orgname}/members/{username}") +async def add_user_to_org(orgname, username): + return Response(status_code=200) + + +@self_signed_app.get("/v1/organizations/{orgname}/errors") +async def org_errors(orgname): + return [] + + +@self_signed_app.put("/v1/organizations/{org_name}") +async def edit_org(org_name): + return Response(status_code=204) + + +@self_signed_app.get("/v1/organizations/{org_name}") +async def get_org_by_name(org_name): + return { + "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg1542@algo.com", + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@self_signed_app.get("/v1/algorithm-environments/edge/languages") +async def get_supported_langs(): + return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", + "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", + "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, + {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", + "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, + {"name": "python2", "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "python3", "display_name": "Python 3.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "r36", "display_name": "R 3.6.x (Environments)", + "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, + {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", + "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] + + +@self_signed_app.get("/v1/algorithm-environments/edge/languages/{language}/environments") +async def get_environments_by_lang(language): + return { + "environments": [ + { + "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", + "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", + "display_name": "Python 2.7", + "description": "Generic Python 2.7 installation", + "created_at": "2020-12-21T21:47:53.239", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "CPU" + }, + { + "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", + "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", + "display_name": "Python 2.7 + GPU support", + "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", + "created_at": "2020-08-14T07:22:32.955", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "GPU" + } + ] + } diff --git a/Test/client_test.py b/Test/client_test.py deleted file mode 100644 index e7d7e8f..0000000 --- a/Test/client_test.py +++ /dev/null @@ -1,226 +0,0 @@ -import os -import shutil -import sys -from datetime import datetime -from random import random -from random import seed - -sys.path = ['../'] + sys.path - -import unittest -import Algorithmia -from uuid import uuid4 - -if sys.version_info.major == 3: - unicode = str - -class ClientTest(unittest.TestCase): - seed(datetime.now().microsecond) - # due to legacy reasons, regular client tests are tested against api.algorithmia.com, whereas admin api tests are run - # against test.algorithmia.com. - admin_username = "a_Mrtest" - admin_org_name = "a_myOrg" - environment_name = "Python 3.9" - - def setUp(self): - self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) - self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) - - self.admin_username = self.admin_username + str(int(random() * 10000)) - self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", - api_key=self.admin_api_key) - self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', - api_key=self.regular_api_key) - - environments = self.regular_client.get_environment("python3") - for environment in environments['environments']: - if environment['display_name'] == self.environment_name: - self.environment_id = environment['id'] - - def test_create_user(self): - response = self.admin_client.create_user( - {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", - "shouldCreateHello": False}) - - if type(response) is dict: - self.assertEqual(self.admin_username, response['username']) - else: - self.assertIsNotNone(response) - - def test_get_org_types(self): - response = self.admin_client.get_org_types() - self.assertTrue(len(response) > 0) - - def test_create_org(self): - response = self.admin_client.create_org( - {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", - "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) - - self.assertEqual(self.admin_org_name, response[u'org_name']) - - def test_get_org(self): - response = self.admin_client.get_org("a_myOrg84") - self.assertEqual("a_myOrg84", response['org_name']) - - def test_get_environment(self): - response = self.admin_client.get_environment("python2") - - if u'error' not in response: - self.assertTrue(response is not None and u'environments' in response) - - def test_get_build_logs(self): - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = unicode('echo') - algo_path = u'%s/%s' % (user, algo) - result = self.regular_client.algo(algo_path).build_logs() - - if u'error' in result: - print(result) - - self.assertTrue(u'error' not in result) - - def test_get_build_logs_no_ssl(self): - client = Algorithmia.client(api_address='https://api.algorithmia.com', - api_key=self.regular_api_key, ca_cert=False) - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = u'Echo' - result = client.algo(user + '/' + algo).build_logs() - if u'error' in result: - print(result) - self.assertTrue("error" not in result) - - def test_edit_org(self): - org_name = "a_myOrg84" - - obj = { - "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", - "org_name": "a_myOrg84", - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": "a_myOrg84@algo.com", - "org_created_at": "2020-11-30T23:51:40", - "org_url": "https://algorithmia.com", - "type_id": "basic", - "resource_type": "organization" - } - - response = self.admin_client.edit_org(org_name, obj) - if type(response) is dict: - print(response) - else: - self.assertEqual(204, response.status_code) - - def test_get_template(self): - filename = "./temptest" - response = self.admin_client.get_template("36fd467e-fbfe-4ea6-aa66-df3f403b7132", filename) - - if type(response) is dict: - self.assertTrue(u'error' in response or u'message' in response) - else: - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - def test_get_supported_languages(self): - response = self.admin_client.get_supported_languages() - self.assertTrue(response is not None) - - if type(response) is not list: - self.assertTrue(u'error' in response) - else: - language_found = any('anaconda3' in languages['name'] for languages in response) - self.assertTrue(response is not None and language_found) - - def test_invite_to_org(self): - response = self.admin_client.invite_to_org("a_myOrg38", "a_Mrtest4") - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(200, response.status_code) - - # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been - # deployed to the remote environment. - def test_get_organization_errors(self): - response = self.admin_client.get_organization_errors(self.admin_org_name) - self.assertTrue(response is not None) - - if type(response) is list: - self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') - - def test_get_user_errors(self): - response = self.admin_client.get_user_errors(self.admin_username) - - self.assertTrue(response is not None) - self.assertEqual(0, len(response)) - - def test_get_algorithm_errors(self): - response = self.admin_client.get_algorithm_errors('hello') - self.assertTrue(response is not None) - - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(404, response.status_code) - - - def test_algorithm_programmatic_create_process(self): - algorithm_name = "algo_" + str(uuid4()).split("-")[-1] - payload = "John" - expected_response = "hello John" - full_path = self.regular_client.username() + "/" + algorithm_name - details = { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - } - settings = { - "source_visibility": "open", - "algorithm_environment": self.environment_id, - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False - } - created_algo = self.regular_client.algo(full_path) - response = created_algo.create(details=details,settings=settings) - self.assertEqual(response.name, algorithm_name, "algorithm creation failed") - - # --- Creation complete, compiling - - response = created_algo.compile() - git_hash = response.version_info.git_hash - algo_with_build = self.regular_client.algo(full_path + "/" + git_hash) - self.assertEqual(response.name, created_algo.algoname) - - # --- compiling complete, now testing algorithm request - response = algo_with_build.pipe(payload).result - self.assertEqual(response, expected_response, "compiling failed") - - # --- testing complete, now publishing new release. - - pub_settings = {"algorithm_callability": "private"} - pub_version_info = { - "release_notes": "created programmatically", - "sample_input": payload, - "version_type": "minor" - } - pub_details = {"label": "testing123"} - - response = algo_with_build.publish( - details=pub_details, - settings=pub_settings, - version_info=pub_version_info - ) - self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", "Publishing failed, semantic version is not correct.") - - # --- publishing complete, getting additional information - - response = created_algo.info(git_hash) - - self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") - - -if __name__ == '__main__': - unittest.main() diff --git a/Test/conftest.py b/Test/conftest.py new file mode 100644 index 0000000..c758814 --- /dev/null +++ b/Test/conftest.py @@ -0,0 +1,15 @@ +import sys +from time import sleep +import os, signal +if sys.version_info.major >= 3: + from Test.api import start_webserver_reg, start_webserver_self_signed + import pytest + + @pytest.fixture(scope='package', autouse=True) + def fastapi_start(): + p_reg = start_webserver_reg() + p_self_signed = start_webserver_self_signed() + sleep(2) + yield p_reg, p_self_signed + os.kill(p_reg.pid, signal.SIGKILL) + os.kill(p_self_signed.pid, signal.SIGKILL) \ No newline at end of file diff --git a/Test/regular/CLI_test.py b/Test/regular/CLI_test.py new file mode 100644 index 0000000..015db70 --- /dev/null +++ b/Test/regular/CLI_test.py @@ -0,0 +1,289 @@ +import sys + +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +import unittest +import os +import json +import Algorithmia +from Algorithmia.CLI import CLI +import argparse +import shutil + +if sys.version_info.major >= 3: + class CLIDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + cls.bearerClient = Algorithmia.client(api_address="http://localhost:8080", bearer_token="simabcd123.token.token") + + def test_run(self): + name = "util/Echo" + inputs = "test" + + parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') + + subparsers = parser.add_subparsers(help='sub cmd', dest='subparser_name') + parser_run = subparsers.add_parser('run', help='algo run [input options] [output options]') + + parser_run.add_argument('algo') + parser_run.add_argument('-d', '--data', action='store', help='detect input type', default=None) + parser_run.add_argument('-t', '--text', action='store', help='treat input as text', default=None) + parser_run.add_argument('-j', '--json', action='store', help='treat input as json data', default=None) + parser_run.add_argument('-b', '--binary', action='store', help='treat input as binary data', default=None) + parser_run.add_argument('-D', '--data-file', action='store', help='specify a path to an input file', + default=None) + parser_run.add_argument('-T', '--text-file', action='store', help='specify a path to a text file', + default=None) + parser_run.add_argument('-J', '--json-file', action='store', help='specify a path to a json file', + default=None) + parser_run.add_argument('-B', '--binary-file', action='store', help='specify a path to a binary file', + default=None) + parser_run.add_argument('--timeout', action='store', type=int, default=300, + help='specify a timeout (seconds)') + parser_run.add_argument('--debug', action='store_true', + help='print the stdout from the algo ') + parser_run.add_argument('--profile', action='store', type=str, default='default') + parser_run.add_argument('-o', '--output', action='store', default=None, type=str) + + args = parser.parse_args(['run', name, '-d', inputs]) + + result = CLI().runalgo(args, self.client) + self.assertEqual(result, inputs) + + def test_run_token(self): + name = "util/Echo" + inputs = "test" + + parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') + + subparsers = parser.add_subparsers(help='sub cmd', dest='subparser_name') + parser_run = subparsers.add_parser('run', help='algo run [input options] [output options]') + + parser_run.add_argument('algo') + parser_run.add_argument('-d', '--data', action='store', help='detect input type', default=None) + parser_run.add_argument('-t', '--text', action='store', help='treat input as text', default=None) + parser_run.add_argument('-j', '--json', action='store', help='treat input as json data', default=None) + parser_run.add_argument('-b', '--binary', action='store', help='treat input as binary data', default=None) + parser_run.add_argument('-D', '--data-file', action='store', help='specify a path to an input file', + default=None) + parser_run.add_argument('-T', '--text-file', action='store', help='specify a path to a text file', + default=None) + parser_run.add_argument('-J', '--json-file', action='store', help='specify a path to a json file', + default=None) + parser_run.add_argument('-B', '--binary-file', action='store', help='specify a path to a binary file', + default=None) + parser_run.add_argument('--timeout', action='store', type=int, default=300, + help='specify a timeout (seconds)') + parser_run.add_argument('--debug', action='store_true', + help='print the stdout from the algo ') + parser_run.add_argument('--profile', action='store', type=str, default='default') + parser_run.add_argument('-o', '--output', action='store', default=None, type=str) + + args = parser.parse_args(['run', name, '-d', inputs]) + + result = CLI().runalgo(args, self.bearerClient) + self.assertEqual(result, inputs) + + +class CLIMainTest(unittest.TestCase): + def setUp(self): + # create a directory to use in testing the cp command + self.client = Algorithmia.client() + CLI().mkdir("data://.my/moredata", self.client) + if not os.path.exists("../TestFiles/"): + os.mkdir("../TestFiles/") + + def test_ls(self): + parentDir = "data://.my/" + newDir = "test" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(result is not None and "moredata" in result and newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + def test_mkdir(self): + + parentDir = "data://.my/" + newDir = "test" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + def test_rmdir(self): + parentDir = "data://.my/" + newDir = "testRmdir" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir not in result) + + def test_cat(self): + file = "data://.my/moredata/test.txt" + localfile = "./../TestFiles/test.txt" + fileContents = "some text in test file" + + CLI().rm(file, self.client) + testfile = open(localfile, "w") + testfile.write(fileContents) + testfile.close() + + CLI().cp([localfile], file, self.client) + + result = CLI().cat([file], self.client) + self.assertEqual(result, fileContents) + + def test_get_build_logs(self): + user = os.environ.get('ALGO_USER_NAME') + algo = "Echo" + + result = json.loads(CLI().getBuildLogs(user, algo, self.client)) + if "error" in result: + print(result) + self.assertTrue("error" not in result) + + # local to remote + def test_cp_L2R(self): + localfile = "./../TestFiles/test.txt" + testfile = open(localfile, "w") + testfile.write("some text") + testfile.close() + + src = [localfile] + dest = "data://.my/moredata/test.txt" + CLI().cp(src, dest, self.client) + + result = CLI().ls("data://.my/moredata/", self.client) + self.assertTrue("test.txt" in result) + + # remote to remote + def test_cp_R2R(self): + + src = ["data://.my/moredata/test.txt"] + dest = "data://.my/moredata/test2.txt" + CLI().cp(src, dest, self.client) + + result = CLI().ls("data://.my/moredata/", self.client) + self.assertTrue("test2.txt" in result) + + # remote to local + def test_cp_R2L(self): + src = ["data://.my/moredata/test.txt"] + dest = "./../test.txt" + + CLI().cp(src, dest, self.client) + self.assertTrue(os.path.isfile(dest)) + + def test_auth(self): + # key for test account + key = os.getenv('ALGORITHMIA_API_KEY') + api_address = "https://api.algorithmia.com" + profile = 'default' + CLI().auth(api_address, key, profile=profile) + resultK = CLI().getAPIkey(profile) + resultA = CLI().getAPIaddress(profile) + self.assertEqual(resultK, key) + self.assertEqual(resultA, api_address) + + def test_auth_cert(self): + + localfile = "./../TestFiles/fakecert.pem" + + testfile = open(localfile, "w") + testfile.write("") + testfile.close() + + # key for test account + key = os.getenv('ALGORITHMIA_API_KEY') + address = 'https://api.algorithmia.com' + cacert = localfile + profile = 'test' + + CLI().auth(address, key, cacert=cacert, profile=profile) + resultK = CLI().getAPIkey(profile) + resultA = CLI().getAPIaddress(profile) + resultC = CLI().getCert(profile) + self.assertEqual(resultK, key) + self.assertEqual(resultA, address) + self.assertEqual(resultC, cacert) + + def test_auth_token(self): + address = 'https://api.algorithmia.com' + bearer = 'testtokenabcd' + profile = 'test' + + CLI().auth(apiaddress=address, bearer=bearer, profile=profile) + resultA = CLI().getAPIaddress(profile) + resultT = CLI().getBearerToken(profile) + self.assertEqual(resultA, address) + self.assertEqual(resultT, bearer) + + def test_get_environment(self): + result = CLI().get_environment_by_language("python2", self.client) + print(result) + if ("error" in result): + print(result) + self.assertTrue(result is not None and "display_name" in result) + + def test_list_languages(self): + result = CLI().list_languages(self.client) + if ("error" in result[0]): + print(result) + self.assertTrue(result is not None and "anaconda3" in result[1]) + + def test_rm(self): + localfile = "./../TestFiles/testRM.txt" + + testfile = open(localfile, "w") + testfile.write("some text") + testfile.close() + + src = [localfile] + dest = "data://.my/moredata/" + CLI().cp(src, dest, self.client) + + result1 = CLI().ls(dest, self.client) + + CLI().rm("data://.my/moredata/testRM.txt", self.client) + + result2 = CLI().ls(dest, self.client) + + self.assertTrue("testRM.txt" in result1 and "testRM.txt" not in result2) + + def test_get_template(self): + filename = "./../temptest" + envid = "36fd467e-fbfe-4ea6-aa66-df3f403b7132" + response = CLI().get_template(envid, filename, self.client) + print(response) + self.assertTrue(response.ok) + try: + shutil.rmtree(filename) + except OSError as e: + print(e) + + def test_api_address_auth(self): + api_key = os.getenv('ALGORITHMIA_API_KEY') + api_address = "https://api.algorithmia.com" + CLI().auth(api_address, api_key) + profile = "default" + + client = Algorithmia.client(CLI().getAPIkey(profile), CLI().getAPIaddress(profile), CLI().getCert(profile)) + result2 = CLI().ls("data://.my", client) + print(result2) + self.assertTrue(result2 != "") + + +if __name__ == '__main__': + unittest.main() diff --git a/Test/regular/__init__.py b/Test/regular/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Test/acl_test.py b/Test/regular/acl_test.py similarity index 100% rename from Test/acl_test.py rename to Test/regular/acl_test.py diff --git a/Test/regular/algo_failure_test.py b/Test/regular/algo_failure_test.py new file mode 100644 index 0000000..0ec4fc2 --- /dev/null +++ b/Test/regular/algo_failure_test.py @@ -0,0 +1,42 @@ +import sys + +if sys.version_info[0] >= 3: + import unittest + import Algorithmia + import uvicorn + import time + from multiprocessing import Process + + # look in ../ BEFORE trying to import Algorithmia. If you append to the + # you will load the version installed on the computer. + sys.path = ['../'] + sys.path + from requests import Response + + class AlgoTest(unittest.TestCase): + error_500 = Response() + error_500.status_code = 500 + error_message = "Non-Algorithm related Failure: " + str(error_500) + + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + + def test_throw_500_error_HTTP_response_on_algo_request(self): + try: + result = self.client.algo('util/500').pipe(bytearray('foo', 'utf-8')) + except Exception as e: + result = e + pass + self.assertEqual(str(self.error_message), str(result)) + + def test_retry_on_400_error_publish(self): + result = self.client.algo("util/failonce").publish() + self.assertEqual(result['version_info']['semantic_version'], "0.1.0") + + def test_throw_on_always_500_publish(self): + try: + result = self.client.algo("util/failalways").publish() + except Exception as e: + result = e + pass + self.assertEqual(str(self.error_message), str(result)) diff --git a/Test/regular/algo_test.py b/Test/regular/algo_test.py new file mode 100644 index 0000000..ab63c0d --- /dev/null +++ b/Test/regular/algo_test.py @@ -0,0 +1,253 @@ +import sys +import os +from Algorithmia.errors import AlgorithmException +from Algorithmia.algorithm import OutputType +import Algorithmia + +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +import unittest + +if sys.version_info.major >= 3: + + class AlgoDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + cls.environment_id = "abcd-123" + + def test_call_customCert(self): + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + + + + def test_normal_call(self): + result = self.client.algo('quality/echo').pipe("foo") + self.assertEqual("text", result.metadata.content_type) + self.assertEqual("foo", result.result) + + def test_async_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEqual("foo", result) + + def test_dict_call(self): + result = self.client.algo('quality/echo').pipe({"foo": "bar"}) + self.assertEqual("json", result.metadata.content_type) + self.assertEqual({"foo": "bar"}, result.result) + + def test_algo_exists(self): + result = self.client.algo('quality/echo').exists() + self.assertEqual(True, result) + + def test_algo_no_exists(self): + result = self.client.algo('quality/not_echo').exists() + self.assertEqual(False, result) + + #TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) + + def test_text_unicode(self): + telephone = u"\u260E" + # Unicode input to pipe() + result1 = self.client.algo('quality/echo').pipe(telephone) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('quality/echo').pipe(result1.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) + + def test_algo_info(self): + result = self.client.algo('quality/echo').info() + self.assertTrue('results' in result) + self.assertTrue('resource_type' in result['results'][0]) + self.assertTrue(result['results'][0]['resource_type'] == "algorithm") + + def test_update_algo(self): + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + version_info = { + "sample_input": "hello" + } + result = self.client.algo('quality/echo').update(details=details, settings=settings, version_info=version_info) + self.assertTrue('id' in result) + + + def test_get_build_by_id(self): + result = self.client.algo("quality/echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('commit_sha' in result) + + def test_get_build_logs(self): + result = self.client.algo("quality/echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('logs' in result) + + def test_get_scm_status(self): + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + + def test_algorithm_programmatic_create_process(self): + algorithm_name = "hello" + payload = "John" + expected_response = "hello John" + full_path = "quality/" + algorithm_name + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + version_info = { + "sample_input": "hello" + } + created_algo = self.client.algo(full_path) + print("about to create algo") + response = created_algo.create(details=details, settings=settings, version_info=version_info) + print("created algo") + self.assertEqual(response['name'], algorithm_name, "algorithm creation failed") + + # --- Creation complete, compiling + + response = created_algo.compile() + git_hash = response['version_info']['git_hash'] + algo_with_build = self.client.algo(full_path + "/" + git_hash) + self.assertEqual(response['name'], created_algo.algoname) + + # --- compiling complete, now testing algorithm request + response = algo_with_build.pipe(payload).result + self.assertEqual(response, expected_response, "compiling failed") + + # --- testing complete, now publishing new release. + + pub_settings = {"algorithm_callability": "private"} + pub_version_info = { + "release_notes": "created programmatically", + "sample_input": payload, + "version_type": "minor" + } + pub_details = {"label": "testing123"} + + response = algo_with_build.publish( + details=pub_details, + settings=pub_settings, + version_info=pub_version_info + ) + self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", + "Publishing failed, semantic version is not correct.") + + # --- publishing complete, getting additional information + + response = created_algo.info(git_hash) + + self.assertEqual(response['version_info']['semantic_version'], "0.1.0", "information is incorrect") + + + def test_set_secret(self): + short_name = "tst" + secret_key = "test_key" + secret_value = "test_value" + description = "loreum epsum" + response = self.client.algo("quality/echo").set_secret(short_name, secret_key, secret_value, description) + self.assertEqual(response['id'], "959af771-7cd8-4981-91c4-70def15bbcdc", "invalid ID for created secret") + + +else: + class AlgoTest(unittest.TestCase): + def setUp(self): + self.client = Algorithmia.client() + + def test_call_customCert(self): + open("./test.pem", 'w') + c = Algorithmia.client(ca_cert="./test.pem") + result = c.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + try: + os.remove("./test.pem") + except OSError as e: + print(e) + + def test_call_binary(self): + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + + def test_async_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEqual("foo", result) + + #TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) + + def test_text_unicode(self): + telephone = u"\u260E" + + # Unicode input to pipe() + result1 = self.client.algo('quality/echo').pipe(telephone) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('quality/echo').pipe(result1.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) + + + def test_get_scm_status(self): + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + +if __name__ == '__main__': + unittest.main() diff --git a/Test/regular/client_test.py b/Test/regular/client_test.py new file mode 100644 index 0000000..9cfc39b --- /dev/null +++ b/Test/regular/client_test.py @@ -0,0 +1,146 @@ +import os +import shutil +import sys +from datetime import datetime +from random import random +from random import seed + +sys.path = ['../'] + sys.path + +import unittest +import Algorithmia +from Algorithmia.errors import AlgorithmException +from uuid import uuid4 + +if sys.version_info.major >= 3: + unicode = str + + + class ClientDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + + admin_username = "a_Mrtest" + admin_org_name = "a_myOrg" + environment_name = "Python 3.9" + + def setUp(self): + self.admin_username = self.admin_username + str(int(random() * 10000)) + self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) + + def test_create_user(self): + response = self.client.create_user( + {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", + "shouldCreateHello": False}) + + if type(response) is dict: + self.assertEqual(self.admin_username, response['username']) + else: + self.assertIsNotNone(response) + + def test_get_org_types(self): + response = self.client.get_org_types() + self.assertTrue(len(response) > 0) + + def test_create_org(self): + response = self.client.create_org( + {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", + "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) + + self.assertEqual(self.admin_org_name, response[u'org_name']) + + def test_get_org(self): + response = self.client.get_org("a_myOrg84") + self.assertEqual("a_myOrg84", response['org_name']) + + def test_get_environment(self): + response = self.client.get_environment("python2") + + if u'error' not in response: + self.assertTrue(response is not None and u'environments' in response) + + def test_get_scms(self): + response = self.client.scms() + results = response['results'] + internal = [result for result in results if result['id'] == 'internal'] + self.assertTrue(len(internal) == 1) + + def test_edit_org(self): + org_name = "a_myOrg84" + + obj = { + "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", + "org_name": "a_myOrg84", + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg84@algo.com", + "org_created_at": "2020-11-30T23:51:40", + "org_url": "https://algorithmia.com", + "type_id": "basic", + "resource_type": "organization" + } + + response = self.client.edit_org(org_name, obj) + if type(response) is dict: + print(response) + else: + self.assertEqual(204, response.status_code) + + def test_get_supported_languages(self): + response = self.client.get_supported_languages() + self.assertTrue(response is not None) + + if type(response) is not list: + self.assertTrue(u'error' in response) + else: + language_found = any('anaconda3' in languages['name'] for languages in response) + self.assertTrue(response is not None and language_found) + + def test_invite_to_org(self): + response = self.client.invite_to_org("a_myOrg38", "a_Mrtest4") + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(200, response.status_code) + + # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been + # deployed to the remote environment. + def test_get_organization_errors(self): + response = self.client.get_organization_errors(self.admin_org_name) + self.assertTrue(response is not None) + + if type(response) is list: + self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') + + def test_get_user_errors(self): + response = self.client.get_user_errors(self.admin_username) + + self.assertTrue(response is not None) + self.assertEqual(0, len(response)) + + def test_get_algorithm_errors(self): + try: + _ = self.client.get_algorithm_errors('hello') + self.assertFalse(True) + except AlgorithmException as e: + self.assertTrue(e.message == "No such algorithm") + + def test_no_auth_client(self): + + key = os.environ.get('ALGORITHMIA_API_KEY', "") + if key != "": + del os.environ['ALGORITHMIA_API_KEY'] + + client = Algorithmia.client(api_address="http://localhost:8080") + error = None + try: + client.algo("demo/hello").pipe("world") + except Exception as e: + error = e + finally: + os.environ['ALGORITHMIA_API_KEY'] = key + self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, + error_type=None))) +if __name__ == '__main__': + unittest.main() diff --git a/Test/datadirectory_test.py b/Test/regular/datadirectory_test.py similarity index 87% rename from Test/datadirectory_test.py rename to Test/regular/datadirectory_test.py index a0d7672..9dc2704 100644 --- a/Test/datadirectory_test.py +++ b/Test/regular/datadirectory_test.py @@ -141,32 +141,34 @@ def test_list_folders(self): testDir.delete(True) - def test_list_files_with_paging(self): - NUM_FILES = 1100 - EXTENSION = '.txt' - dd = DataDirectory(self.client, 'data://.my/pythonLargeDataDirList') - if not dd.exists(): - dd.create() - - for i in range(NUM_FILES): - dd.file(str(i) + EXTENSION).put(str(i)) - - seenFiles = [False] * NUM_FILES - numFiles = 0 - - for f in dd.files(): - numFiles += 1 - name = f.getName() - index = int(name[:-1 * len(EXTENSION)]) - seenFiles[index] = True - - allSeen = True - for cur in seenFiles: - allSeen = (allSeen and cur) - - self.assertEqual(NUM_FILES, numFiles) - self.assertTrue(allSeen) +# TODO: replicate this in Marketplace + # def test_list_files_with_paging(self): + # NUM_FILES = 1100 + # EXTENSION = '.txt' + # + # dd = DataDirectory(self.client, 'data://.my/pythonLargeDataDirList') + # if not dd.exists(): + # dd.create() + # + # for i in range(NUM_FILES): + # dd.file(str(i) + EXTENSION).put(str(i)) + # + # seenFiles = [False] * NUM_FILES + # numFiles = 0 + # + # for f in dd.files(): + # numFiles += 1 + # name = f.getName() + # index = int(name[:-1 * len(EXTENSION)]) + # seenFiles[index] = True + # + # allSeen = True + # for cur in seenFiles: + # allSeen = (allSeen and cur) + # + # self.assertEqual(NUM_FILES, numFiles) + # self.assertTrue(allSeen) def test_data_object(self): dd = DataDirectory(self.client, 'data://foo') diff --git a/Test/datafile_test.py b/Test/regular/datafile_test.py similarity index 100% rename from Test/datafile_test.py rename to Test/regular/datafile_test.py diff --git a/Test/util_test.py b/Test/regular/util_test.py similarity index 100% rename from Test/util_test.py rename to Test/regular/util_test.py diff --git a/Test/resources/cert.cert b/Test/resources/cert.cert new file mode 100644 index 0000000..aeb562f --- /dev/null +++ b/Test/resources/cert.cert @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUTikiwxFBpLW4pC+5VfOis1xCYKcwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjA1MDMxNzE2MjZaFw0yMjA2 +MDIxNzE2MjZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDBUMZkg/bCJurQIB9znskTjv8URtIK6qqvZpYGTbfI +AzY6HiI0o1gPxjINZW7cBky/9MeEV5zyJghC4WoK099cIUNq2TmAWAjlRgIE8iEy +9z7QVfbSMainuw0RTlD5/8FRWtRe5v8qwbqLICMn3qv/KsG6bRezyS7UVihwFJua +E4dki+y6KSha4RrCtC43inbPlncB4om7PfJQyt5nI7N4KxbY2L3BUa5/+x1ux/ni +C/3y808vLJVQ6nLYgTEg/6K6lFrig0mUIMnCuOiBsrms3NmBPuDdRri/z1ulFHJB +WVQVQ5DgWher0f/dMzHwyRj3ffC8bAPlhrvLHwPQtNeRAgMBAAGjUzBRMB0GA1Ud +DgQWBBRoC77Hql6kEzk7WC6BeaPBu82K/jAfBgNVHSMEGDAWgBRoC77Hql6kEzk7 +WC6BeaPBu82K/jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCn +W9acM3+rxsiBBClTYEm2tOiukcXEkI7IzvW/4r7P24SmUiDD3vxVVbZ6nevVkg+P +4/QH+YYE3JUeXaN+xnHYjSy4NKxjd3EHT7BFxLMe0DQaodMj0klHqBtULNzojv8+ +/5tpQsjDLeeeDyOIJNz8r6CU9Gzh7j1EBF8BRdLA1z2UVmt6l6d4o3xOTYpOlZs3 +gI+ASxF9ODQzCCOeMYO2qiuMV3RD0oNdIEHUiMD+iHeC1jFGlxZzaWNeuUzP7Yj/ +MOwbBo8l6Hk2BUuUayLxZFLd0wN28IRkLEU5/SOh3mKz79nfPk6pD9rHUO1a53lI +Ua5xJ5tSwG6bMtNnHYYX +-----END CERTIFICATE----- diff --git a/Test/resources/cert.key b/Test/resources/cert.key new file mode 100644 index 0000000..1746297 --- /dev/null +++ b/Test/resources/cert.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDBUMZkg/bCJurQ +IB9znskTjv8URtIK6qqvZpYGTbfIAzY6HiI0o1gPxjINZW7cBky/9MeEV5zyJghC +4WoK099cIUNq2TmAWAjlRgIE8iEy9z7QVfbSMainuw0RTlD5/8FRWtRe5v8qwbqL +ICMn3qv/KsG6bRezyS7UVihwFJuaE4dki+y6KSha4RrCtC43inbPlncB4om7PfJQ +yt5nI7N4KxbY2L3BUa5/+x1ux/niC/3y808vLJVQ6nLYgTEg/6K6lFrig0mUIMnC +uOiBsrms3NmBPuDdRri/z1ulFHJBWVQVQ5DgWher0f/dMzHwyRj3ffC8bAPlhrvL +HwPQtNeRAgMBAAECggEAPr2OhhTmQ0EKOJ4UVxwTuotQci5CAVUELIUo78bNfNa+ +BMK+60KQVB5JJFvlTPemdS5miqc8wsJhMAOkvPripS4OiWES7nqj+HVuNli3OalQ +86DSyIlhaX6l0RYP5fOBtHu8LUjfS+swNfMqNchpHhmsYmsBpFIJJtUHrsihb7GR +4LpNOZ5go4+LG7FX9KaUE4FvAlS7hi6KLSMua10+3+NAlXggbcVikHr3Uq6eQIvk +z09cs+q2FHaESdTjXSIitmYOfJU5KK3QSfXAr/vaqakjnMvfp8MzQ5dHFsy03HRZ +Sy+LjRKOEOCMCT4DmGIPO4V89i3prbVH4JxixCOaeQKBgQDzuwERWE04JEtvfjxS +OAciQKLIxhfa4t2VB65d3115wxfDPIBYU5Mx5YV4aQyOddNxBwpmX/wYwstx2JDZ +2JM0OjOKLnSvlQfr5UmsY9jUO7CdmgC5HpgbHNhc8uJFw4pd+XypWSjytmVxBSdb +m0+in/iUUQuFNH/+BNLVVgWSiwKBgQDLDBCTEpKQvx2kAc8TEtwrWNhacZILab5D +StQBEL62VfGMdXYaA5dXreo5nqioHfBR3BfAmDecmq3iEFE8/yHJs6pLdcmj0Z1L +034UQedYLCmL9zuAgC6p4SKIMPubnYtMrNJOL3lq0ibogz3rfOhdN2B6S88IYoSL +M6asdoQN0wKBgCd1VPzr4MSAC75nH3joHS+Ma045087Z/6mK7s2/xbBax1QSTWz/ +Sss/L1aJG0FNDgg0bZiZXYTctHcf6oN6Loq8CXALiVSLuhaUrlK8b3QcncFGF2vg +6hspllWl9L/6okIIjAgWqSxyHwYnIXIRONlJMMNCQ60zDK2hNkjXflt1AoGAX0w3 +Tz/NSGBaogozTUlxymp1iOV63R5xLRYmsKVSTTPDHeBXYNhEpOM8ZnS/xb/fdhwt +jbgjib3TVKHB7zXzfr5zc91BmUCdaeRGbW2NDgYULdwIskP3IsZGtdL/lEb6BS+r +uQRxISCnIEPQwQCr8mw2PM/tyIqsmMTSOmmZiv8CgYBAfIC/cNwJyID6AVauZHQo +S3Bii9CPmPnuklBuS7ikX0bmZ93dzv537nqwGr0j9ksxabLWZRcoxx6MhgmXzXVT +dy48TWpqpHiMNorYskB9tcZSrBCl70bu5qKp2owqWHW0d4hqH3lkBNFhfwNWm+qC +54x3T/1fqyaqeapCiE5FGA== +-----END PRIVATE KEY----- diff --git a/Test/resources/manifests/example_manifest.json b/Test/resources/manifests/example_manifest.json new file mode 100644 index 0000000..ba6cbf5 --- /dev/null +++ b/Test/resources/manifests/example_manifest.json @@ -0,0 +1,29 @@ +{ + "required_files" : [ + { "name": "squeezenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + }, + { + "name": "labels", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ], + "optional_files": [ + { + "name": "mobilenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth", + "fail_on_tamper": false, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ] +} \ No newline at end of file diff --git a/Test/self_signed/__init__.py b/Test/self_signed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Test/algo_failure_test.py b/Test/self_signed/algo_failure_test.py similarity index 53% rename from Test/algo_failure_test.py rename to Test/self_signed/algo_failure_test.py index 236857a..aed319d 100644 --- a/Test/algo_failure_test.py +++ b/Test/self_signed/algo_failure_test.py @@ -6,30 +6,25 @@ import uvicorn import time from multiprocessing import Process + # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. sys.path = ['../'] + sys.path from requests import Response - from Test.api import app - - def start_webserver(): - uvicorn.run(app, host="127.0.0.1", port=8080, log_level="debug") class AlgoTest(unittest.TestCase): error_500 = Response() error_500.status_code = 500 + error_message = "Non-Algorithm related Failure: " + str(error_500) + + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) - def setUp(self): - self.client = Algorithmia.client(api_address="http://localhost:8080") - self.uvi_p = Process(target=start_webserver) - self.uvi_p.start() - time.sleep(1) - def tearDown(self): - self.uvi_p.terminate() def test_throw_500_error_HTTP_response_on_algo_request(self): try: - result = self.client.algo('util/Echo').pipe(bytearray('foo','utf-8')) + result = self.client.algo('util/500').pipe(bytearray('foo', 'utf-8')) except Exception as e: result = e pass - self.assertEqual(str(self.error_500), str(result)) + self.assertEqual(str(self.error_message), str(result)) diff --git a/Test/self_signed/algo_test.py b/Test/self_signed/algo_test.py new file mode 100644 index 0000000..b3b377f --- /dev/null +++ b/Test/self_signed/algo_test.py @@ -0,0 +1,97 @@ +import sys +import os +from Algorithmia.errors import AlgorithmException +from Algorithmia.algorithm import OutputType +import Algorithmia + +import unittest + +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +if sys.version_info.major >= 3: + + class AlgoDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) + + def test_call_customCert(self): + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + + def test_normal_call(self): + result = self.client.algo('quality/echo').pipe("foo") + self.assertEquals("text", result.metadata.content_type) + self.assertEquals("foo", result.result) + + def test_async_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEquals("foo", result) + + def test_dict_call(self): + result = self.client.algo('quality/echo').pipe({"foo": "bar"}) + self.assertEquals("json", result.metadata.content_type) + self.assertEquals({"foo": "bar"}, result.result) + + def test_algo_exists(self): + result = self.client.algo('quality/echo').exists() + self.assertEquals(True, result) + + def test_algo_no_exists(self): + result = self.client.algo('quality/not_echo').exists() + self.assertEquals(False, result) + + # TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + + def test_text_unicode(self): + telephone = u"\u260E" + # Unicode input to pipe() + result1 = self.client.algo('quality/echo').pipe(telephone) + self.assertEquals('text', result1.metadata.content_type) + self.assertEquals(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('quality/echo').pipe(result1.result) + self.assertEquals('text', result2.metadata.content_type) + self.assertEquals(telephone, result2.result) + + def test_algo_info(self): + result = self.client.algo('quality/echo').info() + self.assertTrue('results' in result) + self.assertTrue('resource_type' in result['results'][0]) + self.assertTrue(result['results'][0]['resource_type'] == "algorithm") + + def test_get_build_by_id(self): + result = self.client.algo("quality/echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('commit_sha' in result) + + def test_get_build_logs(self): + result = self.client.algo("quality/echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('logs' in result) + + def test_get_scm_status(self): + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + +if __name__ == '__main__': + unittest.main() diff --git a/requirements.txt b/requirements.txt index f12c984..207a7f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.1,<1.2 +algorithmia-adk>=1.2,<1.4 numpy<2 uvicorn==0.14.0 fastapi==0.65.2 diff --git a/requirements27.txt b/requirements27.txt index 3d2b39c..8a118ea 100644 --- a/requirements27.txt +++ b/requirements27.txt @@ -4,5 +4,5 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.1,<1.2 +algorithmia-adk>=1.2,<1.4 numpy<2 diff --git a/setup.py b/setup.py index 3730ce4..0069b73 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'toml', 'argparse', 'algorithmia-api-client==1.5.1', - 'algorithmia-adk>=1.1,<1.2' + 'algorithmia-adk>=1.2,<1.4' ], include_package_data=True, classifiers=[