From 1edb3304a767fdfc1b330517e62ce2fbd9adfa90 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Mon, 25 Oct 2021 11:53:01 -0700 Subject: [PATCH 01/18] Fastapi ci fixtures (#115) * added fastAPI tests to all algorithm tests * added fastAPI tests to all client test cases * fixed test fixtures * added 2/3 switch for conftest, and added back env var gets for python 2 tests * migrated run algo test from CLI main test suite to dummy test suite * added 2 second wait for start of process --- Algorithmia/algorithm.py | 5 - Test/CLI_test.py | 447 +++++++++++++++-------------- Test/algo_failure_test.py | 7 +- Test/algo_test.py | 168 ++++++----- Test/api/__init__.py | 343 +++++++++++++++++++++- Test/client_test.py | 590 +++++++++++++++++++++++++------------- Test/conftest.py | 12 + 7 files changed, 1067 insertions(+), 505 deletions(-) create mode 100644 Test/conftest.py diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 2fe5820..85a6f85 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -69,11 +69,6 @@ def update(self, details={}, settings={}, version_info={}): # Publish an algorithm def publish(self, details={}, settings={}, version_info={}): - # detailsObj = Details(**details) - # settingsObj = SettingsPublish(**settings) - # versionRequestObj = VersionInfoPublish(**version_info) - # publish_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": versionRequestObj} - # version_request = VersionRequest(**publish_parameters) # VersionRequest | Publish Version Request publish_parameters = {"details": details, "settings": settings, "version_info": version_info} url = "/v1/algorithms/"+self.username+"/"+self.algoname + "/versions" print(publish_parameters) diff --git a/Test/CLI_test.py b/Test/CLI_test.py index 35ca486..ae1d546 100644 --- a/Test/CLI_test.py +++ b/Test/CLI_test.py @@ -1,4 +1,5 @@ import sys + # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. sys.path = ['../'] + sys.path @@ -10,225 +11,233 @@ from Algorithmia.CLI import CLI import argparse import shutil -import toml - -class CLITest(unittest.TestCase): - def setUp(self): - # create a directory to use in testing the cp command - self.client = Algorithmia.client() - CLI().mkdir("data://.my/moredata", self.client) - if(not os.path.exists("./TestFiles/")): - os.mkdir("./TestFiles/") - - def test_ls(self): - parentDir = "data://.my/" - newDir = "test" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(result is not None and "moredata" in result and newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - - def test_mkdir(self): - - parentDir = "data://.my/" - newDir = "test" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - def test_rmdir(self): - parentDir = "data://.my/" - newDir = "testRmdir" - - CLI().mkdir(parentDir+newDir, self.client) - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir in result) - - CLI().rmdir(parentDir+newDir, self.client) - - result = CLI().ls(parentDir, self.client) - self.assertTrue(newDir not in result) - - def test_cat(self): - file = "data://.my/moredata/test.txt" - localfile = "./TestFiles/test.txt" - fileContents = "some text in test file" - - CLI().rm(file, self.client) - testfile = open(localfile, "w") - testfile.write(fileContents) - testfile.close() - - CLI().cp([localfile],file,self.client) - - result = CLI().cat([file],self.client) - self.assertEqual(result, fileContents) - - def test_get_build_logs(self): - user=os.environ.get('ALGO_USER_NAME') - algo="Echo" - - result = json.loads(CLI().getBuildLogs(user,algo,self.client)) - if "error" in result: - print(result) - self.assertTrue("error" not in result) - - -#local to remote - def test_cp_L2R(self): - localfile = "./TestFiles/test.txt" - testfile = open(localfile, "w") - testfile.write("some text") - testfile.close() - - src = [localfile] - dest = "data://.my/moredata/test.txt" - CLI().cp(src,dest,self.client) - - result = CLI().ls("data://.my/moredata/",self.client) - self.assertTrue("test.txt" in result) - -#remote to remote - def test_cp_R2R(self): - - src = ["data://.my/moredata/test.txt"] - dest = "data://.my/moredata/test2.txt" - CLI().cp(src,dest,self.client) - - result = CLI().ls("data://.my/moredata/",self.client) - self.assertTrue("test2.txt" in result) - -#remote to local - def test_cp_R2L(self): - src = ["data://.my/moredata/test.txt"] - dest = "./test.txt" - - CLI().cp(src,dest,self.client) - self.assertTrue(os.path.isfile(dest)) - - def test_run(self): - name = "util/Echo" - inputs = "test" - - parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') - - subparsers = parser.add_subparsers(help = 'sub cmd',dest = 'subparser_name') - parser_run = subparsers.add_parser('run', help = 'algo run [input options] [output options]') - - parser_run.add_argument('algo') - parser_run.add_argument('-d','--data', action = 'store', help = 'detect input type', default = None) - parser_run.add_argument('-t','--text', action = 'store', help = 'treat input as text', default = None) - parser_run.add_argument('-j','--json', action = 'store', help = 'treat input as json data', default = None) - parser_run.add_argument('-b','--binary', action = 'store', help = 'treat input as binary data', default = None) - parser_run.add_argument('-D','--data-file', action = 'store', help = 'specify a path to an input file', default = None) - parser_run.add_argument('-T','--text-file', action = 'store', help = 'specify a path to a text file', default = None) - parser_run.add_argument('-J','--json-file', action = 'store', help = 'specify a path to a json file', default = None) - parser_run.add_argument('-B','--binary-file', action = 'store', help = 'specify a path to a binary file', default = None) - parser_run.add_argument('--timeout', action = 'store',type = int, default = 300, help = 'specify a timeout (seconds)') - parser_run.add_argument('--debug', action = 'store_true', help = 'print the stdout from the algo ') - parser_run.add_argument('--profile', action = 'store', type = str, default = 'default') - parser_run.add_argument('-o', '--output', action = 'store', default = None, type = str) - - args = parser.parse_args(['run',name,'-d',inputs]) - - result = CLI().runalgo(args, self.client) - self.assertEqual(result, inputs) - - def test_auth(self): - #key for test account - key = os.getenv('ALGORITHMIA_API_KEY') - address = 'https://api.algorithmia.com' - profile = 'default' - CLI().auth(key,address,profile=profile) - resultK = CLI().getAPIkey(profile) - resultA = CLI().getAPIaddress(profile) - self.assertEqual(resultK, key) - self.assertEqual(resultA, address) - - def test_auth_cert(self): - - localfile = "./TestFiles/fakecert.pem" - - testfile = open(localfile, "w") - testfile.write("") - testfile.close() - - #key for test account - key = os.getenv('ALGORITHMIA_API_KEY') - address = 'https://api.algorithmia.com' - cacert = localfile - profile = 'test' - - CLI().auth(key,address,cacert,profile) - resultK = CLI().getAPIkey(profile) - resultA = CLI().getAPIaddress(profile) - resultC = CLI().getCert(profile) - self.assertEqual(resultK, key) - self.assertEqual(resultA, address) - self.assertEqual(resultC, cacert) - - def test_get_environment(self): - result = CLI().get_environment_by_language("python2",self.client) - print(result) - if("error" in result): - print(result) - self.assertTrue(result is not None and "display_name" in result) - - def test_list_languages(self): - result = CLI().list_languages(self.client) - if("error" in result[0]): - print(result) - self.assertTrue(result is not None and "anaconda3" in result[1]) - - - def test_rm(self): - localfile = "./TestFiles/testRM.txt" - - testfile = open(localfile, "w") - testfile.write("some text") - testfile.close() - - src = [localfile] - dest = "data://.my/moredata/" - CLI().cp(src,dest,self.client) - - result1 = CLI().ls(dest,self.client) - - CLI().rm("data://.my/moredata/testRM.txt",self.client) - - result2 = CLI().ls(dest,self.client) - - self.assertTrue("testRM.txt" in result1 and "testRM.txt" not in result2) - - def test_get_template(self): - filename = "./temptest" - envid = "36fd467e-fbfe-4ea6-aa66-df3f403b7132" - response = CLI().get_template(envid,filename,self.client) - print(response) - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - def test_api_address_auth(self): - api_key = os.getenv('ALGORITHMIA_TEST_API_KEY') - api_address = "https://api.test.algorithmia.com" - CLI().auth(api_key, api_address) - profile = "default" - - client = Algorithmia.client(CLI().getAPIkey(profile), CLI().getAPIaddress(profile), CLI().getCert(profile)) - result2 = CLI().ls("data://.my", client) - print(result2) - self.assertTrue(result2 != "") - + +if sys.version_info.major >= 3: + class CLIDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + + def test_run(self): + name = "util/Echo" + inputs = "test" + + parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') + + subparsers = parser.add_subparsers(help='sub cmd', dest='subparser_name') + parser_run = subparsers.add_parser('run', help='algo run [input options] [output options]') + + parser_run.add_argument('algo') + parser_run.add_argument('-d', '--data', action='store', help='detect input type', default=None) + parser_run.add_argument('-t', '--text', action='store', help='treat input as text', default=None) + parser_run.add_argument('-j', '--json', action='store', help='treat input as json data', default=None) + parser_run.add_argument('-b', '--binary', action='store', help='treat input as binary data', default=None) + parser_run.add_argument('-D', '--data-file', action='store', help='specify a path to an input file', + default=None) + parser_run.add_argument('-T', '--text-file', action='store', help='specify a path to a text file', + default=None) + parser_run.add_argument('-J', '--json-file', action='store', help='specify a path to a json file', + default=None) + parser_run.add_argument('-B', '--binary-file', action='store', help='specify a path to a binary file', + default=None) + parser_run.add_argument('--timeout', action='store', type=int, default=300, + help='specify a timeout (seconds)') + parser_run.add_argument('--debug', action='store_true', + help='print the stdout from the algo ') + parser_run.add_argument('--profile', action='store', type=str, default='default') + parser_run.add_argument('-o', '--output', action='store', default=None, type=str) + + args = parser.parse_args(['run', name, '-d', inputs]) + + result = CLI().runalgo(args, self.client) + self.assertEqual(result, inputs) + + +class CLIMainTest(unittest.TestCase): + def setUp(self): + # create a directory to use in testing the cp command + self.client = Algorithmia.client() + CLI().mkdir("data://.my/moredata", self.client) + if not os.path.exists("./TestFiles/"): + os.mkdir("./TestFiles/") + + def test_ls(self): + parentDir = "data://.my/" + newDir = "test" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(result is not None and "moredata" in result and newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + def test_mkdir(self): + + parentDir = "data://.my/" + newDir = "test" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + def test_rmdir(self): + parentDir = "data://.my/" + newDir = "testRmdir" + + CLI().mkdir(parentDir + newDir, self.client) + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir in result) + + CLI().rmdir(parentDir + newDir, self.client) + + result = CLI().ls(parentDir, self.client) + self.assertTrue(newDir not in result) + + def test_cat(self): + file = "data://.my/moredata/test.txt" + localfile = "./TestFiles/test.txt" + fileContents = "some text in test file" + + CLI().rm(file, self.client) + testfile = open(localfile, "w") + testfile.write(fileContents) + testfile.close() + + CLI().cp([localfile], file, self.client) + + result = CLI().cat([file], self.client) + self.assertEqual(result, fileContents) + + def test_get_build_logs(self): + user = os.environ.get('ALGO_USER_NAME') + algo = "Echo" + + result = json.loads(CLI().getBuildLogs(user, algo, self.client)) + if "error" in result: + print(result) + self.assertTrue("error" not in result) + + # local to remote + def test_cp_L2R(self): + localfile = "./TestFiles/test.txt" + testfile = open(localfile, "w") + testfile.write("some text") + testfile.close() + + src = [localfile] + dest = "data://.my/moredata/test.txt" + CLI().cp(src, dest, self.client) + + result = CLI().ls("data://.my/moredata/", self.client) + self.assertTrue("test.txt" in result) + + # remote to remote + def test_cp_R2R(self): + + src = ["data://.my/moredata/test.txt"] + dest = "data://.my/moredata/test2.txt" + CLI().cp(src, dest, self.client) + + result = CLI().ls("data://.my/moredata/", self.client) + self.assertTrue("test2.txt" in result) + + # remote to local + def test_cp_R2L(self): + src = ["data://.my/moredata/test.txt"] + dest = "./test.txt" + + CLI().cp(src, dest, self.client) + self.assertTrue(os.path.isfile(dest)) + + def test_auth(self): + # key for test account + key = os.getenv('ALGORITHMIA_API_KEY') + address = 'https://api.algorithmia.com' + profile = 'default' + CLI().auth(key, address, profile=profile) + resultK = CLI().getAPIkey(profile) + resultA = CLI().getAPIaddress(profile) + self.assertEqual(resultK, key) + self.assertEqual(resultA, address) + + def test_auth_cert(self): + + localfile = "./TestFiles/fakecert.pem" + + testfile = open(localfile, "w") + testfile.write("") + testfile.close() + + # key for test account + key = os.getenv('ALGORITHMIA_API_KEY') + address = 'https://api.algorithmia.com' + cacert = localfile + profile = 'test' + + CLI().auth(key, address, cacert, profile) + resultK = CLI().getAPIkey(profile) + resultA = CLI().getAPIaddress(profile) + resultC = CLI().getCert(profile) + self.assertEqual(resultK, key) + self.assertEqual(resultA, address) + self.assertEqual(resultC, cacert) + + def test_get_environment(self): + result = CLI().get_environment_by_language("python2", self.client) + print(result) + if ("error" in result): + print(result) + self.assertTrue(result is not None and "display_name" in result) + + def test_list_languages(self): + result = CLI().list_languages(self.client) + if ("error" in result[0]): + print(result) + self.assertTrue(result is not None and "anaconda3" in result[1]) + + def test_rm(self): + localfile = "./TestFiles/testRM.txt" + + testfile = open(localfile, "w") + testfile.write("some text") + testfile.close() + + src = [localfile] + dest = "data://.my/moredata/" + CLI().cp(src, dest, self.client) + + result1 = CLI().ls(dest, self.client) + + CLI().rm("data://.my/moredata/testRM.txt", self.client) + + result2 = CLI().ls(dest, self.client) + + self.assertTrue("testRM.txt" in result1 and "testRM.txt" not in result2) + + def test_get_template(self): + filename = "./temptest" + envid = "36fd467e-fbfe-4ea6-aa66-df3f403b7132" + response = CLI().get_template(envid, filename, self.client) + print(response) + self.assertTrue(response.ok) + try: + shutil.rmtree(filename) + except OSError as e: + print(e) + + def test_api_address_auth(self): + api_key = os.getenv('ALGORITHMIA_TEST_API_KEY') + api_address = "https://api.test.algorithmia.com" + CLI().auth(api_key, api_address) + profile = "default" + + client = Algorithmia.client(CLI().getAPIkey(profile), CLI().getAPIaddress(profile), CLI().getCert(profile)) + result2 = CLI().ls("data://.my", client) + print(result2) + self.assertTrue(result2 != "") if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/Test/algo_failure_test.py b/Test/algo_failure_test.py index 236857a..1e65234 100644 --- a/Test/algo_failure_test.py +++ b/Test/algo_failure_test.py @@ -6,15 +6,18 @@ import uvicorn import time from multiprocessing import Process + # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. sys.path = ['../'] + sys.path from requests import Response from Test.api import app + def start_webserver(): uvicorn.run(app, host="127.0.0.1", port=8080, log_level="debug") + class AlgoTest(unittest.TestCase): error_500 = Response() error_500.status_code = 500 @@ -24,11 +27,13 @@ def setUp(self): self.uvi_p = Process(target=start_webserver) self.uvi_p.start() time.sleep(1) + def tearDown(self): self.uvi_p.terminate() + def test_throw_500_error_HTTP_response_on_algo_request(self): try: - result = self.client.algo('util/Echo').pipe(bytearray('foo','utf-8')) + result = self.client.algo('util/500').pipe(bytearray('foo', 'utf-8')) except Exception as e: result = e pass diff --git a/Test/algo_test.py b/Test/algo_test.py index b14e99b..b22988e 100644 --- a/Test/algo_test.py +++ b/Test/algo_test.py @@ -1,77 +1,117 @@ import sys import os from Algorithmia.errors import AlgorithmException +import Algorithmia # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. sys.path = ['../'] + sys.path import unittest -import Algorithmia +if sys.version_info.major >= 3: + + + class AlgoDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + + def test_call_customCert(self): + result = self.client.algo('util/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + + def test_normal_call(self): + result = self.client.algo('util/echo').pipe("foo") + self.assertEquals("text", result.metadata.content_type) + self.assertEquals("foo", result.result) + + def test_dict_call(self): + result = self.client.algo('util/echo').pipe({"foo": "bar"}) + self.assertEquals("json", result.metadata.content_type) + self.assertEquals({"foo": "bar"}, result.result) + + def test_text_unicode(self): + telephone = u"\u260E" + # Unicode input to pipe() + result1 = self.client.algo('util/Echo').pipe(telephone) + self.assertEquals('text', result1.metadata.content_type) + self.assertEquals(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('util/Echo').pipe(result1.result) + self.assertEquals('text', result2.metadata.content_type) + self.assertEquals(telephone, result2.result) + + def test_get_build_by_id(self): + result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.build_id is not None) + + def test_get_build_logs(self): + result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.logs is not None) + + def test_get_scm_status(self): + result = self.client.algo("J_bragg/Echo").get_scm_status() + self.assertTrue(result.scm_connection_status is not None) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + +else: + class AlgoTest(unittest.TestCase): + def setUp(self): + self.client = Algorithmia.client() + + def test_call_customCert(self): + open("./test.pem", 'w') + c = Algorithmia.client(ca_cert="./test.pem") + result = c.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + try: + os.remove("./test.pem") + except OSError as e: + print(e) + + def test_call_binary(self): + result = self.client.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + + def test_text_unicode(self): + telephone = u"\u260E" + + # Unicode input to pipe() + result1 = self.client.algo('util/Echo').pipe(telephone) + self.assertEquals('text', result1.metadata.content_type) + self.assertEquals(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('util/Echo').pipe(result1.result) + self.assertEquals('text', result2.metadata.content_type) + self.assertEquals(telephone, result2.result) + + def test_get_build_by_id(self): + result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.build_id is not None) + + def test_get_build_logs(self): + result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.logs is not None) + + def test_get_scm_status(self): + result = self.client.algo("J_bragg/Echo").get_scm_status() + self.assertTrue(result.scm_connection_status is not None) -class AlgoTest(unittest.TestCase): - def setUp(self): - self.client = Algorithmia.client() - - def test_call_customCert(self): - open("./test.pem",'w') - c = Algorithmia.client(ca_cert="./test.pem") - result = c.algo('util/Echo').pipe(bytearray('foo','utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo','utf-8'), result.result) - try: - os.remove("./test.pem") - except OSError as e: - print(e) - - def test_call_binary(self): - result = self.client.algo('util/Echo').pipe(bytearray('foo','utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo','utf-8'), result.result) - - def test_text_unicode(self): - telephone = u"\u260E" - - #Unicode input to pipe() - result1 = self.client.algo('util/Echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) - - #Unicode return in .result - result2 = self.client.algo('util/Echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) - - def test_get_build_by_id(self): - result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.build_id is not None) - - def test_get_build_logs(self): - result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.logs is not None) - - def test_get_scm_status(self): - result = self.client.algo("J_bragg/Echo").get_scm_status() - self.assertTrue(result.scm_connection_status is not None) - - def test_exception_ipa_algo(self): - try: - result = self.client.algo('zeryx/raise_exception').pipe("") - except AlgorithmException as e: - self.assertEqual(e.message, "This is an exception") - - # def test_json_unicode(self): - # telephone = [u"\u260E"] - # - # #Unicode input to pipe() - # result1 = self.client.algo('util/Echo').pipe(telephone) - # self.assertEquals('json', result1.metadata.content_type) - # self.assertEquals(telephone, result1.result) - # - # #Unicode return in .result - # result2 = self.client.algo('util/Echo').pipe(result1.result) - # self.assertEquals('json', result2.metadata.content_type) - # self.assertEquals(telephone, result2.result) + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") if __name__ == '__main__': unittest.main() diff --git a/Test/api/__init__.py b/Test/api/__init__.py index 5ca3185..99e7e68 100644 --- a/Test/api/__init__.py +++ b/Test/api/__init__.py @@ -1,16 +1,341 @@ import importlib -from fastapi import FastAPI, Response +from fastapi import FastAPI, Request +from fastapi.responses import Response +import json +import base64 +from multiprocessing import Process +import uvicorn app = FastAPI() -@app.post("/v1/{username}/{algoname}/{version}") -async def throw_error(username, algoname, version): - return Response("Internal Server Error", status_code=500) +def start_webserver(): + def _start_webserver(): + uvicorn.run(app, host="127.0.0.1", port=8080, log_level="debug") -def create_endpoint(algoname): - module = importlib.import_module(algoname) - @app.get("/invocations") - def invocations(data): - return module.apply(data) + p = Process(target=_start_webserver) + p.start() + return p + +@app.post("/v1/algo/{username}/{algoname}") +async def process_algo_req(request: Request, username, algoname): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} + content_type = request.headers['Content-Type'] + request = await request.body() + if algoname == "500": + return Response("Internal Server Error", status_code=500) + elif algoname == "raise_exception": + return {"error": {"message": "This is an exception"}} + else: + if content_type != "application/octet-stream": + request = request.decode('utf-8') + if content_type == "text/plain": + metadata['content_type'] = "text" + elif content_type == "application/json": + request = json.loads(request) + metadata['content_type'] = "json" + else: + metadata['content_type'] = "binary" + request = base64.b64encode(request) + output = {"result": request, "metadata": metadata} + return output + + +@app.post("/v1/algo/{username}/{algoname}/{githash}") +async def process_hello_world(request: Request, username, algoname, githash): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, + 'content_type': "text"} + request = await request.body() + request = request.decode('utf-8') + return {"result": f"hello {request}", "metadata": metadata} + + +### Algorithm Routes +@app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") +async def get_build_id(username, algoname, buildid): + return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", + "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", + "version_info": {"semantic_version": "0.1.1"}} + + +@app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@app.get("/v1/algorithms/{username}/{algoname}/scm/status") +async def get_scm_status(username, algoname): + return {"scm_connection_status": "active"} + + +@app.get("/v1/algorithms/{algo_id}/errors") +async def get_algo_errors(algo_id): + return {"error": {"message": "not found"}} + + +@app.post("/v1/algorithms/{username}") +async def create_algorithm(request: Request, username): + payload = await request.json() + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], + "details": {"label": payload["details"]["label"]}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "resource_type": "algorithm"} + + +@app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + + +@app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "language": "python3", + "environment": "gpu", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "semantic_version": "0.1.0", + "git_hash": algohash, + "release_notes": "created programmatically", + "sample_input": "\"payload\"", + "sample_output": "Exception encountered while running sample input", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "resource_type": "algorithm" + } + + +### Admin Routes +@app.post("/v1/users") +async def create_user(request: Request): + payload = await request.body() + data = json.loads(payload) + username = data['username'] + email = data['email'] + return { + "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", + "username": username, + "email": email, + "fullname": username, + "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" + } + + +@app.get("/v1/users/{user_id}/errors") +async def get_user_errors(user_id): + return [] + + +@app.get("/v1/organization/types") +async def get_org_types(): + return [ + {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, + {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, + {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} + ] + + +@app.post("/v1/organizations") +async def create_org(request: Request): + payload = await request.body() + data = json.loads(payload) + org_name = data["org_name"] + org_email = data["org_email"] + return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": org_email, + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@app.put("/v1/organizations/{orgname}/members/{username}") +async def add_user_to_org(orgname, username): + return Response(status_code=200) + + +@app.get("/v1/organizations/{orgname}/errors") +async def org_errors(orgname): + return [] + + +@app.put("/v1/organizations/{org_name}") +async def edit_org(org_name): + return Response(status_code=204) + + +@app.get("/v1/organizations/{org_name}") +async def get_org_by_name(org_name): + return { + "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg1542@algo.com", + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@app.get("/v1/algorithm-environments/edge/languages") +async def get_supported_langs(): + return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", + "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", + "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, + {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", + "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, + {"name": "python2", "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "python3", "display_name": "Python 3.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "r36", "display_name": "R 3.6.x (Environments)", + "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, + {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", + "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] + + +@app.get("/v1/algorithm-environments/edge/languages/{language}/environments") +async def get_environments_by_lang(language): + return { + "environments": [ + { + "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", + "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", + "display_name": "Python 2.7", + "description": "Generic Python 2.7 installation", + "created_at": "2020-12-21T21:47:53.239", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "CPU" + }, + { + "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", + "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", + "display_name": "Python 2.7 + GPU support", + "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", + "created_at": "2020-08-14T07:22:32.955", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "GPU" + } + ] + } diff --git a/Test/client_test.py b/Test/client_test.py index e7d7e8f..a254c45 100644 --- a/Test/client_test.py +++ b/Test/client_test.py @@ -11,215 +11,391 @@ import Algorithmia from uuid import uuid4 -if sys.version_info.major == 3: +if sys.version_info.major >= 3: unicode = str -class ClientTest(unittest.TestCase): - seed(datetime.now().microsecond) - # due to legacy reasons, regular client tests are tested against api.algorithmia.com, whereas admin api tests are run - # against test.algorithmia.com. - admin_username = "a_Mrtest" - admin_org_name = "a_myOrg" - environment_name = "Python 3.9" - - def setUp(self): - self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) - self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) - - self.admin_username = self.admin_username + str(int(random() * 10000)) - self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", - api_key=self.admin_api_key) - self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', - api_key=self.regular_api_key) - - environments = self.regular_client.get_environment("python3") - for environment in environments['environments']: - if environment['display_name'] == self.environment_name: - self.environment_id = environment['id'] - - def test_create_user(self): - response = self.admin_client.create_user( - {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", - "shouldCreateHello": False}) - - if type(response) is dict: - self.assertEqual(self.admin_username, response['username']) - else: - self.assertIsNotNone(response) - - def test_get_org_types(self): - response = self.admin_client.get_org_types() - self.assertTrue(len(response) > 0) - - def test_create_org(self): - response = self.admin_client.create_org( - {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", - "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) - - self.assertEqual(self.admin_org_name, response[u'org_name']) - - def test_get_org(self): - response = self.admin_client.get_org("a_myOrg84") - self.assertEqual("a_myOrg84", response['org_name']) - - def test_get_environment(self): - response = self.admin_client.get_environment("python2") - - if u'error' not in response: - self.assertTrue(response is not None and u'environments' in response) - - def test_get_build_logs(self): - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = unicode('echo') - algo_path = u'%s/%s' % (user, algo) - result = self.regular_client.algo(algo_path).build_logs() - - if u'error' in result: - print(result) - - self.assertTrue(u'error' not in result) - - def test_get_build_logs_no_ssl(self): - client = Algorithmia.client(api_address='https://api.algorithmia.com', - api_key=self.regular_api_key, ca_cert=False) - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = u'Echo' - result = client.algo(user + '/' + algo).build_logs() - if u'error' in result: - print(result) - self.assertTrue("error" not in result) - - def test_edit_org(self): - org_name = "a_myOrg84" - - obj = { - "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", - "org_name": "a_myOrg84", - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": "a_myOrg84@algo.com", - "org_created_at": "2020-11-30T23:51:40", - "org_url": "https://algorithmia.com", - "type_id": "basic", - "resource_type": "organization" - } - - response = self.admin_client.edit_org(org_name, obj) - if type(response) is dict: - print(response) - else: - self.assertEqual(204, response.status_code) - - def test_get_template(self): - filename = "./temptest" - response = self.admin_client.get_template("36fd467e-fbfe-4ea6-aa66-df3f403b7132", filename) - - if type(response) is dict: - self.assertTrue(u'error' in response or u'message' in response) - else: - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - def test_get_supported_languages(self): - response = self.admin_client.get_supported_languages() - self.assertTrue(response is not None) - - if type(response) is not list: - self.assertTrue(u'error' in response) - else: - language_found = any('anaconda3' in languages['name'] for languages in response) - self.assertTrue(response is not None and language_found) - - def test_invite_to_org(self): - response = self.admin_client.invite_to_org("a_myOrg38", "a_Mrtest4") - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(200, response.status_code) - - # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been - # deployed to the remote environment. - def test_get_organization_errors(self): - response = self.admin_client.get_organization_errors(self.admin_org_name) - self.assertTrue(response is not None) - - if type(response) is list: - self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') - - def test_get_user_errors(self): - response = self.admin_client.get_user_errors(self.admin_username) - - self.assertTrue(response is not None) - self.assertEqual(0, len(response)) - - def test_get_algorithm_errors(self): - response = self.admin_client.get_algorithm_errors('hello') - self.assertTrue(response is not None) - - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(404, response.status_code) - - - def test_algorithm_programmatic_create_process(self): - algorithm_name = "algo_" + str(uuid4()).split("-")[-1] - payload = "John" - expected_response = "hello John" - full_path = self.regular_client.username() + "/" + algorithm_name - details = { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - } - settings = { - "source_visibility": "open", - "algorithm_environment": self.environment_id, - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False - } - created_algo = self.regular_client.algo(full_path) - response = created_algo.create(details=details,settings=settings) - self.assertEqual(response.name, algorithm_name, "algorithm creation failed") - - # --- Creation complete, compiling - - response = created_algo.compile() - git_hash = response.version_info.git_hash - algo_with_build = self.regular_client.algo(full_path + "/" + git_hash) - self.assertEqual(response.name, created_algo.algoname) - - # --- compiling complete, now testing algorithm request - response = algo_with_build.pipe(payload).result - self.assertEqual(response, expected_response, "compiling failed") - - # --- testing complete, now publishing new release. - - pub_settings = {"algorithm_callability": "private"} - pub_version_info = { - "release_notes": "created programmatically", - "sample_input": payload, - "version_type": "minor" - } - pub_details = {"label": "testing123"} - - response = algo_with_build.publish( - details=pub_details, - settings=pub_settings, - version_info=pub_version_info - ) - self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", "Publishing failed, semantic version is not correct.") - - # --- publishing complete, getting additional information - - response = created_algo.info(git_hash) - - self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") + class ClientDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + + admin_username = "a_Mrtest" + admin_org_name = "a_myOrg" + environment_name = "Python 3.9" + + def setUp(self): + self.admin_username = self.admin_username + str(int(random() * 10000)) + self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) + + self.environment_id = "abcd-123" + + def test_create_user(self): + response = self.client.create_user( + {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", + "shouldCreateHello": False}) + + if type(response) is dict: + self.assertEqual(self.admin_username, response['username']) + else: + self.assertIsNotNone(response) + + def test_get_org_types(self): + response = self.client.get_org_types() + self.assertTrue(len(response) > 0) + + def test_create_org(self): + response = self.client.create_org( + {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", + "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) + + self.assertEqual(self.admin_org_name, response[u'org_name']) + + def test_get_org(self): + response = self.client.get_org("a_myOrg84") + self.assertEqual("a_myOrg84", response['org_name']) + + def test_get_environment(self): + response = self.client.get_environment("python2") + + if u'error' not in response: + self.assertTrue(response is not None and u'environments' in response) + + def test_get_build_logs(self): + user = unicode(os.environ.get('ALGO_USER_NAME')) + algo = unicode('echo') + algo_path = u'%s/%s' % (user, algo) + result = self.client.algo(algo_path).build_logs() + + if u'error' in result: + print(result) + + self.assertTrue(u'error' not in result) + + + def test_edit_org(self): + org_name = "a_myOrg84" + + obj = { + "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", + "org_name": "a_myOrg84", + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg84@algo.com", + "org_created_at": "2020-11-30T23:51:40", + "org_url": "https://algorithmia.com", + "type_id": "basic", + "resource_type": "organization" + } + + response = self.client.edit_org(org_name, obj) + if type(response) is dict: + print(response) + else: + self.assertEqual(204, response.status_code) + + def test_get_supported_languages(self): + response = self.client.get_supported_languages() + self.assertTrue(response is not None) + + if type(response) is not list: + self.assertTrue(u'error' in response) + else: + language_found = any('anaconda3' in languages['name'] for languages in response) + self.assertTrue(response is not None and language_found) + + def test_invite_to_org(self): + response = self.client.invite_to_org("a_myOrg38", "a_Mrtest4") + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(200, response.status_code) + + # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been + # deployed to the remote environment. + def test_get_organization_errors(self): + response = self.client.get_organization_errors(self.admin_org_name) + self.assertTrue(response is not None) + + if type(response) is list: + self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') + + def test_get_user_errors(self): + response = self.client.get_user_errors(self.admin_username) + + self.assertTrue(response is not None) + self.assertEqual(0, len(response)) + + def test_get_algorithm_errors(self): + response = self.client.get_algorithm_errors('hello') + self.assertTrue(response is not None) + + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(404, response.status_code) + + def test_algorithm_programmatic_create_process(self): + algorithm_name = "algo_e2d_test" + payload = "John" + expected_response = "hello John" + full_path = "a_Mrtest/" + algorithm_name + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + created_algo = self.client.algo(full_path) + response = created_algo.create(details=details, settings=settings) + self.assertEqual(response.name, algorithm_name, "algorithm creation failed") + + # --- Creation complete, compiling + + response = created_algo.compile() + git_hash = response.version_info.git_hash + algo_with_build = self.client.algo(full_path + "/" + git_hash) + self.assertEqual(response.name, created_algo.algoname) + + # --- compiling complete, now testing algorithm request + response = algo_with_build.pipe(payload).result + self.assertEqual(response, expected_response, "compiling failed") + + # --- testing complete, now publishing new release. + + pub_settings = {"algorithm_callability": "private"} + pub_version_info = { + "release_notes": "created programmatically", + "sample_input": payload, + "version_type": "minor" + } + pub_details = {"label": "testing123"} + + response = algo_with_build.publish( + details=pub_details, + settings=pub_settings, + version_info=pub_version_info + ) + self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", + "Publishing failed, semantic version is not correct.") + + # --- publishing complete, getting additional information + + response = created_algo.info(git_hash) + + self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") +else: + class ClientTest(unittest.TestCase): + seed(datetime.now().microsecond) + # due to legacy reasons, regular client tests are tested against api.algorithmia.com, whereas admin api tests + # are run against test.algorithmia.com. + admin_username = "a_Mrtest" + admin_org_name = "a_myOrg" + environment_name = "Python 3.9" + + def setUp(self): + self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) + self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) + + self.admin_username = self.admin_username + str(int(random() * 10000)) + self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) + self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", + api_key=self.admin_api_key) + self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', + api_key=self.regular_api_key) + + environments = self.regular_client.get_environment("python3") + for environment in environments['environments']: + if environment['display_name'] == self.environment_name: + self.environment_id = environment['id'] + + def test_create_user(self): + response = self.admin_client.create_user( + {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", + "shouldCreateHello": False}) + + if type(response) is dict: + self.assertEqual(self.admin_username, response['username']) + else: + self.assertIsNotNone(response) + + def test_get_org_types(self): + response = self.admin_client.get_org_types() + self.assertTrue(len(response) > 0) + + def test_create_org(self): + response = self.admin_client.create_org( + {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", + "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) + + self.assertEqual(self.admin_org_name, response[u'org_name']) + + def test_get_org(self): + response = self.admin_client.get_org("a_myOrg84") + self.assertEqual("a_myOrg84", response['org_name']) + + def test_get_environment(self): + response = self.admin_client.get_environment("python2") + + if u'error' not in response: + self.assertTrue(response is not None and u'environments' in response) + + def test_get_build_logs(self): + user = unicode(os.environ.get('ALGO_USER_NAME')) + algo = unicode('echo') + algo_path = u'%s/%s' % (user, algo) + result = self.regular_client.algo(algo_path).build_logs() + + if u'error' in result: + print(result) + + self.assertTrue(u'error' not in result) + + def test_get_build_logs_no_ssl(self): + client = Algorithmia.client(api_address='https://api.algorithmia.com', + api_key=self.regular_api_key, ca_cert=False) + user = unicode(os.environ.get('ALGO_USER_NAME')) + algo = u'Echo' + result = client.algo(user + '/' + algo).build_logs() + if u'error' in result: + print(result) + self.assertTrue("error" not in result) + + def test_edit_org(self): + org_name = "a_myOrg84" + + obj = { + "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", + "org_name": "a_myOrg84", + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg84@algo.com", + "org_created_at": "2020-11-30T23:51:40", + "org_url": "https://algorithmia.com", + "type_id": "basic", + "resource_type": "organization" + } + + response = self.admin_client.edit_org(org_name, obj) + if type(response) is dict: + print(response) + else: + self.assertEqual(204, response.status_code) + + def test_get_template(self): + filename = "./temptest" + response = self.admin_client.get_template("36fd467e-fbfe-4ea6-aa66-df3f403b7132", filename) + + if type(response) is dict: + self.assertTrue(u'error' in response or u'message' in response) + else: + self.assertTrue(response.ok) + try: + shutil.rmtree(filename) + except OSError as e: + print(e) + + def test_get_supported_languages(self): + response = self.admin_client.get_supported_languages() + self.assertTrue(response is not None) + + if type(response) is not list: + self.assertTrue(u'error' in response) + else: + language_found = any('anaconda3' in languages['name'] for languages in response) + self.assertTrue(response is not None and language_found) + + def test_invite_to_org(self): + response = self.admin_client.invite_to_org("a_myOrg38", "a_Mrtest4") + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(200, response.status_code) + + # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been + # deployed to the remote environment. + def test_get_organization_errors(self): + response = self.admin_client.get_organization_errors(self.admin_org_name) + self.assertTrue(response is not None) + + if type(response) is list: + self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') + + def test_get_user_errors(self): + response = self.admin_client.get_user_errors(self.admin_username) + + self.assertTrue(response is not None) + self.assertEqual(0, len(response)) + + def test_get_algorithm_errors(self): + response = self.admin_client.get_algorithm_errors('hello') + self.assertTrue(response is not None) + + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(404, response.status_code) + + def test_algorithm_programmatic_create_process(self): + algorithm_name = "algo_" + str(uuid4()).split("-")[-1] + payload = "John" + expected_response = "hello John" + full_path = self.regular_client.username() + "/" + algorithm_name + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + created_algo = self.regular_client.algo(full_path) + response = created_algo.create(details=details, settings=settings) + self.assertEqual(response.name, algorithm_name, "algorithm creation failed") + + # --- Creation complete, compiling + + response = created_algo.compile() + git_hash = response.version_info.git_hash + algo_with_build = self.regular_client.algo(full_path + "/" + git_hash) + self.assertEqual(response.name, created_algo.algoname) + + # --- compiling complete, now testing algorithm request + response = algo_with_build.pipe(payload).result + self.assertEqual(response, expected_response, "compiling failed") + + # --- testing complete, now publishing new release. + + pub_settings = {"algorithm_callability": "private"} + pub_version_info = { + "release_notes": "created programmatically", + "sample_input": payload, + "version_type": "minor" + } + pub_details = {"label": "testing123"} + + response = algo_with_build.publish( + details=pub_details, + settings=pub_settings, + version_info=pub_version_info + ) + self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", + "Publishing failed, semantic version is not correct.") + + # --- publishing complete, getting additional information + + response = created_algo.info(git_hash) + + self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") if __name__ == '__main__': diff --git a/Test/conftest.py b/Test/conftest.py new file mode 100644 index 0000000..71ca978 --- /dev/null +++ b/Test/conftest.py @@ -0,0 +1,12 @@ +import sys +from time import sleep +if sys.version_info.major >= 3: + from Test.api import start_webserver + import pytest + + @pytest.fixture(scope='package', autouse=True) + def fastapi_start(): + p = start_webserver() + sleep(2) + yield p + p.terminate() From 1d26de197314631d59c0335f6933a3f5d40461fa Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 9 Nov 2021 12:26:21 -0800 Subject: [PATCH 02/18] Async Support (#117) * added async algo processing to tests, fixed issue where async response was thought of as error * fixed typo in test casese * fixed async tests, and ensured that the algo endpoint dummy works * also fixed raw output --- Algorithmia/algo_response.py | 7 ++++++- Test/algo_test.py | 19 +++++++++++++++++++ Test/api/__init__.py | 10 +++++++--- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/Algorithmia/algo_response.py b/Algorithmia/algo_response.py index dfce6c1..d5abbc4 100644 --- a/Algorithmia/algo_response.py +++ b/Algorithmia/algo_response.py @@ -1,5 +1,6 @@ import base64 from Algorithmia.errors import raiseAlgoApiError +from Algorithmia.async_response import AsyncResponse import sys @@ -19,8 +20,12 @@ def __repr__(self): @staticmethod def create_algo_response(response): + + # Check if request is async + if 'async_protocol' in response and 'request_id' in response: + return AsyncResponse(response) # Parse response JSON, if it's indeed JSON - if 'error' in response or 'metadata' not in response: + elif 'error' in response or 'metadata' not in response: # Failure raise raiseAlgoApiError(response) else: diff --git a/Test/algo_test.py b/Test/algo_test.py index b22988e..0e3afdd 100644 --- a/Test/algo_test.py +++ b/Test/algo_test.py @@ -1,6 +1,7 @@ import sys import os from Algorithmia.errors import AlgorithmException +from Algorithmia.algorithm import OutputType import Algorithmia # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. @@ -26,6 +27,15 @@ def test_normal_call(self): self.assertEquals("text", result.metadata.content_type) self.assertEquals("foo", result.result) + def test_async_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEquals("foo", result) + def test_dict_call(self): result = self.client.algo('util/echo').pipe({"foo": "bar"}) self.assertEquals("json", result.metadata.content_type) @@ -82,6 +92,15 @@ def test_call_binary(self): self.assertEquals('binary', result.metadata.content_type) self.assertEquals(bytearray('foo', 'utf-8'), result.result) + def test_async_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEquals("foo", result) + def test_text_unicode(self): telephone = u"\u260E" diff --git a/Test/api/__init__.py b/Test/api/__init__.py index 99e7e68..9057bcf 100644 --- a/Test/api/__init__.py +++ b/Test/api/__init__.py @@ -1,5 +1,6 @@ import importlib from fastapi import FastAPI, Request +from typing import Optional from fastapi.responses import Response import json import base64 @@ -19,11 +20,15 @@ def _start_webserver(): @app.post("/v1/algo/{username}/{algoname}") -async def process_algo_req(request: Request, username, algoname): +async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} content_type = request.headers['Content-Type'] request = await request.body() - if algoname == "500": + if output and output == "void": + return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} + elif output and output == "raw": + return Response(request.decode(), status_code=200) + elif algoname == "500": return Response("Internal Server Error", status_code=500) elif algoname == "raise_exception": return {"error": {"message": "This is an exception"}} @@ -41,7 +46,6 @@ async def process_algo_req(request: Request, username, algoname): output = {"result": request, "metadata": metadata} return output - @app.post("/v1/algo/{username}/{algoname}/{githash}") async def process_hello_world(request: Request, username, algoname, githash): metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, From 537945cae718641a755fe181e9d82016b146d7e2 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 23 Nov 2021 20:33:04 -0800 Subject: [PATCH 03/18] updated to track ADK changes (#118) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3730ce4..aae416c 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'toml', 'argparse', 'algorithmia-api-client==1.5.1', - 'algorithmia-adk>=1.1,<1.2' + 'algorithmia-adk>=1.2,<1.3' ], include_package_data=True, classifiers=[ From 276f5b0ff61941ded68f27957686fcd0fee3293f Mon Sep 17 00:00:00 2001 From: John-Bragg <71101029+John-Bragg@users.noreply.github.com> Date: Mon, 29 Nov 2021 09:29:18 -0700 Subject: [PATCH 04/18] Dev 325 (#116) * INSIGHTS-12 Initial structure of insight functionality * INSIGHTS-12 Added todo statements * INSIGHTS-12 Moved Insights out of client * INSIGHTS-12 Adjusted insight methods to reside in the client class. Removed the ability to collect insights before sending, now the every time the user invokes the collectInsights method, it will also send. This prevents any State issues with the algorithm. * INSIGHTS-12 Added a todo. Tests fail for unknown reasons at this time * INSIGHTS-12 Fixed method call. Added a todo to get url from config if necessary. * INSIGHTS-12 Fixed method call. * INSIGHTS-12 added json serialization. might not be needed * INSIGHTS-12 commented test temporarily * INSIGHTS-12 comment updates and json .encode change * INSIGHTS-12 comment update * INSIGHTS-12 changed method signatures to match documentation https://insights1.enthalpy.click/developers/clients/python#publishing-algorithmia-insights * INSIGHTS-12 Added system property for queue reader url * INSIGHTS-12 Fixed URL to not be https * INSIGHTS-12 minor version update * INSIGHTS-12 revert change * INSIGHTS-12 removed todo * INSIGHTS-12 uncommented test. May start failing again in the pipeline. * INSIGHTS-12 commented test. * INSIGHTS-12 changed version. Removed unused import. * INSIGHTS-12 changed url to include /v1/ * Allow listing of non-data:// files on cli * Allow catting non-data:// files on cli * Fix tests * adding jwt support to CLI * refactoring bearertoken method * adding test and simplifying getclient method * test fixes * adding test Co-authored-by: robert-close Co-authored-by: Kenny Daniel Co-authored-by: Kenny Daniel <3903376+kennydaniel@users.noreply.github.com> Co-authored-by: John Bragg Co-authored-by: John-Bragg Co-authored-by: John Bragg --- Algorithmia/CLI.py | 40 ++++++++++++++++++++++--------- Algorithmia/__init__.py | 4 ++-- Algorithmia/__main__.py | 32 ++++++++++++------------- Algorithmia/client.py | 23 +++++++++++++++++- Test/CLI_test.py | 52 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 118 insertions(+), 33 deletions(-) diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 3368915..4551a68 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -12,8 +12,7 @@ class CLI: def __init__(self): self.client = Algorithmia.client() # algo auth - - def auth(self, apikey, apiaddress, cacert="", profile="default"): + def auth(self, apiaddress, apikey="", cacert="", profile="default", bearer=""): # store api key in local config file and read from it each time a client needs to be created key = self.getconfigfile() @@ -24,20 +23,17 @@ def auth(self, apikey, apiaddress, cacert="", profile="default"): config['profiles'][profile]['api_key'] = apikey config['profiles'][profile]['api_server'] = apiaddress config['profiles'][profile]['ca_cert'] = cacert + config['profiles'][profile]['bearer_token'] = bearer else: - config['profiles'][profile] = {'api_key': apikey, 'api_server': apiaddress, 'ca_cert': cacert} + config['profiles'][profile] = {'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert,'bearer_token':bearer} else: - config['profiles'] = {profile: {'api_key': apikey, 'api_server': apiaddress, 'ca_cert': cacert}} + config['profiles'] = {profile:{'api_key':apikey,'api_server':apiaddress,'ca_cert':cacert,'bearer_token':bearer }} with open(key, "w") as key: - toml.dump(config, key) - client = Algorithmia.client( - api_key=self.getAPIkey(profile), - api_address=self.getAPIaddress(profile), - ca_cert=self.getCert(profile) - ) - self.ls(path=None, client=client) + toml.dump(config,key) + + self.ls(path = None,client = CLI().getClient(profile)) # algo run run the the specified algo def runalgo(self, options, client): @@ -366,6 +362,7 @@ def getconfigfile(self): file.write("api_key = ''\n") file.write("api_server = ''\n") file.write("ca_cert = ''\n") + file.write("bearer_token = ''\n") key = keyPath + keyFile @@ -383,6 +380,16 @@ def getAPIkey(self, profile): return config_dict['profiles'][profile]['api_key'] else: return None + + def getBearerToken(self,profile): + key = self.getconfigfile() + config_dict = toml.load(key) + if 'profiles' in config_dict and profile in config_dict['profiles'] and \ + config_dict['profiles'][profile]['bearer_token'] != "": + return config_dict['profiles'][profile]['bearer_token'] + else: + return None + def getAPIaddress(self, profile): key = self.getconfigfile() @@ -401,3 +408,14 @@ def getCert(self, profile): return config_dict['profiles'][profile]['ca_cert'] else: return None + + def getClient(self,profile): + apiAddress = self.getAPIaddress(profile) + apiKey = self.getAPIkey(profile) + caCert = self.getCert(profile) + bearer = None + + if apiKey is None: + bearer = self.getBearerToken(profile) + + return Algorithmia.client(api_key=apiKey,api_address=apiAddress,ca_cert=caCert,bearer_token = bearer) diff --git a/Algorithmia/__init__.py b/Algorithmia/__init__.py index 05ed6dc..38e7ed6 100644 --- a/Algorithmia/__init__.py +++ b/Algorithmia/__init__.py @@ -23,8 +23,8 @@ def file(dataUrl): def dir(dataUrl): return getDefaultClient().dir(dataUrl) -def client(api_key=None, api_address=None, ca_cert=None): - return Client(api_key, api_address, ca_cert) +def client(api_key=None, api_address=None, ca_cert=None, bearer_token=None): + return Client(api_key, api_address, ca_cert, bearer_token) def handler(apply_func, load_func=lambda: None): return Handler(apply_func, load_func) diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index 9e67c5c..1b5f7b5 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -6,6 +6,7 @@ import six from Algorithmia.CLI import CLI import argparse +import re #bind input to raw input try: @@ -145,27 +146,26 @@ def main(): APIkey = input("enter API key: ") CACert = input('(optional) enter path to custom CA certificate: ') - if len(APIkey) == 28 and APIkey.startswith("sim"): - if APIaddress == "" or not APIaddress.startswith("https://api."): - APIaddress = "https://api.algorithmia.com" - - CLI().auth(apikey=APIkey, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + if APIaddress == "" or not APIaddress.startswith("https://api."): + print("invalid API address") else: - print("invalid api key") - + if len(APIkey) == 28 and APIkey.startswith("sim"): + CLI().auth(apikey=APIkey, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + else: + jwt = re.compile(r"^([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_=]+)\.([a-zA-Z0-9_\-\+\/=]*)") + Bearer = input("enter JWT token: ") + if jwt.match(Bearer): + CLI().auth(apikey=APIkey, bearer=Bearer, apiaddress=APIaddress, cacert=CACert, profile=args.profile) + else: + print("invalid authentication") + + + if args.cmd == 'help': parser.parse_args(['-h']) #create a client with the appropreate api address and key - client = Algorithmia.client() - if len(CLI().getAPIaddress(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile)) - elif len(CLI().getAPIaddress(args.profile)) > 1 and len(CLI().getCert(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile),CLI().getCert(args.profile)) - elif len(CLI().getAPIaddress(args.profile)) < 1 and len(CLI().getCert(args.profile)) > 1: - client = Algorithmia.client(CLI().getAPIkey(args.profile), CLI().getAPIaddress(args.profile),CLI().getCert(args.profile)) - else: - client = Algorithmia.client(CLI().getAPIkey(args.profile)) + client = CLI().getClient(args.profile) if args.cmd == 'run': diff --git a/Algorithmia/client.py b/Algorithmia/client.py index ccea0ca..451d6dd 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -21,13 +21,20 @@ class Client(object): apiKey = None apiAddress = None requestSession = None + bearerToken = None - def __init__(self, apiKey=None, apiAddress=None, caCert=None): + + def __init__(self, apiKey = None, apiAddress = None, caCert = None, bearerToken=None): # Override apiKey with environment variable config = None self.requestSession = requests.Session() if apiKey is None and 'ALGORITHMIA_API_KEY' in os.environ: apiKey = os.environ['ALGORITHMIA_API_KEY'] + if apiKey is None: + if bearerToken is None and 'ALGORITHMIA_BEARER_TOKEN' in os.environ: + bearerToken = os.environ['ALGORITHMIA_BEARER_TOKEN'] + self.bearerToken = bearerToken + self.apiKey = apiKey if apiAddress is not None: self.apiAddress = apiAddress @@ -217,6 +224,8 @@ def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + else: + headers['Authorization'] = "Bearer "+ self.bearerToken input_json = None if input_object is None: @@ -244,18 +253,24 @@ def getHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + else: + headers['Authorization'] = 'Bearer '+ self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) def getStreamHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + else: + headers['Authorization'] = 'Bearer '+ self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters, stream=True) def patchHelper(self, url, params): headers = {'content-type': 'application/json'} if self.apiKey is not None: headers['Authorization'] = self.apiKey + else: + headers['Authorization'] = 'Bearer '+ self.bearerToken return self.requestSession.patch(self.apiAddress + url, headers=headers, data=json.dumps(params)) # Used internally to get http head result @@ -263,6 +278,8 @@ def headHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + else: + headers['Authorization'] = 'Bearer '+ self.bearerToken return self.requestSession.head(self.apiAddress + url, headers=headers) # Used internally to http put a file @@ -270,6 +287,8 @@ def putHelper(self, url, data): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + else: + headers['Authorization'] = 'Bearer '+ self.bearerToken if isJson(data): headers['Content-Type'] = 'application/json' @@ -283,6 +302,8 @@ def deleteHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey + else: + headers['Authorization'] = 'Bearer '+ self.bearerToken response = self.requestSession.delete(self.apiAddress + url, headers=headers) if response.reason == "No Content": return response diff --git a/Test/CLI_test.py b/Test/CLI_test.py index ae1d546..99f662e 100644 --- a/Test/CLI_test.py +++ b/Test/CLI_test.py @@ -17,6 +17,7 @@ class CLIDummyTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + cls.bearerClient = Algorithmia.client(api_address="http://localhost:8080", bearer_token="simabcd123.token.token") def test_run(self): name = "util/Echo" @@ -52,6 +53,40 @@ def test_run(self): result = CLI().runalgo(args, self.client) self.assertEqual(result, inputs) + def test_run_token(self): + name = "util/Echo" + inputs = "test" + + parser = argparse.ArgumentParser('CLI for interacting with Algorithmia') + + subparsers = parser.add_subparsers(help='sub cmd', dest='subparser_name') + parser_run = subparsers.add_parser('run', help='algo run [input options] [output options]') + + parser_run.add_argument('algo') + parser_run.add_argument('-d', '--data', action='store', help='detect input type', default=None) + parser_run.add_argument('-t', '--text', action='store', help='treat input as text', default=None) + parser_run.add_argument('-j', '--json', action='store', help='treat input as json data', default=None) + parser_run.add_argument('-b', '--binary', action='store', help='treat input as binary data', default=None) + parser_run.add_argument('-D', '--data-file', action='store', help='specify a path to an input file', + default=None) + parser_run.add_argument('-T', '--text-file', action='store', help='specify a path to a text file', + default=None) + parser_run.add_argument('-J', '--json-file', action='store', help='specify a path to a json file', + default=None) + parser_run.add_argument('-B', '--binary-file', action='store', help='specify a path to a binary file', + default=None) + parser_run.add_argument('--timeout', action='store', type=int, default=300, + help='specify a timeout (seconds)') + parser_run.add_argument('--debug', action='store_true', + help='print the stdout from the algo ') + parser_run.add_argument('--profile', action='store', type=str, default='default') + parser_run.add_argument('-o', '--output', action='store', default=None, type=str) + + args = parser.parse_args(['run', name, '-d', inputs]) + + result = CLI().runalgo(args, self.bearerClient) + self.assertEqual(result, inputs) + class CLIMainTest(unittest.TestCase): def setUp(self): @@ -156,7 +191,7 @@ def test_auth(self): key = os.getenv('ALGORITHMIA_API_KEY') address = 'https://api.algorithmia.com' profile = 'default' - CLI().auth(key, address, profile=profile) + CLI().auth(address, key, profile=profile) resultK = CLI().getAPIkey(profile) resultA = CLI().getAPIaddress(profile) self.assertEqual(resultK, key) @@ -176,13 +211,24 @@ def test_auth_cert(self): cacert = localfile profile = 'test' - CLI().auth(key, address, cacert, profile) + CLI().auth(address, key, cacert=cacert, profile=profile) resultK = CLI().getAPIkey(profile) resultA = CLI().getAPIaddress(profile) resultC = CLI().getCert(profile) self.assertEqual(resultK, key) self.assertEqual(resultA, address) self.assertEqual(resultC, cacert) + + def test_auth_token(self): + address = 'https://api.algorithmia.com' + bearer = 'testtokenabcd' + profile = 'test' + + CLI().auth(apiaddress=address, bearer=bearer, profile=profile) + resultA = CLI().getAPIaddress(profile) + resultT = CLI().getBearerToken(profile) + self.assertEqual(resultA, address) + self.assertEqual(resultT, bearer) def test_get_environment(self): result = CLI().get_environment_by_language("python2", self.client) @@ -230,7 +276,7 @@ def test_get_template(self): def test_api_address_auth(self): api_key = os.getenv('ALGORITHMIA_TEST_API_KEY') api_address = "https://api.test.algorithmia.com" - CLI().auth(api_key, api_address) + CLI().auth(api_address, api_key) profile = "default" client = Algorithmia.client(CLI().getAPIkey(profile), CLI().getAPIaddress(profile), CLI().getCert(profile)) From bf2a033a471cc28ef55f975cbcf54300c8ad6025 Mon Sep 17 00:00:00 2001 From: Ezra Citron <36384768+lemonez@users.noreply.github.com> Date: Mon, 6 Dec 2021 09:20:25 -0800 Subject: [PATCH 05/18] update readme print statements to Python 3 (#119) Co-authored-by: Ezra Citron --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a7f3214..59a80dc 100644 --- a/README.md +++ b/README.md @@ -72,9 +72,9 @@ If the algorithm output is text, then the `result` field of the response will be ```python algo = client.algo('demo/Hello/0.1.1') response = algo.pipe("HAL 9000") -print response.result # Hello, world! -print response.metadata # Metadata(content_type='text',duration=0.0002127) -print response.metadata.duration # 0.0002127 +print(response.result) # Hello, world! +print(response.metadata) # Metadata(content_type='text',duration=0.0002127) +print(response.metadata.duration) # 0.0002127 ``` ### JSON input/output @@ -119,7 +119,7 @@ This includes support for changing the timeout or indicating that the API should ```python from Algorithmia.algorithm import OutputType response = client.algo('util/echo').set_options(timeout=60, stdout=False) -print response.metadata.stdout +print(response.metadata.stdout) ``` Note: `stdout=True` is only supported if you have access to the algorithm source. @@ -186,15 +186,15 @@ foo = client.dir("data://.my/foo") # List files in "foo" for file in foo.files(): - print file.path + " at URL: " + file.url + " last modified " + file.last_modified + print(file.path + " at URL: " + file.url + " last modified " + file.last_modified) # List directories in "foo" for file in foo.dirs(): - print dir.path + " at URL: " + file.url + print(dir.path + " at URL: " + file.url) # List everything in "foo" for entry in foo.list(): - print entry.path + " at URL: " + entry.url + print(entry.path + " at URL: " + entry.url) ``` ### Manage directory permissions @@ -230,7 +230,7 @@ $ algo auth Configuring authentication for profile: 'default' Enter API Endpoint [https://api.algorithmia.com]: Enter API Key: -(optional) enter path to custom CA certificate: +(optional) enter path to custom CA certificate: Profile is ready to use. Test with 'algo ls' ``` @@ -332,7 +332,7 @@ algo auth --profile second_user Configuring authentication for profile: 'second_user' Enter API Endpoint [https://api.algorithmia.com]: Enter API Key: -(optional) enter path to custom CA certificate: +(optional) enter path to custom CA certificate: ``` Now you may use `algo ls --profile second_user` to list files in your `second_user` account. For more information, see the auth command help with `algo auth --help`. @@ -342,7 +342,7 @@ Now you may use `algo ls --profile second_user` to list files in your `second_us When running commands, the Algorithmia CLI will use the default profile unless otherwise specified with the `--profile ` option. See the following example: ```text -$ algo run kenny/factor -d 17 --profile second_user +$ algo run kenny/factor -d 17 --profile second_user [17] ``` From 6d35bbcef49ac811ae788ffa3910257ef82b978a Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Wed, 22 Dec 2021 12:32:27 -0400 Subject: [PATCH 06/18] added the freeze automation to the client object (#120) * added the freeze automation to the client object, pulled from CLI * added test cases for freeze * fixed imports * fixed directory collision --- Algorithmia/CLI.py | 24 +------ Algorithmia/algorithm.py | 63 ++++++++++--------- Algorithmia/client.py | 27 +++++++- Test/client_test.py | 4 ++ .../resources/manifests/example_manifest.json | 29 +++++++++ 5 files changed, 94 insertions(+), 53 deletions(-) create mode 100644 Test/resources/manifests/example_manifest.json diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 4551a68..3acc6ae 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -2,7 +2,6 @@ import os from Algorithmia.errors import DataApiError from Algorithmia.algo_response import AlgoResponse -from Algorithmia.util import md5_for_file, md5_for_str import json, re, requests, six import toml import shutil @@ -244,28 +243,7 @@ def cat(self, path, client): # algo freeze def freezeAlgo(self, client, manifest_path="model_manifest.json"): - if os.path.exists(manifest_path): - with open(manifest_path, 'r') as f: - manifest_file = json.load(f) - manifest_file['timestamp'] = str(time()) - required_files = manifest_file['required_files'] - optional_files = manifest_file['optional_files'] - for i in range(len(required_files)): - uri = required_files[i]['source_uri'] - local_file = client.file(uri).getFile(as_path=True) - md5_checksum = md5_for_file(local_file) - required_files[i]['md5_checksum'] = md5_checksum - for i in range(len(optional_files)): - uri = required_files[i]['source_uri'] - local_file = client.file(uri).getFile(as_path=True) - md5_checksum = md5_for_file(local_file) - required_files[i]['md5_checksum'] = md5_checksum - lock_md5_checksum = md5_for_str(str(manifest_file)) - manifest_file['lock_checksum'] = lock_md5_checksum - with open('model_manifest.json.freeze', 'w') as f: - json.dump(manifest_file, f) - else: - print("Expected to find a model_manifest.json file, none was discovered in working directory") + client.freeze(manifest_path) # algo cp def cp(self, src, dest, client): diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 85a6f85..378e1c0 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -8,10 +8,12 @@ from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError from enum import Enum from algorithmia_api_client.rest import ApiException -from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, SettingsPublish, \ +from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, \ + SettingsPublish, \ CreateRequestVersionInfo, VersionInfo, VersionInfoPublish -OutputType = Enum('OutputType','default raw void') +OutputType = Enum('OutputType', 'default raw void') + class Algorithm(object): def __init__(self, client, algoRef): @@ -32,7 +34,7 @@ def __init__(self, client, algoRef): raise ValueError('Invalid algorithm URI: ' + algoRef) def set_options(self, timeout=300, stdout=False, output=OutputType.default, **query_parameters): - self.query_parameters = {'timeout':timeout, 'stdout':stdout} + self.query_parameters = {'timeout': timeout, 'stdout': stdout} self.output_type = output self.query_parameters.update(query_parameters) return self @@ -42,7 +44,8 @@ def create(self, details={}, settings={}, version_info={}): detailsObj = Details(**details) settingsObj = SettingsMandatory(**settings) createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} + create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, + "version_info": createRequestVersionInfoObj} create_request = CreateRequest(**create_parameters) try: # Create Algorithm @@ -57,7 +60,8 @@ def update(self, details={}, settings={}, version_info={}): detailsObj = Details(**details) settingsObj = Settings(**settings) createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - update_parameters = {"details": detailsObj, "settings": settingsObj, "version_info": createRequestVersionInfoObj} + update_parameters = {"details": detailsObj, "settings": settingsObj, + "version_info": createRequestVersionInfoObj} update_request = UpdateRequest(**update_parameters) try: # Update Algorithm @@ -70,9 +74,10 @@ def update(self, details={}, settings={}, version_info={}): # Publish an algorithm def publish(self, details={}, settings={}, version_info={}): publish_parameters = {"details": details, "settings": settings, "version_info": version_info} - url = "/v1/algorithms/"+self.username+"/"+self.algoname + "/versions" + url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions" print(publish_parameters) - api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, **self.query_parameters) + api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, + **self.query_parameters) return api_response # except ApiException as e: # error_message = json.loads(e.body) @@ -81,7 +86,8 @@ def publish(self, details={}, settings={}, version_info={}): def builds(self, limit=56, marker=None): try: if marker is not None: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, marker=marker) + api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, + marker=marker) else: api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit) return api_response @@ -109,11 +115,10 @@ def get_build_logs(self, build_id): raise raiseAlgoApiError(error_message) def build_logs(self): - url = '/v1/algorithms/'+self.username+'/'+self.algoname+'/builds' + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds' response = json.loads(self.client.getHelper(url).content.decode('utf-8')) return response - def get_scm_status(self): try: api_response = self.client.manageApi.get_algorithm_scm_connection_status(self.username, self.algoname) @@ -157,7 +162,6 @@ def versions(self, limit=None, marker=None, published=None, callable=None): error_message = json.loads(e.body) raise raiseAlgoApiError(error_message) - # Compile an algorithm def compile(self): try: @@ -176,25 +180,26 @@ def pipe(self, input1): elif self.output_type == OutputType.void: return self._postVoidOutput(input1) else: - return AlgoResponse.create_algo_response(self.client.postJsonHelper(self.url, input1, **self.query_parameters)) + return AlgoResponse.create_algo_response( + self.client.postJsonHelper(self.url, input1, **self.query_parameters)) def _postRawOutput(self, input1): - # Don't parse response as json - self.query_parameters['output'] = 'raw' - response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) - # Check HTTP code and throw error as needed - if response.status_code == 400: - # Bad request - raise ApiError(response.text) - elif response.status_code == 500: - raise ApiInternalError(response.text) - else: - return response.text + # Don't parse response as json + self.query_parameters['output'] = 'raw' + response = self.client.postJsonHelper(self.url, input1, parse_response_as_json=False, **self.query_parameters) + # Check HTTP code and throw error as needed + if response.status_code == 400: + # Bad request + raise ApiError(response.text) + elif response.status_code == 500: + raise ApiInternalError(response.text) + else: + return response.text def _postVoidOutput(self, input1): - self.query_parameters['output'] = 'void' - responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) - if 'error' in responseJson: - raise ApiError(responseJson['error']['message']) - else: - return AsyncResponse(responseJson) + self.query_parameters['output'] = 'void' + responseJson = self.client.postJsonHelper(self.url, input1, **self.query_parameters) + if 'error' in responseJson: + raise ApiError(responseJson['error']['message']) + else: + return AsyncResponse(responseJson) diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 451d6dd..7247376 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -6,12 +6,13 @@ from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory, AdvancedDataDirectory from algorithmia_api_client import Configuration, DefaultApi, ApiClient - +from Algorithmia.util import md5_for_file, md5_for_str from tempfile import mkstemp import atexit import json, re, requests, six, certifi import tarfile import os +from time import time class Client(object): @@ -343,6 +344,30 @@ def exit_handler(self): except OSError as e: print(e) + # Used by CI/CD automation for freezing model manifest files, and by the CLI for manual freezing + def freeze(self, manifest_path, manifest_output_dir="."): + if os.path.exists(manifest_path): + with open(manifest_path, 'r') as f: + manifest_file = json.load(f) + manifest_file['timestamp'] = str(time()) + required_files = manifest_file['required_files'] + optional_files = manifest_file['optional_files'] + for i in range(len(required_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + for i in range(len(optional_files)): + uri = required_files[i]['source_uri'] + local_file = self.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + lock_md5_checksum = md5_for_str(str(manifest_file)) + manifest_file['lock_checksum'] = lock_md5_checksum + with open(manifest_output_dir+'/'+'model_manifest.json.freeze', 'w') as f: + json.dump(manifest_file, f) + else: + print("Expected to find a model_manifest.json file, none was discovered in working directory") def isJson(myjson): try: diff --git a/Test/client_test.py b/Test/client_test.py index a254c45..0519266 100644 --- a/Test/client_test.py +++ b/Test/client_test.py @@ -397,6 +397,10 @@ def test_algorithm_programmatic_create_process(self): self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") + def test_algo_freeze(self): + self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests") + + if __name__ == '__main__': unittest.main() diff --git a/Test/resources/manifests/example_manifest.json b/Test/resources/manifests/example_manifest.json new file mode 100644 index 0000000..ba6cbf5 --- /dev/null +++ b/Test/resources/manifests/example_manifest.json @@ -0,0 +1,29 @@ +{ + "required_files" : [ + { "name": "squeezenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + }, + { + "name": "labels", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json", + "fail_on_tamper": true, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ], + "optional_files": [ + { + "name": "mobilenet", + "source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth", + "fail_on_tamper": false, + "metadata": { + "dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266" + } + } + ] +} \ No newline at end of file From 7fd688d75baafe071beb964e250450fd738ca17c Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Wed, 22 Dec 2021 17:33:59 -0400 Subject: [PATCH 07/18] [ALERT] Fix for breaking changes introduced by Bearer Token work (#121) * critical fix to bearer token breakage * corrected issue with header initialization in json helper * removed else exception, no auth is passed by internal algo API processing * added a test case tracking authorization required server responses and system handling no auth provided gracefully * added a false flag to actually invalidate environment variables for client auth * cleaned up client, used environment variable manipulation in the test suite * removing problematic server test --- Algorithmia/client.py | 41 ++++++++++++++++++++--------------------- Test/api/__init__.py | 3 +++ Test/client_test.py | 26 +++++++++++++++++++++----- 3 files changed, 44 insertions(+), 26 deletions(-) diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 7247376..82078d0 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -24,18 +24,16 @@ class Client(object): requestSession = None bearerToken = None - - def __init__(self, apiKey = None, apiAddress = None, caCert = None, bearerToken=None): + def __init__(self, apiKey=None, apiAddress=None, caCert=None, bearerToken=None): # Override apiKey with environment variable config = None self.requestSession = requests.Session() if apiKey is None and 'ALGORITHMIA_API_KEY' in os.environ: apiKey = os.environ['ALGORITHMIA_API_KEY'] - if apiKey is None: - if bearerToken is None and 'ALGORITHMIA_BEARER_TOKEN' in os.environ: - bearerToken = os.environ['ALGORITHMIA_BEARER_TOKEN'] - self.bearerToken = bearerToken + elif bearerToken is None and 'ALGORITHMIA_BEARER_TOKEN' in os.environ: + bearerToken = os.environ['ALGORITHMIA_BEARER_TOKEN'] + self.bearerToken = bearerToken self.apiKey = apiKey if apiAddress is not None: self.apiAddress = apiAddress @@ -225,8 +223,8 @@ def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey - else: - headers['Authorization'] = "Bearer "+ self.bearerToken + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken input_json = None if input_object is None: @@ -254,24 +252,24 @@ def getHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey - else: - headers['Authorization'] = 'Bearer '+ self.bearerToken + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) def getStreamHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey - else: - headers['Authorization'] = 'Bearer '+ self.bearerToken + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters, stream=True) def patchHelper(self, url, params): headers = {'content-type': 'application/json'} if self.apiKey is not None: headers['Authorization'] = self.apiKey - else: - headers['Authorization'] = 'Bearer '+ self.bearerToken + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.patch(self.apiAddress + url, headers=headers, data=json.dumps(params)) # Used internally to get http head result @@ -279,8 +277,8 @@ def headHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey - else: - headers['Authorization'] = 'Bearer '+ self.bearerToken + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.head(self.apiAddress + url, headers=headers) # Used internally to http put a file @@ -288,8 +286,8 @@ def putHelper(self, url, data): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey - else: - headers['Authorization'] = 'Bearer '+ self.bearerToken + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken if isJson(data): headers['Content-Type'] = 'application/json' @@ -303,8 +301,8 @@ def deleteHelper(self, url): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey - else: - headers['Authorization'] = 'Bearer '+ self.bearerToken + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken response = self.requestSession.delete(self.apiAddress + url, headers=headers) if response.reason == "No Content": return response @@ -364,11 +362,12 @@ def freeze(self, manifest_path, manifest_output_dir="."): required_files[i]['md5_checksum'] = md5_checksum lock_md5_checksum = md5_for_str(str(manifest_file)) manifest_file['lock_checksum'] = lock_md5_checksum - with open(manifest_output_dir+'/'+'model_manifest.json.freeze', 'w') as f: + with open(manifest_output_dir + '/' + 'model_manifest.json.freeze', 'w') as f: json.dump(manifest_file, f) else: print("Expected to find a model_manifest.json file, none was discovered in working directory") + def isJson(myjson): try: json_object = json.loads(myjson) diff --git a/Test/api/__init__.py b/Test/api/__init__.py index 9057bcf..ead17d3 100644 --- a/Test/api/__init__.py +++ b/Test/api/__init__.py @@ -23,6 +23,9 @@ def _start_webserver(): async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} content_type = request.headers['Content-Type'] + auth = request.headers.get('Authorization', None) + if auth is None: + return {"error": {"message": "authorization required"}} request = await request.body() if output and output == "void": return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} diff --git a/Test/client_test.py b/Test/client_test.py index 0519266..3be87ad 100644 --- a/Test/client_test.py +++ b/Test/client_test.py @@ -9,11 +9,13 @@ import unittest import Algorithmia +from Algorithmia.errors import AlgorithmException from uuid import uuid4 if sys.version_info.major >= 3: unicode = str + class ClientDummyTest(unittest.TestCase): @classmethod def setUpClass(cls): @@ -71,7 +73,6 @@ def test_get_build_logs(self): self.assertTrue(u'error' not in result) - def test_edit_org(self): org_name = "a_myOrg84" @@ -138,7 +139,7 @@ def test_algorithm_programmatic_create_process(self): algorithm_name = "algo_e2d_test" payload = "John" expected_response = "hello John" - full_path = "a_Mrtest/" + algorithm_name + full_path = "a_Mrtest/" + algorithm_name details = { "summary": "Example Summary", "label": "QA", @@ -189,6 +190,23 @@ def test_algorithm_programmatic_create_process(self): response = created_algo.info(git_hash) self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") + + def test_no_auth_client(self): + + key = os.environ.get('ALGORITHMIA_API_KEY', "") + if key != "": + del os.environ['ALGORITHMIA_API_KEY'] + + client = Algorithmia.client(api_address="http://localhost:8080") + error = None + try: + client.algo("demo/hello").pipe("world") + except Exception as e: + error = e + finally: + os.environ['ALGORITHMIA_API_KEY'] = key + self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, error_type=None))) + else: class ClientTest(unittest.TestCase): seed(datetime.now().microsecond) @@ -201,7 +219,7 @@ class ClientTest(unittest.TestCase): def setUp(self): self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) - + self.admin_username = self.admin_username + str(int(random() * 10000)) self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", @@ -400,7 +418,5 @@ def test_algorithm_programmatic_create_process(self): def test_algo_freeze(self): self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests") - - if __name__ == '__main__': unittest.main() From f911e556fdf631ab82599f35275e4ce848cbe654 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Tue, 1 Mar 2022 12:39:21 -0400 Subject: [PATCH 08/18] improvement for systemic errors (#122) * added a better message when errors are not caused by Algorithms * make failure test compliant with other testcase components --- Algorithmia/errors.py | 2 +- Test/algo_failure_test.py | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/Algorithmia/errors.py b/Algorithmia/errors.py index 9356e24..f23662a 100644 --- a/Algorithmia/errors.py +++ b/Algorithmia/errors.py @@ -47,4 +47,4 @@ def raiseAlgoApiError(result): stacktrace = None return AlgorithmException(message=message, stack_trace=stacktrace, error_type=err_type) else: - return Exception(result) + return Exception("Non-Algorithm related Failure: " + str(result)) diff --git a/Test/algo_failure_test.py b/Test/algo_failure_test.py index 1e65234..7defcc2 100644 --- a/Test/algo_failure_test.py +++ b/Test/algo_failure_test.py @@ -14,22 +14,14 @@ from Test.api import app - def start_webserver(): - uvicorn.run(app, host="127.0.0.1", port=8080, log_level="debug") - - class AlgoTest(unittest.TestCase): error_500 = Response() error_500.status_code = 500 + error_message = "Non-Algorithm related Failure: " + str(error_500) - def setUp(self): - self.client = Algorithmia.client(api_address="http://localhost:8080") - self.uvi_p = Process(target=start_webserver) - self.uvi_p.start() - time.sleep(1) - - def tearDown(self): - self.uvi_p.terminate() + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") def test_throw_500_error_HTTP_response_on_algo_request(self): try: @@ -37,4 +29,4 @@ def test_throw_500_error_HTTP_response_on_algo_request(self): except Exception as e: result = e pass - self.assertEqual(str(self.error_500), str(result)) + self.assertEqual(str(self.error_message), str(result)) From d3e73cb89a575bbb462a0de42f39cfbd15f898eb Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Wed, 2 Mar 2022 13:38:02 -0400 Subject: [PATCH 09/18] update requirements to satisfy actual ADK dependency versions (#123) --- requirements.txt | 2 +- requirements27.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f12c984..f52fe4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.1,<1.2 +algorithmia-adk>=1.2,<1.3 numpy<2 uvicorn==0.14.0 fastapi==0.65.2 diff --git a/requirements27.txt b/requirements27.txt index 3d2b39c..9668467 100644 --- a/requirements27.txt +++ b/requirements27.txt @@ -4,5 +4,5 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.1,<1.2 +algorithmia-adk>=1.2,<1.3 numpy<2 From 12b2b3de4335efc06b2c590fa82d92c456681d9e Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Fri, 6 May 2022 12:03:38 -0700 Subject: [PATCH 10/18] self signed certificate - ssl disabling support w/ testing (#124) * added a ssl failure test, ssl disabling path is now correct * moved resources * kill instead of terminate; actually closes the sockets * swapped from process kill to os.kill path to be compliant with 3.6 * separated normal tests from self signed cert tests where it made sense, split fastAPI to support all endpoints as both regular mode and self signed mode --- .gitignore | 3 +- Algorithmia/client.py | 1 + Test/api/__init__.py | 349 +------------------ Test/api/app.py | 355 +++++++++++++++++++ Test/api/self_signed_app.py | 351 +++++++++++++++++++ Test/conftest.py | 11 +- Test/{ => regular}/CLI_test.py | 14 +- Test/regular/__init__.py | 0 Test/{ => regular}/acl_test.py | 0 Test/{ => regular}/algo_failure_test.py | 2 - Test/{ => regular}/algo_test.py | 0 Test/{ => regular}/client_test.py | 11 - Test/{ => regular}/datadirectory_test.py | 0 Test/{ => regular}/datafile_test.py | 0 Test/{ => regular}/util_test.py | 0 Test/resources/cert.cert | 21 ++ Test/resources/cert.key | 28 ++ Test/self_signed/__init__.py | 0 Test/self_signed/acl_test.py | 38 +++ Test/self_signed/algo_failure_test.py | 30 ++ Test/self_signed/algo_test.py | 136 ++++++++ Test/self_signed/client_test.py | 413 +++++++++++++++++++++++ 22 files changed, 1391 insertions(+), 372 deletions(-) create mode 100644 Test/api/app.py create mode 100644 Test/api/self_signed_app.py rename Test/{ => regular}/CLI_test.py (97%) create mode 100644 Test/regular/__init__.py rename Test/{ => regular}/acl_test.py (100%) rename Test/{ => regular}/algo_failure_test.py (97%) rename Test/{ => regular}/algo_test.py (100%) rename Test/{ => regular}/client_test.py (97%) rename Test/{ => regular}/datadirectory_test.py (100%) rename Test/{ => regular}/datafile_test.py (100%) rename Test/{ => regular}/util_test.py (100%) create mode 100644 Test/resources/cert.cert create mode 100644 Test/resources/cert.key create mode 100644 Test/self_signed/__init__.py create mode 100644 Test/self_signed/acl_test.py create mode 100644 Test/self_signed/algo_failure_test.py create mode 100644 Test/self_signed/algo_test.py create mode 100644 Test/self_signed/client_test.py diff --git a/.gitignore b/.gitignore index e44d1b6..240d5e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ #Eclipse .project - +venv +venv_win TestFiles test.txt diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 82078d0..e72a8f6 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -41,6 +41,7 @@ def __init__(self, apiKey=None, apiAddress=None, caCert=None, bearerToken=None): self.apiAddress = Algorithmia.getApiAddress() if caCert == False: self.requestSession.verify = False + self.requestSession.trust_env = False config = Configuration(use_ssl=False) elif caCert is None and 'REQUESTS_CA_BUNDLE' in os.environ: caCert = os.environ.get('REQUESTS_CA_BUNDLE') diff --git a/Test/api/__init__.py b/Test/api/__init__.py index ead17d3..c5ff73e 100644 --- a/Test/api/__init__.py +++ b/Test/api/__init__.py @@ -1,348 +1,3 @@ -import importlib -from fastapi import FastAPI, Request -from typing import Optional -from fastapi.responses import Response -import json -import base64 -from multiprocessing import Process -import uvicorn +from .app import start_webserver_reg as start_webserver_reg +from .self_signed_app import start_webserver_self_signed as start_webserver_self_signed -app = FastAPI() - - -def start_webserver(): - def _start_webserver(): - uvicorn.run(app, host="127.0.0.1", port=8080, log_level="debug") - - p = Process(target=_start_webserver) - p.start() - return p - - -@app.post("/v1/algo/{username}/{algoname}") -async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): - metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} - content_type = request.headers['Content-Type'] - auth = request.headers.get('Authorization', None) - if auth is None: - return {"error": {"message": "authorization required"}} - request = await request.body() - if output and output == "void": - return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} - elif output and output == "raw": - return Response(request.decode(), status_code=200) - elif algoname == "500": - return Response("Internal Server Error", status_code=500) - elif algoname == "raise_exception": - return {"error": {"message": "This is an exception"}} - else: - if content_type != "application/octet-stream": - request = request.decode('utf-8') - if content_type == "text/plain": - metadata['content_type'] = "text" - elif content_type == "application/json": - request = json.loads(request) - metadata['content_type'] = "json" - else: - metadata['content_type'] = "binary" - request = base64.b64encode(request) - output = {"result": request, "metadata": metadata} - return output - -@app.post("/v1/algo/{username}/{algoname}/{githash}") -async def process_hello_world(request: Request, username, algoname, githash): - metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, - 'content_type': "text"} - request = await request.body() - request = request.decode('utf-8') - return {"result": f"hello {request}", "metadata": metadata} - - -### Algorithm Routes -@app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") -async def get_build_id(username, algoname, buildid): - return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", - "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", - "version_info": {"semantic_version": "0.1.1"}} - - -@app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") -async def get_build_log(username, algoname, buildid): - return {"logs": "This is a log"} - - -@app.get("/v1/algorithms/{username}/{algoname}/scm/status") -async def get_scm_status(username, algoname): - return {"scm_connection_status": "active"} - - -@app.get("/v1/algorithms/{algo_id}/errors") -async def get_algo_errors(algo_id): - return {"error": {"message": "not found"}} - - -@app.post("/v1/algorithms/{username}") -async def create_algorithm(request: Request, username): - payload = await request.json() - return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], - "details": {"label": payload["details"]["label"]}, - "settings": {"algorithm_callability": "private", "source_visibility": "open", - "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", - "pipeline_enabled": False, "insights_enabled": False, - "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, - "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, - "resource_type": "algorithm"} - - -@app.post("/v1/algorithms/{username}/{algoname}/compile") -async def compile_algorithm(username, algoname): - return { - "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", - "name": algoname, - "details": { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - }, - "settings": { - "algorithm_callability": "private", - "source_visibility": "open", - "package_set": "tensorflow-gpu-2.3-python38", - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False, - "insights_enabled": False, - "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" - }, - "version_info": { - "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", - "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" - }, - "source": { - "scm": { - "id": "internal", - "provider": "internal", - "default": True, - "enabled": True - } - }, - "compilation": { - "successful": True, - "output": "" - }, - "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", - "resource_type": "algorithm" - } - - -@app.post("/v1/algorithms/{username}/{algoname}/versions") -async def publish_algorithm(request: Request, username, algoname): - return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, - "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, - "settings": {"algorithm_callability": "private", "source_visibility": "open", - "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", - "pipeline_enabled": False, "insights_enabled": False, - "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, - "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", - "release_notes": "created programmatically", "sample_input": "payload", - "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, - "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, - "compilation": {"successful": True}, - "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", - "resource_type": "algorithm"} - - -@app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") -async def get_algorithm_info(username, algoname, algohash): - return { - "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", - "name": algoname, - "details": { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - }, - "settings": { - "algorithm_callability": "private", - "source_visibility": "open", - "language": "python3", - "environment": "gpu", - "package_set": "tensorflow-gpu-2.3-python38", - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False, - "insights_enabled": False, - "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" - }, - "version_info": { - "semantic_version": "0.1.0", - "git_hash": algohash, - "release_notes": "created programmatically", - "sample_input": "\"payload\"", - "sample_output": "Exception encountered while running sample input", - "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" - }, - "source": { - "scm": { - "id": "internal", - "provider": "internal", - "default": True, - "enabled": True - } - }, - "compilation": { - "successful": True, - "output": "" - }, - "resource_type": "algorithm" - } - - -### Admin Routes -@app.post("/v1/users") -async def create_user(request: Request): - payload = await request.body() - data = json.loads(payload) - username = data['username'] - email = data['email'] - return { - "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", - "username": username, - "email": email, - "fullname": username, - "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" - } - - -@app.get("/v1/users/{user_id}/errors") -async def get_user_errors(user_id): - return [] - - -@app.get("/v1/organization/types") -async def get_org_types(): - return [ - {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, - {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, - {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} - ] - - -@app.post("/v1/organizations") -async def create_org(request: Request): - payload = await request.body() - data = json.loads(payload) - org_name = data["org_name"] - org_email = data["org_email"] - return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", - "org_name": org_name, - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": org_email, - "org_created_at": "2021-10-22T16:41:32", - "org_url": None, - "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", - "stripe_customer_id": None, - "external_admin_group": None, - "external_member_group": None, - "external_id": None, - "owner_ids": None, - "resource_type": "organization", - "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" - } - - -@app.put("/v1/organizations/{orgname}/members/{username}") -async def add_user_to_org(orgname, username): - return Response(status_code=200) - - -@app.get("/v1/organizations/{orgname}/errors") -async def org_errors(orgname): - return [] - - -@app.put("/v1/organizations/{org_name}") -async def edit_org(org_name): - return Response(status_code=204) - - -@app.get("/v1/organizations/{org_name}") -async def get_org_by_name(org_name): - return { - "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", - "org_name": org_name, - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": "a_myOrg1542@algo.com", - "org_created_at": "2021-10-22T16:41:32", - "org_url": None, - "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", - "stripe_customer_id": None, - "external_admin_group": None, - "external_member_group": None, - "external_id": None, - "owner_ids": None, - "resource_type": "organization", - "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" - } - - -@app.get("/v1/algorithm-environments/edge/languages") -async def get_supported_langs(): - return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", - "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, - {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", - "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, - {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", - "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, - {"name": "python2", "display_name": "Python 2.x (Environments)", - "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, - {"name": "python3", "display_name": "Python 3.x (Environments)", - "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, - {"name": "r36", "display_name": "R 3.6.x (Environments)", - "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, - {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", - "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] - - -@app.get("/v1/algorithm-environments/edge/languages/{language}/environments") -async def get_environments_by_lang(language): - return { - "environments": [ - { - "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", - "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", - "display_name": "Python 2.7", - "description": "Generic Python 2.7 installation", - "created_at": "2020-12-21T21:47:53.239", - "language": { - "name": language, - "display_name": "Python 2.x (Environments)", - "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " - " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" - "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," - "\n {\"source\":\"/opt/algorithm\", " - "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " - }, - "machine_type": "CPU" - }, - { - "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", - "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", - "display_name": "Python 2.7 + GPU support", - "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", - "created_at": "2020-08-14T07:22:32.955", - "language": { - "name": language, - "display_name": "Python 2.x (Environments)", - "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " - " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" - "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," - "\n {\"source\":\"/opt/algorithm\", " - "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " - }, - "machine_type": "GPU" - } - ] - } diff --git a/Test/api/app.py b/Test/api/app.py new file mode 100644 index 0000000..99c192b --- /dev/null +++ b/Test/api/app.py @@ -0,0 +1,355 @@ +from fastapi import FastAPI, Request +from typing import Optional +from fastapi.responses import Response +import json +import base64 +from multiprocessing import Process +import uvicorn + + + +regular_app = FastAPI() + + +def start_webserver_reg(): + def _start_webserver(): + uvicorn.run(regular_app, host="127.0.0.1", port=8080, log_level="debug") + + p = Process(target=_start_webserver) + p.start() + return p + +@regular_app.post("/v1/algo/{username}/{algoname}") +async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} + content_type = request.headers['Content-Type'] + auth = request.headers.get('Authorization', None) + if auth is None: + return {"error": {"message": "authorization required"}} + request = await request.body() + if output and output == "void": + return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} + elif output and output == "raw": + return Response(request.decode(), status_code=200) + elif algoname == "500": + return Response("Internal Server Error", status_code=500) + elif algoname == "raise_exception": + return {"error": {"message": "This is an exception"}} + else: + if content_type != "application/octet-stream": + request = request.decode('utf-8') + if content_type == "text/plain": + metadata['content_type'] = "text" + elif content_type == "application/json": + request = json.loads(request) + metadata['content_type'] = "json" + else: + metadata['content_type'] = "binary" + request = base64.b64encode(request) + output = {"result": request, "metadata": metadata} + return output + + +@regular_app.post("/v1/algo/{username}/{algoname}/{githash}") +async def process_hello_world(request: Request, username, algoname, githash): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, + 'content_type': "text"} + request = await request.body() + request = request.decode('utf-8') + return {"result": f"hello {request}", "metadata": metadata} + + +### Algorithm Routes +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") +async def get_build_id(username, algoname, buildid): + return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", + "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", + "version_info": {"semantic_version": "0.1.1"}} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + +@regular_app.get("/v1/algorithms/{username}/{algoname}/scm/status") +async def get_scm_status(username, algoname): + return {"scm_connection_status": "active"} + + +@regular_app.get("/v1/algorithms/{algo_id}/errors") +async def get_algo_errors(algo_id): + return {"error": {"message": "not found"}} + + +@regular_app.post("/v1/algorithms/{username}") +async def create_algorithm(request: Request, username): + payload = await request.json() + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], + "details": {"label": payload["details"]["label"]}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "resource_type": "algorithm"} + + +@regular_app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + + +@regular_app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "language": "python3", + "environment": "gpu", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "semantic_version": "0.1.0", + "git_hash": algohash, + "release_notes": "created programmatically", + "sample_input": "\"payload\"", + "sample_output": "Exception encountered while running sample input", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "resource_type": "algorithm" + } + + +### Admin Routes +@regular_app.post("/v1/users") +async def create_user(request: Request): + payload = await request.body() + data = json.loads(payload) + username = data['username'] + email = data['email'] + return { + "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", + "username": username, + "email": email, + "fullname": username, + "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" + } + + +@regular_app.get("/v1/users/{user_id}/errors") +async def get_user_errors(user_id): + return [] + + +@regular_app.get("/v1/organization/types") +async def get_org_types(): + return [ + {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, + {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, + {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} + ] + + +@regular_app.post("/v1/organizations") +async def create_org(request: Request): + payload = await request.body() + data = json.loads(payload) + org_name = data["org_name"] + org_email = data["org_email"] + return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": org_email, + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@regular_app.put("/v1/organizations/{orgname}/members/{username}") +async def add_user_to_org(orgname, username): + return Response(status_code=200) + + +@regular_app.get("/v1/organizations/{orgname}/errors") +async def org_errors(orgname): + return [] + + +@regular_app.put("/v1/organizations/{org_name}") +async def edit_org(org_name): + return Response(status_code=204) + + +@regular_app.get("/v1/organizations/{org_name}") +async def get_org_by_name(org_name): + return { + "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg1542@algo.com", + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + +@regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@regular_app.get("/v1/algorithm-environments/edge/languages") +async def get_supported_langs(): + return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", + "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", + "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, + {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", + "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, + {"name": "python2", "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "python3", "display_name": "Python 3.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "r36", "display_name": "R 3.6.x (Environments)", + "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, + {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", + "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] + + +@regular_app.get("/v1/algorithm-environments/edge/languages/{language}/environments") +async def get_environments_by_lang(language): + return { + "environments": [ + { + "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", + "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", + "display_name": "Python 2.7", + "description": "Generic Python 2.7 installation", + "created_at": "2020-12-21T21:47:53.239", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "CPU" + }, + { + "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", + "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", + "display_name": "Python 2.7 + GPU support", + "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", + "created_at": "2020-08-14T07:22:32.955", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "GPU" + } + ] + } + + + \ No newline at end of file diff --git a/Test/api/self_signed_app.py b/Test/api/self_signed_app.py new file mode 100644 index 0000000..4d1423f --- /dev/null +++ b/Test/api/self_signed_app.py @@ -0,0 +1,351 @@ +from fastapi import FastAPI, Request +from typing import Optional +from fastapi.responses import Response +import json +import base64 +from multiprocessing import Process +import uvicorn + +self_signed_app = FastAPI() + + +def start_webserver_self_signed(): + def _start_webserver(): + uvicorn.run(self_signed_app, host="127.0.0.1", port=8090, log_level="debug", + ssl_certfile="Test/resources/cert.cert", ssl_keyfile="Test/resources/cert.key") + + p = Process(target=_start_webserver) + p.start() + return p + +@self_signed_app.post("/v1/algo/{username}/{algoname}") +async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} + content_type = request.headers['Content-Type'] + auth = request.headers.get('Authorization', None) + if auth is None: + return {"error": {"message": "authorization required"}} + request = await request.body() + if output and output == "void": + return {"async": "abcd123", "request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725"} + elif output and output == "raw": + return Response(request.decode(), status_code=200) + elif algoname == "500": + return Response("Internal Server Error", status_code=500) + elif algoname == "raise_exception": + return {"error": {"message": "This is an exception"}} + else: + if content_type != "application/octet-stream": + request = request.decode('utf-8') + if content_type == "text/plain": + metadata['content_type'] = "text" + elif content_type == "application/json": + request = json.loads(request) + metadata['content_type'] = "json" + else: + metadata['content_type'] = "binary" + request = base64.b64encode(request) + output = {"result": request, "metadata": metadata} + return output + + +@self_signed_app.post("/v1/algo/{username}/{algoname}/{githash}") +async def process_hello_world(request: Request, username, algoname, githash): + metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, + 'content_type': "text"} + request = await request.body() + request = request.decode('utf-8') + return {"result": f"hello {request}", "metadata": metadata} + + +### Algorithm Routes +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") +async def get_build_id(username, algoname, buildid): + return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", + "started_at": "2021-09-27T22:54:20.786Z", "finished_at": "2021-09-27T22:54:40.898Z", + "version_info": {"semantic_version": "0.1.1"}} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/scm/status") +async def get_scm_status(username, algoname): + return {"scm_connection_status": "active"} + + +@self_signed_app.get("/v1/algorithms/{algo_id}/errors") +async def get_algo_errors(algo_id): + return {"error": {"message": "not found"}} + + +@self_signed_app.post("/v1/algorithms/{username}") +async def create_algorithm(request: Request, username): + payload = await request.json() + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": payload["name"], + "details": {"label": payload["details"]["label"]}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "resource_type": "algorithm"} + + +@self_signed_app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm" + } + + +@self_signed_app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + return { + "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", + "name": algoname, + "details": { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + }, + "settings": { + "algorithm_callability": "private", + "source_visibility": "open", + "language": "python3", + "environment": "gpu", + "package_set": "tensorflow-gpu-2.3-python38", + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False, + "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" + }, + "version_info": { + "semantic_version": "0.1.0", + "git_hash": algohash, + "release_notes": "created programmatically", + "sample_input": "\"payload\"", + "sample_output": "Exception encountered while running sample input", + "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" + }, + "source": { + "scm": { + "id": "internal", + "provider": "internal", + "default": True, + "enabled": True + } + }, + "compilation": { + "successful": True, + "output": "" + }, + "resource_type": "algorithm" + } + + +### Admin Routes +@self_signed_app.post("/v1/users") +async def create_user(request: Request): + payload = await request.body() + data = json.loads(payload) + username = data['username'] + email = data['email'] + return { + "id": "1e5c89ab-3d5c-4bad-b8a3-6c8a294d4418", + "username": username, + "email": email, + "fullname": username, + "self_link": f"http://localhost:8080/v1/users/{username}", "resource_type": "user" + } + + +@self_signed_app.get("/v1/users/{user_id}/errors") +async def get_user_errors(user_id): + return [] + + +@self_signed_app.get("/v1/organization/types") +async def get_org_types(): + return [ + {"id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", "name": "basic"}, + {"id": "d0bff917-ddfa-11ea-a0c8-12a811be4db3", "name": "legacy"}, + {"id": "d0c9d825-ddfa-11ea-a0c8-12a811be4db3", "name": "pro"} + ] + + +@self_signed_app.post("/v1/organizations") +async def create_org(request: Request): + payload = await request.body() + data = json.loads(payload) + org_name = data["org_name"] + org_email = data["org_email"] + return {"id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": org_email, + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + + +@self_signed_app.put("/v1/organizations/{orgname}/members/{username}") +async def add_user_to_org(orgname, username): + return Response(status_code=200) + + +@self_signed_app.get("/v1/organizations/{orgname}/errors") +async def org_errors(orgname): + return [] + + +@self_signed_app.put("/v1/organizations/{org_name}") +async def edit_org(org_name): + return Response(status_code=204) + + +@self_signed_app.get("/v1/organizations/{org_name}") +async def get_org_by_name(org_name): + return { + "id": "55073c92-5f8e-4d7e-a14d-568f94924fd9", + "org_name": org_name, + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg1542@algo.com", + "org_created_at": "2021-10-22T16:41:32", + "org_url": None, + "type_id": "d0c85ea6-ddfa-11ea-a0c8-12a811be4db3", + "stripe_customer_id": None, + "external_admin_group": None, + "external_member_group": None, + "external_id": None, + "owner_ids": None, + "resource_type": "organization", + "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" + } + +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") +async def get_build_log(username, algoname, buildid): + return {"logs": "This is a log"} + + +@self_signed_app.get("/v1/algorithm-environments/edge/languages") +async def get_supported_langs(): + return [{"name": "anaconda3", "display_name": "Conda (Environments) - beta", + "configuration": "{\n \"display_name\": \"Conda (Environments) - beta\",\n \"req_files\": [\n \"environment.yml\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.cache\", \"destination\":\"/home/algo/.cache/\"},\n {\"source\":\"/home/algo/anaconda_environment\", \"destination\": \"/home/algo/anaconda_environment/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "csharp-dotnet-core2", "display_name": "C# .NET Core 2.x+ (Environments)", + "configuration": "{\n \"display_name\": \"C# .NET Core 2.x+ (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/bin/Release/*/*\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/opt/algorithm/resources\", \"destination\":\"/opt/algorithm/resources/\"},\n {\"source\":\"/home/algo/.nuget\", \"destination\":\"/home/algo/.nuget/\"}\n ]\n}\n"}, + {"name": "java11", "display_name": "Java OpenJDK 11.0 (Environments)", + "configuration": "{\n \"display_name\": \"Java OpenJDK 11.0 (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/*.jar\", \"destination\":\"/opt/algorithm/target/algorithm.jar\"},\n {\"source\":\"/opt/algorithm/target/lib\", \"destination\":\"/opt/algorithm/target/lib/\"}\n ]\n}\n"}, + {"name": "python2", "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "python3", "display_name": "Python 3.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 3.x (Environments)\",\n \"req_files\": [\n \"requirements.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"},\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"}\n ]\n}\n"}, + {"name": "r36", "display_name": "R 3.6.x (Environments)", + "configuration": "{\n \"display_name\": \"R 3.6.x (Environments)\",\n \"req_files\": [\n \"packages.txt\"\n ],\n \"artifacts\": [\n {\"source\":\"/opt/algorithm\", \"destination\":\"/opt/algorithm/\"},\n {\"source\":\"/usr/local/lib/R/site-library\", \"destination\":\"/usr/local/lib/R/site-library/\"}\n ]\n}\n\n"}, + {"name": "scala-2", "display_name": "Scala 2.x & sbt 1.3.x (Environments)", + "configuration": "{\n \"display_name\": \"Scala 2.x & sbt 1.3.x (Environments)\",\n \"artifacts\": [\n {\"source\":\"/opt/algorithm/target/universal/stage\", \"destination\":\"/opt/algorithm/stage/\"}\n ]\n}\n\n"}] + + +@self_signed_app.get("/v1/algorithm-environments/edge/languages/{language}/environments") +async def get_environments_by_lang(language): + return { + "environments": [ + { + "id": "717d36e0-222c-44a0-9aa8-06f4ebc1b82a", + "environment_specification_id": "f626effa-e519-431e-9d7a-0d3a7563ae1e", + "display_name": "Python 2.7", + "description": "Generic Python 2.7 installation", + "created_at": "2020-12-21T21:47:53.239", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "CPU" + }, + { + "id": "6f57e041-54e0-4e1a-8b2f-4589bb2c06f8", + "environment_specification_id": "faf81400-eb15-4f64-81c0-3d4ed7181e77", + "display_name": "Python 2.7 + GPU support", + "description": "Python2.7 installation with CUDA 9.0 and CUDNN7", + "created_at": "2020-08-14T07:22:32.955", + "language": { + "name": language, + "display_name": "Python 2.x (Environments)", + "configuration": "{\n \"display_name\": \"Python 2.x (Environments)\",\n \"req_files\": [\n " + " \"requirements.txt\"\n ],\n \"artifacts\": [\n {" + "\"source\":\"/home/algo/.local\", \"destination\":\"/home/algo/.local/\"}," + "\n {\"source\":\"/opt/algorithm\", " + "\"destination\":\"/opt/algorithm/\"}\n ]\n}\n " + }, + "machine_type": "GPU" + } + ] + } diff --git a/Test/conftest.py b/Test/conftest.py index 71ca978..c758814 100644 --- a/Test/conftest.py +++ b/Test/conftest.py @@ -1,12 +1,15 @@ import sys from time import sleep +import os, signal if sys.version_info.major >= 3: - from Test.api import start_webserver + from Test.api import start_webserver_reg, start_webserver_self_signed import pytest @pytest.fixture(scope='package', autouse=True) def fastapi_start(): - p = start_webserver() + p_reg = start_webserver_reg() + p_self_signed = start_webserver_self_signed() sleep(2) - yield p - p.terminate() + yield p_reg, p_self_signed + os.kill(p_reg.pid, signal.SIGKILL) + os.kill(p_self_signed.pid, signal.SIGKILL) \ No newline at end of file diff --git a/Test/CLI_test.py b/Test/regular/CLI_test.py similarity index 97% rename from Test/CLI_test.py rename to Test/regular/CLI_test.py index 99f662e..93a2bb2 100644 --- a/Test/CLI_test.py +++ b/Test/regular/CLI_test.py @@ -93,8 +93,8 @@ def setUp(self): # create a directory to use in testing the cp command self.client = Algorithmia.client() CLI().mkdir("data://.my/moredata", self.client) - if not os.path.exists("./TestFiles/"): - os.mkdir("./TestFiles/") + if not os.path.exists("../TestFiles/"): + os.mkdir("../TestFiles/") def test_ls(self): parentDir = "data://.my/" @@ -132,7 +132,7 @@ def test_rmdir(self): def test_cat(self): file = "data://.my/moredata/test.txt" - localfile = "./TestFiles/test.txt" + localfile = "./../TestFiles/test.txt" fileContents = "some text in test file" CLI().rm(file, self.client) @@ -156,7 +156,7 @@ def test_get_build_logs(self): # local to remote def test_cp_L2R(self): - localfile = "./TestFiles/test.txt" + localfile = "./../TestFiles/test.txt" testfile = open(localfile, "w") testfile.write("some text") testfile.close() @@ -199,7 +199,7 @@ def test_auth(self): def test_auth_cert(self): - localfile = "./TestFiles/fakecert.pem" + localfile = "./../TestFiles/fakecert.pem" testfile = open(localfile, "w") testfile.write("") @@ -244,7 +244,7 @@ def test_list_languages(self): self.assertTrue(result is not None and "anaconda3" in result[1]) def test_rm(self): - localfile = "./TestFiles/testRM.txt" + localfile = "./../TestFiles/testRM.txt" testfile = open(localfile, "w") testfile.write("some text") @@ -263,7 +263,7 @@ def test_rm(self): self.assertTrue("testRM.txt" in result1 and "testRM.txt" not in result2) def test_get_template(self): - filename = "./temptest" + filename = "./../temptest" envid = "36fd467e-fbfe-4ea6-aa66-df3f403b7132" response = CLI().get_template(envid, filename, self.client) print(response) diff --git a/Test/regular/__init__.py b/Test/regular/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Test/acl_test.py b/Test/regular/acl_test.py similarity index 100% rename from Test/acl_test.py rename to Test/regular/acl_test.py diff --git a/Test/algo_failure_test.py b/Test/regular/algo_failure_test.py similarity index 97% rename from Test/algo_failure_test.py rename to Test/regular/algo_failure_test.py index 7defcc2..0804b4a 100644 --- a/Test/algo_failure_test.py +++ b/Test/regular/algo_failure_test.py @@ -11,8 +11,6 @@ # you will load the version installed on the computer. sys.path = ['../'] + sys.path from requests import Response - from Test.api import app - class AlgoTest(unittest.TestCase): error_500 = Response() diff --git a/Test/algo_test.py b/Test/regular/algo_test.py similarity index 100% rename from Test/algo_test.py rename to Test/regular/algo_test.py diff --git a/Test/client_test.py b/Test/regular/client_test.py similarity index 97% rename from Test/client_test.py rename to Test/regular/client_test.py index 3be87ad..8055e51 100644 --- a/Test/client_test.py +++ b/Test/regular/client_test.py @@ -20,7 +20,6 @@ class ClientDummyTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") - admin_username = "a_Mrtest" admin_org_name = "a_myOrg" environment_name = "Python 3.9" @@ -274,16 +273,6 @@ def test_get_build_logs(self): self.assertTrue(u'error' not in result) - def test_get_build_logs_no_ssl(self): - client = Algorithmia.client(api_address='https://api.algorithmia.com', - api_key=self.regular_api_key, ca_cert=False) - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = u'Echo' - result = client.algo(user + '/' + algo).build_logs() - if u'error' in result: - print(result) - self.assertTrue("error" not in result) - def test_edit_org(self): org_name = "a_myOrg84" diff --git a/Test/datadirectory_test.py b/Test/regular/datadirectory_test.py similarity index 100% rename from Test/datadirectory_test.py rename to Test/regular/datadirectory_test.py diff --git a/Test/datafile_test.py b/Test/regular/datafile_test.py similarity index 100% rename from Test/datafile_test.py rename to Test/regular/datafile_test.py diff --git a/Test/util_test.py b/Test/regular/util_test.py similarity index 100% rename from Test/util_test.py rename to Test/regular/util_test.py diff --git a/Test/resources/cert.cert b/Test/resources/cert.cert new file mode 100644 index 0000000..aeb562f --- /dev/null +++ b/Test/resources/cert.cert @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDazCCAlOgAwIBAgIUTikiwxFBpLW4pC+5VfOis1xCYKcwDQYJKoZIhvcNAQEL +BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMjA1MDMxNzE2MjZaFw0yMjA2 +MDIxNzE2MjZaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw +HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDBUMZkg/bCJurQIB9znskTjv8URtIK6qqvZpYGTbfI +AzY6HiI0o1gPxjINZW7cBky/9MeEV5zyJghC4WoK099cIUNq2TmAWAjlRgIE8iEy +9z7QVfbSMainuw0RTlD5/8FRWtRe5v8qwbqLICMn3qv/KsG6bRezyS7UVihwFJua +E4dki+y6KSha4RrCtC43inbPlncB4om7PfJQyt5nI7N4KxbY2L3BUa5/+x1ux/ni +C/3y808vLJVQ6nLYgTEg/6K6lFrig0mUIMnCuOiBsrms3NmBPuDdRri/z1ulFHJB +WVQVQ5DgWher0f/dMzHwyRj3ffC8bAPlhrvLHwPQtNeRAgMBAAGjUzBRMB0GA1Ud +DgQWBBRoC77Hql6kEzk7WC6BeaPBu82K/jAfBgNVHSMEGDAWgBRoC77Hql6kEzk7 +WC6BeaPBu82K/jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCn +W9acM3+rxsiBBClTYEm2tOiukcXEkI7IzvW/4r7P24SmUiDD3vxVVbZ6nevVkg+P +4/QH+YYE3JUeXaN+xnHYjSy4NKxjd3EHT7BFxLMe0DQaodMj0klHqBtULNzojv8+ +/5tpQsjDLeeeDyOIJNz8r6CU9Gzh7j1EBF8BRdLA1z2UVmt6l6d4o3xOTYpOlZs3 +gI+ASxF9ODQzCCOeMYO2qiuMV3RD0oNdIEHUiMD+iHeC1jFGlxZzaWNeuUzP7Yj/ +MOwbBo8l6Hk2BUuUayLxZFLd0wN28IRkLEU5/SOh3mKz79nfPk6pD9rHUO1a53lI +Ua5xJ5tSwG6bMtNnHYYX +-----END CERTIFICATE----- diff --git a/Test/resources/cert.key b/Test/resources/cert.key new file mode 100644 index 0000000..1746297 --- /dev/null +++ b/Test/resources/cert.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDBUMZkg/bCJurQ +IB9znskTjv8URtIK6qqvZpYGTbfIAzY6HiI0o1gPxjINZW7cBky/9MeEV5zyJghC +4WoK099cIUNq2TmAWAjlRgIE8iEy9z7QVfbSMainuw0RTlD5/8FRWtRe5v8qwbqL +ICMn3qv/KsG6bRezyS7UVihwFJuaE4dki+y6KSha4RrCtC43inbPlncB4om7PfJQ +yt5nI7N4KxbY2L3BUa5/+x1ux/niC/3y808vLJVQ6nLYgTEg/6K6lFrig0mUIMnC +uOiBsrms3NmBPuDdRri/z1ulFHJBWVQVQ5DgWher0f/dMzHwyRj3ffC8bAPlhrvL +HwPQtNeRAgMBAAECggEAPr2OhhTmQ0EKOJ4UVxwTuotQci5CAVUELIUo78bNfNa+ +BMK+60KQVB5JJFvlTPemdS5miqc8wsJhMAOkvPripS4OiWES7nqj+HVuNli3OalQ +86DSyIlhaX6l0RYP5fOBtHu8LUjfS+swNfMqNchpHhmsYmsBpFIJJtUHrsihb7GR +4LpNOZ5go4+LG7FX9KaUE4FvAlS7hi6KLSMua10+3+NAlXggbcVikHr3Uq6eQIvk +z09cs+q2FHaESdTjXSIitmYOfJU5KK3QSfXAr/vaqakjnMvfp8MzQ5dHFsy03HRZ +Sy+LjRKOEOCMCT4DmGIPO4V89i3prbVH4JxixCOaeQKBgQDzuwERWE04JEtvfjxS +OAciQKLIxhfa4t2VB65d3115wxfDPIBYU5Mx5YV4aQyOddNxBwpmX/wYwstx2JDZ +2JM0OjOKLnSvlQfr5UmsY9jUO7CdmgC5HpgbHNhc8uJFw4pd+XypWSjytmVxBSdb +m0+in/iUUQuFNH/+BNLVVgWSiwKBgQDLDBCTEpKQvx2kAc8TEtwrWNhacZILab5D +StQBEL62VfGMdXYaA5dXreo5nqioHfBR3BfAmDecmq3iEFE8/yHJs6pLdcmj0Z1L +034UQedYLCmL9zuAgC6p4SKIMPubnYtMrNJOL3lq0ibogz3rfOhdN2B6S88IYoSL +M6asdoQN0wKBgCd1VPzr4MSAC75nH3joHS+Ma045087Z/6mK7s2/xbBax1QSTWz/ +Sss/L1aJG0FNDgg0bZiZXYTctHcf6oN6Loq8CXALiVSLuhaUrlK8b3QcncFGF2vg +6hspllWl9L/6okIIjAgWqSxyHwYnIXIRONlJMMNCQ60zDK2hNkjXflt1AoGAX0w3 +Tz/NSGBaogozTUlxymp1iOV63R5xLRYmsKVSTTPDHeBXYNhEpOM8ZnS/xb/fdhwt +jbgjib3TVKHB7zXzfr5zc91BmUCdaeRGbW2NDgYULdwIskP3IsZGtdL/lEb6BS+r +uQRxISCnIEPQwQCr8mw2PM/tyIqsmMTSOmmZiv8CgYBAfIC/cNwJyID6AVauZHQo +S3Bii9CPmPnuklBuS7ikX0bmZ93dzv537nqwGr0j9ksxabLWZRcoxx6MhgmXzXVT +dy48TWpqpHiMNorYskB9tcZSrBCl70bu5qKp2owqWHW0d4hqH3lkBNFhfwNWm+qC +54x3T/1fqyaqeapCiE5FGA== +-----END PRIVATE KEY----- diff --git a/Test/self_signed/__init__.py b/Test/self_signed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Test/self_signed/acl_test.py b/Test/self_signed/acl_test.py new file mode 100644 index 0000000..2022c65 --- /dev/null +++ b/Test/self_signed/acl_test.py @@ -0,0 +1,38 @@ +import sys +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +import unittest +import Algorithmia +from Algorithmia.acl import AclType, Acl, ReadAcl +from Algorithmia.datadirectory import DataDirectory + +class AclTypeTest(unittest.TestCase): + def test_types(self): + self.assertTrue(AclType.private.acl_string is None) + self.assertEquals(AclType.my_algos.acl_string, 'algo://.my/*') + self.assertEquals(AclType.public.acl_string, 'user://*') + self.assertEquals(AclType.default, AclType.my_algos) + + def test_from_acl_response(self): + self.assertEquals(AclType.from_acl_response([]), AclType.private) + self.assertEquals(AclType.from_acl_response(['algo://.my/*']), AclType.my_algos) + self.assertEquals(AclType.from_acl_response(['user://*']), AclType.public) + + def test_create_acl(self): + c = Algorithmia.client() + dd = DataDirectory(c, 'data://.my/privatePermissions') + if dd.exists(): + dd.delete(True) + dd.create(ReadAcl.private) + + dd_perms = DataDirectory(c, 'data://.my/privatePermissions').get_permissions() + self.assertEquals(dd_perms.read_acl, AclType.private) + + dd.update_permissions(ReadAcl.public) + dd_perms = DataDirectory(c, 'data://.my/privatePermissions').get_permissions() + self.assertEquals(dd_perms.read_acl, AclType.public) + +if __name__ == '__main__': + unittest.main() diff --git a/Test/self_signed/algo_failure_test.py b/Test/self_signed/algo_failure_test.py new file mode 100644 index 0000000..aed319d --- /dev/null +++ b/Test/self_signed/algo_failure_test.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info[0] >= 3: + import unittest + import Algorithmia + import uvicorn + import time + from multiprocessing import Process + + # look in ../ BEFORE trying to import Algorithmia. If you append to the + # you will load the version installed on the computer. + sys.path = ['../'] + sys.path + from requests import Response + + class AlgoTest(unittest.TestCase): + error_500 = Response() + error_500.status_code = 500 + error_message = "Non-Algorithm related Failure: " + str(error_500) + + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) + + def test_throw_500_error_HTTP_response_on_algo_request(self): + try: + result = self.client.algo('util/500').pipe(bytearray('foo', 'utf-8')) + except Exception as e: + result = e + pass + self.assertEqual(str(self.error_message), str(result)) diff --git a/Test/self_signed/algo_test.py b/Test/self_signed/algo_test.py new file mode 100644 index 0000000..a5e0964 --- /dev/null +++ b/Test/self_signed/algo_test.py @@ -0,0 +1,136 @@ +import sys +import os +from Algorithmia.errors import AlgorithmException +from Algorithmia.algorithm import OutputType +import Algorithmia +# look in ../ BEFORE trying to import Algorithmia. If you append to the +# you will load the version installed on the computer. +sys.path = ['../'] + sys.path + +import unittest + +if sys.version_info.major >= 3: + + + class AlgoDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) + + def test_call_customCert(self): + result = self.client.algo('util/echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + + def test_normal_call(self): + result = self.client.algo('util/echo').pipe("foo") + self.assertEquals("text", result.metadata.content_type) + self.assertEquals("foo", result.result) + + def test_async_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEquals("foo", result) + + def test_dict_call(self): + result = self.client.algo('util/echo').pipe({"foo": "bar"}) + self.assertEquals("json", result.metadata.content_type) + self.assertEquals({"foo": "bar"}, result.result) + + def test_text_unicode(self): + telephone = u"\u260E" + # Unicode input to pipe() + result1 = self.client.algo('util/Echo').pipe(telephone) + self.assertEquals('text', result1.metadata.content_type) + self.assertEquals(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('util/Echo').pipe(result1.result) + self.assertEquals('text', result2.metadata.content_type) + self.assertEquals(telephone, result2.result) + + def test_get_build_by_id(self): + result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.build_id is not None) + + def test_get_build_logs(self): + result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.logs is not None) + + def test_get_scm_status(self): + result = self.client.algo("J_bragg/Echo").get_scm_status() + self.assertTrue(result.scm_connection_status is not None) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + +else: + class AlgoTest(unittest.TestCase): + def setUp(self): + self.client = Algorithmia.client() + + def test_call_customCert(self): + open("./test.pem", 'w') + c = Algorithmia.client(ca_cert="./test.pem") + result = c.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + try: + os.remove("./test.pem") + except OSError as e: + print(e) + + def test_call_binary(self): + result = self.client.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) + self.assertEquals('binary', result.metadata.content_type) + self.assertEquals(bytearray('foo', 'utf-8'), result.result) + + def test_async_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") + self.assertTrue(hasattr(result, "async_protocol")) + self.assertTrue(hasattr(result, "request_id")) + + def test_raw_call(self): + result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") + self.assertEquals("foo", result) + + def test_text_unicode(self): + telephone = u"\u260E" + + # Unicode input to pipe() + result1 = self.client.algo('util/Echo').pipe(telephone) + self.assertEquals('text', result1.metadata.content_type) + self.assertEquals(telephone, result1.result) + + # Unicode return in .result + result2 = self.client.algo('util/Echo').pipe(result1.result) + self.assertEquals('text', result2.metadata.content_type) + self.assertEquals(telephone, result2.result) + + def test_get_build_by_id(self): + result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.build_id is not None) + + def test_get_build_logs(self): + result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue(result.logs is not None) + + def test_get_scm_status(self): + result = self.client.algo("J_bragg/Echo").get_scm_status() + self.assertTrue(result.scm_connection_status is not None) + + def test_exception_ipa_algo(self): + try: + result = self.client.algo('zeryx/raise_exception').pipe("") + except AlgorithmException as e: + self.assertEqual(e.message, "This is an exception") + +if __name__ == '__main__': + unittest.main() diff --git a/Test/self_signed/client_test.py b/Test/self_signed/client_test.py new file mode 100644 index 0000000..fc36e5b --- /dev/null +++ b/Test/self_signed/client_test.py @@ -0,0 +1,413 @@ +import os +import shutil +import sys +from datetime import datetime +from random import random +from random import seed + +sys.path = ['../'] + sys.path + +import unittest +import Algorithmia +from Algorithmia.errors import AlgorithmException +from uuid import uuid4 + +if sys.version_info.major >= 3: + unicode = str + + + class ClientDummyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) + + admin_username = "a_Mrtest" + admin_org_name = "a_myOrg" + environment_name = "Python 3.9" + + def setUp(self): + self.admin_username = self.admin_username + str(int(random() * 10000)) + self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) + + self.environment_id = "abcd-123" + + def test_create_user(self): + response = self.client.create_user( + {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", + "shouldCreateHello": False}) + + if type(response) is dict: + self.assertEqual(self.admin_username, response['username']) + else: + self.assertIsNotNone(response) + + def test_get_org_types(self): + response = self.client.get_org_types() + self.assertTrue(len(response) > 0) + + def test_create_org(self): + response = self.client.create_org( + {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", + "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) + + self.assertEqual(self.admin_org_name, response[u'org_name']) + + def test_get_org(self): + response = self.client.get_org("a_myOrg84") + self.assertEqual("a_myOrg84", response['org_name']) + + def test_get_environment(self): + response = self.client.get_environment("python2") + + if u'error' not in response: + self.assertTrue(response is not None and u'environments' in response) + + def test_get_build_logs(self): + user = unicode(os.environ.get('ALGO_USER_NAME')) + algo = unicode('echo') + algo_path = u'%s/%s' % (user, algo) + result = self.client.algo(algo_path).build_logs() + + if u'error' in result: + print(result) + + self.assertTrue(u'error' not in result) + + def test_edit_org(self): + org_name = "a_myOrg84" + + obj = { + "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", + "org_name": "a_myOrg84", + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg84@algo.com", + "org_created_at": "2020-11-30T23:51:40", + "org_url": "https://algorithmia.com", + "type_id": "basic", + "resource_type": "organization" + } + + response = self.client.edit_org(org_name, obj) + if type(response) is dict: + print(response) + else: + self.assertEqual(204, response.status_code) + + def test_get_supported_languages(self): + response = self.client.get_supported_languages() + self.assertTrue(response is not None) + + if type(response) is not list: + self.assertTrue(u'error' in response) + else: + language_found = any('anaconda3' in languages['name'] for languages in response) + self.assertTrue(response is not None and language_found) + + def test_invite_to_org(self): + response = self.client.invite_to_org("a_myOrg38", "a_Mrtest4") + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(200, response.status_code) + + # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been + # deployed to the remote environment. + def test_get_organization_errors(self): + response = self.client.get_organization_errors(self.admin_org_name) + self.assertTrue(response is not None) + + if type(response) is list: + self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') + + def test_get_user_errors(self): + response = self.client.get_user_errors(self.admin_username) + + self.assertTrue(response is not None) + self.assertEqual(0, len(response)) + + + def test_get_algorithm_errors(self): + response = self.client.get_algorithm_errors('hello') + self.assertTrue(response is not None) + + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(404, response.status_code) + + def test_algorithm_programmatic_create_process(self): + algorithm_name = "algo_e2d_test" + payload = "John" + expected_response = "hello John" + full_path = "a_Mrtest/" + algorithm_name + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + created_algo = self.client.algo(full_path) + response = created_algo.create(details=details, settings=settings) + self.assertEqual(response.name, algorithm_name, "algorithm creation failed") + + # --- Creation complete, compiling + + response = created_algo.compile() + git_hash = response.version_info.git_hash + algo_with_build = self.client.algo(full_path + "/" + git_hash) + self.assertEqual(response.name, created_algo.algoname) + + # --- compiling complete, now testing algorithm request + response = algo_with_build.pipe(payload).result + self.assertEqual(response, expected_response, "compiling failed") + + # --- testing complete, now publishing new release. + + pub_settings = {"algorithm_callability": "private"} + pub_version_info = { + "release_notes": "created programmatically", + "sample_input": payload, + "version_type": "minor" + } + pub_details = {"label": "testing123"} + + response = algo_with_build.publish( + details=pub_details, + settings=pub_settings, + version_info=pub_version_info + ) + self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", + "Publishing failed, semantic version is not correct.") + + # --- publishing complete, getting additional information + + response = created_algo.info(git_hash) + + self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") + + def test_no_auth_client(self): + + key = os.environ.get('ALGORITHMIA_API_KEY', "") + if key != "": + del os.environ['ALGORITHMIA_API_KEY'] + + client = Algorithmia.client(api_address="http://localhost:8080") + error = None + try: + client.algo("demo/hello").pipe("world") + except Exception as e: + error = e + finally: + os.environ['ALGORITHMIA_API_KEY'] = key + self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, error_type=None))) + +else: + class ClientTest(unittest.TestCase): + seed(datetime.now().microsecond) + # due to legacy reasons, regular client tests are tested against api.algorithmia.com, whereas admin api tests + # are run against test.algorithmia.com. + admin_username = "a_Mrtest" + admin_org_name = "a_myOrg" + environment_name = "Python 3.9" + + def setUp(self): + self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) + self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) + + self.admin_username = self.admin_username + str(int(random() * 10000)) + self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) + self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", + api_key=self.admin_api_key) + self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', + api_key=self.regular_api_key) + + environments = self.regular_client.get_environment("python3") + for environment in environments['environments']: + if environment['display_name'] == self.environment_name: + self.environment_id = environment['id'] + + def test_create_user(self): + response = self.admin_client.create_user( + {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", + "shouldCreateHello": False}) + + if type(response) is dict: + self.assertEqual(self.admin_username, response['username']) + else: + self.assertIsNotNone(response) + + def test_get_org_types(self): + response = self.admin_client.get_org_types() + self.assertTrue(len(response) > 0) + + def test_create_org(self): + response = self.admin_client.create_org( + {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", + "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) + + self.assertEqual(self.admin_org_name, response[u'org_name']) + + def test_get_org(self): + response = self.admin_client.get_org("a_myOrg84") + self.assertEqual("a_myOrg84", response['org_name']) + + def test_get_environment(self): + response = self.admin_client.get_environment("python2") + + if u'error' not in response: + self.assertTrue(response is not None and u'environments' in response) + + def test_get_build_logs(self): + user = unicode(os.environ.get('ALGO_USER_NAME')) + algo = unicode('echo') + algo_path = u'%s/%s' % (user, algo) + result = self.regular_client.algo(algo_path).build_logs() + + if u'error' in result: + print(result) + + self.assertTrue(u'error' not in result) + + def test_edit_org(self): + org_name = "a_myOrg84" + + obj = { + "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", + "org_name": "a_myOrg84", + "org_label": "some label", + "org_contact_name": "Some owner", + "org_email": "a_myOrg84@algo.com", + "org_created_at": "2020-11-30T23:51:40", + "org_url": "https://algorithmia.com", + "type_id": "basic", + "resource_type": "organization" + } + + response = self.admin_client.edit_org(org_name, obj) + if type(response) is dict: + print(response) + else: + self.assertEqual(204, response.status_code) + + def test_get_template(self): + filename = "./temptest" + response = self.admin_client.get_template("36fd467e-fbfe-4ea6-aa66-df3f403b7132", filename) + + if type(response) is dict: + self.assertTrue(u'error' in response or u'message' in response) + else: + self.assertTrue(response.ok) + try: + shutil.rmtree(filename) + except OSError as e: + print(e) + + def test_get_supported_languages(self): + response = self.admin_client.get_supported_languages() + self.assertTrue(response is not None) + + if type(response) is not list: + self.assertTrue(u'error' in response) + else: + language_found = any('anaconda3' in languages['name'] for languages in response) + self.assertTrue(response is not None and language_found) + + def test_invite_to_org(self): + response = self.admin_client.invite_to_org("a_myOrg38", "a_Mrtest4") + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(200, response.status_code) + + # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been + # deployed to the remote environment. + def test_get_organization_errors(self): + response = self.admin_client.get_organization_errors(self.admin_org_name) + self.assertTrue(response is not None) + + if type(response) is list: + self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') + + def test_get_user_errors(self): + response = self.admin_client.get_user_errors(self.admin_username) + + self.assertTrue(response is not None) + self.assertEqual(0, len(response)) + + def test_get_algorithm_errors(self): + response = self.admin_client.get_algorithm_errors('hello') + self.assertTrue(response is not None) + + if type(response) is dict: + self.assertTrue(u'error' in response) + else: + self.assertEqual(404, response.status_code) + + def test_algorithm_programmatic_create_process(self): + algorithm_name = "algo_" + str(uuid4()).split("-")[-1] + payload = "John" + expected_response = "hello John" + full_path = self.regular_client.username() + "/" + algorithm_name + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + created_algo = self.regular_client.algo(full_path) + response = created_algo.create(details=details, settings=settings) + self.assertEqual(response.name, algorithm_name, "algorithm creation failed") + + # --- Creation complete, compiling + + response = created_algo.compile() + git_hash = response.version_info.git_hash + algo_with_build = self.regular_client.algo(full_path + "/" + git_hash) + self.assertEqual(response.name, created_algo.algoname) + + # --- compiling complete, now testing algorithm request + response = algo_with_build.pipe(payload).result + self.assertEqual(response, expected_response, "compiling failed") + + # --- testing complete, now publishing new release. + + pub_settings = {"algorithm_callability": "private"} + pub_version_info = { + "release_notes": "created programmatically", + "sample_input": payload, + "version_type": "minor" + } + pub_details = {"label": "testing123"} + + response = algo_with_build.publish( + details=pub_details, + settings=pub_settings, + version_info=pub_version_info + ) + self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", + "Publishing failed, semantic version is not correct.") + + # --- publishing complete, getting additional information + + response = created_algo.info(git_hash) + + self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") + + def test_algo_freeze(self): + self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests") + +if __name__ == '__main__': + unittest.main() From 02898a17c79261f0d40c8faf5870ea174c192b67 Mon Sep 17 00:00:00 2001 From: zeryx <1892175+zeryx@users.noreply.github.com> Date: Sun, 8 May 2022 03:47:45 -0300 Subject: [PATCH 11/18] removed all traces of test.algorithmia.com from test cases --- Test/regular/CLI_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Test/regular/CLI_test.py b/Test/regular/CLI_test.py index 93a2bb2..5b7a082 100644 --- a/Test/regular/CLI_test.py +++ b/Test/regular/CLI_test.py @@ -189,13 +189,13 @@ def test_cp_R2L(self): def test_auth(self): # key for test account key = os.getenv('ALGORITHMIA_API_KEY') - address = 'https://api.algorithmia.com' + api_address = "https://api.algorithmia.com" profile = 'default' - CLI().auth(address, key, profile=profile) + CLI().auth(api_address, key, profile=profile) resultK = CLI().getAPIkey(profile) resultA = CLI().getAPIaddress(profile) self.assertEqual(resultK, key) - self.assertEqual(resultA, address) + self.assertEqual(resultA, api_address) def test_auth_cert(self): @@ -274,8 +274,8 @@ def test_get_template(self): print(e) def test_api_address_auth(self): - api_key = os.getenv('ALGORITHMIA_TEST_API_KEY') - api_address = "https://api.test.algorithmia.com" + api_key = os.getenv('ALGORITHMIA_API_KEY') + api_address = "https://api.algorithmia.com" CLI().auth(api_address, api_key) profile = "default" From 23a64d84ebbe7f668dc3dee6111b5752058119df Mon Sep 17 00:00:00 2001 From: zeryx <1892175+zeryx@users.noreply.github.com> Date: Sun, 8 May 2022 04:03:53 -0300 Subject: [PATCH 12/18] removed _all_ test.algorithmia.com references --- Test/regular/CLI_test.py | 2 +- Test/regular/datadirectory_test.py | 52 ++++++++++++++++-------------- Test/self_signed/client_test.py | 4 +-- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/Test/regular/CLI_test.py b/Test/regular/CLI_test.py index 5b7a082..015db70 100644 --- a/Test/regular/CLI_test.py +++ b/Test/regular/CLI_test.py @@ -181,7 +181,7 @@ def test_cp_R2R(self): # remote to local def test_cp_R2L(self): src = ["data://.my/moredata/test.txt"] - dest = "./test.txt" + dest = "./../test.txt" CLI().cp(src, dest, self.client) self.assertTrue(os.path.isfile(dest)) diff --git a/Test/regular/datadirectory_test.py b/Test/regular/datadirectory_test.py index a0d7672..9dc2704 100644 --- a/Test/regular/datadirectory_test.py +++ b/Test/regular/datadirectory_test.py @@ -141,32 +141,34 @@ def test_list_folders(self): testDir.delete(True) - def test_list_files_with_paging(self): - NUM_FILES = 1100 - EXTENSION = '.txt' - dd = DataDirectory(self.client, 'data://.my/pythonLargeDataDirList') - if not dd.exists(): - dd.create() - - for i in range(NUM_FILES): - dd.file(str(i) + EXTENSION).put(str(i)) - - seenFiles = [False] * NUM_FILES - numFiles = 0 - - for f in dd.files(): - numFiles += 1 - name = f.getName() - index = int(name[:-1 * len(EXTENSION)]) - seenFiles[index] = True - - allSeen = True - for cur in seenFiles: - allSeen = (allSeen and cur) - - self.assertEqual(NUM_FILES, numFiles) - self.assertTrue(allSeen) +# TODO: replicate this in Marketplace + # def test_list_files_with_paging(self): + # NUM_FILES = 1100 + # EXTENSION = '.txt' + # + # dd = DataDirectory(self.client, 'data://.my/pythonLargeDataDirList') + # if not dd.exists(): + # dd.create() + # + # for i in range(NUM_FILES): + # dd.file(str(i) + EXTENSION).put(str(i)) + # + # seenFiles = [False] * NUM_FILES + # numFiles = 0 + # + # for f in dd.files(): + # numFiles += 1 + # name = f.getName() + # index = int(name[:-1 * len(EXTENSION)]) + # seenFiles[index] = True + # + # allSeen = True + # for cur in seenFiles: + # allSeen = (allSeen and cur) + # + # self.assertEqual(NUM_FILES, numFiles) + # self.assertTrue(allSeen) def test_data_object(self): dd = DataDirectory(self.client, 'data://foo') diff --git a/Test/self_signed/client_test.py b/Test/self_signed/client_test.py index fc36e5b..fc863ee 100644 --- a/Test/self_signed/client_test.py +++ b/Test/self_signed/client_test.py @@ -213,7 +213,7 @@ class ClientTest(unittest.TestCase): seed(datetime.now().microsecond) # due to legacy reasons, regular client tests are tested against api.algorithmia.com, whereas admin api tests # are run against test.algorithmia.com. - admin_username = "a_Mrtest" + admin_username = "quality" admin_org_name = "a_myOrg" environment_name = "Python 3.9" @@ -223,7 +223,7 @@ def setUp(self): self.admin_username = self.admin_username + str(int(random() * 10000)) self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", + self.admin_client = Algorithmia.client(api_address="https://api.algorithmia.com", api_key=self.admin_api_key) self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', api_key=self.regular_api_key) From d295aeaadf27a7c885c1ac030d237d4d8a1a22f8 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Fri, 13 May 2022 12:03:57 -0700 Subject: [PATCH 13/18] add versions, environments and improve test coverage (#125) * major updates to all paths, removal of class creation from API requests * added more test coverage for versions and info functions * removed unnecessary class conversion function * forgot to remove a dependency, and self signed cert branch wasn't updated * replaced all create/publish/update endpoints to use fixed urls and built-in requests package * simplified builds to follow the normal getJsonHelper pattern * renamed builds to get_builds * added an update endpoint test * fixed the put operation; follows the other helper endpoint systems * removed mutating and cluster specific test cases from client tests (python 2) --- Algorithmia/CLI.py | 9 +- Algorithmia/algorithm.py | 149 +++++------- Algorithmia/client.py | 44 +++- Test/api/app.py | 128 +++++++--- Test/api/self_signed_app.py | 107 +++++---- Test/regular/algo_test.py | 163 ++++++++++--- Test/regular/client_test.py | 269 --------------------- Test/self_signed/acl_test.py | 38 --- Test/self_signed/algo_test.py | 109 +++------ Test/self_signed/client_test.py | 413 -------------------------------- 10 files changed, 423 insertions(+), 1006 deletions(-) delete mode 100644 Test/self_signed/acl_test.py delete mode 100644 Test/self_signed/client_test.py diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 3acc6ae..565f5d0 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -1,11 +1,10 @@ import Algorithmia import os -from Algorithmia.errors import DataApiError +from Algorithmia.errors import DataApiError, AlgorithmException from Algorithmia.algo_response import AlgoResponse import json, re, requests, six import toml import shutil -from time import time class CLI: def __init__(self): @@ -309,12 +308,10 @@ def list_languages(self, client): return table def getBuildLogs(self, user, algo, client): - api_response = client.algo(user + '/' + algo).build_logs() - - if "error" in api_response: - return json.dumps(api_response) + api_response = client.algo(user + '/' + algo).get_builds() return json.dumps(api_response['results'], indent=1) + def getconfigfile(self): if (os.name == "posix"): # if!windows diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 378e1c0..5d3c778 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -1,11 +1,10 @@ 'Algorithmia Algorithm API Client (python)' -import base64 import json import re from Algorithmia.async_response import AsyncResponse from Algorithmia.algo_response import AlgoResponse -from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError +from Algorithmia.errors import ApiError, ApiInternalError, raiseAlgoApiError, AlgorithmException from enum import Enum from algorithmia_api_client.rest import ApiException from algorithmia_api_client import CreateRequest, UpdateRequest, VersionRequest, Details, Settings, SettingsMandatory, \ @@ -40,105 +39,73 @@ def set_options(self, timeout=300, stdout=False, output=OutputType.default, **qu return self # Create a new algorithm - def create(self, details={}, settings={}, version_info={}): - detailsObj = Details(**details) - settingsObj = SettingsMandatory(**settings) - createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - create_parameters = {"name": self.algoname, "details": detailsObj, "settings": settingsObj, - "version_info": createRequestVersionInfoObj} - create_request = CreateRequest(**create_parameters) - try: - # Create Algorithm - api_response = self.client.manageApi.create_algorithm(self.username, create_request) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def create(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + url = "/v1/algorithms/" + self.username + create_parameters = {"name": self.algoname, "details": details, "settings": settings, + "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + + api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) + return api_response # Update the settings in an algorithm - def update(self, details={}, settings={}, version_info={}): - detailsObj = Details(**details) - settingsObj = Settings(**settings) - createRequestVersionInfoObj = CreateRequestVersionInfo(**version_info) - update_parameters = {"details": detailsObj, "settings": settingsObj, - "version_info": createRequestVersionInfoObj} - update_request = UpdateRequest(**update_parameters) - try: - # Update Algorithm - api_response = self.client.manageApi.update_algorithm(self.username, self.algoname, update_request) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def update(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + url = "/v1/algorithms/" + self.username + "/" + self.algoname + update_parameters = {"details": details, "settings": settings, + "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + api_response = self.client.putHelper(url, update_parameters) + return api_response # Publish an algorithm - def publish(self, details={}, settings={}, version_info={}): - publish_parameters = {"details": details, "settings": settings, "version_info": version_info} + def publish(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions" - print(publish_parameters) - api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, - **self.query_parameters) + publish_parameters = {"details": details, "settings": settings, + "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True) return api_response - # except ApiException as e: - # error_message = json.loads(e.body) - # raise raiseAlgoApiError(error_message) - def builds(self, limit=56, marker=None): - try: - if marker is not None: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit, - marker=marker) - else: - api_response = self.client.manageApi.get_algorithm_builds(self.username, self.algoname, limit=limit) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + def get_builds(self, limit=56, marker=None): + kwargs = {"limit": limit, "marker": marker} + url = "/v1/algorithms/" + self.username + "/" + self.algoname + '/builds' + response = self.client.getJsonHelper(url, **kwargs) + return response def get_build(self, build_id): # Get the build object for a given build_id # The build status can have one of the following value: succeeded, failed, in-progress - try: - api_response = self.client.manageApi.get_algorithm_build_by_id(self.username, self.algoname, build_id) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds/' + build_id + response = self.client.getJsonHelper(url) + return response def get_build_logs(self, build_id): # Get the algorithm build logs for a given build_id - try: - api_response = self.client.manageApi.get_algorithm_build_logs(self.username, self.algoname, build_id) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) - - def build_logs(self): - url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds' - response = json.loads(self.client.getHelper(url).content.decode('utf-8')) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/builds/' + build_id + '/logs' + response = self.client.getJsonHelper(url) return response def get_scm_status(self): - try: - api_response = self.client.manageApi.get_algorithm_scm_connection_status(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/scm/status' + response = self.client.getJsonHelper(url) + return response # Get info on an algorithm def info(self, algo_hash=None): + # Get Algorithm + if algo_hash: + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions/' + algo_hash + else: + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions' + response = self.client.getJsonHelper(url) + return response + + # Check if an Algorithm exists + def exists(self): try: - # Get Algorithm - if algo_hash: - api_response = self.client.manageApi.get_algorithm_hash_version(self.username, self.algoname, algo_hash) - else: - api_response = self.client.manageApi.get_algorithm(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + url = '/v1/algorithms/' + self.username + '/' + self.algoname + _ = self.client.getJsonHelper(url) + return True + except AlgorithmException as e: + print(e) + return False # Get all versions of the algorithm, with the given filters def versions(self, limit=None, marker=None, published=None, callable=None): @@ -154,23 +121,17 @@ def versions(self, limit=None, marker=None, published=None, callable=None): if callable: c = callable kwargs["callable"] = str(c).lower() if str(c) in bools else c - try: - # Get Algorithm versions - api_response = self.client.manageApi.get_algorithm_versions(self.username, self.algoname, **kwargs) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + # Get Algorithm versions + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions' + response = self.client.getJsonHelper(url) + return response # Compile an algorithm def compile(self): - try: - # Compile algorithm - api_response = self.client.manageApi.compile_algorithm(self.username, self.algoname) - return api_response - except ApiException as e: - error_message = json.loads(e.body) - raise raiseAlgoApiError(error_message) + # Compile algorithm + url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/compile' + response = self.client.postJsonHelper(url, {}, parse_response_as_json=True) + return response # Pipe an input into this algorithm def pipe(self, input1): diff --git a/Algorithmia/client.py b/Algorithmia/client.py index e72a8f6..30dec03 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -2,6 +2,7 @@ import Algorithmia from Algorithmia.insights import Insights +from Algorithmia.errors import raiseAlgoApiError from Algorithmia.algorithm import Algorithm from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory, AdvancedDataDirectory @@ -15,6 +16,7 @@ from time import time + class Client(object): 'Algorithmia Common Library' @@ -243,10 +245,17 @@ def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query response = self.requestSession.post(self.apiAddress + url, data=input_json, headers=headers, params=query_parameters) - - if parse_response_as_json and response.status_code == 200: - return response.json() - return response + if 200 <= response.status_code <= 299: + if parse_response_as_json: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + return response + else: + raise raiseAlgoApiError(response) # Used internally to http get a file def getHelper(self, url, **query_parameters): @@ -257,6 +266,23 @@ def getHelper(self, url, **query_parameters): headers['Authorization'] = 'Bearer ' + self.bearerToken return self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) + def getJsonHelper(self, url, **query_parameters): + headers = {} + if self.apiKey is not None: + headers['Authorization'] = self.apiKey + elif self.bearerToken is not None: + headers['Authorization'] = 'Bearer ' + self.bearerToken + response = self.requestSession.get(self.apiAddress + url, headers=headers, params=query_parameters) + if 200 <= response.status_code <= 299: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + raise raiseAlgoApiError(response) + + def getStreamHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: @@ -291,11 +317,17 @@ def putHelper(self, url, data): headers['Authorization'] = 'Bearer ' + self.bearerToken if isJson(data): headers['Content-Type'] = 'application/json' - response = self.requestSession.put(self.apiAddress + url, data=data, headers=headers) if response._content == b'': return response - return response.json() + if 200 <= response.status_code <= 299: + response = response.json() + if 'error' in response: + raise raiseAlgoApiError(response) + else: + return response + else: + raise raiseAlgoApiError(response) # Used internally to http delete a file def deleteHelper(self, url): diff --git a/Test/api/app.py b/Test/api/app.py index 99c192b..cd80621 100644 --- a/Test/api/app.py +++ b/Test/api/app.py @@ -6,19 +6,18 @@ from multiprocessing import Process import uvicorn - - regular_app = FastAPI() def start_webserver_reg(): def _start_webserver(): uvicorn.run(regular_app, host="127.0.0.1", port=8080, log_level="debug") - + p = Process(target=_start_webserver) p.start() return p + @regular_app.post("/v1/algo/{username}/{algoname}") async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} @@ -60,6 +59,29 @@ async def process_hello_world(request: Request, username, algoname, githash): ### Algorithm Routes +@regular_app.get('/v1/algorithms/{username}/{algoname}') +async def process_get_algo(request: Request, username, algoname): + if algoname == "echo": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", + "details": {"summary": "", "label": "echo", "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", + "package_set": "python36", "license": "apl", "royalty_microcredits": 0, + "network_access": "full", "pipeline_enabled": True, "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, + "self_link": "https://api.algorithmia.com/v1/algorithms/quality/echo/versions/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"} + else: + return {"error": "No such algorithm"} + + @regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") async def get_build_id(username, algoname, buildid): return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", @@ -71,6 +93,7 @@ async def get_build_id(username, algoname, buildid): async def get_build_log(username, algoname, buildid): return {"logs": "This is a log"} + @regular_app.get("/v1/algorithms/{username}/{algoname}/scm/status") async def get_scm_status(username, algoname): return {"scm_connection_status": "active"} @@ -93,9 +116,8 @@ async def create_algorithm(request: Request, username): "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, "resource_type": "algorithm"} - -@regular_app.post("/v1/algorithms/{username}/{algoname}/compile") -async def compile_algorithm(username, algoname): +@regular_app.put('/v1/algorithms/{username}/{algoname}') +async def update_algorithm(request: Request, username, algoname): return { "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, @@ -135,25 +157,8 @@ async def compile_algorithm(username, algoname): } -@regular_app.post("/v1/algorithms/{username}/{algoname}/versions") -async def publish_algorithm(request: Request, username, algoname): - return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, - "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, - "settings": {"algorithm_callability": "private", "source_visibility": "open", - "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", - "pipeline_enabled": False, "insights_enabled": False, - "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, - "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", - "release_notes": "created programmatically", "sample_input": "payload", - "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, - "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, - "compilation": {"successful": True}, - "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", - "resource_type": "algorithm"} - - -@regular_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") -async def get_algorithm_info(username, algoname, algohash): +@regular_app.post("/v1/algorithms/{username}/{algoname}/compile") +async def compile_algorithm(username, algoname): return { "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, @@ -165,8 +170,6 @@ async def get_algorithm_info(username, algoname, algohash): "settings": { "algorithm_callability": "private", "source_visibility": "open", - "language": "python3", - "environment": "gpu", "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", @@ -175,11 +178,7 @@ async def get_algorithm_info(username, algoname, algohash): "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" }, "version_info": { - "semantic_version": "0.1.0", - "git_hash": algohash, - "release_notes": "created programmatically", - "sample_input": "\"payload\"", - "sample_output": "Exception encountered while running sample input", + "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" }, "source": { @@ -194,10 +193,71 @@ async def get_algorithm_info(username, algoname, algohash): "successful": True, "output": "" }, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", "resource_type": "algorithm" } +@regular_app.post("/v1/algorithms/{username}/{algoname}/versions") +async def publish_algorithm(request: Request, username, algoname): + return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, + "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, + "settings": {"algorithm_callability": "private", "source_visibility": "open", + "package_set": "tensorflow-gpu-2.3-python38", "license": "apl", "network_access": "isolated", + "pipeline_enabled": False, "insights_enabled": False, + "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "e85db9bca2fad519f540b445f30d12523e4dec9c", + "release_notes": "created programmatically", "sample_input": "payload", + "version_uuid": "e85db9bca2fad519f540b445f30d12523e4dec9c"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"http://localhost:8080/v1/algorithms/{username}/{algoname}/versions/e85db9bca2fad519f540b445f30d12523e4dec9c", + "resource_type": "algorithm"} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/versions") +async def versions_of_algorithm(request: Request, username, algoname): + return {"marker": None, "next_link": None, "results": [ + {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "package_set": "python36", + "license": "apl", "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", "started_at": "2022-05-08T22:43:09.050Z", + "finished_at": "2022-05-08T22:43:28.646Z", "version_info": {"semantic_version": "0.1.0"}, + "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"https://api.algorithmia.com/v1/algorithms/{username}/{algoname}/versions" + "/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"}]} + + +@regular_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") +async def get_algorithm_info(username, algoname, algohash): + if algohash == "e85db9bca2fad519f540b445f30d12523e4dec9c": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "language": "python3", + "environment": "cpu", "package_set": "python36", "license": "apl", + "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, "resource_type": "algorithm"} + else: + return {"error": {"message": "not found"}} + + ### Admin Routes @regular_app.post("/v1/users") async def create_user(request: Request): @@ -287,6 +347,7 @@ async def get_org_by_name(org_name): "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" } + @regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") async def get_build_log(username, algoname, buildid): return {"logs": "This is a log"} @@ -350,6 +411,3 @@ async def get_environments_by_lang(language): } ] } - - - \ No newline at end of file diff --git a/Test/api/self_signed_app.py b/Test/api/self_signed_app.py index 4d1423f..693d486 100644 --- a/Test/api/self_signed_app.py +++ b/Test/api/self_signed_app.py @@ -18,6 +18,7 @@ def _start_webserver(): p.start() return p + @self_signed_app.post("/v1/algo/{username}/{algoname}") async def process_algo_req(request: Request, username, algoname, output: Optional[str] = None): metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774} @@ -59,6 +60,29 @@ async def process_hello_world(request: Request, username, algoname, githash): ### Algorithm Routes +@self_signed_app.get('/v1/algorithms/{username}/{algoname}') +async def process_get_algo(request: Request, username, algoname): + if algoname == "echo": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", + "details": {"summary": "", "label": "echo", "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", + "package_set": "python36", "license": "apl", "royalty_microcredits": 0, + "network_access": "full", "pipeline_enabled": True, "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, + "self_link": "https://api.algorithmia.com/v1/algorithms/quality/echo/versions/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"} + else: + return {"error": "No such algorithm"} + + @self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") async def get_build_id(username, algoname, buildid): return {"status": "succeeded", "build_id": buildid, "commit_sha": "bcdadj", @@ -70,6 +94,7 @@ async def get_build_id(username, algoname, buildid): async def get_build_log(username, algoname, buildid): return {"logs": "This is a log"} + @self_signed_app.get("/v1/algorithms/{username}/{algoname}/scm/status") async def get_scm_status(username, algoname): return {"scm_connection_status": "active"} @@ -151,50 +176,47 @@ async def publish_algorithm(request: Request, username, algoname): "resource_type": "algorithm"} +@self_signed_app.get("/v1/algorithms/{username}/{algoname}/versions") +async def versions_of_algorithm(request: Request, username, algoname): + return {"marker": None, "next_link": None, "results": [ + {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "package_set": "python36", + "license": "apl", "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", "started_at": "2022-05-08T22:43:09.050Z", + "finished_at": "2022-05-08T22:43:28.646Z", "version_info": {"semantic_version": "0.1.0"}, + "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True}, + "self_link": f"https://api.algorithmia.com/v1/algorithms/{username}/{algoname}/versions" + "/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "resource_type": "algorithm"}]} + + @self_signed_app.get("/v1/algorithms/{username}/{algoname}/versions/{algohash}") async def get_algorithm_info(username, algoname, algohash): - return { - "id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", - "name": algoname, - "details": { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - }, - "settings": { - "algorithm_callability": "private", - "source_visibility": "open", - "language": "python3", - "environment": "gpu", - "package_set": "tensorflow-gpu-2.3-python38", - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False, - "insights_enabled": False, - "algorithm_environment": "fd980f4f-1f1c-4b2f-a128-d60b40c6567a" - }, - "version_info": { - "semantic_version": "0.1.0", - "git_hash": algohash, - "release_notes": "created programmatically", - "sample_input": "\"payload\"", - "sample_output": "Exception encountered while running sample input", - "version_uuid": "1d9cb91d-11ca-49cb-a7f4-28f67f277654" - }, - "source": { - "scm": { - "id": "internal", - "provider": "internal", - "default": True, - "enabled": True - } - }, - "compilation": { - "successful": True, - "output": "" - }, - "resource_type": "algorithm" - } + if algohash == "e85db9bca2fad519f540b445f30d12523e4dec9c": + return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": algoname, + "details": {"summary": "", "label": algoname, "tagline": ""}, + "settings": {"algorithm_callability": "public", "source_visibility": "closed", "language": "python3", + "environment": "cpu", "package_set": "python36", "license": "apl", + "royalty_microcredits": 0, "network_access": "full", "pipeline_enabled": True, + "insights_enabled": False, + "algorithm_environment": "067110e7-8969-4441-b3d6-5333f18a3db3"}, + "version_info": {"semantic_version": "0.1.0", "git_hash": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "version_uuid": "e06d2808-bb5e-46ae-b7bc-f3d9968e3c6b"}, + "build": {"build_id": "a9ae2c93-6f4e-42c0-ac54-baa4a66e53d3", "status": "succeeded", + "commit_sha": "0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", + "started_at": "2022-05-08T22:43:09.050Z", "finished_at": "2022-05-08T22:43:28.646Z", + "version_info": {"semantic_version": "0.1.0"}, "resource_type": "algorithm_build"}, + "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, + "compilation": {"successful": True, "output": ""}, "resource_type": "algorithm"} + else: + return {"error": {"message": "not found"}} ### Admin Routes @@ -286,6 +308,7 @@ async def get_org_by_name(org_name): "self_link": "http://localhost:8080/v1/organizations/a_myOrg1542" } + @self_signed_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}/logs") async def get_build_log(username, algoname, buildid): return {"logs": "This is a log"} diff --git a/Test/regular/algo_test.py b/Test/regular/algo_test.py index 0e3afdd..b1da4af 100644 --- a/Test/regular/algo_test.py +++ b/Test/regular/algo_test.py @@ -3,6 +3,7 @@ from Algorithmia.errors import AlgorithmException from Algorithmia.algorithm import OutputType import Algorithmia + # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. sys.path = ['../'] + sys.path @@ -11,59 +12,101 @@ if sys.version_info.major >= 3: - class AlgoDummyTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + cls.environment_id = "abcd-123" def test_call_customCert(self): - result = self.client.algo('util/echo').pipe(bytearray('foo', 'utf-8')) + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) self.assertEquals('binary', result.metadata.content_type) self.assertEquals(bytearray('foo', 'utf-8'), result.result) def test_normal_call(self): - result = self.client.algo('util/echo').pipe("foo") + result = self.client.algo('quality/echo').pipe("foo") self.assertEquals("text", result.metadata.content_type) self.assertEquals("foo", result.result) def test_async_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") self.assertTrue(hasattr(result, "async_protocol")) self.assertTrue(hasattr(result, "request_id")) def test_raw_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") self.assertEquals("foo", result) def test_dict_call(self): - result = self.client.algo('util/echo').pipe({"foo": "bar"}) + result = self.client.algo('quality/echo').pipe({"foo": "bar"}) self.assertEquals("json", result.metadata.content_type) self.assertEquals({"foo": "bar"}, result.result) + def test_algo_exists(self): + result = self.client.algo('quality/echo').exists() + self.assertEquals(True, result) + + def test_algo_no_exists(self): + result = self.client.algo('quality/not_echo').exists() + self.assertEquals(False, result) + + #TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + def test_text_unicode(self): telephone = u"\u260E" # Unicode input to pipe() - result1 = self.client.algo('util/Echo').pipe(telephone) + result1 = self.client.algo('quality/echo').pipe(telephone) self.assertEquals('text', result1.metadata.content_type) self.assertEquals(telephone, result1.result) # Unicode return in .result - result2 = self.client.algo('util/Echo').pipe(result1.result) + result2 = self.client.algo('quality/echo').pipe(result1.result) self.assertEquals('text', result2.metadata.content_type) self.assertEquals(telephone, result2.result) + + def test_algo_info(self): + result = self.client.algo('quality/echo').info() + self.assertTrue('results' in result) + self.assertTrue('resource_type' in result['results'][0]) + self.assertTrue(result['results'][0]['resource_type'] == "algorithm") + + def test_update_algo(self): + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + version_info = { + "sample_input": "hello" + } + result = self.client.algo('quality/echo').update(details=details, settings=settings, version_info=version_info) + self.assertTrue('id' in result) + def test_get_build_by_id(self): - result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.build_id is not None) + result = self.client.algo("quality/echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('commit_sha' in result) def test_get_build_logs(self): - result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.logs is not None) + result = self.client.algo("quality/echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('logs' in result) def test_get_scm_status(self): - result = self.client.algo("J_bragg/Echo").get_scm_status() - self.assertTrue(result.scm_connection_status is not None) + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) def test_exception_ipa_algo(self): try: @@ -71,6 +114,67 @@ def test_exception_ipa_algo(self): except AlgorithmException as e: self.assertEqual(e.message, "This is an exception") + def test_algorithm_programmatic_create_process(self): + algorithm_name = "hello" + payload = "John" + expected_response = "hello John" + full_path = "quality/" + algorithm_name + details = { + "summary": "Example Summary", + "label": "QA", + "tagline": "Example Tagline" + } + settings = { + "source_visibility": "open", + "algorithm_environment": self.environment_id, + "license": "apl", + "network_access": "isolated", + "pipeline_enabled": False + } + version_info = { + "sample_input": "hello" + } + created_algo = self.client.algo(full_path) + print("about to create algo") + response = created_algo.create(details=details, settings=settings, version_info=version_info) + print("created algo") + self.assertEqual(response['name'], algorithm_name, "algorithm creation failed") + + # --- Creation complete, compiling + + response = created_algo.compile() + git_hash = response['version_info']['git_hash'] + algo_with_build = self.client.algo(full_path + "/" + git_hash) + self.assertEqual(response['name'], created_algo.algoname) + + # --- compiling complete, now testing algorithm request + response = algo_with_build.pipe(payload).result + self.assertEqual(response, expected_response, "compiling failed") + + # --- testing complete, now publishing new release. + + pub_settings = {"algorithm_callability": "private"} + pub_version_info = { + "release_notes": "created programmatically", + "sample_input": payload, + "version_type": "minor" + } + pub_details = {"label": "testing123"} + + response = algo_with_build.publish( + details=pub_details, + settings=pub_settings, + version_info=pub_version_info + ) + self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", + "Publishing failed, semantic version is not correct.") + + # --- publishing complete, getting additional information + + response = created_algo.info(git_hash) + + self.assertEqual(response['version_info']['semantic_version'], "0.1.0", "information is incorrect") + else: class AlgoTest(unittest.TestCase): def setUp(self): @@ -79,7 +183,7 @@ def setUp(self): def test_call_customCert(self): open("./test.pem", 'w') c = Algorithmia.client(ca_cert="./test.pem") - result = c.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) + result = c.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) self.assertEquals('binary', result.metadata.content_type) self.assertEquals(bytearray('foo', 'utf-8'), result.result) try: @@ -88,43 +192,44 @@ def test_call_customCert(self): print(e) def test_call_binary(self): - result = self.client.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) self.assertEquals('binary', result.metadata.content_type) self.assertEquals(bytearray('foo', 'utf-8'), result.result) def test_async_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") self.assertTrue(hasattr(result, "async_protocol")) self.assertTrue(hasattr(result, "request_id")) def test_raw_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") self.assertEquals("foo", result) + #TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + def test_text_unicode(self): telephone = u"\u260E" # Unicode input to pipe() - result1 = self.client.algo('util/Echo').pipe(telephone) + result1 = self.client.algo('quality/echo').pipe(telephone) self.assertEquals('text', result1.metadata.content_type) self.assertEquals(telephone, result1.result) # Unicode return in .result - result2 = self.client.algo('util/Echo').pipe(result1.result) + result2 = self.client.algo('quality/echo').pipe(result1.result) self.assertEquals('text', result2.metadata.content_type) self.assertEquals(telephone, result2.result) - def test_get_build_by_id(self): - result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.build_id is not None) - - def test_get_build_logs(self): - result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.logs is not None) def test_get_scm_status(self): - result = self.client.algo("J_bragg/Echo").get_scm_status() - self.assertTrue(result.scm_connection_status is not None) + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) def test_exception_ipa_algo(self): try: diff --git a/Test/regular/client_test.py b/Test/regular/client_test.py index 8055e51..3a0e1be 100644 --- a/Test/regular/client_test.py +++ b/Test/regular/client_test.py @@ -28,7 +28,6 @@ def setUp(self): self.admin_username = self.admin_username + str(int(random() * 10000)) self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - self.environment_id = "abcd-123" def test_create_user(self): response = self.client.create_user( @@ -61,17 +60,6 @@ def test_get_environment(self): if u'error' not in response: self.assertTrue(response is not None and u'environments' in response) - def test_get_build_logs(self): - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = unicode('echo') - algo_path = u'%s/%s' % (user, algo) - result = self.client.algo(algo_path).build_logs() - - if u'error' in result: - print(result) - - self.assertTrue(u'error' not in result) - def test_edit_org(self): org_name = "a_myOrg84" @@ -134,61 +122,6 @@ def test_get_algorithm_errors(self): else: self.assertEqual(404, response.status_code) - def test_algorithm_programmatic_create_process(self): - algorithm_name = "algo_e2d_test" - payload = "John" - expected_response = "hello John" - full_path = "a_Mrtest/" + algorithm_name - details = { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - } - settings = { - "source_visibility": "open", - "algorithm_environment": self.environment_id, - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False - } - created_algo = self.client.algo(full_path) - response = created_algo.create(details=details, settings=settings) - self.assertEqual(response.name, algorithm_name, "algorithm creation failed") - - # --- Creation complete, compiling - - response = created_algo.compile() - git_hash = response.version_info.git_hash - algo_with_build = self.client.algo(full_path + "/" + git_hash) - self.assertEqual(response.name, created_algo.algoname) - - # --- compiling complete, now testing algorithm request - response = algo_with_build.pipe(payload).result - self.assertEqual(response, expected_response, "compiling failed") - - # --- testing complete, now publishing new release. - - pub_settings = {"algorithm_callability": "private"} - pub_version_info = { - "release_notes": "created programmatically", - "sample_input": payload, - "version_type": "minor" - } - pub_details = {"label": "testing123"} - - response = algo_with_build.publish( - details=pub_details, - settings=pub_settings, - version_info=pub_version_info - ) - self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", - "Publishing failed, semantic version is not correct.") - - # --- publishing complete, getting additional information - - response = created_algo.info(git_hash) - - self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") def test_no_auth_client(self): @@ -205,207 +138,5 @@ def test_no_auth_client(self): finally: os.environ['ALGORITHMIA_API_KEY'] = key self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, error_type=None))) - -else: - class ClientTest(unittest.TestCase): - seed(datetime.now().microsecond) - # due to legacy reasons, regular client tests are tested against api.algorithmia.com, whereas admin api tests - # are run against test.algorithmia.com. - admin_username = "a_Mrtest" - admin_org_name = "a_myOrg" - environment_name = "Python 3.9" - - def setUp(self): - self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) - self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) - - self.admin_username = self.admin_username + str(int(random() * 10000)) - self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - self.admin_client = Algorithmia.client(api_address="https://test.algorithmia.com", - api_key=self.admin_api_key) - self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', - api_key=self.regular_api_key) - - environments = self.regular_client.get_environment("python3") - for environment in environments['environments']: - if environment['display_name'] == self.environment_name: - self.environment_id = environment['id'] - - def test_create_user(self): - response = self.admin_client.create_user( - {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", - "shouldCreateHello": False}) - - if type(response) is dict: - self.assertEqual(self.admin_username, response['username']) - else: - self.assertIsNotNone(response) - - def test_get_org_types(self): - response = self.admin_client.get_org_types() - self.assertTrue(len(response) > 0) - - def test_create_org(self): - response = self.admin_client.create_org( - {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", - "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) - - self.assertEqual(self.admin_org_name, response[u'org_name']) - - def test_get_org(self): - response = self.admin_client.get_org("a_myOrg84") - self.assertEqual("a_myOrg84", response['org_name']) - - def test_get_environment(self): - response = self.admin_client.get_environment("python2") - - if u'error' not in response: - self.assertTrue(response is not None and u'environments' in response) - - def test_get_build_logs(self): - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = unicode('echo') - algo_path = u'%s/%s' % (user, algo) - result = self.regular_client.algo(algo_path).build_logs() - - if u'error' in result: - print(result) - - self.assertTrue(u'error' not in result) - - def test_edit_org(self): - org_name = "a_myOrg84" - - obj = { - "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", - "org_name": "a_myOrg84", - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": "a_myOrg84@algo.com", - "org_created_at": "2020-11-30T23:51:40", - "org_url": "https://algorithmia.com", - "type_id": "basic", - "resource_type": "organization" - } - - response = self.admin_client.edit_org(org_name, obj) - if type(response) is dict: - print(response) - else: - self.assertEqual(204, response.status_code) - - def test_get_template(self): - filename = "./temptest" - response = self.admin_client.get_template("36fd467e-fbfe-4ea6-aa66-df3f403b7132", filename) - - if type(response) is dict: - self.assertTrue(u'error' in response or u'message' in response) - else: - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - def test_get_supported_languages(self): - response = self.admin_client.get_supported_languages() - self.assertTrue(response is not None) - - if type(response) is not list: - self.assertTrue(u'error' in response) - else: - language_found = any('anaconda3' in languages['name'] for languages in response) - self.assertTrue(response is not None and language_found) - - def test_invite_to_org(self): - response = self.admin_client.invite_to_org("a_myOrg38", "a_Mrtest4") - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(200, response.status_code) - - # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been - # deployed to the remote environment. - def test_get_organization_errors(self): - response = self.admin_client.get_organization_errors(self.admin_org_name) - self.assertTrue(response is not None) - - if type(response) is list: - self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') - - def test_get_user_errors(self): - response = self.admin_client.get_user_errors(self.admin_username) - - self.assertTrue(response is not None) - self.assertEqual(0, len(response)) - - def test_get_algorithm_errors(self): - response = self.admin_client.get_algorithm_errors('hello') - self.assertTrue(response is not None) - - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(404, response.status_code) - - def test_algorithm_programmatic_create_process(self): - algorithm_name = "algo_" + str(uuid4()).split("-")[-1] - payload = "John" - expected_response = "hello John" - full_path = self.regular_client.username() + "/" + algorithm_name - details = { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - } - settings = { - "source_visibility": "open", - "algorithm_environment": self.environment_id, - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False - } - created_algo = self.regular_client.algo(full_path) - response = created_algo.create(details=details, settings=settings) - self.assertEqual(response.name, algorithm_name, "algorithm creation failed") - - # --- Creation complete, compiling - - response = created_algo.compile() - git_hash = response.version_info.git_hash - algo_with_build = self.regular_client.algo(full_path + "/" + git_hash) - self.assertEqual(response.name, created_algo.algoname) - - # --- compiling complete, now testing algorithm request - response = algo_with_build.pipe(payload).result - self.assertEqual(response, expected_response, "compiling failed") - - # --- testing complete, now publishing new release. - - pub_settings = {"algorithm_callability": "private"} - pub_version_info = { - "release_notes": "created programmatically", - "sample_input": payload, - "version_type": "minor" - } - pub_details = {"label": "testing123"} - - response = algo_with_build.publish( - details=pub_details, - settings=pub_settings, - version_info=pub_version_info - ) - self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", - "Publishing failed, semantic version is not correct.") - - # --- publishing complete, getting additional information - - response = created_algo.info(git_hash) - - self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") - - def test_algo_freeze(self): - self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests") - if __name__ == '__main__': unittest.main() diff --git a/Test/self_signed/acl_test.py b/Test/self_signed/acl_test.py deleted file mode 100644 index 2022c65..0000000 --- a/Test/self_signed/acl_test.py +++ /dev/null @@ -1,38 +0,0 @@ -import sys -# look in ../ BEFORE trying to import Algorithmia. If you append to the -# you will load the version installed on the computer. -sys.path = ['../'] + sys.path - -import unittest -import Algorithmia -from Algorithmia.acl import AclType, Acl, ReadAcl -from Algorithmia.datadirectory import DataDirectory - -class AclTypeTest(unittest.TestCase): - def test_types(self): - self.assertTrue(AclType.private.acl_string is None) - self.assertEquals(AclType.my_algos.acl_string, 'algo://.my/*') - self.assertEquals(AclType.public.acl_string, 'user://*') - self.assertEquals(AclType.default, AclType.my_algos) - - def test_from_acl_response(self): - self.assertEquals(AclType.from_acl_response([]), AclType.private) - self.assertEquals(AclType.from_acl_response(['algo://.my/*']), AclType.my_algos) - self.assertEquals(AclType.from_acl_response(['user://*']), AclType.public) - - def test_create_acl(self): - c = Algorithmia.client() - dd = DataDirectory(c, 'data://.my/privatePermissions') - if dd.exists(): - dd.delete(True) - dd.create(ReadAcl.private) - - dd_perms = DataDirectory(c, 'data://.my/privatePermissions').get_permissions() - self.assertEquals(dd_perms.read_acl, AclType.private) - - dd.update_permissions(ReadAcl.public) - dd_perms = DataDirectory(c, 'data://.my/privatePermissions').get_permissions() - self.assertEquals(dd_perms.read_acl, AclType.public) - -if __name__ == '__main__': - unittest.main() diff --git a/Test/self_signed/algo_test.py b/Test/self_signed/algo_test.py index a5e0964..b3b377f 100644 --- a/Test/self_signed/algo_test.py +++ b/Test/self_signed/algo_test.py @@ -3,128 +3,89 @@ from Algorithmia.errors import AlgorithmException from Algorithmia.algorithm import OutputType import Algorithmia + +import unittest + # look in ../ BEFORE trying to import Algorithmia. If you append to the # you will load the version installed on the computer. sys.path = ['../'] + sys.path -import unittest - if sys.version_info.major >= 3: - class AlgoDummyTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) def test_call_customCert(self): - result = self.client.algo('util/echo').pipe(bytearray('foo', 'utf-8')) + result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) self.assertEquals('binary', result.metadata.content_type) self.assertEquals(bytearray('foo', 'utf-8'), result.result) def test_normal_call(self): - result = self.client.algo('util/echo').pipe("foo") + result = self.client.algo('quality/echo').pipe("foo") self.assertEquals("text", result.metadata.content_type) self.assertEquals("foo", result.result) def test_async_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") + result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") self.assertTrue(hasattr(result, "async_protocol")) self.assertTrue(hasattr(result, "request_id")) def test_raw_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") + result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") self.assertEquals("foo", result) def test_dict_call(self): - result = self.client.algo('util/echo').pipe({"foo": "bar"}) + result = self.client.algo('quality/echo').pipe({"foo": "bar"}) self.assertEquals("json", result.metadata.content_type) self.assertEquals({"foo": "bar"}, result.result) - def test_text_unicode(self): - telephone = u"\u260E" - # Unicode input to pipe() - result1 = self.client.algo('util/Echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) - - # Unicode return in .result - result2 = self.client.algo('util/Echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) + def test_algo_exists(self): + result = self.client.algo('quality/echo').exists() + self.assertEquals(True, result) - def test_get_build_by_id(self): - result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.build_id is not None) + def test_algo_no_exists(self): + result = self.client.algo('quality/not_echo').exists() + self.assertEquals(False, result) - def test_get_build_logs(self): - result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.logs is not None) - - def test_get_scm_status(self): - result = self.client.algo("J_bragg/Echo").get_scm_status() - self.assertTrue(result.scm_connection_status is not None) - - def test_exception_ipa_algo(self): - try: - result = self.client.algo('zeryx/raise_exception').pipe("") - except AlgorithmException as e: - self.assertEqual(e.message, "This is an exception") - -else: - class AlgoTest(unittest.TestCase): - def setUp(self): - self.client = Algorithmia.client() - - def test_call_customCert(self): - open("./test.pem", 'w') - c = Algorithmia.client(ca_cert="./test.pem") - result = c.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) - try: - os.remove("./test.pem") - except OSError as e: - print(e) - - def test_call_binary(self): - result = self.client.algo('util/Echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) - - def test_async_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.void).pipe("foo") - self.assertTrue(hasattr(result, "async_protocol")) - self.assertTrue(hasattr(result, "request_id")) - - def test_raw_call(self): - result = self.client.algo('util/echo').set_options(output=OutputType.raw).pipe("foo") - self.assertEquals("foo", result) + # TODO: add more coverage examples to check kwargs + def test_get_versions(self): + result = self.client.algo('quality/echo').versions() + self.assertTrue('results' in result) + self.assertTrue('version_info' in result['results'][0]) + self.assertTrue('semantic_version' in result['results'][0]['version_info']) + self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) def test_text_unicode(self): telephone = u"\u260E" - # Unicode input to pipe() - result1 = self.client.algo('util/Echo').pipe(telephone) + result1 = self.client.algo('quality/echo').pipe(telephone) self.assertEquals('text', result1.metadata.content_type) self.assertEquals(telephone, result1.result) # Unicode return in .result - result2 = self.client.algo('util/Echo').pipe(result1.result) + result2 = self.client.algo('quality/echo').pipe(result1.result) self.assertEquals('text', result2.metadata.content_type) self.assertEquals(telephone, result2.result) + def test_algo_info(self): + result = self.client.algo('quality/echo').info() + self.assertTrue('results' in result) + self.assertTrue('resource_type' in result['results'][0]) + self.assertTrue(result['results'][0]['resource_type'] == "algorithm") + def test_get_build_by_id(self): - result = self.client.algo("J_bragg/Echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.build_id is not None) + result = self.client.algo("quality/echo").get_build("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('commit_sha' in result) def test_get_build_logs(self): - result = self.client.algo("J_bragg/Echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") - self.assertTrue(result.logs is not None) + result = self.client.algo("quality/echo").get_build_logs("1a392e2c-b09f-4bae-a616-56c0830ac8e5") + self.assertTrue('logs' in result) def test_get_scm_status(self): - result = self.client.algo("J_bragg/Echo").get_scm_status() - self.assertTrue(result.scm_connection_status is not None) + result = self.client.algo("quality/echo").get_scm_status() + self.assertTrue('scm_connection_status' in result) def test_exception_ipa_algo(self): try: diff --git a/Test/self_signed/client_test.py b/Test/self_signed/client_test.py deleted file mode 100644 index fc863ee..0000000 --- a/Test/self_signed/client_test.py +++ /dev/null @@ -1,413 +0,0 @@ -import os -import shutil -import sys -from datetime import datetime -from random import random -from random import seed - -sys.path = ['../'] + sys.path - -import unittest -import Algorithmia -from Algorithmia.errors import AlgorithmException -from uuid import uuid4 - -if sys.version_info.major >= 3: - unicode = str - - - class ClientDummyTest(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.client = Algorithmia.client(api_address="https://localhost:8090", api_key="simabcd123", ca_cert=False) - - admin_username = "a_Mrtest" - admin_org_name = "a_myOrg" - environment_name = "Python 3.9" - - def setUp(self): - self.admin_username = self.admin_username + str(int(random() * 10000)) - self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - - self.environment_id = "abcd-123" - - def test_create_user(self): - response = self.client.create_user( - {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", - "shouldCreateHello": False}) - - if type(response) is dict: - self.assertEqual(self.admin_username, response['username']) - else: - self.assertIsNotNone(response) - - def test_get_org_types(self): - response = self.client.get_org_types() - self.assertTrue(len(response) > 0) - - def test_create_org(self): - response = self.client.create_org( - {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", - "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) - - self.assertEqual(self.admin_org_name, response[u'org_name']) - - def test_get_org(self): - response = self.client.get_org("a_myOrg84") - self.assertEqual("a_myOrg84", response['org_name']) - - def test_get_environment(self): - response = self.client.get_environment("python2") - - if u'error' not in response: - self.assertTrue(response is not None and u'environments' in response) - - def test_get_build_logs(self): - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = unicode('echo') - algo_path = u'%s/%s' % (user, algo) - result = self.client.algo(algo_path).build_logs() - - if u'error' in result: - print(result) - - self.assertTrue(u'error' not in result) - - def test_edit_org(self): - org_name = "a_myOrg84" - - obj = { - "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", - "org_name": "a_myOrg84", - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": "a_myOrg84@algo.com", - "org_created_at": "2020-11-30T23:51:40", - "org_url": "https://algorithmia.com", - "type_id": "basic", - "resource_type": "organization" - } - - response = self.client.edit_org(org_name, obj) - if type(response) is dict: - print(response) - else: - self.assertEqual(204, response.status_code) - - def test_get_supported_languages(self): - response = self.client.get_supported_languages() - self.assertTrue(response is not None) - - if type(response) is not list: - self.assertTrue(u'error' in response) - else: - language_found = any('anaconda3' in languages['name'] for languages in response) - self.assertTrue(response is not None and language_found) - - def test_invite_to_org(self): - response = self.client.invite_to_org("a_myOrg38", "a_Mrtest4") - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(200, response.status_code) - - # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been - # deployed to the remote environment. - def test_get_organization_errors(self): - response = self.client.get_organization_errors(self.admin_org_name) - self.assertTrue(response is not None) - - if type(response) is list: - self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') - - def test_get_user_errors(self): - response = self.client.get_user_errors(self.admin_username) - - self.assertTrue(response is not None) - self.assertEqual(0, len(response)) - - - def test_get_algorithm_errors(self): - response = self.client.get_algorithm_errors('hello') - self.assertTrue(response is not None) - - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(404, response.status_code) - - def test_algorithm_programmatic_create_process(self): - algorithm_name = "algo_e2d_test" - payload = "John" - expected_response = "hello John" - full_path = "a_Mrtest/" + algorithm_name - details = { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - } - settings = { - "source_visibility": "open", - "algorithm_environment": self.environment_id, - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False - } - created_algo = self.client.algo(full_path) - response = created_algo.create(details=details, settings=settings) - self.assertEqual(response.name, algorithm_name, "algorithm creation failed") - - # --- Creation complete, compiling - - response = created_algo.compile() - git_hash = response.version_info.git_hash - algo_with_build = self.client.algo(full_path + "/" + git_hash) - self.assertEqual(response.name, created_algo.algoname) - - # --- compiling complete, now testing algorithm request - response = algo_with_build.pipe(payload).result - self.assertEqual(response, expected_response, "compiling failed") - - # --- testing complete, now publishing new release. - - pub_settings = {"algorithm_callability": "private"} - pub_version_info = { - "release_notes": "created programmatically", - "sample_input": payload, - "version_type": "minor" - } - pub_details = {"label": "testing123"} - - response = algo_with_build.publish( - details=pub_details, - settings=pub_settings, - version_info=pub_version_info - ) - self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", - "Publishing failed, semantic version is not correct.") - - # --- publishing complete, getting additional information - - response = created_algo.info(git_hash) - - self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") - - def test_no_auth_client(self): - - key = os.environ.get('ALGORITHMIA_API_KEY', "") - if key != "": - del os.environ['ALGORITHMIA_API_KEY'] - - client = Algorithmia.client(api_address="http://localhost:8080") - error = None - try: - client.algo("demo/hello").pipe("world") - except Exception as e: - error = e - finally: - os.environ['ALGORITHMIA_API_KEY'] = key - self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, error_type=None))) - -else: - class ClientTest(unittest.TestCase): - seed(datetime.now().microsecond) - # due to legacy reasons, regular client tests are tested against api.algorithmia.com, whereas admin api tests - # are run against test.algorithmia.com. - admin_username = "quality" - admin_org_name = "a_myOrg" - environment_name = "Python 3.9" - - def setUp(self): - self.admin_api_key = unicode(os.environ.get('ALGORITHMIA_A_KEY')) - self.regular_api_key = unicode(os.environ.get('ALGORITHMIA_API_KEY')) - - self.admin_username = self.admin_username + str(int(random() * 10000)) - self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - self.admin_client = Algorithmia.client(api_address="https://api.algorithmia.com", - api_key=self.admin_api_key) - self.regular_client = Algorithmia.client(api_address='https://api.algorithmia.com', - api_key=self.regular_api_key) - - environments = self.regular_client.get_environment("python3") - for environment in environments['environments']: - if environment['display_name'] == self.environment_name: - self.environment_id = environment['id'] - - def test_create_user(self): - response = self.admin_client.create_user( - {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", - "shouldCreateHello": False}) - - if type(response) is dict: - self.assertEqual(self.admin_username, response['username']) - else: - self.assertIsNotNone(response) - - def test_get_org_types(self): - response = self.admin_client.get_org_types() - self.assertTrue(len(response) > 0) - - def test_create_org(self): - response = self.admin_client.create_org( - {"org_name": self.admin_org_name, "org_label": "some label", "org_contact_name": "Some owner", - "org_email": self.admin_org_name + "@algo.com", "type_id": "basic"}) - - self.assertEqual(self.admin_org_name, response[u'org_name']) - - def test_get_org(self): - response = self.admin_client.get_org("a_myOrg84") - self.assertEqual("a_myOrg84", response['org_name']) - - def test_get_environment(self): - response = self.admin_client.get_environment("python2") - - if u'error' not in response: - self.assertTrue(response is not None and u'environments' in response) - - def test_get_build_logs(self): - user = unicode(os.environ.get('ALGO_USER_NAME')) - algo = unicode('echo') - algo_path = u'%s/%s' % (user, algo) - result = self.regular_client.algo(algo_path).build_logs() - - if u'error' in result: - print(result) - - self.assertTrue(u'error' not in result) - - def test_edit_org(self): - org_name = "a_myOrg84" - - obj = { - "id": "b85d8c4e-7f3c-40b9-9659-6adc2cb0e16f", - "org_name": "a_myOrg84", - "org_label": "some label", - "org_contact_name": "Some owner", - "org_email": "a_myOrg84@algo.com", - "org_created_at": "2020-11-30T23:51:40", - "org_url": "https://algorithmia.com", - "type_id": "basic", - "resource_type": "organization" - } - - response = self.admin_client.edit_org(org_name, obj) - if type(response) is dict: - print(response) - else: - self.assertEqual(204, response.status_code) - - def test_get_template(self): - filename = "./temptest" - response = self.admin_client.get_template("36fd467e-fbfe-4ea6-aa66-df3f403b7132", filename) - - if type(response) is dict: - self.assertTrue(u'error' in response or u'message' in response) - else: - self.assertTrue(response.ok) - try: - shutil.rmtree(filename) - except OSError as e: - print(e) - - def test_get_supported_languages(self): - response = self.admin_client.get_supported_languages() - self.assertTrue(response is not None) - - if type(response) is not list: - self.assertTrue(u'error' in response) - else: - language_found = any('anaconda3' in languages['name'] for languages in response) - self.assertTrue(response is not None and language_found) - - def test_invite_to_org(self): - response = self.admin_client.invite_to_org("a_myOrg38", "a_Mrtest4") - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(200, response.status_code) - - # This test will require updating after the /v1/organizations/{org_name}/errors endpoint has been - # deployed to the remote environment. - def test_get_organization_errors(self): - response = self.admin_client.get_organization_errors(self.admin_org_name) - self.assertTrue(response is not None) - - if type(response) is list: - self.assertEqual(0, len(response), 'Received unexpected result, should have been 0.') - - def test_get_user_errors(self): - response = self.admin_client.get_user_errors(self.admin_username) - - self.assertTrue(response is not None) - self.assertEqual(0, len(response)) - - def test_get_algorithm_errors(self): - response = self.admin_client.get_algorithm_errors('hello') - self.assertTrue(response is not None) - - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(404, response.status_code) - - def test_algorithm_programmatic_create_process(self): - algorithm_name = "algo_" + str(uuid4()).split("-")[-1] - payload = "John" - expected_response = "hello John" - full_path = self.regular_client.username() + "/" + algorithm_name - details = { - "summary": "Example Summary", - "label": "QA", - "tagline": "Example Tagline" - } - settings = { - "source_visibility": "open", - "algorithm_environment": self.environment_id, - "license": "apl", - "network_access": "isolated", - "pipeline_enabled": False - } - created_algo = self.regular_client.algo(full_path) - response = created_algo.create(details=details, settings=settings) - self.assertEqual(response.name, algorithm_name, "algorithm creation failed") - - # --- Creation complete, compiling - - response = created_algo.compile() - git_hash = response.version_info.git_hash - algo_with_build = self.regular_client.algo(full_path + "/" + git_hash) - self.assertEqual(response.name, created_algo.algoname) - - # --- compiling complete, now testing algorithm request - response = algo_with_build.pipe(payload).result - self.assertEqual(response, expected_response, "compiling failed") - - # --- testing complete, now publishing new release. - - pub_settings = {"algorithm_callability": "private"} - pub_version_info = { - "release_notes": "created programmatically", - "sample_input": payload, - "version_type": "minor" - } - pub_details = {"label": "testing123"} - - response = algo_with_build.publish( - details=pub_details, - settings=pub_settings, - version_info=pub_version_info - ) - self.assertEqual(response["version_info"]["semantic_version"], "0.1.0", - "Publishing failed, semantic version is not correct.") - - # --- publishing complete, getting additional information - - response = created_algo.info(git_hash) - - self.assertEqual(response.version_info.semantic_version, "0.1.0", "information is incorrect") - - def test_algo_freeze(self): - self.regular_client.freeze("Test/resources/manifests/example_manifest.json", "Test/resources/manifests") - -if __name__ == '__main__': - unittest.main() From e23c8a7de56149732082ac4690cb3c0a344b4d12 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Fri, 13 May 2022 13:45:32 -0700 Subject: [PATCH 14/18] fixes non 2xx responses (handles failures better) (#126) --- Algorithmia/algorithm.py | 8 ++++++-- Algorithmia/client.py | 5 +++-- Algorithmia/errors.py | 2 +- Test/api/app.py | 18 +++++++++++------- Test/regular/client_test.py | 12 +++++------- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 5d3c778..b181efa 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -104,8 +104,12 @@ def exists(self): _ = self.client.getJsonHelper(url) return True except AlgorithmException as e: - print(e) - return False + if "404" in str(e) or "No such algorithm" in str(e): + return False + elif "403" in str(e): + raise Exception("unable to check exists on algorithms you don't own.") + else: + raise e # Get all versions of the algorithm, with the given filters def versions(self, limit=None, marker=None, published=None, callable=None): diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 30dec03..74d88fa 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -214,8 +214,7 @@ def get_algorithm_errors(self, algorithm_id): """ url = '/v1/algorithms/%s/errors' % algorithm_id - response = self.getHelper(url) - return response.json() + return self.getJsonHelper(url) # Used to send insight data to Algorithm Queue Reader in cluster def report_insights(self, insights): @@ -280,6 +279,8 @@ def getJsonHelper(self, url, **query_parameters): else: return response else: + if response.content is not None: + response = response.json() raise raiseAlgoApiError(response) diff --git a/Algorithmia/errors.py b/Algorithmia/errors.py index f23662a..22ad68e 100644 --- a/Algorithmia/errors.py +++ b/Algorithmia/errors.py @@ -36,7 +36,7 @@ def raiseAlgoApiError(result): if 'message' in result['error']: message = result['error']['message'] else: - message = None + message = result['error'] if 'error_type' in result['error']: err_type = result['error']['error_type'] else: diff --git a/Test/api/app.py b/Test/api/app.py index cd80621..8871d91 100644 --- a/Test/api/app.py +++ b/Test/api/app.py @@ -1,6 +1,6 @@ -from fastapi import FastAPI, Request +from fastapi import FastAPI, Request, status from typing import Optional -from fastapi.responses import Response +from fastapi.responses import Response, JSONResponse import json import base64 from multiprocessing import Process @@ -49,7 +49,7 @@ async def process_algo_req(request: Request, username, algoname, output: Optiona return output -@regular_app.post("/v1/algo/{username}/{algoname}/{githash}") +@regular_app.post("/v1/algo/{username}/{algoname}/{githash}", status_code=status.HTTP_200_OK) async def process_hello_world(request: Request, username, algoname, githash): metadata = {"request_id": "req-55c0480d-6af3-4a21-990a-5c51d29f5725", "duration": 0.000306774, 'content_type': "text"} @@ -61,7 +61,7 @@ async def process_hello_world(request: Request, username, algoname, githash): ### Algorithm Routes @regular_app.get('/v1/algorithms/{username}/{algoname}') async def process_get_algo(request: Request, username, algoname): - if algoname == "echo": + if algoname == "echo" and username == 'quality': return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", "details": {"summary": "", "label": "echo", "tagline": ""}, "settings": {"algorithm_callability": "public", "source_visibility": "closed", @@ -78,8 +78,11 @@ async def process_get_algo(request: Request, username, algoname): "compilation": {"successful": True, "output": ""}, "self_link": "https://api.algorithmia.com/v1/algorithms/quality/echo/versions/0cfd7a6600f1fa05f9fe93016e661a9332c4ded2", "resource_type": "algorithm"} + elif algoname == "echo": + return JSONResponse(content={"error": {"id": "1cfb98c5-532e-4cbf-9192-fdd45b86969c", "code": 2001, + "message": "Caller is not authorized to perform the operation"}}, status_code=403) else: - return {"error": "No such algorithm"} + return JSONResponse(content={"error": "No such algorithm"}, status_code=404) @regular_app.get("/v1/algorithms/{username}/{algoname}/builds/{buildid}") @@ -101,7 +104,7 @@ async def get_scm_status(username, algoname): @regular_app.get("/v1/algorithms/{algo_id}/errors") async def get_algo_errors(algo_id): - return {"error": {"message": "not found"}} + return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) @regular_app.post("/v1/algorithms/{username}") @@ -116,6 +119,7 @@ async def create_algorithm(request: Request, username): "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, "resource_type": "algorithm"} + @regular_app.put('/v1/algorithms/{username}/{algoname}') async def update_algorithm(request: Request, username, algoname): return { @@ -255,7 +259,7 @@ async def get_algorithm_info(username, algoname, algohash): "source": {"scm": {"id": "internal", "provider": "internal", "default": True, "enabled": True}}, "compilation": {"successful": True, "output": ""}, "resource_type": "algorithm"} else: - return {"error": {"message": "not found"}} + return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) ### Admin Routes diff --git a/Test/regular/client_test.py b/Test/regular/client_test.py index 3a0e1be..c7ed9f2 100644 --- a/Test/regular/client_test.py +++ b/Test/regular/client_test.py @@ -114,13 +114,11 @@ def test_get_user_errors(self): self.assertEqual(0, len(response)) def test_get_algorithm_errors(self): - response = self.client.get_algorithm_errors('hello') - self.assertTrue(response is not None) - - if type(response) is dict: - self.assertTrue(u'error' in response) - else: - self.assertEqual(404, response.status_code) + try: + _ = self.client.get_algorithm_errors('hello') + self.assertFalse(True) + except AlgorithmException as e: + self.assertTrue(e.message == "No such algorithm") def test_no_auth_client(self): From c59038523e01073a7d6715b39732bd9330ae5108 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Mon, 16 May 2022 19:19:25 -0700 Subject: [PATCH 15/18] kwarg added (#127) --- Algorithmia/algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index b181efa..b4f75a1 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -127,7 +127,7 @@ def versions(self, limit=None, marker=None, published=None, callable=None): kwargs["callable"] = str(c).lower() if str(c) in bools else c # Get Algorithm versions url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/versions' - response = self.client.getJsonHelper(url) + response = self.client.getJsonHelper(url, **kwargs) return response # Compile an algorithm From 73348a3ae08855c8191bf88c75a4b6bc1d55e5a4 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Wed, 22 Jun 2022 09:44:22 -0700 Subject: [PATCH 16/18] get available SCMs support (#128) * added an scm get operation in the client class * added testing app endpoint, fixed client.create for external scms * added very basic test --- Algorithmia/algorithm.py | 11 ++++++++--- Algorithmia/client.py | 5 +++++ Test/api/app.py | 31 ++++++++++++++++++++++++++++++- Test/regular/client_test.py | 12 +++++++++--- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index b4f75a1..7d2adcd 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -39,10 +39,15 @@ def set_options(self, timeout=300, stdout=False, output=OutputType.default, **qu return self # Create a new algorithm - def create(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + def create(self, details, settings, version_info=None, source=None, scmsCredentials=None): url = "/v1/algorithms/" + self.username - create_parameters = {"name": self.algoname, "details": details, "settings": settings, - "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + create_parameters = {"name": self.algoname, "details": details, "settings": settings} + if version_info: + create_parameters['version_info'] = version_info + if source: + create_parameters['source'] = source + if scmsCredentials: + create_parameters['scmsCredentials'] = scmsCredentials api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) return api_response diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 74d88fa..ffc3f03 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -71,6 +71,11 @@ def username(self): username = next(self.dir("").list()).path return username + def scms(self): + url = "/v1/scms" + response = self.getJsonHelper(url) + return response + def file(self, dataUrl, cleanup=False): if dataUrl.startswith('file://'): return LocalDataFile(self, dataUrl) diff --git a/Test/api/app.py b/Test/api/app.py index 8871d91..825af29 100644 --- a/Test/api/app.py +++ b/Test/api/app.py @@ -80,7 +80,8 @@ async def process_get_algo(request: Request, username, algoname): "resource_type": "algorithm"} elif algoname == "echo": return JSONResponse(content={"error": {"id": "1cfb98c5-532e-4cbf-9192-fdd45b86969c", "code": 2001, - "message": "Caller is not authorized to perform the operation"}}, status_code=403) + "message": "Caller is not authorized to perform the operation"}}, + status_code=403) else: return JSONResponse(content={"error": "No such algorithm"}, status_code=404) @@ -102,6 +103,34 @@ async def get_scm_status(username, algoname): return {"scm_connection_status": "active"} +@regular_app.get("/v1/scms") +async def get_scms(): + return {'results': [{'default': True, 'enabled': True, 'id': 'internal', 'name': '', 'provider': 'internal'}, + {'default': False, 'enabled': True, 'id': 'github', 'name': 'https://github.com', + 'provider': 'github', 'scm': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'oauth': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'urls': {'web': 'https://github.com', 'api': 'https://api.github.com', + 'ssh': 'ssh://git@github.com'}}, + {'default': False, 'enabled': True, 'id': 'aadebe70-007f-48ff-ba38-49007c6e0377', + 'name': 'https://gitlab.com', 'provider': 'gitlab', + 'scm': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'oauth': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': True, 'id': '24ad1496-5a1d-43e2-9d96-42fce8e5484f', + 'name': 'IQIVA Public GitLab', 'provider': 'gitlab', + 'scm': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'oauth': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': False, 'id': '83cd96ae-b1f4-4bd9-b9ca-6f7f25c37708', + 'name': 'GitlabTest', 'provider': 'gitlab', + 'scm': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'oauth': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}]} + + @regular_app.get("/v1/algorithms/{algo_id}/errors") async def get_algo_errors(algo_id): return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) diff --git a/Test/regular/client_test.py b/Test/regular/client_test.py index c7ed9f2..9cfc39b 100644 --- a/Test/regular/client_test.py +++ b/Test/regular/client_test.py @@ -20,6 +20,7 @@ class ClientDummyTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + admin_username = "a_Mrtest" admin_org_name = "a_myOrg" environment_name = "Python 3.9" @@ -28,7 +29,6 @@ def setUp(self): self.admin_username = self.admin_username + str(int(random() * 10000)) self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - def test_create_user(self): response = self.client.create_user( {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", @@ -60,6 +60,12 @@ def test_get_environment(self): if u'error' not in response: self.assertTrue(response is not None and u'environments' in response) + def test_get_scms(self): + response = self.client.scms() + results = response['results'] + internal = [result for result in results if result['id'] == 'internal'] + self.assertTrue(len(internal) == 1) + def test_edit_org(self): org_name = "a_myOrg84" @@ -120,7 +126,6 @@ def test_get_algorithm_errors(self): except AlgorithmException as e: self.assertTrue(e.message == "No such algorithm") - def test_no_auth_client(self): key = os.environ.get('ALGORITHMIA_API_KEY', "") @@ -135,6 +140,7 @@ def test_no_auth_client(self): error = e finally: os.environ['ALGORITHMIA_API_KEY'] = key - self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, error_type=None))) + self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, + error_type=None))) if __name__ == '__main__': unittest.main() From 5e2a8a8e0cd3a1c112380895c509dfda3447e9f9 Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Fri, 22 Jul 2022 12:54:44 -0700 Subject: [PATCH 17/18] added get and set secrets endpoints for python client (#129) * added get and set secrets endpoints for python client * bumped version support to 1.4x of adk * adjusted description field to always be present * added better test coverage, removed get test (verified works on marketplace) --- Algorithmia/algorithm.py | 41 ++++++++++++++++ Algorithmia/client.py | 5 ++ Test/api/app.py | 98 ++++++++++++++++++++++++++++++++++++++- Test/regular/algo_test.py | 60 ++++++++++++++---------- requirements.txt | 2 +- requirements27.txt | 2 +- setup.py | 2 +- 7 files changed, 182 insertions(+), 28 deletions(-) diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 7d2adcd..04dfbaf 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -38,6 +38,47 @@ def set_options(self, timeout=300, stdout=False, output=OutputType.default, **qu self.query_parameters.update(query_parameters) return self + def get_algorithm_id(self): + url = '/v1/algorithms/' + self.username + '/' + self.algoname + print(url) + api_response = self.client.getJsonHelper(url) + if 'id' in api_response: + return api_response['id'] + else: + raise Exception("field 'id' not found in response: ", api_response) + + + def get_secrets(self): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + api_response = self.client.getJsonHelper(url) + return api_response + + + def set_secret(self, short_name, secret_key, secret_value, description=None): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + secret_providers = self.client.get_secret_providers() + provider_id = secret_providers[0]['id'] + + create_parameters = { + "owner_type": "algorithm", + "owner_id": algorithm_id, + "short_name": short_name, + "provider_id": provider_id, + "secret_key": secret_key, + "secret_value": secret_value, + } + if description: + create_parameters['description'] = description + else: + create_parameters['description'] = " " + + print(create_parameters) + api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) + return api_response + + # Create a new algorithm def create(self, details, settings, version_info=None, source=None, scmsCredentials=None): url = "/v1/algorithms/" + self.username diff --git a/Algorithmia/client.py b/Algorithmia/client.py index ffc3f03..a5121f1 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -177,6 +177,11 @@ def get_supported_languages(self): response = self.getHelper(url) return response.json() + def get_secret_providers(self): + url = "/v1/secret-provider" + api_response = self.getJsonHelper(url) + return api_response + def get_organization_errors(self, org_name): """Gets the errors for the organization. diff --git a/Test/api/app.py b/Test/api/app.py index 825af29..0384ae6 100644 --- a/Test/api/app.py +++ b/Test/api/app.py @@ -60,7 +60,7 @@ async def process_hello_world(request: Request, username, algoname, githash): ### Algorithm Routes @regular_app.get('/v1/algorithms/{username}/{algoname}') -async def process_get_algo(request: Request, username, algoname): +async def process_get_algo(username, algoname): if algoname == "echo" and username == 'quality': return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", "details": {"summary": "", "label": "echo", "tagline": ""}, @@ -444,3 +444,99 @@ async def get_environments_by_lang(language): } ] } + + +@regular_app.get("/v1/secret-provider") +async def get_service_providers(): + return [ + { + "id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "name": "algorithmia_internal_secret_provider", + "description": "Internal Secret Provider", + "moduleName": "module", + "factoryClassName": "com.algorithmia.plugin.sqlsecretprovider.InternalSecretProviderFactory", + "interfaceVersion": "1.0", + "isEnabled": True, + "isDefault": True, + "created": "2021-03-11T20:42:23Z", + "modified": "2021-03-11T20:42:23Z" + } + ] + + +@regular_app.get("/v1/algorithms/{algorithm_id}/secrets") +async def get_secrets_for_algorithm(algorithm_id): + return { + "secrets": [ + { + "id": "45e97c47-3ae6-46be-87ee-8ab23746706b", + "short_name": "MLOPS_SERVICE_URL", + "description": "", + "secret_key": "MLOPS_SERVICE_URL", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-22T14:36:01Z", + "modified": "2022-07-22T14:36:01Z" + }, + { + "id": "50dca60e-317f-4582-8854-5b83b4d182d0", + "short_name": "deploy_id", + "description": "", + "secret_key": "DEPLOYMENT_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "5a75cdc8-ecc8-4715-8c4b-8038991f1608", + "short_name": "model_path", + "description": "", + "secret_key": "MODEL_PATH", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "80e51ed3-f6db-419d-9349-f59f4bbfdcbb", + "short_name": "model_id", + "description": "", + "secret_key": "MODEL_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:30Z", + "modified": "2022-07-21T19:04:30Z" + }, + { + "id": "8773c654-ea2f-4ac5-9ade-55dfc47fec9d", + "short_name": "datarobot_api_token", + "description": "", + "secret_key": "DATAROBOT_MLOPS_API_TOKEN", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + } + ] + } + + +@regular_app.post("/v1/algorithms/{algorithm_id}/secrets") +async def set_algorithm_secret(algorithm_id): + return { + "id":"959af771-7cd8-4981-91c4-70def15bbcdc", + "short_name":"tst", + "description":"", + "secret_key":"test", + "owner_type":"algorithm", + "owner_id":"fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id":"dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created":"2022-07-22T18:28:42Z", + "modified":"2022-07-22T18:28:42Z" +} \ No newline at end of file diff --git a/Test/regular/algo_test.py b/Test/regular/algo_test.py index b1da4af..ab63c0d 100644 --- a/Test/regular/algo_test.py +++ b/Test/regular/algo_test.py @@ -20,13 +20,15 @@ def setUpClass(cls): def test_call_customCert(self): result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + + def test_normal_call(self): result = self.client.algo('quality/echo').pipe("foo") - self.assertEquals("text", result.metadata.content_type) - self.assertEquals("foo", result.result) + self.assertEqual("text", result.metadata.content_type) + self.assertEqual("foo", result.result) def test_async_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") @@ -35,20 +37,20 @@ def test_async_call(self): def test_raw_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") - self.assertEquals("foo", result) + self.assertEqual("foo", result) def test_dict_call(self): result = self.client.algo('quality/echo').pipe({"foo": "bar"}) - self.assertEquals("json", result.metadata.content_type) - self.assertEquals({"foo": "bar"}, result.result) + self.assertEqual("json", result.metadata.content_type) + self.assertEqual({"foo": "bar"}, result.result) def test_algo_exists(self): result = self.client.algo('quality/echo').exists() - self.assertEquals(True, result) + self.assertEqual(True, result) def test_algo_no_exists(self): result = self.client.algo('quality/not_echo').exists() - self.assertEquals(False, result) + self.assertEqual(False, result) #TODO: add more coverage examples to check kwargs def test_get_versions(self): @@ -56,19 +58,19 @@ def test_get_versions(self): self.assertTrue('results' in result) self.assertTrue('version_info' in result['results'][0]) self.assertTrue('semantic_version' in result['results'][0]['version_info']) - self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) def test_text_unicode(self): telephone = u"\u260E" # Unicode input to pipe() result1 = self.client.algo('quality/echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) # Unicode return in .result result2 = self.client.algo('quality/echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) def test_algo_info(self): result = self.client.algo('quality/echo').info() @@ -175,6 +177,16 @@ def test_algorithm_programmatic_create_process(self): self.assertEqual(response['version_info']['semantic_version'], "0.1.0", "information is incorrect") + + def test_set_secret(self): + short_name = "tst" + secret_key = "test_key" + secret_value = "test_value" + description = "loreum epsum" + response = self.client.algo("quality/echo").set_secret(short_name, secret_key, secret_value, description) + self.assertEqual(response['id'], "959af771-7cd8-4981-91c4-70def15bbcdc", "invalid ID for created secret") + + else: class AlgoTest(unittest.TestCase): def setUp(self): @@ -184,8 +196,8 @@ def test_call_customCert(self): open("./test.pem", 'w') c = Algorithmia.client(ca_cert="./test.pem") result = c.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) try: os.remove("./test.pem") except OSError as e: @@ -193,8 +205,8 @@ def test_call_customCert(self): def test_call_binary(self): result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) def test_async_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") @@ -203,7 +215,7 @@ def test_async_call(self): def test_raw_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") - self.assertEquals("foo", result) + self.assertEqual("foo", result) #TODO: add more coverage examples to check kwargs def test_get_versions(self): @@ -211,20 +223,20 @@ def test_get_versions(self): self.assertTrue('results' in result) self.assertTrue('version_info' in result['results'][0]) self.assertTrue('semantic_version' in result['results'][0]['version_info']) - self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) def test_text_unicode(self): telephone = u"\u260E" # Unicode input to pipe() result1 = self.client.algo('quality/echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) # Unicode return in .result result2 = self.client.algo('quality/echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) def test_get_scm_status(self): diff --git a/requirements.txt b/requirements.txt index f52fe4f..207a7f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.2,<1.3 +algorithmia-adk>=1.2,<1.4 numpy<2 uvicorn==0.14.0 fastapi==0.65.2 diff --git a/requirements27.txt b/requirements27.txt index 9668467..8a118ea 100644 --- a/requirements27.txt +++ b/requirements27.txt @@ -4,5 +4,5 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.2,<1.3 +algorithmia-adk>=1.2,<1.4 numpy<2 diff --git a/setup.py b/setup.py index aae416c..0069b73 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'toml', 'argparse', 'algorithmia-api-client==1.5.1', - 'algorithmia-adk>=1.2,<1.3' + 'algorithmia-adk>=1.2,<1.4' ], include_package_data=True, classifiers=[ From 31f0e9047483bea72b4fdec9ef42b9174b2aa2dc Mon Sep 17 00:00:00 2001 From: James Sutton <1892175+zeryx@users.noreply.github.com> Date: Wed, 3 Aug 2022 09:30:46 -0700 Subject: [PATCH 18/18] added a retry mechanic to PostJsonHelper to avoid 400 error issues (#130) * added a retry mechanic to PostJsonHelper to avoid 400 error issues * added test cases to verify the retry once system --- Algorithmia/algorithm.py | 4 ++-- Algorithmia/client.py | 6 +++--- Test/api/app.py | 7 +++++++ Test/regular/algo_failure_test.py | 12 ++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index 04dfbaf..40be378 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -106,7 +106,7 @@ def publish(self, details={}, settings={}, version_info={}, source={}, scmsCrede url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions" publish_parameters = {"details": details, "settings": settings, "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} - api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True) + api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, retry=True) return api_response def get_builds(self, limit=56, marker=None): @@ -180,7 +180,7 @@ def versions(self, limit=None, marker=None, published=None, callable=None): def compile(self): # Compile algorithm url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/compile' - response = self.client.postJsonHelper(url, {}, parse_response_as_json=True) + response = self.client.postJsonHelper(url, {}, parse_response_as_json=True, retry=True) return response # Pipe an input into this algorithm diff --git a/Algorithmia/client.py b/Algorithmia/client.py index a5121f1..dc26e1a 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -16,7 +16,6 @@ from time import time - class Client(object): 'Algorithmia Common Library' @@ -231,7 +230,7 @@ def report_insights(self, insights): return Insights(insights) # Used internally to post json to the api and parse json response - def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query_parameters): + def postJsonHelper(self, url, input_object, parse_response_as_json=True, retry=False, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey @@ -263,6 +262,8 @@ def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query return response else: return response + elif retry: + return self.postJsonHelper(url, input_object, parse_response_as_json, False, **query_parameters) else: raise raiseAlgoApiError(response) @@ -293,7 +294,6 @@ def getJsonHelper(self, url, **query_parameters): response = response.json() raise raiseAlgoApiError(response) - def getStreamHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: diff --git a/Test/api/app.py b/Test/api/app.py index 0384ae6..db7efd2 100644 --- a/Test/api/app.py +++ b/Test/api/app.py @@ -230,9 +230,16 @@ async def compile_algorithm(username, algoname): "resource_type": "algorithm" } +fail_cnt = 0 @regular_app.post("/v1/algorithms/{username}/{algoname}/versions") async def publish_algorithm(request: Request, username, algoname): + global fail_cnt + if "failonce" == algoname and fail_cnt == 0: + fail_cnt +=1 + return JSONResponse(content="This is an expected failure mode, try again", status_code=400) + elif "failalways" == algoname: + return JSONResponse(status_code=500) return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, "settings": {"algorithm_callability": "private", "source_visibility": "open", diff --git a/Test/regular/algo_failure_test.py b/Test/regular/algo_failure_test.py index 0804b4a..0ec4fc2 100644 --- a/Test/regular/algo_failure_test.py +++ b/Test/regular/algo_failure_test.py @@ -28,3 +28,15 @@ def test_throw_500_error_HTTP_response_on_algo_request(self): result = e pass self.assertEqual(str(self.error_message), str(result)) + + def test_retry_on_400_error_publish(self): + result = self.client.algo("util/failonce").publish() + self.assertEqual(result['version_info']['semantic_version'], "0.1.0") + + def test_throw_on_always_500_publish(self): + try: + result = self.client.algo("util/failalways").publish() + except Exception as e: + result = e + pass + self.assertEqual(str(self.error_message), str(result))