From 3f7127c233e74d9768afc73670fb96e7f522f320 Mon Sep 17 00:00:00 2001 From: Inbal Tako Date: Sun, 25 Oct 2020 10:55:03 +0200 Subject: [PATCH 1/5] Support custom headers extraction --- README.md | 29 +++++++++++++++++--- VERSION | 2 +- securenative/config/configuration_manager.py | 4 ++- securenative/config/securenative_options.py | 5 +++- securenative/context/securenative_context.py | 4 +-- securenative/securenative.py | 5 ++++ securenative/utils/request_utils.py | 11 +++++++- securenative/utils/version_utils.py | 2 +- tests/context_builder_test.py | 4 +-- tests/encryption_utils_test.py | 9 ------ tests/request_utils_test.py | 20 ++++++++++++++ 11 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 tests/request_utils_test.py diff --git a/README.md b/README.md index 7b9552a..b83807c 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,6 @@ You can also create request context from requests: ```python from securenative.securenative import SecureNative -from securenative.context.securenative_context import SecureNativeContext from securenative.models.event_options import EventOptions from securenative.enums.event_types import EventTypes from securenative.models.user_traits import UserTraits @@ -120,7 +119,7 @@ from securenative.models.user_traits import UserTraits def track(request): securenative = SecureNative.get_instance() - context = SecureNativeContext.from_http_request(request) + context = SecureNative.from_http_request(request) event_options = EventOptions(event=EventTypes.LOG_IN, user_id="1234", user_traits=UserTraits("Your Name", "name@gmail.com", "+1234567890"), @@ -137,7 +136,6 @@ def track(request): ```python from securenative.securenative import SecureNative from securenative.models.event_options import EventOptions -from securenative.context.securenative_context import SecureNativeContext from securenative.enums.event_types import EventTypes from securenative.models.user_traits import UserTraits @@ -145,7 +143,7 @@ from securenative.models.user_traits import UserTraits def verify(request): securenative = SecureNative.get_instance() - context = SecureNativeContext.from_http_request(request) + context = SecureNative.from_http_request(request) event_options = EventOptions(event=EventTypes.LOG_IN, user_id="1234", user_traits=UserTraits("Your Name", "name@gmail.com", "+1234567890"), @@ -173,3 +171,26 @@ def webhook_endpoint(request): is_verified = securenative.verify_request_payload(request) ``` +## Extract proxy headers from Cloudflare + +You can specify custom header keys to allow extraction of client ip from different providers. +This example demonstrates the usage of proxy headers for ip extraction from Cloudflare. + +### Option 1: Using config file +```ini +SECURENATIVE_API_KEY: dsbe27fh3437r2yd326fg3fdg36f43 +SECURENATIVE_PROXY_HEADERS: ["CF-Connecting-IP"] +``` + +Initialize sdk as shown above. + +### Options 2: Using ConfigurationBuilder + +```python +from securenative.securenative import SecureNative +from securenative.config.securenative_options import SecureNativeOptions + + +options = SecureNativeOptions(api_key="YOUR_API_KEY", max_events=10, log_level="ERROR", proxy_headers=['CF-Connecting-IP']) +securenative = SecureNative.init_with_options(options) +``` \ No newline at end of file diff --git a/VERSION b/VERSION index a2268e2..9fc80f9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 \ No newline at end of file +0.3.2 \ No newline at end of file diff --git a/securenative/config/configuration_manager.py b/securenative/config/configuration_manager.py index fc4bb21..881af09 100644 --- a/securenative/config/configuration_manager.py +++ b/securenative/config/configuration_manager.py @@ -63,4 +63,6 @@ def load_config(cls, resource_path): options.log_level), fail_over_strategy=cls._get_env_or_default(properties, "SECURENATIVE_FAILOVER_STRATEGY", - options.fail_over_strategy)) + options.fail_over_strategy), + proxy_headers=cls._get_env_or_default(properties, "SECURENATIVE_PROXY_HEADERS", + options.proxy_headers)) diff --git a/securenative/config/securenative_options.py b/securenative/config/securenative_options.py index 36329f3..11ef9a5 100644 --- a/securenative/config/securenative_options.py +++ b/securenative/config/securenative_options.py @@ -5,8 +5,10 @@ class SecureNativeOptions(object): def __init__(self, api_key=None, api_url="https://api.securenative.com/collector/api/v1", interval=1000, max_events=1000, timeout=1500, auto_send=True, disable=False, log_level="CRITICAL", - fail_over_strategy=FailOverStrategy.FAIL_OPEN.value): + fail_over_strategy=FailOverStrategy.FAIL_OPEN.value, proxy_headers=None): + if proxy_headers is None: + proxy_headers = [] if fail_over_strategy != FailOverStrategy.FAIL_OPEN.value and \ fail_over_strategy != FailOverStrategy.FAIL_CLOSED.value: self.fail_over_strategy = FailOverStrategy.FAIL_OPEN.value @@ -21,3 +23,4 @@ def __init__(self, api_key=None, api_url="https://api.securenative.com/collector self.auto_send = auto_send self.disable = disable self.log_level = log_level + self.proxy_headers = proxy_headers diff --git a/securenative/context/securenative_context.py b/securenative/context/securenative_context.py index 25c072b..0ede738 100644 --- a/securenative/context/securenative_context.py +++ b/securenative/context/securenative_context.py @@ -14,7 +14,7 @@ def __init__(self, client_token=None, ip=None, remote_ip=None, headers=None, url self.body = body @staticmethod - def from_http_request(request): + def from_http_request(request, options): try: client_token = request.cookies[RequestUtils.SECURENATIVE_COOKIE] except Exception: @@ -28,6 +28,6 @@ def from_http_request(request): if Utils.is_null_or_empty(client_token): client_token = RequestUtils.get_secure_header_from_request(headers) - return SecureNativeContext(client_token, RequestUtils.get_client_ip_from_request(request), + return SecureNativeContext(client_token, RequestUtils.get_client_ip_from_request(request, options), RequestUtils.get_remote_ip_from_request(request), headers, request.url, request.method, None) diff --git a/securenative/securenative.py b/securenative/securenative.py index b10cc7b..d6f28a0 100644 --- a/securenative/securenative.py +++ b/securenative/securenative.py @@ -1,6 +1,7 @@ from securenative.api_manager import ApiManager from securenative.config.configuration_manager import ConfigurationManager from securenative.config.securenative_options import SecureNativeOptions +from securenative.context.securenative_context import SecureNativeContext from securenative.event_manager import EventManager from securenative.exceptions.securenative_config_exception import SecureNativeConfigException from securenative.exceptions.securenative_sdk_Illegal_state_exception import SecureNativeSDKIllegalStateException @@ -75,6 +76,10 @@ def verify(self, event_options): def _flush(cls): cls._securenative = None + @classmethod + def from_http_request(cls, request): + return SecureNativeContext.from_http_request(request, SecureNative._options) + def verify_request_payload(self, request): request_signature = request.header[SignatureUtils.SignatureHeader] body = request.body diff --git a/securenative/utils/request_utils.py b/securenative/utils/request_utils.py index 0b42ebd..2a75689 100644 --- a/securenative/utils/request_utils.py +++ b/securenative/utils/request_utils.py @@ -10,7 +10,16 @@ def get_secure_header_from_request(headers): return "" @staticmethod - def get_client_ip_from_request(request): + def get_client_ip_from_request(request, options): + if options and len(options.proxy_headers) > 0: + for header in options.proxy_headers: + try: + if request.environ.get(header) is not None: + return request.environ.get(header) + except AttributeError: + if request.headers[header] is not None: + return request.headers[header] + try: x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') if x_forwarded_for: diff --git a/securenative/utils/version_utils.py b/securenative/utils/version_utils.py index 7bebfd0..dc26337 100644 --- a/securenative/utils/version_utils.py +++ b/securenative/utils/version_utils.py @@ -2,4 +2,4 @@ class VersionUtils(object): @staticmethod def get_version(): - return "0.3.1" + return "0.3.2" diff --git a/tests/context_builder_test.py b/tests/context_builder_test.py index 4f48bb2..3c44373 100644 --- a/tests/context_builder_test.py +++ b/tests/context_builder_test.py @@ -21,7 +21,7 @@ def test_create_context_from_request(self): request.headers = { "x-securenative": "71532c1fad2c7f56118f7969e401f3cf080239140d208e7934e6a530818c37e544a0c2330a487bcc6fe4f662a57f265a3ed9f37871e80529128a5e4f2ca02db0fb975ded401398f698f19bb0cafd68a239c6caff99f6f105286ab695eaf3477365bdef524f5d70d9be1d1d474506b433aed05d7ed9a435eeca357de57817b37c638b6bb417ffb101eaf856987615a77a"} - context = SecureNativeContext.from_http_request(request) + context = SecureNativeContext.from_http_request(request, None) self.assertEqual(context.client_token, "71532c1fad2c7f56118f7969e401f3cf080239140d208e7934e6a530818c37e544a0c2330a487bcc6fe4f662a57f265a3ed9f37871e80529128a5e4f2ca02db0fb975ded401398f698f19bb0cafd68a239c6caff99f6f105286ab695eaf3477365bdef524f5d70d9be1d1d474506b433aed05d7ed9a435eeca357de57817b37c638b6bb417ffb101eaf856987615a77a") @@ -47,7 +47,7 @@ def test_create_context_from_request_with_cookie(self): request.cookies = {"_sn": "71532c1fad2c7f56118f7969e401f3cf080239140d208e7934e6a530818c37e544a0c2330a487bcc6fe4f662a57f265a3ed9f37871e80529128a5e4f2ca02db0fb975ded401398f698f19bb0cafd68a239c6caff99f6f105286ab695eaf3477365bdef524f5d70d9be1d1d474506b433aed05d7ed9a435eeca357de57817b37c638b6bb417ffb101eaf856987615a77a"} - context = SecureNativeContext.from_http_request(request) + context = SecureNativeContext.from_http_request(request, None) self.assertEqual(context.client_token, "71532c1fad2c7f56118f7969e401f3cf080239140d208e7934e6a530818c37e544a0c2330a487bcc6fe4f662a57f265a3ed9f37871e80529128a5e4f2ca02db0fb975ded401398f698f19bb0cafd68a239c6caff99f6f105286ab695eaf3477365bdef524f5d70d9be1d1d474506b433aed05d7ed9a435eeca357de57817b37c638b6bb417ffb101eaf856987615a77a") diff --git a/tests/encryption_utils_test.py b/tests/encryption_utils_test.py index eeed235..fedf1e4 100644 --- a/tests/encryption_utils_test.py +++ b/tests/encryption_utils_test.py @@ -1,7 +1,5 @@ import unittest -import pytest - from securenative.utils.encryption_utils import EncryptionUtils @@ -19,10 +17,3 @@ def test_decrypt(self): self.assertEqual(result.cid, self.CID) self.assertEqual(result.fp, self.FP) - - @pytest.mark.skip("Differences in crypto version fails this test when in reality it's passing") - def test_encrypt(self): - result = EncryptionUtils.encrypt(self.PAYLOAD, self.SECRET_KEY) - - self.assertIsNotNone(result) - self.assertGreater(len(self.PAYLOAD), len(result)) diff --git a/tests/request_utils_test.py b/tests/request_utils_test.py new file mode 100644 index 0000000..964ff29 --- /dev/null +++ b/tests/request_utils_test.py @@ -0,0 +1,20 @@ +import unittest + +import requests_mock + +from securenative.config.securenative_options import SecureNativeOptions +from securenative.utils.request_utils import RequestUtils + + +class RequestUtilsTest(unittest.TestCase): + + def test_proxy_headers_extraction_from_request(self): + options = SecureNativeOptions(proxy_headers=['CF-Connecting-IP']) + + with requests_mock.Mocker(real_http=True) as request: + request.headers = { + "CF-Connecting-IP": "203.0.113.1"} + + client_ip = RequestUtils.get_client_ip_from_request(request, options) + + self.assertEqual(client_ip, "203.0.113.1") From 93a4eb279de259623ab6d9c8720767a383c7ef1c Mon Sep 17 00:00:00 2001 From: Inbal Tako Date: Sun, 25 Oct 2020 11:49:35 +0200 Subject: [PATCH 2/5] Minor fixes --- securenative/config/configuration_manager.py | 4 ++-- securenative/securenative.py | 5 ++--- securenative/utils/request_utils.py | 3 ++- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/securenative/config/configuration_manager.py b/securenative/config/configuration_manager.py index 881af09..f945db3 100644 --- a/securenative/config/configuration_manager.py +++ b/securenative/config/configuration_manager.py @@ -2,7 +2,7 @@ from configparser import ConfigParser from securenative.config.securenative_options import SecureNativeOptions -from securenative.exceptions.securenative_config_exception import SecureNativeConfigException +from securenative.logger import Logger class ConfigurationManager(object): @@ -15,7 +15,7 @@ def read_resource_file(cls, resource_path): try: cls.config.read(resource_path) except Exception as e: - raise SecureNativeConfigException("Invalid config file; %s", e) + Logger.debug("Invalid config file; {}, using default options".format(e)) properties = {} for key, value in cls.config.defaults().items(): diff --git a/securenative/securenative.py b/securenative/securenative.py index d6f28a0..c6d407a 100644 --- a/securenative/securenative.py +++ b/securenative/securenative.py @@ -76,9 +76,8 @@ def verify(self, event_options): def _flush(cls): cls._securenative = None - @classmethod - def from_http_request(cls, request): - return SecureNativeContext.from_http_request(request, SecureNative._options) + def from_http_request(self, request): + return SecureNativeContext.from_http_request(request, self._options) def verify_request_payload(self, request): request_signature = request.header[SignatureUtils.SignatureHeader] diff --git a/securenative/utils/request_utils.py b/securenative/utils/request_utils.py index 2a75689..14156fd 100644 --- a/securenative/utils/request_utils.py +++ b/securenative/utils/request_utils.py @@ -16,9 +16,10 @@ def get_client_ip_from_request(request, options): try: if request.environ.get(header) is not None: return request.environ.get(header) - except AttributeError: if request.headers[header] is not None: return request.headers[header] + except Exception: + continue try: x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') From d24d67004a428aec3dc8ed783af2a98f97c27d4f Mon Sep 17 00:00:00 2001 From: Inbal Tako Date: Sun, 25 Oct 2020 11:56:14 +0200 Subject: [PATCH 3/5] Fix tests --- README.md | 4 ++-- securenative/utils/request_utils.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b83807c..2d68cb3 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ from securenative.models.user_traits import UserTraits def track(request): securenative = SecureNative.get_instance() - context = SecureNative.from_http_request(request) + context = securenative.from_http_request(request) event_options = EventOptions(event=EventTypes.LOG_IN, user_id="1234", user_traits=UserTraits("Your Name", "name@gmail.com", "+1234567890"), @@ -143,7 +143,7 @@ from securenative.models.user_traits import UserTraits def verify(request): securenative = SecureNative.get_instance() - context = SecureNative.from_http_request(request) + context = securenative.from_http_request(request) event_options = EventOptions(event=EventTypes.LOG_IN, user_id="1234", user_traits=UserTraits("Your Name", "name@gmail.com", "+1234567890"), diff --git a/securenative/utils/request_utils.py b/securenative/utils/request_utils.py index 14156fd..1aaa4ee 100644 --- a/securenative/utils/request_utils.py +++ b/securenative/utils/request_utils.py @@ -19,7 +19,11 @@ def get_client_ip_from_request(request, options): if request.headers[header] is not None: return request.headers[header] except Exception: - continue + try: + if request.headers[header] is not None: + return request.headers[header] + except Exception: + continue try: x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') @@ -27,9 +31,13 @@ def get_client_ip_from_request(request, options): ip = x_forwarded_for.split(',')[-1].strip() else: ip = request.META.get('REMOTE_ADDR') + + if ip is None or ip == "": + ip = request.environ.get('HTTP_X_FORWARDED_FOR', request.environ.get('REMOTE_ADDR', "")) + return ip except Exception: - return request.environ.get('HTTP_X_FORWARDED_FOR', request.environ.get('REMOTE_ADDR', "")) + return "" @staticmethod def get_remote_ip_from_request(request): From 04be5e9935447e4c5e73da2a76ac639c8231b2b0 Mon Sep 17 00:00:00 2001 From: Inbal Tako Date: Mon, 26 Oct 2020 17:08:56 +0200 Subject: [PATCH 4/5] Align timeout and default response --- securenative/api_manager.py | 2 +- securenative/event_manager.py | 16 ++++----------- securenative/http/securenative_http_client.py | 7 ++++++- tests/api_manager_test.py | 20 ++++++++++++++++++- tests/event_manager_test.py | 6 +++--- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/securenative/api_manager.py b/securenative/api_manager.py index 0aaa716..23d07ea 100644 --- a/securenative/api_manager.py +++ b/securenative/api_manager.py @@ -23,7 +23,7 @@ def verify(self, event_options): Logger.debug("Verify event call") event = SDKEvent(event_options, self.options) try: - res = json.loads(self.event_manager.send_sync(event, ApiRoute.VERIFY.value, False).text) + res = json.loads(self.event_manager.send_sync(event, ApiRoute.VERIFY.value).text) return VerifyResult(res["riskLevel"], res["score"], res["triggers"]) except Exception as e: Logger.debug("Failed to call verify; {}".format(e)) diff --git a/securenative/event_manager.py b/securenative/event_manager.py index 3d249f5..e569cf9 100644 --- a/securenative/event_manager.py +++ b/securenative/event_manager.py @@ -54,7 +54,7 @@ def flush(self): for item in self.queue: self.http_client.post(item.url, item.body) - def send_sync(self, event, resource_path, retry): + def send_sync(self, event, resource_path): if self.options.disable: Logger.warning("SDK is disabled. no operation will be performed") return @@ -64,17 +64,9 @@ def send_sync(self, event, resource_path, retry): resource_path, json.dumps(EventManager.serialize(event)) ) - if res.status_code != 200: - Logger.info("SecureNative failed to call endpoint {} with event {}. adding back to queue".format( - resource_path, event)) - item = QueueItem( - resource_path, - json.dumps(EventManager.serialize(event)), - retry - ) - self.queue.append(item) - if self._is_queue_full(): - self.queue = self.queue[:len(self.queue - 1)] + if res is None or res.status_code != 200: + Logger.info("SecureNative failed to call endpoint {} with event {}.".format(resource_path, event)) + return res def _is_queue_full(self): diff --git a/securenative/http/securenative_http_client.py b/securenative/http/securenative_http_client.py index e68699b..d0523b5 100644 --- a/securenative/http/securenative_http_client.py +++ b/securenative/http/securenative_http_client.py @@ -1,4 +1,5 @@ import requests +from requests import Timeout from securenative.utils.version_utils import VersionUtils @@ -24,4 +25,8 @@ def _headers(self): def post(self, path, body): url = "{}/{}".format(self.options.api_url, path) - return requests.post(url=url, data=body, headers=self._headers()) + try: + res = requests.post(url=url, data=body, headers=self._headers(), timeout=self.options.timeout / 1000) + return res + except Timeout: + return None diff --git a/tests/api_manager_test.py b/tests/api_manager_test.py index 6e76a8e..ef6fd8e 100644 --- a/tests/api_manager_test.py +++ b/tests/api_manager_test.py @@ -8,7 +8,6 @@ from securenative.enums.event_types import EventTypes from securenative.enums.risk_level import RiskLevel from securenative.event_manager import EventManager -from securenative.exceptions.securenative_invalid_options_exception import SecureNativeInvalidOptionsException from securenative.models.event_options import EventOptions from securenative.models.user_traits import UserTraits from securenative.models.verify_result import VerifyResult @@ -49,6 +48,25 @@ def test_track_event(self): finally: event_manager.stop_event_persist() + @responses.activate + def test_should_timeout_on_post(self): + options = SecureNativeOptions(api_key="YOUR_API_KEY", auto_send=True, timeout=-1, + api_url="https://api.securenative-stg.com/collector/api/v1") + + responses.add(responses.POST, "https://api.securenative-stg.com/collector/api/v1/verify", + json={"event": "SOME_EVENT_NAME"}, status=408) + + event_manager = EventManager(options) + event_manager.start_event_persist() + api_manager = ApiManager(event_manager, options) + + verify_result = VerifyResult(RiskLevel.LOW.value, 0, None) + res = api_manager.verify(self.event_options) + + self.assertEqual(res.risk_level, verify_result.risk_level) + self.assertEqual(res.score, verify_result.score) + self.assertEqual(res.triggers, verify_result.triggers) + @responses.activate def test_verify_event(self): options = SecureNativeOptions(api_key="YOUR_API_KEY", diff --git a/tests/event_manager_test.py b/tests/event_manager_test.py index 48a397d..f6d32da 100644 --- a/tests/event_manager_test.py +++ b/tests/event_manager_test.py @@ -37,7 +37,7 @@ def test_should_successfully_send_sync_event_with_status_code_200(self): json=json.loads(res_body), status=200) event_manager = EventManager(options) - data = event_manager.send_sync(self.event, "some-path/to-api", False) + data = event_manager.send_sync(self.event, "some-path/to-api") self.assertEqual(res_body, data.text) @responses.activate @@ -49,7 +49,7 @@ def test_should_send_sync_event_and_fail_when_status_code_401(self): json={}, status=401) event_manager = EventManager(options) - res = event_manager.send_sync(self.event, "some-path/to-api", False) + res = event_manager.send_sync(self.event, "some-path/to-api") self.assertEqual(res.status_code, 401) @@ -62,6 +62,6 @@ def test_should_send_sync_event_and_fail_when_status_code_500(self): json={}, status=500) event_manager = EventManager(options) - res = event_manager.send_sync(self.event, "some-path/to-api", False) + res = event_manager.send_sync(self.event, "some-path/to-api") self.assertEqual(res.status_code, 500) From b9fc52b6afe7e07013b67edf725362c9967d87ee Mon Sep 17 00:00:00 2001 From: Inbal Tako Date: Mon, 26 Oct 2020 17:13:30 +0200 Subject: [PATCH 5/5] Align timeout and default response --- securenative/api_manager.py | 4 ++-- tests/api_manager_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/securenative/api_manager.py b/securenative/api_manager.py index 23d07ea..63ac9d6 100644 --- a/securenative/api_manager.py +++ b/securenative/api_manager.py @@ -28,5 +28,5 @@ def verify(self, event_options): except Exception as e: Logger.debug("Failed to call verify; {}".format(e)) if self.options.fail_over_strategy is FailOverStrategy.FAIL_OPEN.value: - return VerifyResult(RiskLevel.LOW.value, 0, None) - return VerifyResult(RiskLevel.HIGH.value, 1, None) + return VerifyResult(RiskLevel.LOW.value, 0, []) + return VerifyResult(RiskLevel.HIGH.value, 1, []) diff --git a/tests/api_manager_test.py b/tests/api_manager_test.py index ef6fd8e..d8216c7 100644 --- a/tests/api_manager_test.py +++ b/tests/api_manager_test.py @@ -50,7 +50,7 @@ def test_track_event(self): @responses.activate def test_should_timeout_on_post(self): - options = SecureNativeOptions(api_key="YOUR_API_KEY", auto_send=True, timeout=-1, + options = SecureNativeOptions(api_key="YOUR_API_KEY", auto_send=True, timeout=0, api_url="https://api.securenative-stg.com/collector/api/v1") responses.add(responses.POST, "https://api.securenative-stg.com/collector/api/v1/verify", @@ -60,7 +60,7 @@ def test_should_timeout_on_post(self): event_manager.start_event_persist() api_manager = ApiManager(event_manager, options) - verify_result = VerifyResult(RiskLevel.LOW.value, 0, None) + verify_result = VerifyResult(RiskLevel.LOW.value, 0, []) res = api_manager.verify(self.event_options) self.assertEqual(res.risk_level, verify_result.risk_level)