From bb8643aca1680ecc05317ff0602aee5a043280d7 Mon Sep 17 00:00:00 2001 From: Nick Schouten Date: Wed, 20 Nov 2024 15:07:53 +0100 Subject: [PATCH 1/7] Allow httphandler to have auth and other mediatype --- papermill/iorw.py | 22 +++++++++- papermill/tests/test_iorw.py | 82 ++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/papermill/iorw.py b/papermill/iorw.py index 14a0122c..ba7943d0 100644 --- a/papermill/iorw.py +++ b/papermill/iorw.py @@ -165,9 +165,27 @@ def get_handler(self, path, extensions=None): class HttpHandler: + @classmethod + def _get_auth_kwargs(cls): + username = os.environ.get('PAPERMILL_HTTP_AUTH_USERNAME', None) + password = os.environ.get('PAPERMILL_HTTP_AUTH_PASSWORD', None) + if username or password: + return {'auth': requests.auth.HTTPBasicAuth(username or '', password or '')} + return {} + + @classmethod + def _get_read_kwargs(cls): + kwargs = {} + kwargs['headers'] = { + 'Accept': os.environ.get('PAPERMILL_HTTP_ACCEPT_TYPE', 'application/json') + } + return kwargs | cls._get_auth_kwargs() + + @classmethod def read(cls, path): - return requests.get(path, headers={'Accept': 'application/json'}).text + r = requests.get(path, **cls._get_read_kwargs()) + return r.text @classmethod def listdir(cls, path): @@ -175,7 +193,7 @@ def listdir(cls, path): @classmethod def write(cls, buf, path): - result = requests.put(path, json=json.loads(buf)) + result = requests.put(path, json=json.loads(buf), **cls._get_auth_kwargs()) result.raise_for_status() @classmethod diff --git a/papermill/tests/test_iorw.py b/papermill/tests/test_iorw.py index ab09f01a..6b939360 100644 --- a/papermill/tests/test_iorw.py +++ b/papermill/tests/test_iorw.py @@ -320,6 +320,68 @@ def test_read(self): self.assertEqual(HttpHandler.read(path), text) mock_get.assert_called_once_with(path, headers={'Accept': 'application/json'}) + def test_read_with_auth(self): + """ + Tests that the `read` function performs a request to the giving path + with authentication from the environment variables and returns the response. + """ + path = 'http://example.com' + text = 'request test response' + username = 'testusername' + password = 'testpassword' + auth_token = 'token' + + with patch('papermill.iorw.requests.auth.HTTPBasicAuth') as mock_auth,\ + patch.dict(os.environ, clear=True) as env,\ + patch('papermill.iorw.requests.get') as mock_get: + mock_auth.return_value = auth_token + env['PAPERMILL_HTTP_AUTH_USERNAME'] = username + env['PAPERMILL_HTTP_AUTH_PASSWORD'] = password + mock_get.return_value = Mock(text=text) + + self.assertEqual(HttpHandler.read(path), text) + mock_auth.assert_called_once_with(username, password) + mock_get.assert_called_once_with(path, headers={'Accept': 'application/json'}, auth=auth_token) + + def test_read_with_token(self): + """ + Tests that the `read` function performs a request to the giving path + with just a password as authentication and returns the response. + """ + path = 'http://example.com' + text = 'request test response' + password = 'testpassword' + auth_token = 'token' + + with patch('papermill.iorw.requests.auth.HTTPBasicAuth') as mock_auth,\ + patch.dict(os.environ, clear=True) as env,\ + patch('papermill.iorw.requests.get') as mock_get: + mock_auth.return_value = auth_token + env['PAPERMILL_HTTP_AUTH_PASSWORD'] = password + mock_get.return_value = Mock(text=text) + + self.assertEqual(HttpHandler.read(path), text) + mock_auth.assert_called_once_with('', password) + mock_get.assert_called_once_with(path, headers={'Accept': 'application/json'}, auth=auth_token) + + + def test_read_with_accept_type(self): + """ + Tests that the `read` function performs a request to the giving path + with an accept type from env variables and returns the response. + """ + path = 'http://example.com' + text = 'request test response' + accept_type = 'test accept type' + + with patch.dict(os.environ, clear=True) as env,\ + patch('papermill.iorw.requests.get') as mock_get: + env['PAPERMILL_HTTP_ACCEPT_TYPE'] = accept_type + mock_get.return_value = Mock(text=text) + + self.assertEqual(HttpHandler.read(path), text) + mock_get.assert_called_once_with(path, headers={'Accept': accept_type}) + def test_write(self): """ Tests that the `write` function performs a put request to the given @@ -332,6 +394,26 @@ def test_write(self): HttpHandler.write(buf, path) mock_put.assert_called_once_with(path, json=json.loads(buf)) + def test_write_with_username(self): + """ + Tests that the `write` function performs a put request to the given + path with just a username as authentication from env variables. + """ + path = 'http://example.com' + buf = '{"papermill": true}' + username = 'testusername' + auth_token = 'token' + + with patch('papermill.iorw.requests.auth.HTTPBasicAuth') as mock_auth,\ + patch.dict(os.environ, clear=True) as env,\ + patch('papermill.iorw.requests.put') as mock_put: + mock_auth.return_value = auth_token + env['PAPERMILL_HTTP_AUTH_USERNAME'] = username + + HttpHandler.write(buf, path) + mock_auth.assert_called_once_with(username, '') + mock_put.assert_called_once_with(path, json=json.loads(buf), auth=auth_token) + def test_write_failure(self): """ Tests that the `write` function raises on failure to put the buffer. From 022aefe002f40412c0b9c7bc9f61d5ce10473976 Mon Sep 17 00:00:00 2001 From: Nick Schouten Date: Wed, 20 Nov 2024 15:57:19 +0100 Subject: [PATCH 2/7] refactored the auth header and tests --- papermill/iorw.py | 18 ++++++------ papermill/tests/test_iorw.py | 56 ++++++++---------------------------- 2 files changed, 21 insertions(+), 53 deletions(-) diff --git a/papermill/iorw.py b/papermill/iorw.py index ba7943d0..49e4be93 100644 --- a/papermill/iorw.py +++ b/papermill/iorw.py @@ -167,20 +167,20 @@ def get_handler(self, path, extensions=None): class HttpHandler: @classmethod def _get_auth_kwargs(cls): - username = os.environ.get('PAPERMILL_HTTP_AUTH_USERNAME', None) - password = os.environ.get('PAPERMILL_HTTP_AUTH_PASSWORD', None) - if username or password: - return {'auth': requests.auth.HTTPBasicAuth(username or '', password or '')} + """Gets the Authorization header from PAPERMILL_HTTP_AUTH_HEADER. + A valid value could be Basic dW5hbWU6cGFzc3dvcmQK""" + auth_header = os.environ.get('PAPERMILL_HTTP_AUTH_HEADER', None) + if auth_header: + return {'headers': {'Authorization': auth_header}} return {} @classmethod def _get_read_kwargs(cls): - kwargs = {} - kwargs['headers'] = { - 'Accept': os.environ.get('PAPERMILL_HTTP_ACCEPT_TYPE', 'application/json') + kwargs = cls._get_auth_kwargs() or {'headers': {}} + kwargs['headers'] |= { + 'Accept': os.environ.get('PAPERMILL_HTTP_ACCEPT_HEADER', 'application/json') } - return kwargs | cls._get_auth_kwargs() - + return kwargs @classmethod def read(cls, path): diff --git a/papermill/tests/test_iorw.py b/papermill/tests/test_iorw.py index 6b939360..fd2fe9c1 100644 --- a/papermill/tests/test_iorw.py +++ b/papermill/tests/test_iorw.py @@ -327,45 +327,17 @@ def test_read_with_auth(self): """ path = 'http://example.com' text = 'request test response' - username = 'testusername' - password = 'testpassword' - auth_token = 'token' + auth = 'Basic dW5hbWU6cGFzc3dvcmQK' - with patch('papermill.iorw.requests.auth.HTTPBasicAuth') as mock_auth,\ - patch.dict(os.environ, clear=True) as env,\ - patch('papermill.iorw.requests.get') as mock_get: - mock_auth.return_value = auth_token - env['PAPERMILL_HTTP_AUTH_USERNAME'] = username - env['PAPERMILL_HTTP_AUTH_PASSWORD'] = password - mock_get.return_value = Mock(text=text) - - self.assertEqual(HttpHandler.read(path), text) - mock_auth.assert_called_once_with(username, password) - mock_get.assert_called_once_with(path, headers={'Accept': 'application/json'}, auth=auth_token) - - def test_read_with_token(self): - """ - Tests that the `read` function performs a request to the giving path - with just a password as authentication and returns the response. - """ - path = 'http://example.com' - text = 'request test response' - password = 'testpassword' - auth_token = 'token' - - with patch('papermill.iorw.requests.auth.HTTPBasicAuth') as mock_auth,\ - patch.dict(os.environ, clear=True) as env,\ + with patch.dict(os.environ, clear=True) as env,\ patch('papermill.iorw.requests.get') as mock_get: - mock_auth.return_value = auth_token - env['PAPERMILL_HTTP_AUTH_PASSWORD'] = password + env['PAPERMILL_HTTP_AUTH_HEADER'] = auth mock_get.return_value = Mock(text=text) self.assertEqual(HttpHandler.read(path), text) - mock_auth.assert_called_once_with('', password) - mock_get.assert_called_once_with(path, headers={'Accept': 'application/json'}, auth=auth_token) - + mock_get.assert_called_once_with(path, headers={'Accept': 'application/json', 'Authorization': auth}) - def test_read_with_accept_type(self): + def test_read_with_accept_header(self): """ Tests that the `read` function performs a request to the giving path with an accept type from env variables and returns the response. @@ -376,7 +348,7 @@ def test_read_with_accept_type(self): with patch.dict(os.environ, clear=True) as env,\ patch('papermill.iorw.requests.get') as mock_get: - env['PAPERMILL_HTTP_ACCEPT_TYPE'] = accept_type + env['PAPERMILL_HTTP_ACCEPT_HEADER'] = accept_type mock_get.return_value = Mock(text=text) self.assertEqual(HttpHandler.read(path), text) @@ -394,25 +366,21 @@ def test_write(self): HttpHandler.write(buf, path) mock_put.assert_called_once_with(path, json=json.loads(buf)) - def test_write_with_username(self): + def test_write_with_auth(self): """ Tests that the `write` function performs a put request to the given - path with just a username as authentication from env variables. + path with authentication from env variables. """ path = 'http://example.com' buf = '{"papermill": true}' - username = 'testusername' - auth_token = 'token' + auth = 'token' - with patch('papermill.iorw.requests.auth.HTTPBasicAuth') as mock_auth,\ - patch.dict(os.environ, clear=True) as env,\ + with patch.dict(os.environ, clear=True) as env,\ patch('papermill.iorw.requests.put') as mock_put: - mock_auth.return_value = auth_token - env['PAPERMILL_HTTP_AUTH_USERNAME'] = username + env['PAPERMILL_HTTP_AUTH_HEADER'] = auth HttpHandler.write(buf, path) - mock_auth.assert_called_once_with(username, '') - mock_put.assert_called_once_with(path, json=json.loads(buf), auth=auth_token) + mock_put.assert_called_once_with(path, json=json.loads(buf), headers={'Authorization': auth}) def test_write_failure(self): """ From df525ba2be98cfcc355399a318072640a9e0306a Mon Sep 17 00:00:00 2001 From: Nick Schouten Date: Wed, 20 Nov 2024 15:58:10 +0100 Subject: [PATCH 3/7] undo changes for debugging --- papermill/iorw.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/papermill/iorw.py b/papermill/iorw.py index 49e4be93..87ada9a4 100644 --- a/papermill/iorw.py +++ b/papermill/iorw.py @@ -184,8 +184,7 @@ def _get_read_kwargs(cls): @classmethod def read(cls, path): - r = requests.get(path, **cls._get_read_kwargs()) - return r.text + return requests.get(path, **cls._get_read_kwargs()) @classmethod def listdir(cls, path): From 3f3e2430b5249b75c7b88b0dcb8f5873978dc6ec Mon Sep 17 00:00:00 2001 From: Nick Schouten Date: Wed, 20 Nov 2024 15:58:38 +0100 Subject: [PATCH 4/7] .text still needed --- papermill/iorw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papermill/iorw.py b/papermill/iorw.py index 87ada9a4..43a848ae 100644 --- a/papermill/iorw.py +++ b/papermill/iorw.py @@ -184,7 +184,7 @@ def _get_read_kwargs(cls): @classmethod def read(cls, path): - return requests.get(path, **cls._get_read_kwargs()) + return requests.get(path, **cls._get_read_kwargs()).text @classmethod def listdir(cls, path): From 9a28376d388660cf87e1cb4dee572de4633cce21 Mon Sep 17 00:00:00 2001 From: Nick Schouten Date: Wed, 20 Nov 2024 15:59:48 +0100 Subject: [PATCH 5/7] less strange docs, the base64 is a useless token --- papermill/iorw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papermill/iorw.py b/papermill/iorw.py index 43a848ae..934f1b2e 100644 --- a/papermill/iorw.py +++ b/papermill/iorw.py @@ -168,7 +168,7 @@ class HttpHandler: @classmethod def _get_auth_kwargs(cls): """Gets the Authorization header from PAPERMILL_HTTP_AUTH_HEADER. - A valid value could be Basic dW5hbWU6cGFzc3dvcmQK""" + A valid example value Basic dW5hbWU6cGFzc3dvcmQK""" auth_header = os.environ.get('PAPERMILL_HTTP_AUTH_HEADER', None) if auth_header: return {'headers': {'Authorization': auth_header}} From 20d75866ad3c06aeb6a4e2a77e0295ba987b2796 Mon Sep 17 00:00:00 2001 From: Nick Schouten Date: Wed, 20 Nov 2024 16:10:10 +0100 Subject: [PATCH 6/7] apparently you want to still support 3.8 :o who does that. It is deprecated! :angry: --- papermill/iorw.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/papermill/iorw.py b/papermill/iorw.py index 934f1b2e..7feeb0e7 100644 --- a/papermill/iorw.py +++ b/papermill/iorw.py @@ -177,9 +177,7 @@ def _get_auth_kwargs(cls): @classmethod def _get_read_kwargs(cls): kwargs = cls._get_auth_kwargs() or {'headers': {}} - kwargs['headers'] |= { - 'Accept': os.environ.get('PAPERMILL_HTTP_ACCEPT_HEADER', 'application/json') - } + kwargs['headers']['Accept'] = os.environ.get('PAPERMILL_HTTP_ACCEPT_HEADER', 'application/json') return kwargs @classmethod From 630b373bbf12f44e82e3dd3b008fed48604b99e0 Mon Sep 17 00:00:00 2001 From: Nick Schouten Date: Wed, 20 Nov 2024 22:16:35 +0100 Subject: [PATCH 7/7] blacked my tests --- papermill/tests/test_iorw.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/papermill/tests/test_iorw.py b/papermill/tests/test_iorw.py index fd2fe9c1..7d80caff 100644 --- a/papermill/tests/test_iorw.py +++ b/papermill/tests/test_iorw.py @@ -329,8 +329,7 @@ def test_read_with_auth(self): text = 'request test response' auth = 'Basic dW5hbWU6cGFzc3dvcmQK' - with patch.dict(os.environ, clear=True) as env,\ - patch('papermill.iorw.requests.get') as mock_get: + with patch.dict(os.environ, clear=True) as env, patch('papermill.iorw.requests.get') as mock_get: env['PAPERMILL_HTTP_AUTH_HEADER'] = auth mock_get.return_value = Mock(text=text) @@ -346,8 +345,7 @@ def test_read_with_accept_header(self): text = 'request test response' accept_type = 'test accept type' - with patch.dict(os.environ, clear=True) as env,\ - patch('papermill.iorw.requests.get') as mock_get: + with patch.dict(os.environ, clear=True) as env, patch('papermill.iorw.requests.get') as mock_get: env['PAPERMILL_HTTP_ACCEPT_HEADER'] = accept_type mock_get.return_value = Mock(text=text) @@ -375,8 +373,7 @@ def test_write_with_auth(self): buf = '{"papermill": true}' auth = 'token' - with patch.dict(os.environ, clear=True) as env,\ - patch('papermill.iorw.requests.put') as mock_put: + with patch.dict(os.environ, clear=True) as env, patch('papermill.iorw.requests.put') as mock_put: env['PAPERMILL_HTTP_AUTH_HEADER'] = auth HttpHandler.write(buf, path)