Skip to content

Commit 3be383c

Browse files
EmadMokhtarEmadMokhtar
committed
Decode the error response if it is in byte format
1 parent 438e30d commit 3be383c

File tree

2 files changed

+114
-68
lines changed

2 files changed

+114
-68
lines changed

‎pydruid/client.py

+4
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import json
2020
import re
2121

22+
from six import binary_type
2223
from six.moves import urllib
2324

2425
from pydruid.query import QueryBuilder
@@ -557,6 +558,9 @@ def _post(self, query):
557558
if e.code == 500:
558559
# has Druid returned an error?
559560
try:
561+
if isinstance(err, binary_type):
562+
# Decode the error before serialize it to JSON
563+
err = err.decode("utf-8")
560564
err = json.loads(err)
561565
except ValueError:
562566
if HTML_ERROR.search(err):

‎tests/test_client.py

+110-68
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
import pytest
55
from mock import patch, Mock
66
from six.moves import urllib
7-
from six import StringIO
7+
from six import StringIO, binary_type, BytesIO
88

99
from pydruid.client import PyDruid
1010
from pydruid.query import Query
@@ -17,23 +17,22 @@ def create_client():
1717

1818

1919
def create_blank_query():
20-
return Query({}, 'none')
20+
return Query({}, "none")
2121

2222

23-
def _http_error(code, msg, data = ''):
23+
def _http_error(code, msg, data=""):
2424
# Need a file-like object for the response data
25-
fp = StringIO(data)
25+
if isinstance(data, binary_type):
26+
fp = BytesIO(data)
27+
else:
28+
fp = StringIO(data)
2629
return urllib.error.HTTPError(
27-
url='http://fakeurl:8080/druid/v2/',
28-
hdrs={},
29-
code=code,
30-
msg=msg,
31-
fp=fp,
30+
url="http://fakeurl:8080/druid/v2/", hdrs={}, code=code, msg=msg, fp=fp,
3231
)
3332

3433

3534
class TestPyDruid:
36-
@patch('pydruid.client.urllib.request.urlopen')
35+
@patch("pydruid.client.urllib.request.urlopen")
3736
def test_druid_returns_error(self, mock_urlopen):
3837
# given
3938
mock_urlopen.side_effect = _http_error(500, "Druid error")
@@ -42,52 +41,63 @@ def test_druid_returns_error(self, mock_urlopen):
4241
# when / then
4342
with pytest.raises(IOError):
4443
client.topn(
45-
datasource="testdatasource",
46-
granularity="all",
47-
intervals="2015-12-29/pt1h",
48-
aggregations={"count": doublesum("count")},
49-
dimension="user_name",
50-
metric="count",
51-
filter=Dimension("user_lang") == "en",
52-
threshold=1,
53-
context={"timeout": 1000})
54-
55-
@patch('pydruid.client.urllib.request.urlopen')
44+
datasource="testdatasource",
45+
granularity="all",
46+
intervals="2015-12-29/pt1h",
47+
aggregations={"count": doublesum("count")},
48+
dimension="user_name",
49+
metric="count",
50+
filter=Dimension("user_lang") == "en",
51+
threshold=1,
52+
context={"timeout": 1000},
53+
)
54+
55+
@patch("pydruid.client.urllib.request.urlopen")
5656
def test_druid_returns_html_error(self, mock_urlopen):
5757
# given
58-
message = textwrap.dedent("""
58+
message = textwrap.dedent(
59+
"""
5960
<html>
6061
<head>
61-
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
62+
<meta http-equiv="Content-Type" content="text/html;
63+
charset=ISO-8859-1"/>
6264
<title>Error 500 </title>
6365
</head>
6466
<body>
6567
<h2>HTTP ERROR: 500</h2>
6668
<p>Problem accessing /druid/v2/. Reason:
67-
<pre> javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
68-
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.19.v20170502</a><hr/>
69+
<pre> javax.servlet.ServletException:
70+
java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
71+
<hr /><a href="http://eclipse.org/jetty">
72+
Powered by Jetty:// 9.3.19.v20170502</a><hr/>
6973
</body>
7074
</html>
71-
""").strip()
72-
mock_urlopen.side_effect = _http_error(500, 'Internal Server Error', message)
75+
"""
76+
).strip()
77+
mock_urlopen.side_effect = _http_error(500, "Internal Server Error", message)
7378
client = create_client()
7479

7580
# when / then
7681
with pytest.raises(IOError) as e:
7782
client.topn(
78-
datasource="testdatasource",
79-
granularity="all",
80-
intervals="2015-12-29/pt1h",
81-
aggregations={"count": doublesum("count")},
82-
dimension="user_name",
83-
metric="count",
84-
filter=Dimension("user_lang") == "en",
85-
threshold=1,
86-
context={"timeout": 1000})
87-
88-
assert str(e.value) == textwrap.dedent("""
89-
HTTP Error 500: Internal Server Error
90-
Druid Error: javax.servlet.ServletException: java.lang.OutOfMemoryError: GC overhead limit exceeded
83+
datasource="testdatasource",
84+
granularity="all",
85+
intervals="2015-12-29/pt1h",
86+
aggregations={"count": doublesum("count")},
87+
dimension="user_name",
88+
metric="count",
89+
filter=Dimension("user_lang") == "en",
90+
threshold=1,
91+
context={"timeout": 1000},
92+
)
93+
94+
assert (
95+
str(e.value)
96+
== textwrap.dedent(
97+
"""
98+
HTTP Error 500: Internal Server Error
99+
Druid Error: javax.servlet.ServletException:
100+
java.lang.OutOfMemoryError: GC overhead limit exceeded
91101
Query is: {
92102
"aggregations": [
93103
{
@@ -112,9 +122,34 @@ def test_druid_returns_html_error(self, mock_urlopen):
112122
"queryType": "topN",
113123
"threshold": 1
114124
}
115-
""").strip()
125+
"""
126+
).strip()
127+
)
128+
129+
@patch("pydruid.client.urllib.request.urlopen")
130+
def test_druid_returns_string_error_bytes_error_response(self, mock_urlopen):
131+
# given
132+
message = b"Error as bytes, please decode me"
133+
mock_urlopen.side_effect = _http_error(500, "Internal Server Error", message)
134+
client = create_client()
116135

117-
@patch('pydruid.client.urllib.request.urlopen')
136+
# when / then
137+
with pytest.raises(IOError) as e:
138+
client.topn(
139+
datasource="testdatasource",
140+
granularity="all",
141+
intervals="2015-12-29/pt1h",
142+
aggregations={"count": doublesum("count")},
143+
dimension="user_name",
144+
metric="count",
145+
filter=Dimension("user_lang") == "en",
146+
threshold=1,
147+
context={"timeout": 1000},
148+
)
149+
150+
assert "Error as bytes, please decode me" in str(e.value)
151+
152+
@patch("pydruid.client.urllib.request.urlopen")
118153
def test_druid_returns_results(self, mock_urlopen):
119154
# given
120155
response = Mock()
@@ -126,28 +161,31 @@ def test_druid_returns_results(self, mock_urlopen):
126161
"metric" : 100
127162
} ]
128163
} ]
129-
""".encode("utf-8")
164+
""".encode(
165+
"utf-8"
166+
)
130167
mock_urlopen.return_value = response
131168
client = create_client()
132169

133170
# when
134171
top = client.topn(
135-
datasource="testdatasource",
136-
granularity="all",
137-
intervals="2015-12-29/pt1h",
138-
aggregations={"count": doublesum("count")},
139-
dimension="user_name",
140-
metric="count",
141-
filter=Dimension("user_lang") == "en",
142-
threshold=1,
143-
context={"timeout": 1000})
172+
datasource="testdatasource",
173+
granularity="all",
174+
intervals="2015-12-29/pt1h",
175+
aggregations={"count": doublesum("count")},
176+
dimension="user_name",
177+
metric="count",
178+
filter=Dimension("user_lang") == "en",
179+
threshold=1,
180+
context={"timeout": 1000},
181+
)
144182

145183
# then
146184
assert top is not None
147185
assert len(top.result) == 1
148-
assert len(top.result[0]['result']) == 1
186+
assert len(top.result[0]["result"]) == 1
149187

150-
@patch('pydruid.client.urllib.request.urlopen')
188+
@patch("pydruid.client.urllib.request.urlopen")
151189
def test_client_allows_to_export_last_query(self, mock_urlopen):
152190
# given
153191
response = Mock()
@@ -159,29 +197,33 @@ def test_client_allows_to_export_last_query(self, mock_urlopen):
159197
"metric" : 100
160198
} ]
161199
} ]
162-
""".encode("utf-8")
200+
""".encode(
201+
"utf-8"
202+
)
163203
mock_urlopen.return_value = response
164204
client = create_client()
165205
client.topn(
166-
datasource="testdatasource",
167-
granularity="all",
168-
intervals="2015-12-29/pt1h",
169-
aggregations={"count": doublesum("count")},
170-
dimension="user_name",
171-
metric="count",
172-
filter=Dimension("user_lang") == "en",
173-
threshold=1,
174-
context={"timeout": 1000})
206+
datasource="testdatasource",
207+
granularity="all",
208+
intervals="2015-12-29/pt1h",
209+
aggregations={"count": doublesum("count")},
210+
dimension="user_name",
211+
metric="count",
212+
filter=Dimension("user_lang") == "en",
213+
threshold=1,
214+
context={"timeout": 1000},
215+
)
175216

176217
# when / then
177-
# assert that last_query.export_tsv method was called (it should throw an exception, given empty path)
218+
# assert that last_query.export_tsv method was called (it should throw
219+
# an exception, given empty path)
178220
with pytest.raises(TypeError):
179221
client.export_tsv(None)
180222

181-
@patch('pydruid.client.urllib.request.urlopen')
223+
@patch("pydruid.client.urllib.request.urlopen")
182224
def test_client_auth_creds(self, mock_urlopen):
183225
client = create_client()
184226
query = create_blank_query()
185-
client.set_basic_auth_credentials('myUsername', 'myPassword')
227+
client.set_basic_auth_credentials("myUsername", "myPassword")
186228
headers, _, _ = client._prepare_url_headers_and_body(query)
187-
assert headers['Authorization'] == "Basic bXlVc2VybmFtZTpteVBhc3N3b3Jk"
229+
assert headers["Authorization"] == "Basic bXlVc2VybmFtZTpteVBhc3N3b3Jk"

0 commit comments

Comments
 (0)