Skip to content

Commit

Permalink
Python 3: Correctly read results from s3. Fixes #94.
Browse files Browse the repository at this point in the history
  • Loading branch information
Rohit Agarwal committed Mar 30, 2015
1 parent a3d61b9 commit 65eefb2
Showing 1 changed file with 16 additions and 2 deletions.
18 changes: 16 additions & 2 deletions qds_sdk/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,11 @@ def get_results(self, fp=sys.stdout, inline=True, delim=None, fetch=True):
#fetch latest value of num_result_dir
num_result_dir = Command.find(self.id).num_result_dir
for s3_path in r['result_location']:
# In Python 3, in this case, `fp` should always be binary mode.
# In Python 3,
# If the delim is None, fp should be in binary mode because
# boto expects it to be.
# If the delim is not None, then both text and binary modes
# work.
_download_to_local(boto_conn, s3_path, fp, num_result_dir, delim=delim)
else:
fp.write(",".join(r['result_location']))
Expand Down Expand Up @@ -1040,7 +1044,17 @@ def _read_iteratively(key_instance, fp, delim):
try:
# Default buffer size is 8192 bytes
data = next(key_instance)
fp.write(str(data).replace(chr(1), delim))
if sys.version_info < (3, 0, 0):
fp.write(str(data).replace(chr(1), delim))
else:
import io
if isinstance(fp, io.TextIOBase):
fp.buffer.write(data.decode('utf-8').replace(chr(1), delim).encode('utf8'))

This comment has been minimized.

Copy link
@amelio-vazquez-reina

amelio-vazquez-reina Oct 16, 2015

I think this is the line that is breaking the queries that we mention here

elif isinstance(fp, io.BufferedIOBase) or isinstance(fp, io.RawIOBase):
fp.write(data.decode('utf8').replace(chr(1), delim).encode('utf8'))
else:
# Can this happen? Don't know what's the right thing to do in this case.
pass
except StopIteration:
# Stream closes itself when the exception is raised
return
Expand Down

0 comments on commit 65eefb2

Please sign in to comment.