From cf5783077d593a92a686c5ab8fe9ce96b6b13b52 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Tue, 3 Mar 2020 16:16:42 +0000 Subject: [PATCH 01/10] Add pipeto --format=mimepart option. The most notable use case is piping html to a browser without extra scripts such as those shared in #789. --- alot/commands/thread.py | 9 ++++++++- docs/source/usage/modes/thread.rst | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/alot/commands/thread.py b/alot/commands/thread.py index 433ba6a62..f02b65e4a 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -32,6 +32,8 @@ from ..db.utils import extract_headers from ..db.utils import clear_my_address from ..db.utils import ensure_unique_address +from ..db.utils import remove_cte +from ..db.utils import string_sanitize from ..db.envelope import Envelope from ..db.attachment import Attachment from ..db.errors import DatabaseROError @@ -632,7 +634,8 @@ def apply(self, ui): (['cmd'], {'help': 'shellcommand to pipe to', 'nargs': '+'}), (['--all'], {'action': 'store_true', 'help': 'pass all messages'}), (['--format'], {'help': 'output format', 'default': 'raw', - 'choices': ['raw', 'decoded', 'id', 'filepath']}), + 'choices': [ + 'raw', 'decoded', 'id', 'filepath', 'mimepart']}), (['--separately'], {'action': 'store_true', 'help': 'call command once for each message'}), (['--background'], {'action': 'store_true', @@ -671,6 +674,7 @@ def __init__(self, cmd, all=False, separately=False, background=False, 'decoded': message content, decoded quoted printable, 'id': message ids, separated by newlines, 'filepath': paths to message files on disk + 'mimepart': only pipe the currently selected mime part :type format: str :param add_tags: add 'Tags' header to the message :type add_tags: bool @@ -742,6 +746,9 @@ async def apply(self, ui): bodytext = msg.get_body_text() msgtext = '%s\n\n%s' % (headertext, bodytext) pipestrings.append(msgtext) + elif self.output_format == 'mimepart': + pipestrings.append(string_sanitize(remove_cte( + msg.mime_part, as_string=True))) if not self.separately: pipestrings = [separator.join(pipestrings)] diff --git a/docs/source/usage/modes/thread.rst b/docs/source/usage/modes/thread.rst index cc389d5de..62a8765a8 100644 --- a/docs/source/usage/modes/thread.rst +++ b/docs/source/usage/modes/thread.rst @@ -72,7 +72,7 @@ The following commands are available in thread mode: optional arguments :---all: pass all messages - :---format: output format; valid choices are: 'raw','decoded','id','filepath' (defaults to: 'raw') + :---format: output format; valid choices are: 'raw','decoded','id','filepath','mimepart' (defaults to: 'raw') :---separately: call command once for each message :---background: don't stop the interface :---add_tags: add 'Tags' header to the message From 19b8201df714a5cad0632c7c0d883c3e969e04a1 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Thu, 5 Mar 2020 19:29:22 +0000 Subject: [PATCH 02/10] Pipeto specific mime part format. While `pipeto --format mimepart` pipes whichever part is currently selected, the 'html' and 'plain' formats override the selected mime part (and preferences in settings). This is useful because the mime type you want displayed in alot isn't necessarily the one you want piped and also for setting keybindings to pipe specific mime types to specific applications (plain -> text editor, html -> web browser). --- alot/commands/thread.py | 7 +++++-- docs/source/usage/modes/thread.rst | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/alot/commands/thread.py b/alot/commands/thread.py index f02b65e4a..a33452df4 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -635,7 +635,8 @@ def apply(self, ui): (['--all'], {'action': 'store_true', 'help': 'pass all messages'}), (['--format'], {'help': 'output format', 'default': 'raw', 'choices': [ - 'raw', 'decoded', 'id', 'filepath', 'mimepart']}), + 'raw', 'decoded', 'id', 'filepath', 'mimepart', + 'plain', 'html']}), (['--separately'], {'action': 'store_true', 'help': 'call command once for each message'}), (['--background'], {'action': 'store_true', @@ -746,7 +747,9 @@ async def apply(self, ui): bodytext = msg.get_body_text() msgtext = '%s\n\n%s' % (headertext, bodytext) pipestrings.append(msgtext) - elif self.output_format == 'mimepart': + elif self.output_format in ['mimepart', 'plain', 'html']: + if self.output_format in ['plain', 'html']: + mimepart = get_body_part(mail, self.output_format) pipestrings.append(string_sanitize(remove_cte( msg.mime_part, as_string=True))) diff --git a/docs/source/usage/modes/thread.rst b/docs/source/usage/modes/thread.rst index 62a8765a8..155b50dd0 100644 --- a/docs/source/usage/modes/thread.rst +++ b/docs/source/usage/modes/thread.rst @@ -72,7 +72,7 @@ The following commands are available in thread mode: optional arguments :---all: pass all messages - :---format: output format; valid choices are: 'raw','decoded','id','filepath','mimepart' (defaults to: 'raw') + :---format: output format; valid choices are: 'raw','decoded','id','filepath','mimepart','plain','html' (defaults to: 'raw') :---separately: call command once for each message :---background: don't stop the interface :---add_tags: add 'Tags' header to the message From 48ca7670499c0c6a94c4a514af1409ba86ed0ed6 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Wed, 11 Mar 2020 15:52:18 +0000 Subject: [PATCH 03/10] Pipe focused mime part from the mime tree. When the mimetree is toggled on and a mime part is focused, pipeto should pipe the focused mime part rather than the currently selected part. This is only applicable to --format's that pipe a single part, which are decoded and mimepart. --- alot/commands/thread.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/alot/commands/thread.py b/alot/commands/thread.py index a33452df4..4da6f7f08 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -29,6 +29,7 @@ from ..db.utils import decode_header from ..db.utils import formataddr from ..db.utils import get_body_part +from ..db.utils import extract_body_part from ..db.utils import extract_headers from ..db.utils import clear_my_address from ..db.utils import ensure_unique_address @@ -738,20 +739,22 @@ async def apply(self, ui): else: for msg in to_print: mail = msg.get_email() + mimepart = (getattr(ui.get_deep_focus(), 'mimepart', False) + or msg.get_mime_part()) if self.add_tags: mail.add_header('Tags', ', '.join(msg.get_tags())) if self.output_format == 'raw': pipestrings.append(mail.as_string()) elif self.output_format == 'decoded': headertext = extract_headers(mail) - bodytext = msg.get_body_text() + bodytext = extract_body_part(mimepart) msgtext = '%s\n\n%s' % (headertext, bodytext) pipestrings.append(msgtext) elif self.output_format in ['mimepart', 'plain', 'html']: if self.output_format in ['plain', 'html']: mimepart = get_body_part(mail, self.output_format) pipestrings.append(string_sanitize(remove_cte( - msg.mime_part, as_string=True))) + mimepart, as_string=True))) if not self.separately: pipestrings = [separator.join(pipestrings)] From 1cc95117984cbecd751354225948bea4dcb5abcb Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sun, 9 Feb 2020 08:10:43 +0000 Subject: [PATCH 04/10] Call ExternalCommand from pipeto. Resolve #715. --- alot/commands/envelope.py | 2 +- alot/commands/globals.py | 13 ++++++------ alot/commands/thread.py | 42 +++++++++++---------------------------- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/alot/commands/envelope.py b/alot/commands/envelope.py index 9293d66f6..e233ea591 100644 --- a/alot/commands/envelope.py +++ b/alot/commands/envelope.py @@ -364,7 +364,7 @@ async def apply(self, ui): edit_headers = edit_headers - blacklist logging.info('editable headers: %s', edit_headers) - def openEnvelopeFromTmpfile(): + def openEnvelopeFromTmpfile(*args): # This parses the input from the tempfile. # we do this ourselves here because we want to be able to # just type utf-8 encoded stuff into the tempfile and let alot diff --git a/alot/commands/globals.py b/alot/commands/globals.py index aa1fd4368..ff1b19b55 100644 --- a/alot/commands/globals.py +++ b/alot/commands/globals.py @@ -249,9 +249,8 @@ async def apply(self, ui): if self.stdin is not None: # wrap strings in StrinIO so that they behaves like a file if isinstance(self.stdin, str): - # XXX: is utf-8 always safe to use here, or do we need to check - # the terminal encoding first? - stdin = BytesIO(self.stdin.encode('utf-8')) + stdin = BytesIO( + self.stdin.encode(urwid.util.detected_encoding)) else: stdin = self.stdin @@ -279,7 +278,8 @@ async def apply(self, ui): except OSError as e: ret = str(e) else: - _, err = await proc.communicate(stdin.read() if stdin else None) + out, err = await proc.communicate( + stdin.read() if stdin else None) if proc.returncode == 0: ret = 'success' elif err: @@ -294,7 +294,8 @@ async def apply(self, ui): except OSError as e: ret = str(e) else: - _, err = proc.communicate(stdin.read() if stdin else None) + out, err = proc.communicate( + stdin.read() if stdin else None) if proc.returncode == 0: ret = 'success' elif err: @@ -302,7 +303,7 @@ async def apply(self, ui): if ret == 'success': if self.on_success is not None: - self.on_success() + self.on_success(out) else: msg = "editor has exited with error code {} -- {}".format( proc.returncode, diff --git a/alot/commands/thread.py b/alot/commands/thread.py index 4da6f7f08..2935d55d2 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -6,14 +6,12 @@ import logging import mailcap import os -import subprocess import tempfile import email import email.policy from email.utils import getaddresses, parseaddr from email.message import Message -import urwid from io import BytesIO from . import Command, registerCommand @@ -762,34 +760,18 @@ async def apply(self, ui): self.cmd = [' '.join(self.cmd)] # do the monkey - for mail in pipestrings: - encoded_mail = mail.encode(urwid.util.detected_encoding) - if self.background: - logging.debug('call in background: %s', self.cmd) - proc = subprocess.Popen(self.cmd, - shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = proc.communicate(encoded_mail) - if self.notify_stdout: - ui.notify(out) - else: - with ui.paused(): - logging.debug('call: %s', self.cmd) - # if proc.stdout is defined later calls to communicate - # seem to be non-blocking! - proc = subprocess.Popen(self.cmd, shell=True, - stdin=subprocess.PIPE, - # stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - out, err = proc.communicate(encoded_mail) - if err: - ui.notify(err, priority='error') - return + def callback(out): + if self.notify_stdout: + ui.notify(out) + if self.done_msg: + ui.notify(self.done_msg) - # display 'done' message - if self.done_msg: - ui.notify(self.done_msg) + for mail in pipestrings: + await ui.apply_command(ExternalCommand(self.cmd, + stdin=mail, + shell=True, + thread=self.background, + on_success=callback)) @registerCommand(MODE, 'remove', arguments=[ @@ -993,7 +975,7 @@ async def apply(self, ui): tempfile_name = tmpfile.name self.attachment.write(tmpfile) - def afterwards(): + def afterwards(*args): os.unlink(tempfile_name) else: handler_stdin = BytesIO() From a03a29e77424399a6bc820ea48e169d238d65835 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Sun, 9 Feb 2020 22:07:00 +0000 Subject: [PATCH 05/10] Don't try to redefine ret if an exception occurred If an exception occurs ret is already defined and proc is not. --- alot/commands/globals.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alot/commands/globals.py b/alot/commands/globals.py index ff1b19b55..bf835462b 100644 --- a/alot/commands/globals.py +++ b/alot/commands/globals.py @@ -296,10 +296,10 @@ async def apply(self, ui): else: out, err = proc.communicate( stdin.read() if stdin else None) - if proc.returncode == 0: - ret = 'success' - elif err: - ret = err.decode(urwid.util.detected_encoding) + if proc.returncode == 0: + ret = 'success' + elif err: + ret = err.decode(urwid.util.detected_encoding) if ret == 'success': if self.on_success is not None: From c06f47a266b34aa092ee6f1c6bb886244af3e5ed Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Thu, 13 Feb 2020 05:29:48 +0000 Subject: [PATCH 06/10] Add pipeto --as_file option. Resolve #703. There just doesn't seem to be any way to invoke a usable editor with a subprocess that's passed stdin. Therefore this option doesn't literally pipe mail into the command but writes mail to a temporary file and passes the filename at the end of the external command. --- alot/commands/thread.py | 37 +++++++++++++++++++++++------- docs/source/usage/modes/thread.rst | 1 + 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/alot/commands/thread.py b/alot/commands/thread.py index 2935d55d2..2287f5217 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -12,6 +12,7 @@ from email.utils import getaddresses, parseaddr from email.message import Message +import urwid from io import BytesIO from . import Command, registerCommand @@ -636,6 +637,8 @@ def apply(self, ui): 'choices': [ 'raw', 'decoded', 'id', 'filepath', 'mimepart', 'plain', 'html']}), + (['--as_file'], {'action': 'store_true', + 'help': 'pass mail as a file to the given application'}), (['--separately'], {'action': 'store_true', 'help': 'call command once for each message'}), (['--background'], {'action': 'store_true', @@ -653,7 +656,7 @@ class PipeCommand(Command): repeatable = True def __init__(self, cmd, all=False, separately=False, background=False, - shell=False, notify_stdout=False, format='raw', + shell=False, notify_stdout=False, format='raw', as_file=False, add_tags=False, noop_msg='no command specified', confirm_msg='', done_msg=None, **kwargs): """ @@ -676,6 +679,8 @@ def __init__(self, cmd, all=False, separately=False, background=False, 'filepath': paths to message files on disk 'mimepart': only pipe the currently selected mime part :type format: str + :param as_file: pass mail as a file to the given application + :type as_file: bool :param add_tags: add 'Tags' header to the message :type add_tags: bool :param noop_msg: error notification to show if `cmd` is empty @@ -696,6 +701,7 @@ def __init__(self, cmd, all=False, separately=False, background=False, self.shell = shell self.notify_stdout = notify_stdout self.output_format = format + self.as_file = as_file self.add_tags = add_tags self.noop_msg = noop_msg self.confirm_msg = confirm_msg @@ -760,14 +766,29 @@ async def apply(self, ui): self.cmd = [' '.join(self.cmd)] # do the monkey - def callback(out): - if self.notify_stdout: - ui.notify(out) - if self.done_msg: - ui.notify(self.done_msg) - for mail in pipestrings: - await ui.apply_command(ExternalCommand(self.cmd, + cmd = self.cmd + + # Pass mail as temporary file rather than piping through stdin. + if self.as_file: + with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + tmpfile.write(mail.encode(urwid.util.detected_encoding)) + tempfile_name = tmpfile.name + mail = None + if self.shell: + cmd = [' '.join([cmd[0], tempfile_name])] + else: + cmd.append(tempfile_name) + + def callback(out): + if self.as_file: + os.unlink(tempfile_name) + if self.notify_stdout: + ui.notify(out) + if self.done_msg: + ui.notify(self.done_msg) + + await ui.apply_command(ExternalCommand(cmd, stdin=mail, shell=True, thread=self.background, diff --git a/docs/source/usage/modes/thread.rst b/docs/source/usage/modes/thread.rst index 155b50dd0..c8d6a3099 100644 --- a/docs/source/usage/modes/thread.rst +++ b/docs/source/usage/modes/thread.rst @@ -73,6 +73,7 @@ The following commands are available in thread mode: optional arguments :---all: pass all messages :---format: output format; valid choices are: 'raw','decoded','id','filepath','mimepart','plain','html' (defaults to: 'raw') + :---as_file: pass mail as a file to the given application :---separately: call command once for each message :---background: don't stop the interface :---add_tags: add 'Tags' header to the message From b03cb70a953f655e4f80261382b11650cc96de5a Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Wed, 4 Mar 2020 02:25:13 +0000 Subject: [PATCH 07/10] Enable piping html to browser without scripts. Many browsers (including chromium) require that files end in the '.html' extension in order to render them. See . When using `pipeto --format=mimepart --as_file`, on an html mime part, let's automatically set the file extension. This eliminates the need for additional user scripts referenced in issues such as #789 and #1153. --- alot/commands/thread.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/alot/commands/thread.py b/alot/commands/thread.py index 2287f5217..020ab693b 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -771,7 +771,9 @@ async def apply(self, ui): # Pass mail as temporary file rather than piping through stdin. if self.as_file: - with tempfile.NamedTemporaryFile(delete=False) as tmpfile: + suffix = {'html': '.html'}.get(mimepart.get_content_subtype()) + with tempfile.NamedTemporaryFile( + delete=False, suffix=suffix) as tmpfile: tmpfile.write(mail.encode(urwid.util.detected_encoding)) tempfile_name = tmpfile.name mail = None From 60f3e898f43e53c583ad76ddb1d4b68d10f1a664 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Mon, 11 May 2020 00:28:33 +0000 Subject: [PATCH 08/10] Delay tempfile deletion in pipeto --as_file. This is an asynchronous method so there's little harm in sleeping. This fixes two different use cases in which the file could otherwise be removed before it is opened: - When piping into a tab of an existing browser process (for example firefox), the command does not block. - When using pipeto's --background flag, the command will be invoked asynchronously. --- alot/commands/thread.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alot/commands/thread.py b/alot/commands/thread.py index 020ab693b..12b3c597c 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -7,6 +7,7 @@ import mailcap import os import tempfile +import time import email import email.policy from email.utils import getaddresses, parseaddr @@ -784,6 +785,8 @@ async def apply(self, ui): def callback(out): if self.as_file: + # Wait to remove file in case it's opened asynchronously. + time.sleep(5) os.unlink(tempfile_name) if self.notify_stdout: ui.notify(out) From d6172702e1ca8745366e282a229846c30b80903a Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Mon, 11 May 2020 00:57:23 +0000 Subject: [PATCH 09/10] Update external command error message. This message could now display for several different external commands besides editor. --- alot/commands/globals.py | 5 +++-- tests/commands/test_global.py | 15 ++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/alot/commands/globals.py b/alot/commands/globals.py index bf835462b..b9a17be62 100644 --- a/alot/commands/globals.py +++ b/alot/commands/globals.py @@ -305,9 +305,10 @@ async def apply(self, ui): if self.on_success is not None: self.on_success(out) else: - msg = "editor has exited with error code {} -- {}".format( + msg = ( + "external command has exited with error code {} -- {}".format( proc.returncode, - ret or "No stderr output") + ret or "No stderr output")) ui.notify(msg, priority='error') if self.refocus and callerbuffer in ui.buffers: logging.info('refocussing') diff --git a/tests/commands/test_global.py b/tests/commands/test_global.py index 775a822ce..7ef1421a6 100644 --- a/tests/commands/test_global.py +++ b/tests/commands/test_global.py @@ -105,6 +105,9 @@ def test_get_template_decode(self): class TestExternalCommand(unittest.TestCase): + NO_STDERR_MSG = ( + 'external command has exited with error code 1 -- No stderr output' + ) @utilities.async_test async def test_no_spawn_no_stdin_success(self): @@ -134,18 +137,14 @@ async def test_no_spawn_stdin_attached(self): cmd = g_commands.ExternalCommand( "test -t 0", stdin='0', refocus=False) await cmd.apply(ui) - ui.notify.assert_called_once_with( - 'editor has exited with error code 1 -- No stderr output', - priority='error') + ui.notify.assert_called_once_with(self.NO_STDERR_MSG, priority='error') @utilities.async_test async def test_no_spawn_failure(self): ui = utilities.make_ui() cmd = g_commands.ExternalCommand('false', refocus=False) await cmd.apply(ui) - ui.notify.assert_called_once_with( - 'editor has exited with error code 1 -- No stderr output', - priority='error') + ui.notify.assert_called_once_with(self.NO_STDERR_MSG, priority='error') @utilities.async_test @mock.patch( @@ -177,9 +176,7 @@ async def test_spawn_failure(self): ui = utilities.make_ui() cmd = g_commands.ExternalCommand('false', refocus=False, spawn=True) await cmd.apply(ui) - ui.notify.assert_called_once_with( - 'editor has exited with error code 1 -- No stderr output', - priority='error') + ui.notify.assert_called_once_with(self.NO_STDERR_MSG, priority='error') class TestCallCommand(unittest.TestCase): From aa04a72f0a93286f66a910117be4d4b47e38fe42 Mon Sep 17 00:00:00 2001 From: ryneeverett Date: Mon, 11 May 2020 01:54:05 +0000 Subject: [PATCH 10/10] PipeCommand: pass shell arg to ExternalCommand. Prior to this branch, `shell=True` was passed to Popen regardless of whether `--shell` was passed. I'm not sure why this was the case but it wasn't a problem. Now that we're passing to ExternalCommand, this doesn't work because ExternalCommand does it's own manipulation of cmd when shell=False which is incompatible with a list-style cmd. --- alot/commands/thread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alot/commands/thread.py b/alot/commands/thread.py index 12b3c597c..c439d654d 100644 --- a/alot/commands/thread.py +++ b/alot/commands/thread.py @@ -795,7 +795,7 @@ def callback(out): await ui.apply_command(ExternalCommand(cmd, stdin=mail, - shell=True, + shell=self.shell, thread=self.background, on_success=callback))