fix(jira2markdown): process windows line breaks as a Unix line breaks

jira2markdown/markup/advanced.py

@@ -58,8 +58,10 @@ def action(self, tokens: ParseResults) -> str:
         else:
             prefix = ""
 
-        text = self.markup.transform_string("\n".join([line.lstrip() for line in tokens.text.strip().splitlines()]))
-        return prefix + "\n".join([f"> {line}" for line in text.splitlines()])
+        text = self.markup.transform_string(
+            "".join([line.lstrip() for line in tokens.text.strip().splitlines(keepends=True)]),
+        )
+        return prefix + "".join([f"> {line}" for line in text.splitlines(keepends=True)])
 
     @property
     def expr(self) -> ParserElement:

jira2markdown/markup/headings.py

@@ -1,6 +1,7 @@
-from pyparsing import Combine, LineEnd, LineStart, Optional, ParserElement, ParseResults, SkipTo, StringEnd, White, Word
+from pyparsing import Combine, LineStart, Optional, ParserElement, ParseResults, SkipTo, StringEnd, White, Word
 
 from jira2markdown.markup.base import AbstractMarkup
+from jira2markdown.tokens import UniversalLineEnd
 
 
 class Headings(AbstractMarkup):
@@ -17,6 +18,6 @@ def expr(self) -> ParserElement:
             + Combine(
                 Word("h", "123456", exact=2).set_results_name("level")
                 + ". "
-                + SkipTo(LineEnd() | StringEnd()).set_results_name("text"),
+                + SkipTo(UniversalLineEnd() | StringEnd()).set_results_name("text"),
             ).set_parse_action(self.action)
         )

jira2markdown/markup/lists.py

@@ -3,7 +3,6 @@
 from pyparsing import (
     Char,
     Combine,
-    LineEnd,
     LineStart,
     Literal,
     MatchFirst,
@@ -20,6 +19,7 @@
 from jira2markdown.markup.advanced import Panel
 from jira2markdown.markup.base import AbstractMarkup
 from jira2markdown.markup.text_effects import BlockQuote, Color
+from jira2markdown.tokens import UniversalLineEnd
 
 
 class ListIndentState:
@@ -63,7 +63,7 @@ def __init__(self, nested_token: str, nested_indent: int, tokens: str, indent: i
         self.indent_state = ListIndentState()
 
     def action(self, tokens: ParseResults) -> str:
-        result = []
+        result = ""
 
         for line in tokens:
             bullets, text = line.split(" ", maxsplit=1)
@@ -77,20 +77,17 @@ def action(self, tokens: ParseResults) -> str:
 
             line_padding = " " * count
             item_padding = " " * (count - self.indent) + self.bullet + " "
-            text = self.markup.transform_string(text).splitlines() or [""]
+            text = self.markup.transform_string(text).splitlines(keepends=True) or [""]
 
-            result.append(
-                "\n".join([item_padding + line if i == 0 else line_padding + line for i, line in enumerate(text)]),
-            )
+            result += "".join([item_padding + line if i == 0 else line_padding + line for i, line in enumerate(text)])
 
         self.indent_state.reset()
-        text_end = "\n" if (tokens[-1][-1] == "\n") else ""
-        return "\n".join(result) + text_end
+        return result
 
     @property
     def expr(self) -> ParserElement:
         WHITESPACE = Regex(r"[ \t]+", flags=re.UNICODE)
-        EOL = LineEnd()
+        EOL = UniversalLineEnd()
         LIST_BREAK = EOL + Optional(WHITESPACE) + EOL | StringEnd()
         IGNORE = BlockQuote(**self.init_kwargs).expr | Panel(**self.init_kwargs).expr | Color(**self.init_kwargs).expr
         ROW = (
@@ -118,7 +115,7 @@ def action(self, tokens: ParseResults) -> str:
         result = super().action(tokens)
         first_line = (result.splitlines() or [""])[0].strip()
 
-        # Text with dashed below it turns into a heading. To prevent this
+        # Text with dashes below it turns into a heading. To prevent this
         # add a line break before an empty list.
         if first_line == "-":
             return "\n" + result

jira2markdown/markup/tables.py

@@ -2,7 +2,6 @@
 
 from pyparsing import (
     Group,
-    LineEnd,
     LineStart,
     Literal,
     OneOrMore,
@@ -19,22 +18,25 @@
 from jira2markdown.markup.base import AbstractMarkup
 from jira2markdown.markup.images import Image
 from jira2markdown.markup.links import Link, MailTo, Mention
+from jira2markdown.tokens import UniversalLineEnd
 
 
 class Table(AbstractMarkup):
     is_inline_element = False
 
     def action(self, tokens: ParseResults) -> str:
-        lines = [line for line in tokens if len(line) > 0]
-        max_columns_count = max(len(row) for row in tokens)
+        eol = tokens[0].eol or "\n"
+        stripped_tokens = [row[:-1] if row[-1] == eol else row for row in tokens]
+        max_columns_count = max(len(row) for row in stripped_tokens)
+        lines = [row for row in stripped_tokens if len(row) > 0]
 
         # Converts multiline text to one line,
         # because markdown doesn't support multiline text in table cells
         output = [
             "|"
             + "|".join(
                 map(
-                    lambda cell: cell.replace("\n", "<br>"),
+                    lambda cell: cell.replace(eol, "<br>"),
                     map(self.markup.transform_string, row),
                 ),
             )
@@ -48,11 +50,11 @@ def action(self, tokens: ParseResults) -> str:
         # Insert header delimiter after the first row
         output.insert(1, "|" + "---|" * max(max_columns_count, 1))
 
-        return "\n".join(output) + "\n"
+        return eol.join(output) + eol
 
     @property
     def expr(self) -> ParserElement:
-        NL = LineEnd().suppress()
+        NL = UniversalLineEnd().suppress()
         SEP = (Literal("||") | Literal("|")).suppress()
         ROW_BREAK = NL + SEP | NL + NL | StringEnd()
         IGNORE = (
@@ -62,14 +64,18 @@ def expr(self) -> ParserElement:
             | Mention(**self.init_kwargs).expr
         )
 
-        ROW = SEP + ZeroOrMore(
-            SkipTo(SEP | ROW_BREAK, ignore=IGNORE) + Optional(SEP),
-            stop_on=ROW_BREAK | NL + ~SEP,
+        ROW = (
+            SEP
+            + ZeroOrMore(
+                SkipTo(SEP | ROW_BREAK, ignore=IGNORE) + Optional(SEP),
+                stop_on=ROW_BREAK | NL + ~SEP,
+            )
+            + UniversalLineEnd().set_results_name("eol")
         )
 
-        EMPTY_LINE = LineStart() + Optional(Regex(r"[ \t]+", flags=re.UNICODE)) + LineEnd()
+        EMPTY_LINE = LineStart() + Optional(Regex(r"[ \t]+", flags=re.UNICODE)) + UniversalLineEnd()
         return (
             (StringStart() ^ Optional(EMPTY_LINE, default="\n"))
-            + OneOrMore(LineStart() + Group(ROW) + NL).set_parse_action(self.action)
+            + OneOrMore(LineStart() + Group(ROW)).set_parse_action(self.action)
             + (StringEnd() | Optional(EMPTY_LINE, default="\n"))
         )

jira2markdown/markup/text_breaks.py

@@ -1,6 +1,7 @@
-from pyparsing import Keyword, LineEnd, LineStart, Optional, ParserElement, White, WordEnd, WordStart, replaceWith
+from pyparsing import Keyword, LineStart, Optional, ParserElement, White, WordEnd, WordStart, replaceWith
 
 from jira2markdown.markup.base import AbstractMarkup
+from jira2markdown.tokens import UniversalLineEnd
 
 
 class LineBreak(AbstractMarkup):
@@ -33,5 +34,5 @@ def expr(self) -> ParserElement:
             + (Optional(White()) + Keyword("----", ident_chars="-") + Optional(White())).set_parse_action(
                 replaceWith("\n----"),
             )
-            + LineEnd()
+            + UniversalLineEnd()
         )

jira2markdown/markup/text_effects.py

@@ -5,7 +5,6 @@
     Char,
     Combine,
     FollowedBy,
-    LineEnd,
     LineStart,
     Literal,
     OneOrMore,
@@ -30,6 +29,7 @@
 from jira2markdown.markup.base import AbstractMarkup
 from jira2markdown.markup.images import Image
 from jira2markdown.markup.links import Attachment, Link, Mention
+from jira2markdown.tokens import UniversalLineEnd
 
 
 class QuotedElement(AbstractMarkup):
@@ -53,7 +53,7 @@ def expr(self) -> ParserElement:
         ELEMENT = Combine(
             TOKEN
             + (~White() & ~Char(self.TOKEN))
-            + SkipTo(TOKEN, ignore=IGNORE, fail_on="\n")
+            + SkipTo(TOKEN, ignore=IGNORE, fail_on=UniversalLineEnd())
             + TOKEN
             + FollowedBy(NON_ALPHANUMS | StringEnd()),
         )
@@ -159,7 +159,7 @@ def action(self, tokens: ParseResults) -> str:
 
     @property
     def expr(self) -> ParserElement:
-        NL = LineEnd()
+        NL = UniversalLineEnd()
         EMPTY_LINE = LineStart() + Optional(Regex(r"[ \t]+", flags=re.UNICODE)) + NL
         ROW = (
             LineStart()
@@ -173,8 +173,10 @@ def expr(self) -> ParserElement:
 
 class BlockQuote(AbstractMarkup):
     def action(self, tokens: ParseResults) -> str:
-        text = self.markup.transform_string("\n".join([line.lstrip() for line in tokens[0].strip().splitlines()]))
-        return "\n".join([f"> {line}" for line in text.splitlines()])
+        text = self.markup.transform_string(
+            "".join([line.lstrip() for line in tokens[0].strip().splitlines(keepends=True)]),
+        )
+        return "".join([f"> {line}" for line in text.splitlines(keepends=True)])
 
     @property
     def expr(self) -> ParserElement:

jira2markdown/tokens.py

@@ -0,0 +1,20 @@
+from pyparsing import LineEnd, ParseException, ParseImplReturnType
+
+
+class UniversalLineEnd(LineEnd):
+    def __init__(self):
+        super().__init__()
+        self.whiteChars.discard("\r\n")
+
+    def parseImpl(self, instring, loc, do_actions=True) -> ParseImplReturnType:
+        if loc < len(instring):
+            if instring.startswith("\r\n", loc):
+                return loc + 2, "\r\n"
+            elif instring[loc] in ("\n", "\r"):
+                return loc + 1, "\n"
+            else:
+                raise ParseException(instring, loc, self.errmsg, self)
+        elif loc == len(instring):
+            return loc + 1, []
+        else:
+            raise ParseException(instring, loc, self.errmsg, self)