From fb0d115388ed1403476bc61fc732f92d328022aa Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Mon, 23 Oct 2023 23:58:27 +0900 Subject: [PATCH 1/6] refactor(test): use forceDeleteOnExit for TestCore Signed-off-by: Hiroshi Miura --- test/fixtures/org/omegat/core/TestCore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixtures/org/omegat/core/TestCore.java b/test/fixtures/org/omegat/core/TestCore.java index b98eb9ae10..4a0340a6de 100644 --- a/test/fixtures/org/omegat/core/TestCore.java +++ b/test/fixtures/org/omegat/core/TestCore.java @@ -681,6 +681,6 @@ public void activateEntry() { @After public final void tearDownCore() throws Exception { - FileUtils.deleteDirectory(configDir); + FileUtils.forceDeleteOnExit(configDir); } } From 4cf85a1a4ee9c9712594dabadb85ab3fc168c1c1 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Mon, 23 Oct 2023 21:56:38 +0900 Subject: [PATCH 2/6] fix: PoFilter to handle eol properly - use system line-separator when write - Enable unit test on Windows Signed-off-by: Hiroshi Miura --- config/checkstyle/suppressions.xml | 2 +- src/org/omegat/filters2/po/PoFilter.java | 46 +++++++++++-------- test/src/org/omegat/filters/POFilterTest.java | 11 +---- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/config/checkstyle/suppressions.xml b/config/checkstyle/suppressions.xml index b286bf1f14..654ad9fdc2 100644 --- a/config/checkstyle/suppressions.xml +++ b/config/checkstyle/suppressions.xml @@ -160,7 +160,7 @@ - + diff --git a/src/org/omegat/filters2/po/PoFilter.java b/src/org/omegat/filters2/po/PoFilter.java index 29ff1ee8d2..1be90620c5 100644 --- a/src/org/omegat/filters2/po/PoFilter.java +++ b/src/org/omegat/filters2/po/PoFilter.java @@ -10,6 +10,7 @@ 2011 Didier Briel 2013-1014 Alex Buloichik, Enrique Estevez 2017 Didier Briel + 2023 Hiroshi Miura Home page: https://www.omegat.org/ Support center: https://omegat.org/support @@ -89,6 +90,8 @@ public class PoFilter extends AbstractFilter { public static final String OPTION_AUTO_FILL_IN_PLURAL_STATEMENT = "autoFillInPluralStatement"; public static final String OPTION_FORMAT_MONOLINGUAL = "monolingualFormat"; + private static final String BR = System.lineSeparator(); + private static class PluralInfo { public int plurals; public String expression; @@ -610,7 +613,7 @@ private void processPoFile(BufferedReader in, FilterContext fc) throws IOExcepti protected void eol(String s) throws IOException { if (out != null) { out.write(s); - out.write('\n'); + out.write(BR); } } @@ -639,7 +642,7 @@ protected void parseOrAlign(int pair) { if (references.length() > 0) { c += OStrings.getString("POFILTER_REFERENCES") + "\n" + unescape(references.toString() + "\n"); } - if (c.length() == 0) { + if (c.isEmpty()) { c = null; } parseOrAlign(s, t, c, pathSuffix); @@ -726,7 +729,7 @@ protected void flushTranslation(MODE currentMode, FilterContext fc) throws IOExc if (out != null) { // Header is always written - out.write("msgstr " + getTranslation(null, targets[0], false, true, fc, 0) + "\n"); + out.write("msgstr " + getTranslation(null, targets[0], false, true, fc, 0) + BR); } else { parseHeader(targets[0].toString(), fc); } @@ -740,10 +743,10 @@ protected void flushTranslation(MODE currentMode, FilterContext fc) throws IOExc if (formatMonolingual) { out.write("msgstr " + getTranslation(sources[0].toString(), targets[0], allowBlank, false, fc, 0) - + "\n"); + + BR); } else { out.write("msgstr " + getTranslation(null, sources[0], allowBlank, false, fc, 0) - + "\n"); + + BR); } } else { parseOrAlign(0); @@ -751,10 +754,10 @@ protected void flushTranslation(MODE currentMode, FilterContext fc) throws IOExc } else { // plurals if (out != null) { - out.write("msgstr[0] " + getTranslation(null, sources[0], allowBlank, false, fc, 0) + "\n"); + out.write("msgstr[0] " + getTranslation(null, sources[0], allowBlank, false, fc, 0) + BR); for (int i = 1; i < plurals; i++) { out.write("msgstr[" + i + "] " + getTranslation(null, sources[1], allowBlank, false, fc, i) - + "\n"); + + BR); } } else { parseOrAlign(0); @@ -820,7 +823,7 @@ private String getTranslation(String id, StringBuilder en, boolean allowNull, bo } // Do real translation - String translation = null; + String translation; if (isHeader) { entry = autoFillInPluralStatement(entry, fc); } @@ -907,18 +910,23 @@ private String escape(String translation) { * necessary for the translation of the po-header anyway) We can also honor the no-wrap instruction at * least by letting the first line of a multi-line translation not be on the same line as 'msgstr'. */ - // Interprets newline chars. 'blah
blah' becomes - // 'blah\n"
"blah' - translation = translation.replace("\n", "\\n\"\n\""); - // don't make empty new line at the end (in case the last 'blah' is - // empty string) - if (translation.endsWith("\"\n\"")) { - translation = translation.substring(0, translation.length() - 3); - } - if (nowrap && translation.contains("\n")) { - // start with empty string, to align all lines of translation - translation = "\"\n\"" + translation; + + // Interprets newline chars. + if (translation.contains("\n")) { + final String newLine = "\"" + BR + "\""; + // 'blah
blah' becomes 'blah\n"
"blah' + translation = translation.replace("\n", "\\n" + newLine); + // don't make empty new line at the end (in case the last 'blah' is + // empty string) + if (translation.endsWith(newLine)) { + translation = translation.substring(0, translation.length() - newLine.length()); + } + if (nowrap) { + // start with empty string, to align all lines of translation + translation = newLine + translation; + } } + // Interprets tab chars. 'blahblah' becomes 'blah\tblah' // ( representing the tab character '\u0009') translation = translation.replace("\t", "\\t"); diff --git a/test/src/org/omegat/filters/POFilterTest.java b/test/src/org/omegat/filters/POFilterTest.java index 9d37caff5b..a1d314fb85 100644 --- a/test/src/org/omegat/filters/POFilterTest.java +++ b/test/src/org/omegat/filters/POFilterTest.java @@ -33,27 +33,20 @@ import java.util.Map; import java.util.TreeMap; -import org.junit.Before; import org.junit.Test; import org.omegat.core.data.ExternalTMX; import org.omegat.core.data.ITMXEntry; import org.omegat.filters2.po.PoFilter; import org.omegat.util.OStrings; -import org.omegat.util.Platform; import org.omegat.util.StringUtil; public class POFilterTest extends TestFilterBase { - @Before - public void condition() { - org.junit.Assume.assumeFalse(Platform.isWindows); - } - @Test public void testParse() throws Exception { - Map data = new TreeMap(); - Map tmx = new TreeMap(); + Map data = new TreeMap<>(); + Map tmx = new TreeMap<>(); parse2(new PoFilter(), "test/data/filters/po/file-POFilter-be.po", data, tmx); From 9ec1320ab25196455c623cf3c86bea7a1e745827 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Tue, 24 Oct 2023 10:47:42 +0900 Subject: [PATCH 3/6] style: suppress checkstyle specific rules Signed-off-by: Hiroshi Miura --- config/checkstyle/checkstyle.xml | 20 ++++-- src/org/omegat/filters2/po/PoFilter.java | 78 ++++++++++++++---------- 2 files changed, 62 insertions(+), 36 deletions(-) diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml index fe1869fb09..cbe6e3bbd6 100644 --- a/config/checkstyle/checkstyle.xml +++ b/config/checkstyle/checkstyle.xml @@ -98,14 +98,24 @@ - - + + + + + + + + + + + + + - - + + diff --git a/src/org/omegat/filters2/po/PoFilter.java b/src/org/omegat/filters2/po/PoFilter.java index 1be90620c5..b57fe1de7d 100644 --- a/src/org/omegat/filters2/po/PoFilter.java +++ b/src/org/omegat/filters2/po/PoFilter.java @@ -102,10 +102,10 @@ private static class PluralInfo { } } - // CHECKSTYLE:OFF + // CHECKSTYLE.OFF: LineLength private static final Map PLURAL_INFOS; static { - HashMap info = new HashMap(); + HashMap info = new HashMap<>(); // list taken from http://translate.sourceforge.net/wiki/l10n/pluralforms d.d. 14-09-2012 // See also http://unicode.org/repos/cldr-tmp/trunk/diff/supplemental/language_plural_rules.html info.put("ach", new PluralInfo(2, "(n > 1)")); @@ -247,7 +247,7 @@ private static class PluralInfo { info.put("zh", new PluralInfo(1, "0 ")); PLURAL_INFOS = Collections.unmodifiableMap(info); } - // CHECKSTYLE:ON + // CHECKSTYLE.ON: LineLength /** * If true, non-translated segments will contain the source text in ms @@ -311,7 +311,7 @@ public String getFileFormatName() { public Instance[] getDefaultInstances() { return new Instance[] { new Instance("*.po", StandardCharsets.UTF_8.name(), StandardCharsets.UTF_8.name()), - new Instance("*.pot", StandardCharsets.UTF_8.name(), StandardCharsets.UTF_8.name()) }; + new Instance("*.pot", StandardCharsets.UTF_8.name(), StandardCharsets.UTF_8.name()) }; } /** @@ -319,7 +319,7 @@ public Instance[] getDefaultInstances() { * @param inFile * The source file. * @param inEncoding - * Encoding of the input file, if the filter supports it. Otherwise null. + * Encoding of the input file, if the filter supports it. Otherwise, null. * @return * @throws IOException */ @@ -397,17 +397,19 @@ public void processFile(File inFile, File outFile, FilterContext fc) throws IOEx } @Override - protected void alignFile(BufferedReader sourceFile, BufferedReader translatedFile, FilterContext fc) throws Exception { - this.out = null; + protected void alignFile(BufferedReader sourceFile, BufferedReader translatedFile, FilterContext fc) + throws Exception { + out = null; processPoFile(translatedFile, fc); } @Override - public void processFile(BufferedReader in, BufferedWriter out, FilterContext fc) throws IOException { - this.out = out; + public void processFile(BufferedReader in, BufferedWriter writer, FilterContext fc) throws IOException { + out = writer; processPoFile(in, fc); } + // CHECKSTYLE.OFF: MethodLength private void processPoFile(BufferedReader in, FilterContext fc) throws IOException { fuzzy = false; fuzzyTrue = false; @@ -419,7 +421,8 @@ private void processPoFile(BufferedReader in, FilterContext fc) throws IOExcepti sources = new StringBuilder[2]; sources[0] = new StringBuilder(); sources[1] = new StringBuilder(); - // can be overridden when header has been read and the number of plurals is different. + // can be overridden when header has been read and the number of + // plurals is different. targets = new StringBuilder[2]; targets[0] = new StringBuilder(); targets[1] = new StringBuilder(); @@ -433,8 +436,8 @@ private void processPoFile(BufferedReader in, FilterContext fc) throws IOExcepti String s; while ((s = in.readLine()) != null) { - // We trim trailing spaces, otherwise the regexps could fail, thus making some segments - // invisible to OmegaT + // We trim trailing spaces, otherwise the regexps could fail, thus + // making some segments invisible to OmegaT s = s.trim(); // We have a real fuzzy @@ -452,7 +455,8 @@ private void processPoFile(BufferedReader in, FilterContext fc) throws IOExcepti } /* - * Removing the fuzzy markers, as it has no meanings after being processed by omegat + * Removing the fuzzy markers, as it has no meanings after being + * processed by omegat */ if (COMMENT_FUZZY.matcher(s).matches()) { currentPlural = 0; @@ -506,9 +510,10 @@ private void processPoFile(BufferedReader in, FilterContext fc) throws IOExcepti Matcher mStr = MSG_STR.matcher(s); if (mStr.matches()) { - // Hack to be able to translate empty segments - // If the source segment is empty and there is a reference then - // it copies the reference of the segment and the localization note into the source segment + // Hack to be able to translate empty segments. + // If the source segment is empty and there is a reference, then + // it copies the reference of the segment and the localization + // note into the source segment if (allowEditingBlankSegment && sources[0].length() == 0 && references.length() > 0 && headerProcessed) { String aux = references + extractedComments.toString(); sources[0].append(aux); @@ -600,6 +605,8 @@ private void processPoFile(BufferedReader in, FilterContext fc) throws IOExcepti case MSGCTX: eol(s); break; + default: + throw new IllegalArgumentException(); } continue; } @@ -609,6 +616,7 @@ private void processPoFile(BufferedReader in, FilterContext fc) throws IOExcepti } flushTranslation(currentMode, fc); } + // CHECKSTYLE.ON: MethodLength protected void eol(String s) throws IOException { if (out != null) { @@ -650,12 +658,12 @@ protected void parseOrAlign(int pair) { /** * - * @param source - * @param translation - * @param comments + * @param source unescaped source string. + * @param translation unescaped translated string in po file. + * @param comments unescaped comment in po file. * @param pathSuffix - * suffix for path to distinguish plural forms. It will be empty for first one, and [1],[2],... - * for next + * suffix for a path to distinguish plural forms. It will be empty for the first one, + * and [1], [2], ... for next */ protected void parseOrAlign(String source, String translation, String comments, String pathSuffix) { if (translation.isEmpty()) { @@ -675,7 +683,8 @@ protected void parseOrAlign(String source, String translation, String comments, entryParseCallback.addEntryWithProperties(null, sourceFuzzyTrue.toString(), translation, false, props, path + pathSuffix, this, null); fuzzyTrue = false; - fuzzy = false; // Do not load false fuzzy when there is a real one + // Do not load false fuzzy when there is a real one + fuzzy = false; translation = null; } entryParseCallback.addEntry(null, source, translation, fuzzy, comments, path + pathSuffix, this, @@ -686,6 +695,11 @@ protected void parseOrAlign(String source, String translation, String comments, } } + /** + * Parse PO file header. + * @param header header block in the file. + * @param fc filter context to process. + */ protected void parseHeader(String header, FilterContext fc) { if (entryParseCallback != null && !skipHeader) { header = unescape(autoFillInPluralStatement(header, fc)); @@ -704,7 +718,8 @@ protected void flushTranslation(MODE currentMode, FilterContext fc) throws IOExc } else { // header - // check existing plural statement. If it contains the number of plurals, then use it! + // Check an existing plural statement. If it contains the + // number of plurals, then use it! StringBuilder targets0 = targets[0]; String header = targets[0].toString(); Matcher pluralMatcher = PLURAL_FORMS.matcher(header); @@ -712,7 +727,7 @@ protected void flushTranslation(MODE currentMode, FilterContext fc) throws IOExc String nrOfPluralsString = header.substring(pluralMatcher.start(1), pluralMatcher.end(1)); plurals = Integer.parseInt(nrOfPluralsString); } else { - //else use predefined number of plurals, if it exists + // else use predefined number of plurals if it exists Language targetLang = fc.getTargetLang(); String lang = targetLang.getLanguageCode().toLowerCase(Locale.ENGLISH); PluralInfo pluralInfo = PLURAL_INFOS.get(lang); @@ -790,17 +805,17 @@ protected void flushTranslation(MODE currentMode, FilterContext fc) throws IOExc * The given entry is interpreted to a string (e.g. escaped quotes are unescaped, '\n' is translated into newline * character, '\t' into tab character.) then translated and then returned as a PO-string-notation (e.g. double * quotes escaped, newline characters represented as '\n' and surrounded by double quotes, possibly split up over - * multiple lines)
+ * multiple lines)
* Long translations are not split up over multiple lines as some PO editors do, but when there are newline - * characters in a translation, it is split up at the newline markers.
+ * characters in a translation, it is split up at the newline markers.
* If the nowrap parameter is true, a translation that exists of multiple lines starts with an empty string-line to * left-align all lines. [With nowrap set to true, long lines are also never wrapped (except for at newline - * characters), but that was already not done without nowrap.] [ 1869069 ] Escape support for PO + * characters), but that was yet not done without nowrap.] [ 1869069 ] Escape support for PO * * @param en * The entire source text * @param allowNull - * Allow to output a blank translation in msgstr + * Allow outputting a blank translation in msgstr * @param isHeader * is the given string the PO-header string? * @param fc @@ -833,7 +848,8 @@ private String getTranslation(String id, StringBuilder en, boolean allowNull, bo translation = entryTranslateCallback.getTranslation(id, entry, path + pathSuffix); } - if (translation == null && !allowNull) { // We write the source in translation + if (translation == null && !allowNull) { + // We write the source in translation translation = entry; } @@ -873,7 +889,7 @@ private String unescape(String entry) { // of backslashes before \". Replace only the \" with " and keep the // other escaped backslashes ) entry = R1.matcher(entry).replaceAll("$1\""); - // Interprets newline sequence, except when preceded by \ + // Interprets a newline sequence, except when preceded by \ // \n becomes Linefeed, unless the \ was escaped itself. // The number of preceding slashes before \n should not be odd, // else the \ is escaped and not part of \n. @@ -884,7 +900,7 @@ private String unescape(String entry) { entry = R2.matcher(entry).replaceAll("$1\n"); // same for \t, the tab character entry = R3.matcher(entry).replaceAll("$1\t"); - // Interprets newline sequence at the beginning of a line + // Interprets a newline sequence at the beginning of a line entry = R4.matcher(entry).replaceAll("\\\n"); // Removes escape from backslash entry = entry.replace("\\\\", "\\"); From 7190dc32b8f174ea1b12b58b55cae2e14ada541c Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Thu, 16 Nov 2023 09:32:47 +0900 Subject: [PATCH 4/6] fix: PoFilter: fallback default encoding to be UTF-8 Signed-off-by: Hiroshi Miura --- src/org/omegat/filters2/po/PoFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/omegat/filters2/po/PoFilter.java b/src/org/omegat/filters2/po/PoFilter.java index b57fe1de7d..ab04a4d08f 100644 --- a/src/org/omegat/filters2/po/PoFilter.java +++ b/src/org/omegat/filters2/po/PoFilter.java @@ -332,7 +332,7 @@ protected BufferedReader createReader(File inFile, String inEncoding) throws IOE if (bom != null) { charset = bom.getCharsetName(); } else if (inEncoding == null) { - charset = Charset.defaultCharset().name(); + charset = StandardCharsets.UTF_8.name(); } else { charset = inEncoding; } From 37a0f3e31a30b3666535c778a1b9700bfac61cbb Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Thu, 16 Nov 2023 10:15:46 +0900 Subject: [PATCH 5/6] fix: PoFilter: fallback default encoding to be UTF-8 Signed-off-by: Hiroshi Miura --- src/org/omegat/filters2/po/PoFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/omegat/filters2/po/PoFilter.java b/src/org/omegat/filters2/po/PoFilter.java index ab04a4d08f..7d15b8e5e4 100644 --- a/src/org/omegat/filters2/po/PoFilter.java +++ b/src/org/omegat/filters2/po/PoFilter.java @@ -349,7 +349,7 @@ protected BufferedWriter createWriter(File outFile, String outEncoding) if (outEncoding != null) { charset = Charset.forName(outEncoding); } else { - charset = Charset.defaultCharset(); + charset = StandardCharsets.UTF_8; } return Files.newBufferedWriter(outFile.toPath(), charset); } From 011e544036c2eb8addcb4931f75763cc27809d43 Mon Sep 17 00:00:00 2001 From: Hiroshi Miura Date: Thu, 16 Nov 2023 23:43:25 +0900 Subject: [PATCH 6/6] docs: update changes.txt Signed-off-by: Hiroshi Miura --- release/changes.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release/changes.txt b/release/changes.txt index 0bd5cb6b5e..53ae84ce37 100644 --- a/release/changes.txt +++ b/release/changes.txt @@ -2,7 +2,7 @@ OmegaT 6.1.0 ---------------------------------------------------------------------- 26 Enhancement - 35 Bug fixes + 36 Bug fixes ---------------------------------------------------------------------- 6.1.0 vs 6.0.0 @@ -89,6 +89,9 @@ Bug fixes: + Unit tests failed on Windows + - https://sourceforge.net/p/omegat/bugs/1223/ + - OmegaT does not handle line breaks correctly on Windows https://sourceforge.net/p/omegat/bugs/1219/