From d1c4b639717f1efd26209278140264a702308858 Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Thu, 25 Jul 2019 17:15:39 +0200 Subject: [PATCH 1/9] Add empty script to perform imports from Lionbridge I want to automate this or it'll be a lot of work. Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 scripts/lionbridge_import.py diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py new file mode 100644 index 0000000000..a876a90cfd --- /dev/null +++ b/scripts/lionbridge_import.py @@ -0,0 +1,26 @@ +# Copyright (c) 2019 Ultimaker B.V. +# Cura is released under the terms of the LGPLv3 or higher. + +import argparse #To get the source directory from command line arguments. +import os.path #To find files from the source and the destination path. + +## Imports translation files from Lionbridge. +# +# Lionbridge has a bit of a weird export feature. It exports it to the same +# file type as what we imported, so that's a .pot file. However this .pot file +# only contains the translations, so the header is completely empty. We need +# to merge those translations into our existing files so that the header is +# preserved. +def lionbridge_import(source: str) -> None: + print("Importing from:", source) + print("Importing to:", destination()) + +## Gets the destination path to copy the translations to. +def destination() -> str: + return os.path.abspath(os.path.join(__file__, "..", "..", "resources", "i18n")) + +if __name__ == "__main__": + argparser = argparse.ArgumentParser(description = "Import translation files from Lionbridge.") + argparser.add_argument("source") + args = argparser.parse_args() + lionbridge_import(args.source) \ No newline at end of file From 5a03ace239a31929731550006196c30677802b7e Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 13:18:35 +0200 Subject: [PATCH 2/9] Also discover Uranium translation directory Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index a876a90cfd..dae1fe416c 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -2,6 +2,7 @@ # Cura is released under the terms of the LGPLv3 or higher. import argparse #To get the source directory from command line arguments. +import os #To find files from the source. import os.path #To find files from the source and the destination path. ## Imports translation files from Lionbridge. @@ -13,12 +14,26 @@ import os.path #To find files from the source and the destination path. # preserved. def lionbridge_import(source: str) -> None: print("Importing from:", source) - print("Importing to:", destination()) + print("Importing to Cura:", destination_cura()) + print("Importing to Uranium:", destination_uranium()) -## Gets the destination path to copy the translations to. -def destination() -> str: +## Gets the destination path to copy the translations for Cura to. +def destination_cura() -> str: return os.path.abspath(os.path.join(__file__, "..", "..", "resources", "i18n")) +## Gets the destination path to copy the translations for Uranium to. +def destination_uranium() -> str: + try: + import UM + except ImportError: + relative_path = os.path.join(__file__, "..", "..", "..", "Uranium", "resources", "i18n", "uranium.pot") + print(os.path.abspath(relative_path)) + if os.path.exists(relative_path): + return os.path.abspath(relative_path) + else: + raise Exception("Can't find Uranium. Please put UM on the PYTHONPATH or put the Uranium folder next to the Cura folder.") + return os.path.abspath(os.path.join(UM.__file__, "..", "..", "resources", "i18n")) + if __name__ == "__main__": argparser = argparse.ArgumentParser(description = "Import translation files from Lionbridge.") argparser.add_argument("source") From 3f7bd2ac23eef19d67e1e0bae70b45525dc72295 Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 13:48:10 +0200 Subject: [PATCH 3/9] Handle iterating over source files and finding correct destination file Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index dae1fe416c..f2f505e52f 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -5,6 +5,9 @@ import argparse #To get the source directory from command line arguments. import os #To find files from the source. import os.path #To find files from the source and the destination path. +cura_files = {"cura", "fdmprinter.def.json", "fdmextruder.def.json"} +uranium_files = {"uranium"} + ## Imports translation files from Lionbridge. # # Lionbridge has a bit of a weird export feature. It exports it to the same @@ -17,6 +20,27 @@ def lionbridge_import(source: str) -> None: print("Importing to Cura:", destination_cura()) print("Importing to Uranium:", destination_uranium()) + for language in (directory for directory in os.listdir(source) if os.path.isdir(os.path.join(source, directory))): + print("================ Processing language:", language, "================") + directory = os.path.join(source, language) + for file_pot in (file for file in os.listdir(directory) if file.endswith(".pot")): + source_file = file_pot[:-4] #Strip extension. + if source_file in cura_files: + destination_file = os.path.join(destination_cura(), language.replace("-", "_"), source_file + ".po") + print("Merging", source_file, "(Cura) into", destination_file) + elif source_file in uranium_files: + destination_file = os.path.join(destination_uranium(), language.replace("-", "_"), source_file + ".po") + print("Merging", source_file, "(Uranium) into", destination_file) + else: + raise Exception("Unknown file: " + source_file + "... Is this Cura or Uranium?") + + with open(os.path.join(directory, file_pot)) as f: + source_str = f.read() + with open(destination_file) as f: + destination_str = f.read() + result = merge(source_str, destination_str) + print(result) #DEBUG! Instead we should write this to a file. + ## Gets the destination path to copy the translations for Cura to. def destination_cura() -> str: return os.path.abspath(os.path.join(__file__, "..", "..", "resources", "i18n")) @@ -34,6 +58,9 @@ def destination_uranium() -> str: raise Exception("Can't find Uranium. Please put UM on the PYTHONPATH or put the Uranium folder next to the Cura folder.") return os.path.abspath(os.path.join(UM.__file__, "..", "..", "resources", "i18n")) +def merge(source: str, destination: str) -> str: + return "TODO" + if __name__ == "__main__": argparser = argparse.ArgumentParser(description = "Import translation files from Lionbridge.") argparser.add_argument("source") From a3611404d62b2607c8f84599e99a4696c98f7d83 Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 14:20:06 +0200 Subject: [PATCH 4/9] Find translations in source file So it can add the translations to the destination file. Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 60 +++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index f2f505e52f..2ee9c0a937 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -59,7 +59,65 @@ def destination_uranium() -> str: return os.path.abspath(os.path.join(UM.__file__, "..", "..", "resources", "i18n")) def merge(source: str, destination: str) -> str: - return "TODO" + last_destination = { + "msgctxt": "", + "msgid": "", + "msgstr": "" + } + + current_state = "none" + for line in destination.split("\n"): + if line.startswith("msgctxt \""): + current_state = "msgctxt" + line = line[8:] + last_destination[current_state] = "" + elif line.startswith("msgid \""): + current_state = "msgid" + line = line[6:] + last_destination[current_state] = "" + elif line.startswith("msgstr \""): + current_state = "msgstr" + line = line[7:] + last_destination[current_state] = "" + + if line.startswith("\"") and line.endswith("\""): + last_destination[current_state] += line[1:-1] + else: #White lines trigger us to search for the translation in the source file. + if last_destination["msgstr"] == "" and last_destination["msgid"] != "": #No translation for this yet! + translation = find_translation(source, last_destination["msgctxt"], last_destination["msgid"]) + +def find_translation(source: str, msgctxt, msgid): + last_source = { + "msgctxt": "", + "msgid": "", + "msgstr": "" + } + + current_state = "none" + for line in source.split("\n"): + if line.startswith("msgctxt \""): + current_state = "msgctxt" + line = line[8:] + last_source[current_state] = "" + elif line.startswith("msgid \""): + current_state = "msgid" + line = line[6:] + last_source[current_state] = "" + elif line.startswith("msgstr \""): + current_state = "msgstr" + line = line[7:] + last_source[current_state] = "" + + if line.startswith("\"") and line.endswith("\""): + last_source[current_state] += line[1:-1] + else: #White lines trigger us to process this translation. Is it the correct one? + if last_source["msgctxt"] == msgctxt and last_source["msgid"] == msgid: + if last_source["msgstr"] == "": + print("!!! Empty translation for {" + msgctxt + "}", msgid, "!!!") + return last_source["msgstr"] + + #Still here? Then the entire msgctxt+msgid combination was not found at all. + print("!!! Missing translation for {" + msgctxt + "}", msgid, "!!!") if __name__ == "__main__": argparser = argparse.ArgumentParser(description = "Import translation files from Lionbridge.") From 58c32f97a1ff34b99962d3422d71a5641ca000a1 Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 14:23:04 +0200 Subject: [PATCH 5/9] Add documentation Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index 2ee9c0a937..64def0a220 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -42,10 +42,12 @@ def lionbridge_import(source: str) -> None: print(result) #DEBUG! Instead we should write this to a file. ## Gets the destination path to copy the translations for Cura to. +# \return Destination path for Cura. def destination_cura() -> str: return os.path.abspath(os.path.join(__file__, "..", "..", "resources", "i18n")) ## Gets the destination path to copy the translations for Uranium to. +# \return Destination path for Uranium. def destination_uranium() -> str: try: import UM @@ -58,6 +60,10 @@ def destination_uranium() -> str: raise Exception("Can't find Uranium. Please put UM on the PYTHONPATH or put the Uranium folder next to the Cura folder.") return os.path.abspath(os.path.join(UM.__file__, "..", "..", "resources", "i18n")) +## Merges translations from the source file into the destination file if they +# were missing in the destination file. +# \param source The contents of the source .po file. +# \param destination The contents of the destination .po file. def merge(source: str, destination: str) -> str: last_destination = { "msgctxt": "", @@ -86,7 +92,11 @@ def merge(source: str, destination: str) -> str: if last_destination["msgstr"] == "" and last_destination["msgid"] != "": #No translation for this yet! translation = find_translation(source, last_destination["msgctxt"], last_destination["msgid"]) -def find_translation(source: str, msgctxt, msgid): +## Finds a translation in the source file. +# \param source The contents of the source .po file. +# \param msgctxt The ctxt of the translation to find. +# \param msgid The id of the translation to find. +def find_translation(source: str, msgctxt: str, msgid: str) -> str: last_source = { "msgctxt": "", "msgid": "", From 23826d3404ab6dce11deb2169422d690849598c7 Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 15:07:26 +0200 Subject: [PATCH 6/9] Write resulting file It seems to fail now for plurals. I didn't think of that yet. Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 41 ++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index 64def0a220..dc9a24ead5 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -39,7 +39,8 @@ def lionbridge_import(source: str) -> None: with open(destination_file) as f: destination_str = f.read() result = merge(source_str, destination_str) - print(result) #DEBUG! Instead we should write this to a file. + with open(destination_file, "w") as f: + f.write(result) ## Gets the destination path to copy the translations for Cura to. # \return Destination path for Cura. @@ -65,10 +66,11 @@ def destination_uranium() -> str: # \param source The contents of the source .po file. # \param destination The contents of the destination .po file. def merge(source: str, destination: str) -> str: + result_lines = [] last_destination = { - "msgctxt": "", - "msgid": "", - "msgstr": "" + "msgctxt": "\"\"\n", + "msgid": "\"\"\n", + "msgstr": "\"\"\n" } current_state = "none" @@ -87,10 +89,22 @@ def merge(source: str, destination: str) -> str: last_destination[current_state] = "" if line.startswith("\"") and line.endswith("\""): - last_destination[current_state] += line[1:-1] - else: #White lines trigger us to search for the translation in the source file. - if last_destination["msgstr"] == "" and last_destination["msgid"] != "": #No translation for this yet! - translation = find_translation(source, last_destination["msgctxt"], last_destination["msgid"]) + last_destination[current_state] += line + "\n" + else: #White lines or comment lines trigger us to search for the translation in the source file. + if last_destination["msgstr"] == "\"\"\n" and last_destination["msgid"] != "\"\"\n": #No translation for this yet! + last_destination["msgstr"] = find_translation(source, last_destination["msgctxt"], last_destination["msgid"]) #Actually place the translation in. + if last_destination["msgctxt"] != "\"\"\n" or last_destination["msgid"] != "\"\"\n" or last_destination["msgstr"] != "\"\"\n": + result_lines.append("msgctxt {msgctxt}".format(msgctxt = last_destination["msgctxt"][:-1])) #The [:-1] to strip the last newline. + result_lines.append("msgid {msgid}".format(msgid = last_destination["msgid"][:-1])) + result_lines.append("msgstr {msgstr}".format(msgstr = last_destination["msgstr"][:-1])) + last_destination = { + "msgctxt": "\"\"\n", + "msgid": "\"\"\n", + "msgstr": "\"\"\n" + } + + result_lines.append(line) #This line itself. + return "\n".join(result_lines) ## Finds a translation in the source file. # \param source The contents of the source .po file. @@ -98,9 +112,9 @@ def merge(source: str, destination: str) -> str: # \param msgid The id of the translation to find. def find_translation(source: str, msgctxt: str, msgid: str) -> str: last_source = { - "msgctxt": "", - "msgid": "", - "msgstr": "" + "msgctxt": "\"\"\n", + "msgid": "\"\"\n", + "msgstr": "\"\"\n" } current_state = "none" @@ -119,15 +133,16 @@ def find_translation(source: str, msgctxt: str, msgid: str) -> str: last_source[current_state] = "" if line.startswith("\"") and line.endswith("\""): - last_source[current_state] += line[1:-1] + last_source[current_state] += line + "\n" else: #White lines trigger us to process this translation. Is it the correct one? if last_source["msgctxt"] == msgctxt and last_source["msgid"] == msgid: - if last_source["msgstr"] == "": + if last_source["msgstr"] == "\"\"\n": print("!!! Empty translation for {" + msgctxt + "}", msgid, "!!!") return last_source["msgstr"] #Still here? Then the entire msgctxt+msgid combination was not found at all. print("!!! Missing translation for {" + msgctxt + "}", msgid, "!!!") + return "\"\"\n" if __name__ == "__main__": argparser = argparse.ArgumentParser(description = "Import translation files from Lionbridge.") From 9e059b7a1ad559413197214e888ce6d94a5ef809 Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 15:31:42 +0200 Subject: [PATCH 7/9] Fix writing files with plural forms Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 39 +++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index dc9a24ead5..95f55da2e7 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -70,7 +70,8 @@ def merge(source: str, destination: str) -> str: last_destination = { "msgctxt": "\"\"\n", "msgid": "\"\"\n", - "msgstr": "\"\"\n" + "msgstr": "\"\"\n", + "msgid_plural": "\"\"\n" } current_state = "none" @@ -87,21 +88,30 @@ def merge(source: str, destination: str) -> str: current_state = "msgstr" line = line[7:] last_destination[current_state] = "" + elif line.startswith("msgid_plural \""): + current_state = "msgid_plural" + line = line[13:] + last_destination[current_state] = "" if line.startswith("\"") and line.endswith("\""): last_destination[current_state] += line + "\n" else: #White lines or comment lines trigger us to search for the translation in the source file. if last_destination["msgstr"] == "\"\"\n" and last_destination["msgid"] != "\"\"\n": #No translation for this yet! last_destination["msgstr"] = find_translation(source, last_destination["msgctxt"], last_destination["msgid"]) #Actually place the translation in. - if last_destination["msgctxt"] != "\"\"\n" or last_destination["msgid"] != "\"\"\n" or last_destination["msgstr"] != "\"\"\n": - result_lines.append("msgctxt {msgctxt}".format(msgctxt = last_destination["msgctxt"][:-1])) #The [:-1] to strip the last newline. + if last_destination["msgctxt"] != "\"\"\n" or last_destination["msgid"] != "\"\"\n" or last_destination["msgid_plural"] != "\"\"\n" or last_destination["msgstr"] != "\"\"\n": + if last_destination["msgctxt"] != "\"\"\n": + result_lines.append("msgctxt {msgctxt}".format(msgctxt = last_destination["msgctxt"][:-1])) #The [:-1] to strip the last newline. result_lines.append("msgid {msgid}".format(msgid = last_destination["msgid"][:-1])) - result_lines.append("msgstr {msgstr}".format(msgstr = last_destination["msgstr"][:-1])) - last_destination = { - "msgctxt": "\"\"\n", - "msgid": "\"\"\n", - "msgstr": "\"\"\n" - } + if last_destination["msgid_plural"] != "\"\"\n": + result_lines.append("msgid_plural {msgid_plural}".format(msgid_plural = last_destination["msgid_plural"][:-1])) + else: + result_lines.append("msgstr {msgstr}".format(msgstr = last_destination["msgstr"][:-1])) + last_destination = { + "msgctxt": "\"\"\n", + "msgid": "\"\"\n", + "msgstr": "\"\"\n", + "msgid_plural": "\"\"\n" + } result_lines.append(line) #This line itself. return "\n".join(result_lines) @@ -135,13 +145,18 @@ def find_translation(source: str, msgctxt: str, msgid: str) -> str: if line.startswith("\"") and line.endswith("\""): last_source[current_state] += line + "\n" else: #White lines trigger us to process this translation. Is it the correct one? - if last_source["msgctxt"] == msgctxt and last_source["msgid"] == msgid: + #Process the source and destination keys for comparison independent of newline technique. + source_ctxt = "".join((line.strip()[1:-1].strip() for line in last_source["msgctxt"].split("\n"))) + source_id = "".join((line.strip()[1:-1].strip() for line in last_source["msgid"].split("\n"))) + dest_ctxt = "".join((line.strip()[1:-1].strip() for line in msgctxt.split("\n"))) + dest_id = "".join((line.strip()[1:-1].strip() for line in msgid.split("\n"))) + if source_ctxt == dest_ctxt and source_id == dest_id: if last_source["msgstr"] == "\"\"\n": - print("!!! Empty translation for {" + msgctxt + "}", msgid, "!!!") + print("!!! Empty translation for {" + dest_ctxt + "}", dest_id, "!!!") return last_source["msgstr"] #Still here? Then the entire msgctxt+msgid combination was not found at all. - print("!!! Missing translation for {" + msgctxt + "}", msgid, "!!!") + print("!!! Missing translation for {" + msgctxt.strip() + "}", msgid.strip(), "!!!") return "\"\"\n" if __name__ == "__main__": From 79d57ec10ad79402267ab06a04d9cf03a25b597b Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 15:43:19 +0200 Subject: [PATCH 8/9] Fix comparisons when source is single-line but dest is multi-line Or vice-versa. It's now agnostic to number of lines. Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index 95f55da2e7..7b4d510639 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -146,10 +146,11 @@ def find_translation(source: str, msgctxt: str, msgid: str) -> str: last_source[current_state] += line + "\n" else: #White lines trigger us to process this translation. Is it the correct one? #Process the source and destination keys for comparison independent of newline technique. - source_ctxt = "".join((line.strip()[1:-1].strip() for line in last_source["msgctxt"].split("\n"))) - source_id = "".join((line.strip()[1:-1].strip() for line in last_source["msgid"].split("\n"))) - dest_ctxt = "".join((line.strip()[1:-1].strip() for line in msgctxt.split("\n"))) - dest_id = "".join((line.strip()[1:-1].strip() for line in msgid.split("\n"))) + source_ctxt = "".join((line.strip()[1:-1] for line in last_source["msgctxt"].split("\n"))) + source_id = "".join((line.strip()[1:-1] for line in last_source["msgid"].split("\n"))) + dest_ctxt = "".join((line.strip()[1:-1] for line in msgctxt.split("\n"))) + dest_id = "".join((line.strip()[1:-1] for line in msgid.split("\n"))) + if source_ctxt == dest_ctxt and source_id == dest_id: if last_source["msgstr"] == "\"\"\n": print("!!! Empty translation for {" + dest_ctxt + "}", dest_id, "!!!") From b1cbaaef72e6f68f539872a849d76626a8a3193b Mon Sep 17 00:00:00 2001 From: Ghostkeeper Date: Fri, 26 Jul 2019 15:53:42 +0200 Subject: [PATCH 9/9] Fix warning message appearing for plural forms There is no msgstr then so it thinks it's empty. Contributes to issue CURA-6663. --- scripts/lionbridge_import.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/lionbridge_import.py b/scripts/lionbridge_import.py index 7b4d510639..07c5d5b66a 100644 --- a/scripts/lionbridge_import.py +++ b/scripts/lionbridge_import.py @@ -124,7 +124,8 @@ def find_translation(source: str, msgctxt: str, msgid: str) -> str: last_source = { "msgctxt": "\"\"\n", "msgid": "\"\"\n", - "msgstr": "\"\"\n" + "msgstr": "\"\"\n", + "msgid_plural": "\"\"\n" } current_state = "none" @@ -141,6 +142,10 @@ def find_translation(source: str, msgctxt: str, msgid: str) -> str: current_state = "msgstr" line = line[7:] last_source[current_state] = "" + elif line.startswith("msgid_plural \""): + current_state = "msgid_plural" + line = line[13:] + last_source[current_state] = "" if line.startswith("\"") and line.endswith("\""): last_source[current_state] += line + "\n" @@ -152,10 +157,17 @@ def find_translation(source: str, msgctxt: str, msgid: str) -> str: dest_id = "".join((line.strip()[1:-1] for line in msgid.split("\n"))) if source_ctxt == dest_ctxt and source_id == dest_id: - if last_source["msgstr"] == "\"\"\n": + if last_source["msgstr"] == "\"\"\n" and last_source["msgid_plural"] == "\"\"\n": print("!!! Empty translation for {" + dest_ctxt + "}", dest_id, "!!!") return last_source["msgstr"] + last_source = { + "msgctxt": "\"\"\n", + "msgid": "\"\"\n", + "msgstr": "\"\"\n", + "msgid_plural": "\"\"\n" + } + #Still here? Then the entire msgctxt+msgid combination was not found at all. print("!!! Missing translation for {" + msgctxt.strip() + "}", msgid.strip(), "!!!") return "\"\"\n"