From 4688496b1ad4a01ee9173bebdc82493c5c95be0e Mon Sep 17 00:00:00 2001 From: luboslenco Date: Sat, 20 Jan 2024 14:41:53 +0100 Subject: [PATCH] Use js for extract_locales --- README.md | 4 +- base/Tools/extract_locales.js | 64 +++++++++++++++++++++ base/Tools/extract_locales.py | 105 ---------------------------------- 3 files changed, 66 insertions(+), 107 deletions(-) create mode 100644 base/Tools/extract_locales.js delete mode 100644 base/Tools/extract_locales.py diff --git a/README.md b/README.md index 17bfc18d..60380edf 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ git submodule update --init --recursive **Generating a locale file** ```bash -pip install typing_extensions -t base/tools -python ./base/tools/extract_locales.py +export ARM_LOCALE= +./armorcore/Kinc/make --from base/Tools --kfile extract_locales.js # Generates a `base/Assets/locale/.json` file ``` diff --git a/base/Tools/extract_locales.js b/base/Tools/extract_locales.js new file mode 100644 index 00000000..49de0d47 --- /dev/null +++ b/base/Tools/extract_locales.js @@ -0,0 +1,64 @@ +// Extracts localizable strings from a set of source files and writes them to JSON files. +// This script can create new translations or update existing ones. +// Usage: +// `export ARM_LOCALE=` +// `../../armorcore/Kinc/make --kfile extract_locales.js` +// Generates a `base/Assets/locale/.json` file + +const fs = require('fs'); + +if (!process.env.ARM_LOCALE) { + console.log("ARM_LOCALE env variable not set!"); +} + +let locale = process.env.ARM_LOCALE; +let locale_path = "./base/Assets/locale/" + locale + ".json"; + +let out = {}; +let old = {}; +if (fs.existsSync(locale_path)) { + old = JSON.parse(fs.readFileSync(locale_path).toString()); +} + +let source_paths = [ + "base/Sources", "base/Sources/nodes", + "armorpaint/Sources", "armorpaint/Sources/nodes", + "armorlab/Sources", "armorlab/Sources/nodes", + "armorsculpt/Sources", "armorsculpt/Sources/nodes", + "armorforge/Sources", "armorforge/Sources/nodes" +]; + +for (let path of source_paths) { + if (!fs.existsSync(path)) continue; + + let files = fs.readdirSync(path); + for (let file of files) { + if (!file.endsWith(".ts")) continue; + + let data = fs.readFileSync(path + "/" + file).toString(); + let start = 0; + while (true) { + start = data.indexOf('tr("', start); + if (start == -1) break; + start += 4; // tr(" + + let end_a = data.indexOf('")', start); + let end_b = data.indexOf('",', start); + if (end_a == -1) end_a = end_b; + if (end_b == -1) end_b = end_a; + let end = end_a < end_b ? end_a : end_b; + + let val = data.substring(start, end); + val = val.replaceAll("\\n", "\n"); + if (old.hasOwnProperty(val)) { + out[val] = old[val]; + } + else { + out[val] = ""; + } + start = end; + } + } +} + +fs.writeFileSync(locale_path, JSON.stringify(out, Object.keys(out).sort(), 4)); diff --git a/base/Tools/extract_locales.py b/base/Tools/extract_locales.py deleted file mode 100644 index 0cd831ee..00000000 --- a/base/Tools/extract_locales.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -# Extracts localizable strings from a set of source files and writes them to JSON files. -# This script can create new translations or update existing ones. -# Usage: ./extract_locales.py - -import fnmatch -import json -import os -import sys -from typing import Any, Dict, IO, List -from typing_extensions import Final - -unique_str: List[str] = [] - - -def process_file(f: IO[Any], fname: str, template_data: Dict[str, str]) -> None: - line = f.readline() - while line: - patterns = ['tr("'] - idx = 0 - pos = 0 - while pos >= 0: - pos = line.find(patterns[idx], pos) - if pos == -1: - if idx < len(patterns) - 1: - idx += 1 - pos = 0 - continue - pos += len(patterns[idx]) - - msg = "" - while pos < len(line) and (line[pos] != '"' or line[pos - 1] == "\\"): - msg += line[pos] - pos += 1 - - # Only add each unique string once. - if msg not in unique_str: - # Empty keys are considered untranslated by the i18n library. - # Fix newlines so they're not escaped anymore. Otherwise, - # they won't match the source strings. - template_data[msg.replace("\\n", "\n")] = "" - unique_str.append(msg) - - line = f.readline() - - -def main() -> None: - if len(sys.argv) != 2: - sys.exit(f"Usage: {sys.argv[0]} ") - - # Change to the directory where the script is located, - # so that the script can be run from any location. - os.chdir(os.path.dirname(os.path.realpath(__file__)) + "/../../") - - output_path: Final = f"base/Assets/locale/{sys.argv[1]}.json" - - if not os.path.exists(os.path.join("armorpaint", "Sources")): - sys.exit( - "ERROR: Couldn't find the Sources folder in the folder where this script is located." - ) - - if not os.path.exists(os.path.join("armorlab", "Sources")): - sys.exit( - "ERROR: Couldn't find the Sources folder in the folder where this script is located." - ) - - matches: List[str] = [] - for folder in [os.path.join("armorpaint", "Sources"), os.path.join("armarlab", "Sources"), "Libraries", "base"]: - for root, dirnames, filenames in os.walk(folder): - dirnames[:] = [d for d in dirnames] - for filename in fnmatch.filter(filenames, "*.hx"): - matches.append(os.path.join(root, filename)) - matches.sort() - - template_data: Dict[str, str] = {} - for filename in matches: - with open(filename, "r", encoding="utf8") as f: - # Read source files for localizable strings. - process_file(f, filename, template_data) - - if os.path.exists(output_path): - print(f'Updating the translation at "{output_path}"...') - with open(output_path, "r", encoding="utf8") as f: - existing_data = json.loads(f.read()) - # Remove obsolete translations (i.e. translations that are no longer - # present in the generated data). - existing_data_no_obsolete = { - key: value - for key, value in existing_data.items() - if key in template_data - } - # Merge existing data with the generated template data (so we keep - # existing translations). - template_data = {**template_data, **existing_data_no_obsolete} - - with open(output_path, "w", encoding="utf8") as f: - json.dump(template_data, f, ensure_ascii=False, indent=4, sort_keys=True) - else: - print(f'Creating new translation template at "{output_path}"...') - with open(output_path, "w", encoding="utf8") as f: - json.dump(template_data, f, ensure_ascii=False, indent=4, sort_keys=True) - - -if __name__ == "__main__": - main()