[translation] Make smarter, find ::translateFile calls etc.

5 files changed, 126 insertions, 32 deletions

diff --git a/modules-available/translation/lang/de/messages.json b/modules-available/translation/lang/de/messages.json
index e95c7750..d0e678fe 100644
--- a/modules-available/translation/lang/de/messages.json
+++ b/modules-available/translation/lang/de/messages.json
@@ -4,7 +4,6 @@
     "invalid-custom-handler": "Ung\u00fcltiger \"custom handler\" {{0}}",
     "invalid-section": "Ung\u00fcltige Auswahl",
     "invalid-template": "Ausgew\u00e4hlte Template ist nicht g\u00fcltig",
-    "no-custom-handlers": "Modul hat keine \"custom handlers\"",
     "no-module-given": "Kein Modul angegeben",
     "updated-tags": "Tags wurden aktualisiert"
 }
\ No newline at end of file
diff --git a/modules-available/translation/lang/de/template-tags.json b/modules-available/translation/lang/de/template-tags.json
index cb10d3b4..dd1b771d 100644
--- a/modules-available/translation/lang/de/template-tags.json
+++ b/modules-available/translation/lang/de/template-tags.json
@@ -1,6 +1,6 @@
 {
+    "lang_clear": "L\u00e4utern",
     "lang_createTag": "Tag erstellen",
-    "lang_clear": "Läutern",
     "lang_global": "Global",
     "lang_globalTooltip": "Dieser Tag ist global verf\u00fcgbar und braucht normalerweise nicht explizit f\u00fcr dieses Modul \u00fcbersetzt zu werden",
     "lang_menuCategories": "Men\u00fckategorien",
@@ -14,7 +14,6 @@
     "lang_status": "Status",
     "lang_tag": "Tag",
     "lang_tags": "Tags",
-    "lang_templateAdminHelp": "Hier k\u00f6nnen Sie die verwendeten Texte und S\u00e4tze \u00fcbersetzen.",
     "lang_templateHint": "Hinweis: Gelbe Linien zeigen an, dass eine \u00dcbersetzung fehlt, rote Linien, dass ein Tag nicht von dem jeweiligen Template verwendet wird.",
     "lang_templates": "Templates",
     "lang_translation": "\u00dcbersetzung",
diff --git a/modules-available/translation/lang/en/messages.json b/modules-available/translation/lang/en/messages.json
index b2389fee..a86bbe8f 100644
--- a/modules-available/translation/lang/en/messages.json
+++ b/modules-available/translation/lang/en/messages.json
@@ -4,7 +4,6 @@
     "invalid-custom-handler": "Invalid custom handler {{0}}",
     "invalid-section": "Invalid selection",
     "invalid-template": "Selected template is not valid",
-    "no-custom-handlers": "Module has no custom handlers",
     "no-module-given": "No module given",
     "updated-tags": "Tags have been updated"
 }
\ No newline at end of file
diff --git a/modules-available/translation/lang/en/template-tags.json b/modules-available/translation/lang/en/template-tags.json
index c0c37cb5..6fca2624 100644
--- a/modules-available/translation/lang/en/template-tags.json
+++ b/modules-available/translation/lang/en/template-tags.json
@@ -1,6 +1,6 @@
 {
-    "lang_createTag": "Create Tag",
     "lang_clear": "Clear",
+    "lang_createTag": "Create Tag",
     "lang_global": "Global",
     "lang_globalTooltip": "This tag is global; usually there is no need to translate it explicitly for this module",
     "lang_menuCategories": "Menu categories",
@@ -14,7 +14,6 @@
     "lang_status": "Status",
     "lang_tag": "Tag",
     "lang_tags": "Tags",
-    "lang_templateAdminHelp": "Here you can translate and edit phrases and texts.",
     "lang_templateHint": "Hint: Yellow lines indicate a translation is missing and red lines indicate a tag is not being used by the template.",
     "lang_templates": "Templates",
     "lang_translation": "Translation",
diff --git a/modules-available/translation/page.inc.php b/modules-available/translation/page.inc.php
index 4563b8e5..5f915782 100644
--- a/modules-available/translation/page.inc.php
+++ b/modules-available/translation/page.inc.php
@@ -211,7 +211,7 @@ class Page_Translation extends Page
 		$modules = Module::getAll();
 		
 		foreach ($modules as $module) {
-			$msgs = $this->checkModuleTranslation($module);
+			$msgs = $this->buildStatusStringForOverview($module);
 			$table[] = array(
 				'module' => $module->getIdentifier(),
 				'depfail' => $module->hasMissingDependencies(),
@@ -241,6 +241,8 @@ class Page_Translation extends Page
 		$this->showModuleStrings();
 		// Menu categories
 		$this->showModuleMenuCategories();
+		// Dict::translateFile calls
+		$this->getModuleOtherFiles();
 		// Module specific
 		$this->showModuleCustom();
 		Render::closeTag('div');
@@ -321,7 +323,7 @@ class Page_Translation extends Page
 		$data = array('module' => $this->module->getIdentifier());
 		$data['tagcount'] = count($moduleTags);
 		foreach (Dictionary::getLanguages(true) as $lang) {
-			list($missing, $unused) = $this->getModuleTranslationStatus($lang['cc'], 'menucategory', true, $moduleTags);
+			list($missing, $unused) = $this->getModuleTranslationStatus($lang['cc'], 'categories', true, $moduleTags);
 			$data['langs'][] = array(
 				'cc' => $lang['cc'],
 				'name' => $lang['name'],
@@ -467,7 +469,7 @@ class Page_Translation extends Page
 			$allFiles = array_merge($allFiles, $this->getAllFiles('apis', '.php'), $this->getAllFiles('inc', '.php'));
 			$allFiles[] = 'index.php';
 		}
-		$full = $this->loadTagsFromPhp('/Message\s*::\s*add\w+\s*\(\s*[\'"](?<tag>[^\'"\.]*\.[^\'"]*)[\'"]\s*(?<data>\)|\,.*)/i',
+		$full = $this->loadTagsFromPhp(['/Message\s*::\s*add\w+\s*\(\s*[\'"](?<tag>[^\'"\.]*\.[^\'"]*)[\'"]\s*(?<data>\)|\,.*)/i'],
 			$allFiles);
 		$tags = [];
 		// Filter out tags that don't refer to this module
@@ -480,7 +482,7 @@ class Page_Translation extends Page
 				$tags[$p[1]] = $tag;
 			}
 		}
-		return $this->loadTagsFromPhp('/Message\s*::\s*add\w+\s*\(\s*[\'"](?<tag>[^\'"\.]*)[\'"]\s*(?<data>\)|\,.*)/i',
+		return $this->loadTagsFromPhp(['/Message\s*::\s*add\w+\s*\(\s*[\'"](?<tag>[^\'"\.]*)[\'"]\s*(?<data>\)|\,.*)/i'],
 			$this->getModulePhpFiles($module), $tags);
 	}
 
@@ -494,14 +496,19 @@ class Page_Translation extends Page
 		if ($module === null) {
 			$module = $this->module;
 		}
-		$tags = $this->loadTagsFromPhp('/Dictionary\s*::\s*translate\s*\(\s*[\'"](?<tag>[^\'"\.]*)[\'"]\s*[\),]/i',
+		$emod = preg_quote($module->getIdentifier(), '/');
+		$tags = $this->loadTagsFromPhp([
+			'/Dictionary\s*::\s*translate\s*\(\s*[\'"](?<tag>[^\'"\.]*)[\'"]\s*[\),]/i',
+			'/Dictionary\s*::\s*translateFile\s*\(\s*[\'"]module[\'"]\s*,\s*[\'"](?<tag>[^\'"\.]*)[\'"]\s*[\),]/i',
+			'/Dictionary\s*::\s*translateFileModule\s*\(\s*[\'"]'.$emod.'[\'"]\s*,\s*[\'"]module[\'"]\s*,\s*[\'"](?<tag>[^\'"\.]*)[\'"]\s*[\),]/i',
+		],
 			$this->getModulePhpFiles($module));
 		foreach ($tags as &$tag) {
 			$tag = true;
 		}
 		unset($tag);
 		// Fixup special tags
-		if ($module->getCategory() === null) {
+		if (!file_exists('modules/' . $module->getIdentifier() . '/page.inc.php')) {
 			unset($tags['module_name']);
 			unset($tags['page_title']);
 		} else {
@@ -533,13 +540,21 @@ class Page_Translation extends Page
 	 * the value of each entry can be false if the tag is optional.
 	 *
 	 * @param string $subsection Name of subsection
-	 * @return array|false List of tags as KEYS of array
+	 * @return ?array List of tags as KEYS of array
 	 */
-	private function loadUsedCustomTags(string $subsection)
+	private function loadUsedCustomTags(string $subsection): ?array
 	{
-		if (!isset($this->customHandler['grep_'.$subsection]))
-			return false;
-		return $this->customHandler['grep_'.$subsection]($this->module);
+		if (isset($this->customHandler['grep_'.$subsection])) {
+			return $this->customHandler['grep_' . $subsection]($this->module);
+		}
+		$byFile = $this->getDictTranslateFileArray($this->module);
+		if (isset($byFile[$subsection])) {
+			foreach ($byFile[$subsection] as &$tag) {
+				$tag = true;
+			}
+			return $byFile[$subsection];
+		}
+		return null;
 	}
 	
 	private function getTagsFromTemplate(string $templateFile)
@@ -564,12 +579,83 @@ class Page_Translation extends Page
 	 * @param string $lang lang to use
 	 * @param array $tags Array of tags, where the tag names are the keys
 	 * @param Module $module the module to work with, defaults to the currently edited module
-	 * @return array [missingCount, unusedCount]
+	 * @return array{0: int, 1 :int} (missing, unused)
 	 */
 	private function getModuleTemplateStatus(string $lang, array $tags, Module $module = null): array
 	{
 		return $this->getModuleTranslationStatus($lang, 'template-tags', true, $tags, $module);
 	}
+
+	private function getDictTranslateFileArray(Module $module): array
+	{
+		$tags = $this->loadTagsFromPhp(
+			['/Dictionary\s*::\s*translateFile\s*\(\s*[\'"](?<json>[^\'".]*)[\'"]\s*,\s*[\'"](?<tag>[^\'".]*)[\'"]\s*[),]/i'],
+			$this->getModulePhpFiles($module));
+		$byFile = [];
+		foreach ($tags as $tag) {
+			if ($tag['json'] === 'messages' || $tag['json'] === 'module' || $tag['json'] === 'permissions' || $tag['json'] === 'template-tags')
+				continue;
+			if (!isset($byFile[$tag['json']])) {
+				$byFile[$tag['json']] = [];
+			}
+			$byFile[$tag['json']][$tag['tag']] = $tag;
+		}
+		return $byFile;
+	}
+
+	/**  @return array{0: int, 1 :int} (missing, unused) */
+	private function getModuleOtherFileStatus(string $lang, Module $module = null, array $byFile = null): array
+	{
+		if ($module === null) {
+			$module = $this->module;
+		}
+		if ($byFile === null) {
+			$byFile = $this->getDictTranslateFileArray($module);
+		}
+		$missing = $unused = 0;
+		foreach ($byFile as $file => $list) {
+			$translation = Dictionary::getArray($module->getIdentifier(), $file, $lang);
+			foreach ($list as $tag) {
+				$tag = $tag['tag'];
+				if (isset($translation[$tag])) {
+					unset($translation[$tag]);
+				} else {
+					$missing++;
+				}
+			}
+			$unused += count($translation);
+		}
+		return [$missing, $unused];
+	}
+
+	/**  @return array{missingCount: int, unusedCount :int} */
+	private function getModuleOtherFiles(): void
+	{
+		$module = $this->module;
+		$this->renderDictFileOneLang($module);
+	}
+
+	private function renderDictFileOneLang(Module $module): void
+	{
+		$byFile = $this->getDictTranslateFileArray($module);
+		foreach ($byFile as $file => $list) {
+			$data = [
+				'subsection' => $file,
+				'module' => $this->module->getIdentifier(),
+				'tagcount' => count($list),
+			];
+			foreach (Dictionary::getLanguages(true) as $lang) {
+				list($missing, $unused) = $this->getModuleOtherFileStatus($lang['cc'], $module, $byFile);
+				$data['langs'][] = array(
+					'cc' => $lang['cc'],
+					'name' => $lang['name'],
+					'missing' => $missing,
+					'unused' => $unused
+				);
+			}
+			Render::addTemplate('custom-list', $data);
+		}
+	}
 	
 	/**
 	 * Get missing and unused counters for given translation unit.
@@ -608,6 +694,7 @@ class Page_Translation extends Page
 					$unused++;
 				}
 			} else {
+				// Only if mandatory, since $expected is also calculated accordingly
 				if ($tags[$key] !== false) {
 					$matches++;
 				}
@@ -618,7 +705,7 @@ class Page_Translation extends Page
 		return array($missing, $unused);
 	}
 
-	private function checkModuleTranslation(Module $module): string
+	private function buildStatusStringForOverview(Module $module): string
 	{
 		$templateTags = $this->loadUsedTemplateTags($module);
 		$messageTags = $this->loadUsedMessageTags($module);
@@ -628,8 +715,9 @@ class Page_Translation extends Page
 			list($m1, $u1) = $this->getModuleTemplateStatus($lang, $templateTags, $module);
 			list($m2, $u2) = $this->getModuleTranslationStatus($lang, 'messages', true, $messageTags, $module);
 			list($m3, $u3) = $this->getModuleTranslationStatus($lang, 'module', true, $moduleTags, $module);
-			$missing = $m1 + $m2 + $m3;
-			$unused = $u1 + $u2 + $u3;
+			list($m4, $u4) = $this->getModuleOtherFileStatus($lang, $module);
+			$missing = $m1 + $m2 + $m3 + $m4;
+			$unused = $u1 + $u2 + $u3 + $u4;
 
 			$msg = "";
 			if ($missing > 0) {
@@ -753,6 +841,10 @@ class Page_Translation extends Page
 	private function loadCustomEditArray(): array
 	{
 		$tags = $this->loadUsedCustomTags($this->subsection);
+		if ($tags === null) {
+			Message::addError('invalid-custom-handler', $this->subsection);
+			$this->redirect(1);
+		}
 		return $this->buildTranslationTable($this->subsection, array_keys($tags), true);
 	}
 	
@@ -823,19 +915,26 @@ class Page_Translation extends Page
 	 *	    fileN => count
 	 * )).
 	 *
-	 * @param string $regexp regular expression
+	 * @param string[] $regexp regular expressions
 	 * @param string[] $files list of files to scan
 	 * @param string[] $tags existing tag array to append to
 	 * @return array of all tags found, where the tag is the key, and the value is as described above
 	 */
-	private function loadTagsFromPhp(string $regexp, array $files, array $tags = []): array
+	private function loadTagsFromPhp(array $regexps, array $files, array $tags = []): array
 	{
 		// Get all php files, so we can find all strings that need to be translated
 		// Now find all tags in all php files. Only works for literal usage, not something like $foo = 'bar'; Dictionary::translate($foo);
 		foreach ($files as $file) {
 			$content = file_get_contents($file);
-			if ($content === false || preg_match_all($regexp, $content, $out, PREG_SET_ORDER) < 1)
+			if ($content === false)
 				continue;
+			$out = [];
+			foreach ($regexps as $regexp) {
+				$tmp = [];
+				if (preg_match_all($regexp, $content, $tmp, PREG_SET_ORDER) < 1)
+					continue;
+				$out = array_merge($out, $tmp);
+			}
 			foreach ($out as $set) {
 				if (!isset($tags[$set['tag']])) {
 					$tags[$set['tag']] = $set;
@@ -953,15 +1052,15 @@ class Page_Translation extends Page
 		}
 		// Custom submodule
 		if ($this->section === 'custom') {
-			if ($this->customHandler === null || !isset($this->customHandler['subsections'])) {
-				Message::addError('no-custom-handlers');
-				$this->redirect(1);
+			if (isset($this->customHandler['subsections']) && in_array($this->subsection, $this->customHandler['subsections'], true)) {
+				return $prefix . '/' . $this->subsection . '.json';
 			}
-			if (!in_array($this->subsection, $this->customHandler['subsections'], true)) {
-				Message::addError('invalid-custom-handler', $this->subsection);
-				$this->redirect(1);
+			$byFile = $this->getDictTranslateFileArray($this->module);
+			if (isset($byFile[$this->subsection])) {
+				return $prefix . '/' . $this->subsection . '.json';
 			}
-			return $prefix . '/' . $this->subsection . '.json';
+			Message::addError('invalid-custom-handler', $this->subsection);
+			$this->redirect(1);
 		}
 		Message::addError('invalid-section', $this->section);
 		$this->redirect(1);
@@ -984,7 +1083,6 @@ class Page_Translation extends Page
 		//find the tag requests to change the file
 		$tags = Request::post('langtag', array(), 'array');
 		foreach ($tags as $tag => $value) {
-			error_log($tag . '=' . $value);
 			$tag = trim($tag);
 			if (empty($tag)) {
 				Message::addWarning('i18n-empty-tag');