count($fa), 'bCount' => count($fb), 'diff' => count($fa) - count($fb), ); unused var */ $missing = array(); $notrans = array(); $ka = array_keys($fa); $kb = array_keys($fb); $missing = array_diff($ka, $kb); $extra = array_diff($kb, $ka); // search for untranslated strings foreach ($fa as $k => $v) { if (array_key_exists($k, $fb)) { if ($v == $fb[$k] || '' == $fb[$k]) { $notrans[] = $k; } } } return array( 'a_name' => $a, 'b_name' => $b, 'a' => count($fa), 'b' => count($fb), 'missing' => $missing, 'notrans' => $notrans, 'extra' => $extra, 'dup_str' => $duplicates, ); } /** * Diff pot and po files, to get: * - source (pot) strings count * - missing strings in target * - untranslated strings in target * - empty array for extra and duplicate strings for backward compatibility * * @param string $locale locale name ('sl') * @param string $resource file name ('about/license') * @param array $source_l array with source file strings (to avoid duplicated parsing) * @param string $path directly passed path for nonlocal files * @param array $compared array with strings for comparing * * @return array */ function _po_diff($locale, $resource, $source_l = NULL, $path = NULL, $compared = NULL) { if (is_null($path)) { if (NULL == $source_l) { $source_l = read_translation_file('en', $resource); if (FALSE == $source_l) { $source_l = array(array()); } } $target_l = read_translation_file($locale, $resource); } else { if (NULL == $source_l) { $source_path_filename = sprintf('%s/%s.pot', $path, $resource); $source_l = phpmo_parse_po_file($source_path_filename); if (FALSE == $source_l) { $source_l = array(array()); } } if ('en' == $locale) { $target_l = $source_l; } else { $locale = locale_hyphen_underscore($locale, true); $target_path_filename = sprintf('%s/%s.po', $path, $locale); $target_l = phpmo_parse_po_file($target_path_filename); if (FALSE == $target_l) { $target_l = array(array()); } } } unset($source_l['']); // filter out header unset($target_l['']); // filter out header // process $source_l, $target_l and $compared $num_of_original_strings = 0; $fuzzy_or_missing = array(); $untrans = array(); $differences = array(); foreach ($source_l as $msgid => $subarray) { foreach ($subarray as $context_or_num => $msgstr_subarray) { $num_of_original_strings++; // Note: count plural msgid strings also as one if (array_key_exists('msgid_plural', $msgstr_subarray)) { $msgid_plural = $msgstr_subarray['msgid_plural']; } else { $msgid_plural = NULL; } // process $target_l translation if (!isset($target_l[$msgid])) { // adding fuzzy or missing string $fuzzy_or_missing[] = $msgid; $msgstr_target_l[0] = ''; } else { $msgstr_target_l = $target_l[$msgid][$context_or_num]; // are there any plurals untranslated? $untranslated_plural_target_l = FALSE; if (!is_null($msgid_plural)) { foreach ($msgstr_target_l as $nplural) { if (empty($nplural)) { $untranslated_plural_target_l = TRUE; } } } // untranslated string if ($untranslated_plural_target_l || empty($msgstr_target_l[0])) { $untrans[] = $msgid; // adding untranslated string } } // process $compared translation if (!is_null($compared)) { unset($compared['']); // filter out header // if there is a translation for this string in $compared if (isset($compared[$msgid][$context_or_num])) { $msgstr_compared = $compared[$msgid][$context_or_num]; } else { $msgstr_compared[0] = ''; } if ($msgstr_target_l != $msgstr_compared) { $differences[$msgid][$context_or_num]['target_l'] = $msgstr_target_l; $differences[$msgid][$context_or_num]['compared'] = $msgstr_compared; } } } } return array( 'a' => $num_of_original_strings, // # of original strings // 'b' => count($po_strings), // # of target strings 'source_strings' => $source_l, // array of original strings 'fuzzy_or_missing' => $fuzzy_or_missing, // array of fuzzy or missing strings 'notrans' => $untrans, // array of untranslated strings 'differences' => $differences, // array of different strings from two translations 'extra' => array(), 'dup_str' => array(), ); } /** * Diff English and translated ts files, to get: * - source (ts) strings count * - missing strings in target * - untranslated strings in target * - empty array for extra and duplicate strings for backward compatibility * * @param string $locale locale name ('sl') * @param string $resource file name ('mageiaSync') * @param array $source_l array with source file strings (to avoid duplicated parsing) * @param string $path directly passed path for nonlocal files * @param array $compared array with strings for comparing * * @return array */ function _ts_diff($locale, $resource, $source_l = NULL, $path = NULL, $compared = NULL) { $source_path_filename = sprintf('%s%s_%s.ts', $path, $resource, $locale); // mageiaSync_sl.ts $source_strings = array(); $untranslated_strings = array(); $obsoleted_strings = array(); $num_of_original_str = 0; // read .ts file $file_handle = @fopen($source_path_filename, 'r'); if ($file_handle === false) { // Could not open file resource return false; } // iterate over lines while(($line = fgets($file_handle, 65536)) !== false) { // store context name if (false !== strpos($line, '')) { preg_match_all("/()(.+)(<\/name>)/", $line, $context_name); } // store source lines else if (false !== strpos($line, '')) { preg_match_all("/()(.+)(<\/source>)/", $line, $source_string); $num_of_original_str++; } // store translations else if (false !== strpos($line, '')) { preg_match_all("/()(.+)(<\/translation>)/", $line, $translation_string); $source_strings[$source_string[2][0]][$context_name[2][0]] = $translation_string[2]; } // store untranslated string else if (false !== strpos($line, ' $num_of_original_str - count($obsoleted_strings), // # of original strings // 'b' => , // # of target strings 'source_strings' => $source_strings, // array of original strings 'fuzzy_or_missing' => array(), // array of fuzzy or missing strings 'notrans' => $untranslated_strings, // array of untranslated strings 'differences' => $differences, // array of different strings from two translations 'extra' => $obsoleted_strings, 'dup_str' => array(), ); } /*function _lang_diff_stats($a, $b) { $diff = _lang_diff($a, $b); $diff['missing'] = count($diff['missing']); $diff['notrans'] = count($diff['notrans']); $diff['extra'] = count($diff['extra']); $diff['ok'] = (($diff['b'] - $diff['a']) == 0) ? true : false; $diff['correct'] = $diff['b'] - $diff['notrans'] - $diff['missing']; return $diff; } /**/ if ( ! function_exists('glob_recursive')) { // Does not support flag GLOB_BRACE function glob_recursive($pattern, $flags = 0) { $files = glob($pattern, $flags); // removing dirs from $files as they are not files ;) $files_wo_dirs = array(); foreach ($files as $single_file) { $single_file_as_string = str_split($single_file); $last_sign = array_pop($single_file_as_string); if($last_sign != '/') { $files_wo_dirs[] = $single_file; }; } $files = $files_wo_dirs; foreach (glob(dirname($pattern).'/*', GLOB_ONLYDIR|GLOB_NOSORT) as $dir) { $files = array_merge($files, glob_recursive($dir.'/'.basename($pattern), $flags)); } return $files; } } /** * Create 'sl/about/license.sl.lang' * from 'en/about/license.en.lang' * or ../_nav/langs/sl.lang * from ../_nav/langs/en.lang * * @param string $s file name with path * @param string $l locale name * * @return string */ function _lang_file_switch($s, $l) { $s = str_replace('en.lang', $l . '.lang', $s); return str_replace('en/', $l . '/', $s); } /** * Create 'sl/about/license.po' * from 'en/about/license.pot' * or ../_nav/langs/sl.po * from ../_nav/langs/en.pot * * @param string $s file name with path * @param string $l locale name * * @return string */ function _po_file_switch($s, $l) { if($l != 'en') { $s = str_replace('.pot', '.po', $s); } $s = str_replace('/en.', '/' . $l . '.', $s); return str_replace('en/', $l . '/', $s); } /** * Create 'about/license' * from 'en/about/license.pot' or 'en/about/license.en.lang' * * @param string $source_file file name with path * @param string $extension file extension to remove * * @return string */ function _extract_resource($source_file, $extension = '.pot') { $resource = str_replace($extension, '', $source_file); return str_replace('en/', '', $resource); } function get_lang_references($pattern = '*') { return glob_recursive('en/' . $pattern, GLOB_MARK); } function get_other_langs() { $ls = glob('*'); $re = array(); foreach ($ls as $l) { if (!is_dir($l)) continue; if ($l == 'en') continue; $re[] = $l; } array_unshift($re, 'en'); return $re; } function aproximate_number_of_untranslated_constitution_lines($app_root, $lang, $unique_lines_in_eng_constitution = array()) { $constitution_readable = FALSE; $dest_constitution = sprintf('%s/%s/%s/%s_%s.md', $app_root, $lang, 'about/constitution', 'mageia.org_statutes', $lang); $number_of_unique_lines_in_eng_constitution = count($unique_lines_in_eng_constitution); $aproximate_number_of_untranslated_lines = 0; if(is_readable($dest_constitution)) { $unique_lines_in_constitution = array_unique(file($dest_constitution)); $number_of_unique_lines_in_constitution = count($unique_lines_in_constitution); $constitution_readable = TRUE; if ($lang == 'en') { $aproximate_number_of_untranslated_lines = $number_of_unique_lines_in_constitution; $untranslated_lines_in_constitution = array(); } else { $untranslated_lines_in_constitution = array_intersect($unique_lines_in_eng_constitution, $unique_lines_in_constitution); $number_of_nonunique_lines_lang_constitution = count($untranslated_lines_in_constitution); $ratio = $number_of_nonunique_lines_lang_constitution / $number_of_unique_lines_in_eng_constitution; $limit_ratio = 0.15; // limit ratio of "allowed" untranslated lines if ($ratio > $limit_ratio) { // add aproximate number of untranslated constitution lines $aproximate_number_of_untranslated_lines = $number_of_nonunique_lines_lang_constitution - round($limit_ratio * $number_of_unique_lines_in_eng_constitution); } } } else { $unique_lines_in_constitution = $unique_lines_in_eng_constitution; $aproximate_number_of_untranslated_lines = $number_of_unique_lines_in_eng_constitution; $untranslated_lines_in_constitution = $unique_lines_in_eng_constitution; } return array( 'unique_lines_in_constitution' => $unique_lines_in_constitution, 'constitution_readable' => $constitution_readable, 'untranslated_lines_in_constitution' => $untranslated_lines_in_constitution, 'aproximate_number_of_untranslated_lines' => $aproximate_number_of_untranslated_lines, ); } function build_language_and_resource_summary($report, $all_languages_only_one_resource = FALSE, $one_resource = NULL, $one_language_all_resources = FALSE, $one_language = NULL) { $total_num_of_strings = 0; // total of all source strings $language_summary = array(); $resource_summary = array(); foreach ($report as $resource_data) { if ($resource_data['language'] == 'en') { $total_num_of_strings += $resource_data['num_of_all_strings']; } // don't add if there is a need to store languages only for one resource if(!$all_languages_only_one_resource || $resource_data['resource_filename'] == $one_resource) { $key_exists = recursive_array_search($resource_data['language'], $language_summary); // is language already in the $language_summary array? if($resource_data['resource_filename'] == $one_resource) { $temp_var[0]['language'] = $resource_data['language']; $temp_var[0]['num_of_all_strings'] = $resource_data['num_of_all_strings']; $temp_var[0]['num_of_fuzzy_or_missing_strings'] = $resource_data['num_of_fuzzy_or_missing_strings']; $temp_var[0]['num_of_untranslated_strings'] = $resource_data['num_of_untranslated_strings']; $temp_var[0]['references'] = $resource_data['references']; $language_summary[] = $temp_var[0]; unset($temp_var[0]); // clear var } else { if ($key_exists !== FALSE) { $language_summary[$key_exists]['num_of_all_strings'] += $resource_data['num_of_all_strings']; $language_summary[$key_exists]['num_of_fuzzy_or_missing_strings'] += $resource_data['num_of_fuzzy_or_missing_strings']; $language_summary[$key_exists]['num_of_untranslated_strings'] += $resource_data['num_of_untranslated_strings']; } else { if($key_exists === FALSE) { $key_exists = count($language_summary); } $language_summary[$key_exists]['language'] = $resource_data['language']; $language_summary[$key_exists]['num_of_all_strings'] = $resource_data['num_of_all_strings']; $language_summary[$key_exists]['num_of_fuzzy_or_missing_strings'] = $resource_data['num_of_fuzzy_or_missing_strings']; $language_summary[$key_exists]['num_of_untranslated_strings'] = $resource_data['num_of_untranslated_strings']; $language_summary[$key_exists]['references'] = $resource_data['references']; } } } // don't add if there is a need to store resources only for one language if(!$one_language_all_resources || ($resource_data['language'] == $one_language || $resource_data['language'] == 'en')) { $key_exists = recursive_array_search($resource_data['resource_filename'], $resource_summary); // is resource already in the $resource_summary array? if($resource_data['language'] == 'en') { $temp_var[0]['resource_filename'] = $resource_data['resource_filename']; $temp_var[0]['num_of_all_strings'] = $resource_data['num_of_all_strings']; $temp_var[0]['num_of_fuzzy_or_missing_strings'] = $resource_data['num_of_fuzzy_or_missing_strings']; $temp_var[0]['num_of_untranslated_strings'] = $resource_data['num_of_untranslated_strings']; $temp_var[0]['references'] = $resource_data['references']; $resource_summary[] = $temp_var[0]; unset($temp_var[0]); // clear var } else { if($one_language_all_resources) { if($key_exists === FALSE) { $key_exists = count($resource_summary); } $resource_summary[$key_exists]['num_of_fuzzy_or_missing_strings'] = $resource_data['num_of_fuzzy_or_missing_strings']; $resource_summary[$key_exists]['num_of_untranslated_strings'] = $resource_data['num_of_untranslated_strings']; $resource_summary[$key_exists]['references'] = $resource_data['references']; } else if ($key_exists !== FALSE) { $resource_summary[$key_exists]['num_of_all_strings'] += $resource_data['num_of_all_strings']; $resource_summary[$key_exists]['num_of_fuzzy_or_missing_strings'] += $resource_data['num_of_fuzzy_or_missing_strings']; $resource_summary[$key_exists]['num_of_untranslated_strings'] += $resource_data['num_of_untranslated_strings']; } } } } foreach ($language_summary as &$single_language_summary) { $single_language_summary['num_of_translated_strings'] = $single_language_summary['num_of_all_strings'] - $single_language_summary['num_of_fuzzy_or_missing_strings'] - $single_language_summary['num_of_untranslated_strings']; } unset($single_language_summary); // foreach by reference foreach ($resource_summary as &$single_resource_summary) { $single_resource_summary['num_of_translated_strings'] = $single_resource_summary['num_of_all_strings'] - $single_resource_summary['num_of_fuzzy_or_missing_strings'] - $single_resource_summary['num_of_untranslated_strings']; } unset($single_resource_summary); // foreach by reference return array( 'total_num_of_strings' => $total_num_of_strings, // total of all source strings 'language_summary' => $language_summary, 'resource_summary' => $resource_summary, ); } /** * Transifex API implementation in php detailed on http://docs.transifex.com/developer/api/projects * * Returns $tx_result_array from Transifex request * * @param string $tx_request request * @param string $project default project * * @return array */ function tx_call($tx_request, $project = 'project/mageia/') { global $errors; $tx_url_prefix = "www.transifex.com/api/2/"; $tx_url = $tx_url_prefix . $project . $tx_request; $user = "filip_mageia"; $pass = "report"; $tx_result = @file_get_contents("https://$user:$pass@$tx_url"); $tx_result_array = json_decode($tx_result, TRUE); $json_last_error = json_last_error(); if (JSON_ERROR_NONE !== $json_last_error) { $error = "There was an error during API call to Transifex $tx_request ($json_last_error)."; $errors['tx_json_error'] = "$error Please reload the page later and report this on mailing list if it persist."; $tx_result_array = array(); } else if (FALSE === $tx_result) { $error = "API call to Transifex $tx_request failed."; $errors['tx_call'] = "$error Please reload the page later and report this on mailing list if it persist."; $tx_result_array = array(); } return $tx_result_array; } /** * Convert git resource name to Transifex one or reverse * * @param string $resource_name like 'about/constitution' (git name) * @param string $category like 'Webpages', 'Cauldron' or 'Documentation' * @param boolean $tx_to_git_name_conversion direction of conversion * * @return string $resource_name like 'page-about-constitution' (tx name) */ function resource_name_conversion($resource_name, $category = '') { if ('Webpages' == $category) { $tx_names = array('nav', '-'); $git_names = array('_nav/langs/en', '/'); $resource_name = 'page-' . str_replace($git_names, $tx_names, $resource_name); } else if ('Cauldron' == $category) { $tx_names = array('mageia-welcome', 'identity-catdap'); $git_names = array('Mageia%20Welcome', 'Identity%20(CatDap)'); $resource_name = str_replace($git_names, $tx_names, $resource_name); } return $resource_name; } /** * Build Transifex and git links with numbers of untranslated strings * * Returns text string * * @param string $git_resource_name like 'about/constitution' * @param string $language_code like 'sl' * @param string $resource_type like 'Webpages' * @param array $stat_data with numbers of untranslated strings * * @return string */ function build_links($git_resource_name, $language_code, $resource_type, $stat_data) { $tx_resource_name = resource_name_conversion($git_resource_name); $locale_name = locale_underscore_to_hyphen($language_code); if ('Webpages' == $resource_type) { if ('nav' == $git_resource_name) { $git_link = sprintf('http://gitweb.mageia.org/web/nav/tree/langs/%s.po', $locale_name); } else { $git_link = sprintf('http://gitweb.mageia.org/web/www/tree/langs/%s/%s.po', $locale_name, $git_resource_name); } } else { // fixing exceptions as there is a different naming convention between Tx and git for some lanuages: if ('system-config-printer' == $git_resource_name && 'sr@latin' == $language_code) { $lang_code = 'sr@latin'; } else if ('identity-catdap' == $git_resource_name && 'en_GB' == $language_code) { $lang_code = 'en_gb'; } else { $tx_array = array('sr@latin', 'uz@Cyrl'); $git_array = array('sr@Latn', 'uz@cyrillic'); $lang_code = str_replace($tx_array, $git_array, $language_code); } // treat TS files differently if (false !== strpos($stat_data['pot_name'], '_en.ts')) { $partial_git_resource_name = substr($stat_data['pot_name'], 0, -6); // cuts '_en.ts' $git_link = sprintf('%s%s_%s.ts', $stat_data['git_path'], $partial_git_resource_name, $lang_code); // mageiaSync_sl.ts } else { $git_link = sprintf('%s/%s.po', $stat_data['git_path'], $lang_code); } } $links_and_num = build_transifex_link($language_code, 'Tx', $resource_type, $tx_resource_name) . ": "; $links_and_num .= $stat_data['tx_untran']; $links_and_num .= (0 == $stat_data['tx_untran'] ? ' - full' : ''); $links_and_num .= ', git: '; $links_and_num .= $stat_data['git_untran']; $links_and_num .= ($stat_data['num_of_all'] == $stat_data['git_untran'] ? ' - empty' : ''); return $links_and_num; } /** * Build customized Transifex link * * @param string $tx_language_code like 'pt_BR' * @param string $link_name is the visible link name * @param string $tx_category like 'Webpages' * @param string $tx_resource_name like 'page-4' * * @return string customized Transifex link */ function build_transifex_link($tx_language_code, $link_name = NULL, $tx_category = NULL, $tx_resource_name = NULL) { // Transifex cripled direct link access in a new version in a way that limiting // the result by the category or resource name is no longer possible // $prefix = ""; if(is_null($link_name)) { $transifex_url .= $tx_language_code; } else { $transifex_url .= $link_name; } $transifex_url .= ""; return $transifex_url; } /** * Generating report about git resources statistics * * @param array $language_codes list * @param array $resource_names list * @param string $path to the git, otherwise NULL * * @return array */ function generating_report($language_codes, $resource_names, $path = NULL, $pot_name = NULL, $compared = NULL) { $report = array(); foreach ($resource_names as $f) { $source_strings = NULL; foreach ($language_codes as $l) { $references = ''; if (is_null($pot_name)) { $resource = _extract_resource($f); } else { $resource = $pot_name; } $langF = _po_file_switch($f, $l); if (strstr($f, '../_nav/langs/en.') !== FALSE) { $langF = '../_nav/langs/' . $l . '.po' . (($l == 'en') ? 't' : ''); } if (!is_null($path) || file_exists($langF)) { // treat TS files differently if (false !== strpos($pot_name, '_en.ts')) { $partial_pot_name = substr($pot_name, 0, -6); // cuts '_en.ts' from mageiaSync_en.ts $stat = _ts_diff($l, $partial_pot_name, $source_strings, $path); } else { $stat = _po_diff($l, $resource, $source_strings, $path, $compared); } $num_of_fuzzy_or_missing = count($stat['fuzzy_or_missing']); } else { // file $langF doesn't exists in 'Webpages' $resource_type $stat = _po_diff('en', $resource, $source_strings); $num_of_fuzzy_or_missing = 0; } $num_of_untranslated = count($stat['notrans']); $source_strings = $stat['source_strings']; // unify resource names, navigation is a special exception $resource_name = str_replace(array('../_nav/langs/en', 'en/', '.pot'), array('nav', '', ''), $f); // fixing exceptions as there is a different naming convention between Tx and git for some lanuages: if ('system-config-printer' == $resource_name && 'sr@latin' == $l) { $web_language_code = 'sr@latin'; } else if ('identity-catdap' == $resource_name && 'en_gb' == $l) { $web_language_code = 'en_GB'; } else { $git_array = array('sr@Latn', 'uz@cyrillic'); $tx_array = array('sr@latin', 'uz@Cyrl'); // create pt_BR from pt-br and alike to unify languages $web_language_code = locale_hyphen_underscore($l, true); $web_language_code = str_replace($git_array, $tx_array, $web_language_code); } $num_of_not_fully_trans = $num_of_fuzzy_or_missing + $num_of_untranslated; $report[] = array( 'num_of_all_strings' => $stat['a'], // 'fuzzy_or_missing_str' => $stat['fuzzy_or_missing'], // 'untranslated_strings' => $stat['notrans'], // 'source_strings' => $stat['source_strings'], 'differences' => $stat['differences'], 'resource_name' => $resource_name, 'web_language_code' => $web_language_code, 'num_of_not_fully_trans' => $num_of_not_fully_trans, 'webgit_path' => $path, 'pot_name' => $pot_name, ); } } return $report; } /** * Returns native language name from Mageia web site if exists otherwise English name from Transifex * * @param string $language_code for language * * @return string */ function get_language_name($language_code) { global $langs; static $tx_languages_details = NULL; $web_language_code = locale_underscore_to_hyphen($language_code); if (array_key_exists($web_language_code, $langs)) { $language_name = $langs[$web_language_code]; } else { if (is_null($tx_languages_details)) { $tx_languages_details = tx_call("languages", ''); } // is language code in the $tx_languages_details array? $key_exists = recursive_array_search($language_code, $tx_languages_details); if ($key_exists !== FALSE) { $language_name = $tx_languages_details[$key_exists]['name']; } else { $language_name = $language_code; } } return $language_name; } /** * from http://www.php.net/manual/en/function.array-search.php#91365 * * copyright (c) the PHP Documentation * covered by the Creative Commons Attribution 3.0 License (http://creativecommons.org/licenses/by/3.0/legalcode) */ function recursive_array_search($needle, $haystack) { foreach ($haystack as $key => $value) { $current_key = $key; if ($needle === $value OR (is_array($value) && recursive_array_search($needle, $value) !== FALSE)) { return $current_key; } } return FALSE; }