From 9ce6d646ff718cd62f6434c9a4c94e450436612a Mon Sep 17 00:00:00 2001 From: Thomas Backlund Date: Sun, 12 Oct 2014 12:27:45 +0300 Subject: Revert "Updated Estonian translation" This reverts commit a0b2d09125d7911656fe4cb286f20643c4432281. --- tools/README | 17 ++++ tools/extract2gettext.php | 139 ++++++++++++++++++++++++++++++++ tools/extract2lang.php | 105 ++++++++++++++++++++++++ tools/lookup_for_second_argument.php | 61 ++++++++++++++ tools/lookup_for_second_argument.sh | 18 +++++ tools/pa2lang.php | 108 +++++++++++++++++++++++++ tools/rebuild_gettext_catalogs.sh | 149 +++++++++++++++++++++++++++++++++++ tools/translated_converter.py | 107 +++++++++++++++++++++++++ tools/update-mirrors-list.php | 8 ++ tools/web_projects.dat | 25 ++++++ 10 files changed, 737 insertions(+) create mode 100644 tools/README create mode 100644 tools/extract2gettext.php create mode 100644 tools/extract2lang.php create mode 100644 tools/lookup_for_second_argument.php create mode 100755 tools/lookup_for_second_argument.sh create mode 100644 tools/pa2lang.php create mode 100755 tools/rebuild_gettext_catalogs.sh create mode 100644 tools/translated_converter.py create mode 100644 tools/update-mirrors-list.php create mode 100644 tools/web_projects.dat (limited to 'tools') diff --git a/tools/README b/tools/README new file mode 100644 index 000000000..9b10f0a8b --- /dev/null +++ b/tools/README @@ -0,0 +1,17 @@ +REBUILDING A GETTEXT DICTIONARIES + +To rebuild a gettext dictionaries and refresing the po files you need to run rebuild_gettext_catalogs.sh script in the root of main web server: + +It cals extract2gettext.php for every translatable resource. It picks the database for that from web_projects.dat file. + +Python script translated_converter.py is used in conversion of lang files to gettext po files. +It comes from http://gitweb.mageia.org/software/i18n/tools/tree/websites where you can find it's license and some other informations. + + +LEGACY UTILITIES +1. There is also useful script lookup_for_second_argument.sh used as a preparation for transition. +It cals lookup_for_second_argument.php for every translatable resource. It picks the database for that from web_projects.dat file. + +2. File extract2lang.php is used for old lang l10n system. + +3. In the transition from php arrays to lang l10n system file pa2lang.php was used. diff --git a/tools/extract2gettext.php b/tools/extract2gettext.php new file mode 100644 index 000000000..73e873333 --- /dev/null +++ b/tools/extract2gettext.php @@ -0,0 +1,139 @@ + $strs) { + $f[] = '# gettext catalog for ' . $domain . ' web page(s)'; + $f[] = '# Copyright (C) 2014 - ' . date('Y') . ' Mageia'; + $f[] = '# This file is distributed under the same license as'; + $f[] = '# the content of the corresponding web page(s).'; + $f[] = '#'; + $cur_date = date('Y-m-d H:i:sO'); + $f[] = '# Generated by extract2gettext.php'; + $f[] = sprintf('# Domain: %s', $domain); + $f[] = '#'; + $f[] = '# include translation strings from:'; + $files = array_unique($files); + foreach ($files as $source) { + $f[] = sprintf('# %s', $source); + } + $f[] = '#'; + $f[] = 'msgid ""'; + $f[] = 'msgstr ""'; + $f[] = '"Project-Id-Version: ' . $domain . '\n"'; + $f[] = '"Report-Msgid-Bugs-To: mageia-i18n@mageia.org\n"'; + $f[] = '"POT-Creation-Date: ' . $cur_date . '\n"'; + $f[] = '"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"'; + $f[] = '"Last-Translator: FULL NAME \n"'; + $f[] = '"Language-Team: LANGUAGE \n"'; + $f[] = '"Language: \n"'; + $f[] = '"MIME-Version: 1.0\n"'; + $f[] = '"Content-Type: text/plain; charset=UTF-8\n"'; // CHARSET + $f[] = '"Content-Transfer-Encoding: 8bit\n"'; + $f[] = ''; + + foreach ($strs as $str => $info) { + $str = str_replace(array("\'", '"'), array("'", '\"'), $str); + $f[] = sprintf('#: "%s"', $info[0]); // location of string in the source (#: reference...) + $f[] = 'msgid "' . $str . '"'; + $f[] = 'msgstr ""'; + $f[] = ''; + } + $f = implode("\n", $f); + if ($domain == 'mognase') { + $dest = sprintf('%s/_nav/langs/en.pot', APP_ROOT); + } else { + $dest = sprintf('%s/langs/en/%s.pot', APP_ROOT, $domain); + } + $dir = dirname($dest); + + if (!is_dir($dir)) { + if(!$quiet) { echo "making $dir\n"; } + mkdir($dir, 0755, true); + } + if(!$quiet) { echo sprintf("saved %d strings in %s\n", count($strs), $dest); } + if (FALSE === file_put_contents($dest, $f)) { + if(!$quiet) { echo "Failed to write $f to $dest.\n"; } + $error = 30; + } +} +if(!$quiet) { echo "Done. thxbye.\n"; } +exit($error); diff --git a/tools/extract2lang.php b/tools/extract2lang.php new file mode 100644 index 000000000..17d8417ff --- /dev/null +++ b/tools/extract2lang.php @@ -0,0 +1,105 @@ + $strs) { + $f[] = sprintf('# Generated by extract2lang.php on %s', date('c')); + $f[] = sprintf('# Domain %s', $domain); + $f[] = '# include translation strings from:'; + $files = array_unique($files); + foreach ($files as $source) { + $f[] = sprintf('# %s', $source); + } + + foreach ($strs as $str => $info) { + $str = str_replace(array("\'", "\""), array("'", '"'), $str); + $f[] = ''; + $f[] = sprintf('# %s', $info[0]); + $f[] = ';' . $str; + $f[] = $str; + $f[] = ''; + } + $f = implode("\n", $f); + $dest = sprintf('%s/langs/%s/%s.%s.lang', APP_ROOT, 'en', $domain, 'en'); + $dir = dirname($dest); + + if (!is_dir($dir)) { + echo "making $dir\n"; + mkdir($dir, 0755, true); + } + echo sprintf("saved %d strings in %s\n", count($strs), $dest); + if (FALSE === file_put_contents($dest, $f)) + echo "Failed to write.\n"; +} +echo "Done. kthxbye.\n"; +exit($error); diff --git a/tools/lookup_for_second_argument.php b/tools/lookup_for_second_argument.php new file mode 100644 index 000000000..97ce33728 --- /dev/null +++ b/tools/lookup_for_second_argument.php @@ -0,0 +1,61 @@ + $v) { + + echo sprintf("> %s has %d translated strings, adding the rest %d untranslated.", $k, count($v), ($number_of_eng_strings - count($v))); + $v = array_merge($eng_array, $v); // add untranslated strings + + $f = array(); + $f[] = sprintf('# Generated by pa2lang.php on %s', date('c')); + $f[] = sprintf('# from %s $%s', $php_source, $var); + + foreach ($v as $s0 => $s1) { + $s0 = str_replace("\n", ' ', $s0); + $s0 = preg_replace('/\s+/', ' ', $s0); // remove multiple spaces + $s0 = str_replace("\'", "'", $s0); // remove also backslashes + + if (is_array($s1)) { + echo "\nWe have an array for string $k:'$s0':\n"; + foreach ($s1 as $sv) + echo " * ", $sv, "\n"; + + echo "\nThis can't be inserted into this .lang file. Fix this upstream and come back.\n\n"; + exit(1); + } + $s1 = str_replace("\n", ' ', $s1); + $s1 = preg_replace('/\s+/', ' ', $s1); + $s1 = str_replace("\'", "'", $s1); + + $f[] = ''; + $f[] = ';' . $s0; + $f[] = $s1; + $f[] = "\n"; + } + + echo " ok\n"; + $files[$k] = implode("\n", $f); +} + +echo "Saving those into ...\n"; +foreach ($files as $k => $data) { + $file = str_replace('{LOC}', $k, $dest); +// file_put_contents($file, $data); +} + +echo "Done.\n"; +exit(0); \ No newline at end of file diff --git a/tools/rebuild_gettext_catalogs.sh b/tools/rebuild_gettext_catalogs.sh new file mode 100755 index 000000000..1c7b9e34e --- /dev/null +++ b/tools/rebuild_gettext_catalogs.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# Filip Komar, 2014 +# @license GPL v2 +# @author Filip (rebuilding a gettext dictionaries) +# @copyright 2014/03 +# inspired by check_for_translation_work.sh + +error=0 +declare -A resources +declare -A errors +if [ -f ./tools/web_projects.dat ]; then + source ./tools/web_projects.dat +else + echo Critical error!!! It was not possible to load web projects database! +fi + +for resource in "${!resources[@]}" +do + difference= + if [ "$resource" = "mognase" ]; then # is it about navigation? + path_and_filename=./_nav/langs/en + else + path_and_filename=./langs/en/$resource + fi + if [ -f $path_and_filename.pot ]; then # do a temporary copy + cp $path_and_filename.pot $path_and_filename.copy + fi + php_source=${resources[$resource]} + php tools/extract2gettext.php "$php_source" $resource true + php_error_level=$? # catch php error level + errors[$php_source]=$php_error_level + if [ -f $path_and_filename.pot ]; then + # msgmerge check of generated pot file + msgmerge --quiet --no-wrap $path_and_filename.pot $path_and_filename.pot --output-file=$path_and_filename.tmp + msgmerge_error_level=$? # catch msgmerge error level + if [ $msgmerge_error_level -gt 0 ]; then # on failure stop the process for this resource + errors[$php_source]=$msgmerge_error_level + rm $path_and_filename.tmp + fi + if [ -f $path_and_filename.copy ] && [ -f $path_and_filename.tmp ]; then + diff --ignore-matching-lines='^"POT-Creation-Date:' $path_and_filename.tmp $path_and_filename.copy + if [ $? -eq 1 ]; then # there are differences + errors[$php_source]=0 + mv $path_and_filename.tmp $path_and_filename.pot + rm $path_and_filename.copy + echo Changes in $path_and_filename.pot + else # Cleanup as there are no changes in $resource + errors[$php_source]=5 + rm $path_and_filename.tmp + mv $path_and_filename.copy $path_and_filename.pot + difference=none + fi + else # diff not possible + if [ -f $path_and_filename.tmp ]; then + mv $path_and_filename.tmp $path_and_filename.pot + if [ -f $path_and_filename.copy ]; then + rm $path_and_filename.copy + fi + else + rm $path_and_filename.pot + if [ -f $path_and_filename.copy ]; then + mv $path_and_filename.copy $path_and_filename.pot + difference=none + fi + fi + fi + if [ "$resource" = "mognase" ]; then # is it about navigation? + # create po files if needed from lang files if available + if [ -f ./_nav/langs/en.lang ]; then + python tools/translated_converter.py --filename ./_nav/langs/ + python_error_level=$? # catch python error level + if [ $python_error_level -gt 0 ]; then + errors[$resource]=50 + fi + if [ -f ./_nav/langs/en.po ]; then + rm ./_nav/langs/en.po + fi + fi + + for tr_file in ./_nav/langs/*.po + do + if [ -z $difference ]; then # update po file if needed +# msgfmt --statistics --verbose -c $tr_file -o /dev/null # usefull for debuging (msgctxt "/web/en/ or msgctxt "en/) + echo merging $tr_file + msgmerge --quiet --no-wrap $tr_file ./_nav/langs/en.pot --output-file=$tr_file.tmp + msgmerge_error_level=$? # catch msgmerge error level + if [ $msgmerge_error_level -gt 0 ]; then + errors[$directory/$resource]=60 + echo "$directory"/$resource.po could not be created succesfully + if [ -f $tr_file.tmp ]; then + rm $tr_file.tmp + fi + else + mv $tr_file.tmp $tr_file +# msgfmt --statistics --verbose -c $tr_file -o /dev/null # usefull for debuging (msgctxt "/web/en/ or msgctxt "en/) + fi + fi + done + else + for directory in ./langs/* + do + # in each language directory except source + if [ -d $directory/ ] && [ $directory != ./langs/en ]; then + # create po file if needed from lang file if available + if [ ! -f $directory/$resource.po ] && [ -f $directory/$resource.*.lang ]; then +# echo $directory # usefull for debuging of non UTF-8 files or other python errors + python tools/translated_converter.py --filename $directory/$resource + python_error_level=$? # catch python error level + if [ $python_error_level -gt 0 ]; then + errors[$directory/$resource]=50 + fi + fi + if [ -f $directory/$resource.po ] && [ -z $difference ]; then # update po file if it exists +# msgfmt --statistics --verbose -c $directory/$resource.po -o /dev/null # usefull for debuging (msgctxt "/web/en/ or msgctxt "en/) + echo merging $directory/$resource.po + msgmerge --quiet --no-wrap $directory/$resource.po ./langs/en/$resource.pot --output-file=$directory/$resource.tmp + msgmerge_error_level=$? # catch msgmerge error level + if [ $msgmerge_error_level -gt 0 ]; then + errors[$directory/$resource]=60 + echo "$directory"/$resource.po could not be created succesfully + if [ -f $tr_file.tmp ]; then + rm $directory/$resource.tmp + fi + else + mv $directory/$resource.tmp $directory/$resource.po +# msgfmt --statistics --verbose -c $directory/$resource.po -o /dev/null # usefull for debuging (msgctxt "/web/en/ or msgctxt "en/) + fi + fi + fi + done + fi + fi +done + +echo '' +#printf "%s\n" "${errors[@]}" +for error_in_source in "${!errors[@]}" +do + case ${errors[$error_in_source]} in + 0) echo "Done lookup for _g() and _r() in $error_in_source. Gettext catalogs (pot files) rebuilded. Existing po files updated." ;; + 1) echo "Unkown error!!" ;; + 5) echo "There are no changes in $error_in_source." ;; + 10) echo "No strings to save from $error_in_source!" ;; + 20) echo "!!!! Parse error, please fix $error_in_source first !!!! See details above." ;; + 30) echo "!!!! Failed to write $error_in_source !!!! See details above." ;; + 50) echo "!!!! Script translated_converter.py encounter an error in $error_in_source !!!!" ;; + 60) echo "$error_in_source.po could not be created succesfully!! Duplicates?" ;; + esac +done diff --git a/tools/translated_converter.py b/tools/translated_converter.py new file mode 100644 index 000000000..568e3fb43 --- /dev/null +++ b/tools/translated_converter.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This file is free software. It come without any warranty, to the extent +# permitted by applicable law. You can redistribute it and/or modify them under +# the terms of the Do What The Fuck You Want To Public License, Version 2, as +# published by Sam Hocevar. See http://sam.zoy.org/wtfpl/COPYING for more details. + +# author yurchor +# http://gitweb.mageia.org/software/i18n/tools/tree/websites +# adaptation by filip + +import errno, glob, polib, re, os, getopt, sys +from time import strftime + +def usage(): + print '\nUsage: python %s [OPTION]' %os.path.basename(sys.argv[0]) + print ' generates po file for existing lang translations' + print 'Options: -h, --help : usage' + print ' -f filename, --filename filename : target filename' + sys.exit(2) +try: + opts, args = getopt.getopt(sys.argv[1:], "hf:", ["help", "filename="]) +except getopt.GetoptError: + usage() # print help information and exit + +filename='' +for o,a in opts: + if o in ("-h", "--help"): + usage() + if o in ("-f", "--filename"): + filename=a + +if filename == '': + sys.exit('No filename given') + +def po_append(message_id, message_str, langfile): + # Strip ';' from msgid + message_id = message_id.lstrip(';') + if message_str == message_id: + message_str='' + message_str = message_str.replace('{ok}', '').rstrip() + potentry = polib.POEntry( + msgid = message_id.decode('utf-8'), + msgstr = message_str.decode('utf-8'), + occurrences=[(langfile,'')] + ) + po.append(potentry) + +file_mask = filename + '.*.lang' +if filename == './_nav/langs/': + file_mask = './_nav/langs/*.lang' +for langfile in glob.glob(file_mask): +# print langfile # useful for debuging + nofilename = langfile.replace(filename + '.','') + language = nofilename.replace('.lang','') + language = language.replace('./_nav/langs/','') + #open lang file + text = open(langfile,"r").read()+"\n" + #Remove trailing spaces from lines + spaces=' {1,}\n' + spattern=re.compile(spaces,re.DOTALL) + for emptyline in spattern.findall(text): + text = text.replace(emptyline,'\n') + text = text.replace('\n\n','\n\n\n\n')+'\n' + + # Write PO file + po = polib.POFile(wrapwidth=999) # increase default wrap limit + pocreationtime = strftime('%Y-%m-%d %H:%M%z') + po.metadata = { + 'Project-Id-Version': langfile, + 'Report-Msgid-Bugs-To': 'mageia-i18n@mageia.org', + 'POT-Creation-Date': pocreationtime, + 'PO-Revision-Date': pocreationtime, + 'Last-Translator': 'Duffy Duck ', + 'Language-Team': 'LANGUAGE ', + 'Language': language, + 'MIME-Version': '1.0', + 'Content-Type': 'text/plain; charset=UTF-8', + 'Content-Transfer-Encoding': '8bit', + } + + # Parse contents and add them to PO + messagetemplate='\n\n#\ .*?\n\n' + mpattern=re.compile(messagetemplate,re.DOTALL) + for mblock in mpattern.findall(text): + message_comment, message_text = mblock.strip('\n').split('\n;',1) + # Strip '# ' from comments + message_comment = message_comment.lstrip('# ') + message_id, message_str = message_text.split('\n',1) +# print message_id # useful for debuging + po_append(message_id, message_str, langfile) + messagetemplate='\n\n;.*?\n\n' + mpattern=re.compile(messagetemplate,re.DOTALL) + for mblock in mpattern.findall(text): + message_id, message_str = mblock.strip('\n').split('\n',1) +# print message_id # useful for debuging + po_append(message_id, message_str, langfile) + if text[0] == ';': + message_id, message_str = text.partition('\n\n')[0].strip('\n').split('\n',1) +# print message_id # useful for debuging + po_append(message_id, message_str, langfile) + if filename == './_nav/langs/': + file_name = filename + language + else: + file_name = filename + po.save(file_name + '.po') diff --git a/tools/update-mirrors-list.php b/tools/update-mirrors-list.php new file mode 100644 index 000000000..d9be85226 --- /dev/null +++ b/tools/update-mirrors-list.php @@ -0,0 +1,8 @@ +