add testing for new resources & adding 10 of them + some bugfixes

- GitHub resources are now also supported

3 files changed, 116 insertions, 22 deletions

diff --git a/langs/lib.php b/langs/lib.php
index a5c408038..ca5ff48cc 100644
--- a/langs/lib.php
+++ b/langs/lib.php
@@ -623,9 +623,9 @@ function tx_call($tx_request, $project = 'project/mageia/')
 function resource_name_conversion($resource_name, $category = '')
 {
     if ('Webpages' == $category) {
-        $tx_names  = array('nav', '-');
-        $git_names = array('_nav/langs/en', '/');
-        $resource_name = 'page-' . str_replace($git_names, $tx_names, $resource_name);
+        $tx_names  = array('page-nav', 'page-', '-', '');
+        $git_names = array('../_nav/langs/en.pot', 'en/', '/', '.pot');
+        $resource_name = str_replace($git_names, $tx_names, $resource_name);
     } else if ('Cauldron' == $category) {
         $tx_names  = array('mageia-welcome', 'identity-catdap');
         $git_names = array('Mageia%20Welcome', 'Identity%20(CatDap)');
diff --git a/langs/report_tx_git.php b/langs/report_tx_git.php
index 30b8ecefb..d06369df6 100644
--- a/langs/report_tx_git.php
+++ b/langs/report_tx_git.php
@@ -32,6 +32,9 @@
     <p>Languages translation progress report for web pages is available <a href="./report.php">here</a>.</p>
 
 <?php
+// TODO:
+// fix one lang or one resource in $wanted_category: Webpages
+
 define('HLANG', TRUE);
 include 'lib.php';
 
@@ -65,6 +68,7 @@ include 'resources_list.php';
 // get resources data from TX
 $tx_resources = tx_call("resources");
 
+
 $details_wanted = FALSE;
 if (!empty($wanted_language) && !empty($wanted_resource_name)) {
     $details_wanted = TRUE;
@@ -87,20 +91,21 @@ if ($details_wanted) {
 
 // create separate array ($tx_resources_info) and add statistics to it from TX
 $tx_resources_info     = array();
+$tx_resources_list     = array();
 $tx_cauldron_resources = array();
 $tx_stat_for_resource  = array();
 foreach ($tx_resources as $one_resource) {
     $tx_resource_name = $one_resource['slug'];
-    $category         = $one_resource['categories'][0];
+    $categories       = explode(" ", $one_resource['categories'][0]);
     $add_data         = FALSE;
-    if ('Cauldron' == $category) {
+    if (in_array('Cauldron', $categories)) {
         $tx_cauldron_resources[] = $tx_resource_name;
     }
     // limit resource type to one
-    if ($category == $resource_type) {
-        if ('Cauldron' == $category) { // add data for Cauldron category
+    if (in_array($resource_type, $categories)) {
+        if (in_array('Cauldron', $categories)) { // add data for Cauldron category
             // if name is required
-            if (!empty($wanted_resource) && $tx_resource_name == resource_name_conversion($wanted_resource_name, $category)) {
+            if (!empty($wanted_resource) && $tx_resource_name == resource_name_conversion($wanted_resource_name, $wanted_category)) {
                 $add_data = TRUE;
             // if only language is required
             } else if (!empty($wanted_language) && empty($wanted_resource)) {
@@ -112,9 +117,10 @@ foreach ($tx_resources as $one_resource) {
             }
         }
         // add data if details are needed
-        if ($details_wanted && $tx_resource_name == resource_name_conversion($wanted_resource_name, $category)) {
+        if ($details_wanted && $tx_resource_name == resource_name_conversion($wanted_resource_name, $wanted_category)) {
             $add_data = TRUE;
         }
+        $tx_resources_list[] = $tx_resource_name;
     }
     if ($add_data) {
         // add statistic
@@ -146,14 +152,22 @@ if (empty($wanted_language)) {
 
 // create array ($report) with statistics from git
 $report = array();
+$git_resources_list = array();
 if ('Webpages' == $resource_type) {
     if (empty($wanted_resource) && empty($wanted_language)) {
         $enFiles = array_merge(array('../_nav/langs/en.pot'), get_lang_references('*.pot')); // added navigation file
         $report = generating_report(get_other_langs(), $enFiles);
-    } else {
+    } else if (!empty($wanted_resource)) {
         $enFiles = array($wanted_resource);
+        $report = generating_report(get_other_langs(), $enFiles);
+    } else if (!empty($wanted_language)) { // TODO
+        $enFiles = array_merge(array('../_nav/langs/en.pot'), get_lang_references('*.pot'));
         $report = generating_report(array($wanted_language), $enFiles, NULL, NULL, $parsed_tx_file_content);
     }
+    foreach ($enFiles as $git_resource_webpage_name) {
+        $git_webpage_name = resource_name_conversion($git_resource_webpage_name, $wanted_category, TRUE); // TRUE
+        $git_resources_list[] = $git_webpage_name;
+    }
 } else {
     $tx_resources_not_yet_in_git = array();
     $translated_in_tx = ' translated in Transifex';
@@ -164,18 +178,26 @@ if ('Webpages' == $resource_type) {
         }
         $git_path = $git_resource_details['git_path'];
         $pot_name = $git_resource_details['pot_name'];
-        $raw_html_dump = file_get_contents($git_path);
-        if (FALSE === $raw_html_dump) {
-            $errors['file_get_contents_failed'] = "Access to $git_path failed.";
-        }
-        // list all po files from links within $raw_html_dump
-        if (false !== strpos($pot_name, '_en.ts')) { // treat TS files differently
-            $first_part_pot_name = substr($pot_name, 0, -5); // cuts 'en.ts' from mageiaSync_en.ts
-            preg_match_all("/('>$first_part_pot_name)([a-z_A-Z0-9@-]+)(\.ts<)/", $raw_html_dump, $language_codes);
+        if (isset($git_resource_details['github_path'])) { // treat resources on GitHub differently
+            $github_raw_path = $git_resource_details['github_path'];
+            $github_raw_html_dump = file_get_contents($github_raw_path);
+            preg_match_all('/\"(\w*)\.po\"/m', $github_raw_html_dump, $language_codes);
+            $git_language_codes = $language_codes[1];
         } else {
-            preg_match_all("/('>)([a-z_A-Z0-9@-]+)(\.po<)/", $raw_html_dump, $language_codes);
+            $raw_html_dump = file_get_contents($git_path);
+            if (FALSE === $raw_html_dump) {
+                $errors['file_get_contents_failed'] = "Access to $git_path failed.";
+            }
+            // list all po files from links within $raw_html_dump
+            if (false !== strpos($pot_name, '_en.ts')) { // treat TS files differently
+                $first_part_pot_name = substr($pot_name, 0, -5); // cuts 'en.ts' from mageiaSync_en.ts
+                preg_match_all("/'>$first_part_pot_name([a-z_A-Z0-9@-]+)\.ts</", $raw_html_dump, $language_codes);
+            } else {
+                preg_match_all("/'>([a-z_A-Z0-9@-]+)\.po</", $raw_html_dump, $language_codes);
+            }
+            $git_language_codes = $language_codes[1];
         }
-        $git_language_codes = $language_codes[2];
+
         if (empty($wanted_language)) {
             $report_language_codes = $git_language_codes;
         } else {
@@ -193,6 +215,7 @@ if ('Webpages' == $resource_type) {
         }
         $one_report = generating_report(array_values($report_language_codes), array($tx_name), $git_path, $pot_name, $parsed_tx_file_content);
         $report = array_merge($one_report, $report);
+        $git_resources_list[] = $tx_name;
     }
 }
 
@@ -304,6 +327,8 @@ $tx_only_resources  = array_diff($tx_all_resources_above_treshold, $git_compare_
 $git_only_resources = array_diff($git_compare_resources, $tx_all_resource_names);
 $tx_only_languages  = array_diff(array_keys($tx_all_langs_above_treshold), $git_all_languages);
 $git_only_languages = array_diff($git_all_languages, $tx_all_languages);
+$new_resources_in_tx  = array_diff($git_resources_list, $tx_resources_list);
+$new_resources_in_git = array_diff($tx_resources_list, $git_resources_list);
 
 // preparing text for languages not present yet in our git repositories
 $tx_only_languages_details = '';
@@ -346,6 +371,15 @@ if (0 < count($git_only_languages)) {
     $error                        = "Note that some languages (" . implode(", ", $git_only_languages) . ") are present in our git";
     $errors['git_only_languages'] = "$error but they are below treshold ($tx_lang_completed_treshold %) in Transifex.";
 }
+if (0 < count($new_resources_in_tx) && (empty($wanted_resource))) {
+    $error                        = "Some resources (" . implode(", ", $new_resources_in_tx) . ") are new in Transifex";
+    $errors['new_resources_in_tx'] = "$error but not present in resources_list.php file so not processed. Please report that.";
+}
+
+if (0 < count($new_resources_in_git) && (empty($wanted_resource))) {
+    $error                        = "Some resources (" . implode(", ", $new_resources_in_git) . ") are new in our git";
+    $errors['new_resources_in_git'] = "$error but not present in Transifex so not processed. Please report that.";
+}
 
 if (0 < count($git_only_resources)) {
     $error                        = "Some resources (" . implode(", ", $git_only_resources) . ") are present only in our git";
@@ -587,14 +621,15 @@ $num_of_tx_cauldron_resources = count($resources_links);
 $resources_link = implode(", ", $resources_links);
 
 $reports_links  = '<p>You can also see reports about differences in ' . implode(" or ", $links) . '.</p>';
-$reports_links .= "<p>For software translations please choose one of it's $num_of_tx_cauldron_resources resources ($resources_link) or one of a $num_of_tx_langs languages ($languages_link).</p>";
+$reports_links .= "<p>For software translations please choose one of it's $num_of_tx_cauldron_resources resources ($resources_link) or ";
+$reports_links .= "one of a $num_of_tx_langs languages ($languages_link) currently present in Transifex.</p>";
 $reports_links .= '<p>There is a nice ';
 $reports_links .= '<a href="https://wiki.mageia.org/en/Transifex_and_git_syncing">wiki page about Transifex and git synchronising</a>.</p>';
 echo $reports_links . PHP_EOL;
 
 // print out any errors
 if (0 < count($errors)) {
-    $error_text  = '<h3>There seems to be some errors during processing this time :(</h3>';
+    $error_text  = '<h3>There seems to be some errors, warnings or notices during processing this time :(</h3>';
     $error_text .= '<ul>';
     foreach ($errors as $single_error) {
         $error_text .= '<li class="bold">' . $single_error . '</li>';
diff --git a/langs/resources_list.php b/langs/resources_list.php
index 5c5f0114c..86d6b15ed 100644
--- a/langs/resources_list.php
+++ b/langs/resources_list.php
@@ -1,4 +1,8 @@
 <?php
+/*
+ * In Transifex you can find links on dialog "Automatic update of source files"
+ * 
+ * */
 
 if ('Documentation' == $resource_type) {
     $git_resources = array(
@@ -22,6 +26,11 @@ if ('Documentation' == $resource_type) {
             'tx_name'  => 'bootiso-help',
             'git_path' => 'http://gitweb.mageia.org/software/i18n/tools/plain/docs/docs/stable/bootiso',
         ),
+        array(
+            'pot_name' => 'mageia_docs_desktop', // Mageia documentation desktop files translation
+            'tx_name'  => 'mageia_docs_desktop',
+            'git_path' => 'http://gitweb.mageia.org/software/i18n/tools/plain/docs/desktop/po',
+        ),
     );
 } else if ('Cauldron' == $resource_type) {
     $git_resources = array(
@@ -36,6 +45,18 @@ if ('Documentation' == $resource_type) {
             'git_path' => 'http://gitweb.mageia.org/software/desktop/common-data/plain/po',
         ),
         array(
+            'pot_name' => 'dnfdragora',
+            'tx_name'  => 'dnfdragora',
+            'git_path' => 'https://raw.githubusercontent.com/manatools/dnfdragora/master/po',
+            'github_path' => 'https://github.com/manatools/dnfdragora/tree/master/po',
+        ),
+        array(
+            'pot_name' => 'dnfdragora_desktop',
+            'tx_name'  => 'dnfdragora_desktop',
+            'git_path' => 'https://raw.githubusercontent.com/manatools/dnfdragora/master/po/desktop',
+            'github_path' => 'https://github.com/manatools/dnfdragora/tree/master/po/desktop',
+        ),
+        array(
             'pot_name' => 'drak3d',
             'tx_name'  => 'drak3d',
             'git_path' => 'http://gitweb.mageia.org/software/drak3d/plain/po',
@@ -46,6 +67,11 @@ if ('Documentation' == $resource_type) {
             'git_path' => 'http://gitweb.mageia.org/software/drakguard/plain/po',
         ),
         array(
+            'pot_name' => 'bootloader',
+            'tx_name'  => 'drakiso',
+            'git_path' => 'http://gitweb.mageia.org/software/build-system/drakiso/plain/bootloader/po',
+        ), /**/
+        array(
             'pot_name' => 'draklive-install',
             'tx_name'  => 'draklive-install',
             'git_path' => 'http://gitweb.mageia.org/software/draklive-install/plain/po',
@@ -56,6 +82,11 @@ if ('Documentation' == $resource_type) {
             'git_path' => 'http://gitweb.mageia.org/software/drakpxelinux/plain/po',
         ),
         array(
+            'pot_name' => 'draksnapshot',
+            'tx_name'  => 'draksnapshot',
+            'git_path' => 'http://gitweb.mageia.org/software/draksnapshot/plain/po',
+        ),
+        array(
             'pot_name' => 'DrakX',
             'tx_name'  => 'drakx_install',
             'git_path' => 'http://gitweb.mageia.org/software/drakx/plain/perl-install/install/share/po',
@@ -76,6 +107,11 @@ if ('Documentation' == $resource_type) {
             'git_path' => 'http://gitweb.mageia.org/software/drakwizard/plain/po',
         ),
         array(
+            'pot_name' => 'grub2-theme',
+            'tx_name'  => 'grub2-theme',
+            'git_path' => 'http://gitweb.mageia.org/software/design/grub2-theme/plain/po',
+        ),
+        array(
             'pot_name' => 'libDrakX',
             'tx_name'  => 'drakx_share',
             'git_path' => 'http://gitweb.mageia.org/software/drakx/plain/perl-install/share/po',
@@ -86,6 +122,18 @@ if ('Documentation' == $resource_type) {
             'git_path' => 'http://gitweb.mageia.org/software/drakx/plain/perl-install/standalone/po',
         ),
         array(
+            'pot_name' => 'manafirewall',
+            'tx_name'  => 'manafirewall',
+            'git_path' => 'https://raw.githubusercontent.com/manatools/manafirewall/master/po',
+            'github_path' => 'https://github.com/manatools/manafirewall/tree/master/po',
+        ),
+        array(
+            'pot_name' => 'manafirewall_desktop',
+            'tx_name'  => 'manafirewall_desktop',
+            'git_path' => 'https://raw.githubusercontent.com/manatools/manafirewall/master/po/desktop',
+            'github_path' => 'https://github.com/manatools/manafirewall/tree/master/po/desktop',
+        ),
+        array(
             'pot_name' => 'messages',
             'tx_name'  => 'identity-catdap',
             'git_path' => 'http://gitweb.mageia.org/web/identity/plain/lib/CatDap/I18N',
@@ -153,6 +201,17 @@ if ('Documentation' == $resource_type) {
             'git_path' => 'http://gitweb.mageia.org/software/net_monitor/plain/po',
         ),
         array(
+            'pot_name' => 'json',
+            'tx_name'  => 'plasma-json',
+            'git_path' => 'http://gitweb.mageia.org/software/i18n/tools/plain/plasma_json/po/',
+        ),
+        array(
+            'pot_name' => 'python-manatools',
+            'tx_name'  => 'python-manatools',
+            'git_path' => 'https://raw.githubusercontent.com/manatools/python-manatools/master/po',
+            'github_path' => 'https://github.com/manatools/python-manatools/tree/master/po',
+        ),
+        array(
             'pot_name' => 'rpmdrake',
             'tx_name'  => 'rpmdrake',
             'git_path' => 'http://gitweb.mageia.org/software/rpmdrake/plain/po',