diff options
Diffstat (limited to 'phpBB/phpbb')
-rw-r--r-- | phpBB/phpbb/console/command/reparser/reparse.php | 99 | ||||
-rw-r--r-- | phpBB/phpbb/cron/task/text_reparser/reparser.php | 168 | ||||
-rw-r--r-- | phpBB/phpbb/db/driver/driver.php | 43 | ||||
-rw-r--r-- | phpBB/phpbb/db/migration/data/v320/text_reparser.php | 101 | ||||
-rw-r--r-- | phpBB/phpbb/install/controller/helper.php | 2 | ||||
-rw-r--r-- | phpBB/phpbb/install/module/update_filesystem/task/show_file_status.php | 2 | ||||
-rw-r--r-- | phpBB/phpbb/install/module/update_filesystem/task/update_files.php | 10 | ||||
-rw-r--r-- | phpBB/phpbb/textreparser/manager.php | 128 |
8 files changed, 469 insertions, 84 deletions
diff --git a/phpBB/phpbb/console/command/reparser/reparse.php b/phpBB/phpbb/console/command/reparser/reparse.php index 63124b4b8c..ddc97a1d1d 100644 --- a/phpBB/phpbb/console/command/reparser/reparse.php +++ b/phpBB/phpbb/console/command/reparser/reparse.php @@ -13,6 +13,7 @@ namespace phpbb\console\command\reparser; +use phpbb\exception\runtime_exception; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputOption; @@ -22,11 +23,6 @@ use Symfony\Component\Console\Style\SymfonyStyle; class reparse extends \phpbb\console\command\command { /** - * @var \phpbb\config\db_text - */ - protected $config_text; - - /** * @var InputInterface */ protected $input; @@ -42,12 +38,22 @@ class reparse extends \phpbb\console\command\command protected $output; /** + * @var \phpbb\lock\db + */ + protected $reparse_lock; + + /** + * @var \phpbb\textreparser\manager + */ + protected $reparser_manager; + + /** * @var \phpbb\di\service_collection */ protected $reparsers; /** - * @var array Reparser names as keys, and their last $current ID as values + * @var array The reparser's last $current ID as values */ protected $resume_data; @@ -55,14 +61,16 @@ class reparse extends \phpbb\console\command\command * Constructor * * @param \phpbb\user $user + * @param \phpbb\lock\db $reparse_lock + * @param \phpbb\textreparser\manager $reparser_manager * @param \phpbb\di\service_collection $reparsers - * @param \phpbb\config\db_text $config_text */ - public function __construct(\phpbb\user $user, \phpbb\di\service_collection $reparsers, \phpbb\config\db_text $config_text) + public function __construct(\phpbb\user $user, \phpbb\lock\db $reparse_lock, \phpbb\textreparser\manager $reparser_manager, \phpbb\di\service_collection $reparsers) { require_once __DIR__ . '/../../../../includes/functions_content.php'; - $this->config_text = $config_text; + $this->reparse_lock = $reparse_lock; + $this->reparser_manager = $reparser_manager; $this->reparsers = $reparsers; parent::__construct($user); } @@ -163,7 +171,11 @@ class reparse extends \phpbb\console\command\command $this->input = $input; $this->output = $output; $this->io = new SymfonyStyle($input, $output); - $this->load_resume_data(); + + if (!$this->reparse_lock->acquire()) + { + throw new runtime_exception('REPARSE_LOCK_ERROR', array(), null, 1); + } $name = $input->getArgument('reparser-name'); if (isset($name)) @@ -185,6 +197,8 @@ class reparse extends \phpbb\console\command\command $this->io->success($this->user->lang('CLI_REPARSER_REPARSE_SUCCESS')); + $this->reparse_lock->release(); + return 0; } @@ -194,36 +208,18 @@ class reparse extends \phpbb\console\command\command * Will use the last saved value if --resume is set and the option was not specified * on the command line * - * @param string $reparser_name Reparser name * @param string $option_name Option name * @return integer */ - protected function get_option($reparser_name, $option_name) + protected function get_option($option_name) { // Return the option from the resume_data if applicable - if ($this->input->getOption('resume') && isset($this->resume_data[$reparser_name][$option_name]) && !$this->input->hasParameterOption('--' . $option_name)) + if ($this->input->getOption('resume') && isset($this->resume_data[$option_name]) && !$this->input->hasParameterOption('--' . $option_name)) { - return $this->resume_data[$reparser_name][$option_name]; + return $this->resume_data[$option_name]; } - $value = $this->input->getOption($option_name); - - // range-max has no default value, it must be computed for each reparser - if ($option_name === 'range-max' && $value === null) - { - $value = $this->reparsers[$reparser_name]->get_max_id(); - } - - return $value; - } - - /** - * Load the resume data from the database - */ - protected function load_resume_data() - { - $resume_data = $this->config_text->get('reparser_resume'); - $this->resume_data = (empty($resume_data)) ? array() : unserialize($resume_data); + return $this->input->getOption($option_name); } /** @@ -234,6 +230,7 @@ class reparse extends \phpbb\console\command\command protected function reparse($name) { $reparser = $this->reparsers[$name]; + $this->resume_data = $this->reparser_manager->get_resume_data($name); if ($this->input->getOption('dry-run')) { $reparser->disable_save(); @@ -244,9 +241,15 @@ class reparse extends \phpbb\console\command\command } // Start at range-max if specified or at the highest ID otherwise - $max = $this->get_option($name, 'range-max'); - $min = $this->get_option($name, 'range-min'); - $size = $this->get_option($name, 'range-size'); + $max = $this->get_option('range-max'); + $min = $this->get_option('range-min'); + $size = $this->get_option('range-size'); + + // range-max has no default value, it must be computed for each reparser + if ($max == null) + { + $max = $reparser->get_max_id(); + } if ($max < $min) { @@ -272,34 +275,10 @@ class reparse extends \phpbb\console\command\command $current = $start - 1; $progress->setProgress($max + 1 - $start); - $this->update_resume_data($name, $current); + $this->reparser_manager->update_resume_data($name, $min, $current, $size, !$this->input->getOption('dry-run')); } $progress->finish(); $this->io->newLine(2); } - - /** - * Save the resume data to the database - */ - protected function save_resume_data() - { - $this->config_text->set('reparser_resume', serialize($this->resume_data)); - } - - /** - * Save the resume data to the database - * - * @param string $name Reparser name - * @param string $current Current ID - */ - protected function update_resume_data($name, $current) - { - $this->resume_data[$name] = array( - 'range-min' => $this->get_option($name, 'range-min'), - 'range-max' => $current, - 'range-size' => $this->get_option($name, 'range-size'), - ); - $this->save_resume_data(); - } } diff --git a/phpBB/phpbb/cron/task/text_reparser/reparser.php b/phpBB/phpbb/cron/task/text_reparser/reparser.php new file mode 100644 index 0000000000..aa644de827 --- /dev/null +++ b/phpBB/phpbb/cron/task/text_reparser/reparser.php @@ -0,0 +1,168 @@ +<?php +/** + * + * This file is part of the phpBB Forum Software package. + * + * @copyright (c) phpBB Limited <https://www.phpbb.com> + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbb\cron\task\text_reparser; + +/** + * Reparse text cron task + */ +class reparser extends \phpbb\cron\task\base +{ + const MIN = 1; + const SIZE = 100; + + /** + * @var \phpbb\config\config + */ + protected $config; + + /** + * @var \phpbb\config\db_text + */ + protected $config_text; + + /** + * @var \phpbb\lock\db + */ + protected $reparse_lock; + + /** + * @var \phpbb\textreparser\manager + */ + protected $reparser_manager; + + /** + * @var string + */ + protected $reparser_name; + + /** + * @var \phpbb\di\service_collection + */ + protected $reparsers; + + /** + * @var array + */ + protected $resume_data; + + /** + * Constructor + * + * @param \phpbb\config\config $config + * @param \phpbb\config\db_text $config_text + * @param \phpbb\lock\db $reparse_lock + * @param \phpbb\textreparser\manager $reparser_manager + * @param \phpbb\di\service_collection $reparsers + */ + public function __construct(\phpbb\config\config $config, \phpbb\config\db_text $config_text, \phpbb\lock\db $reparse_lock, \phpbb\textreparser\manager $reparser_manager, \phpbb\di\service_collection $reparsers) + { + $this->config = $config; + $this->config_text = $config_text; + $this->reparse_lock = $reparse_lock; + $this->reparser_manager = $reparser_manager; + $this->reparsers = $reparsers; + } + + /** + * Sets the reparser for this cron task + * + * @param string $reparser + */ + public function set_reparser($reparser) + { + $this->reparser_name = (!isset($this->reparsers[$reparser]) ? 'text_reparser.' : '') . $reparser; + + if ($this->resume_data === null) + { + $this->reparser_manager->get_resume_data($this->reparser_name); + } + } + + /** + * {@inheritdoc} + */ + public function is_runnable() + { + if ($this->resume_data === null) + { + $this->reparser_manager->get_resume_data($this->reparser_name); + } + + if (empty($this->resume_data['range-max']) || $this->resume_data['range-max'] >= $this->resume_data['range-min']) + { + return true; + } + + return false; + } + + /** + * {@inheritdoc} + */ + public function should_run() + { + if (!empty($this->config['reparse_lock'])) + { + $last_run = explode(' ', $this->config['reparse_lock']); + + if ($last_run[0] + 3600 >= time()) + { + return false; + } + } + + if ($this->config[$this->reparser_name . '_cron_interval']) + { + return $this->config[$this->reparser_name . '_last_cron'] < time() - $this->config[$this->reparser_name . '_cron_interval']; + } + + return false; + } + + /** + * {@inheritdoc} + */ + public function run() + { + if ($this->reparse_lock->acquire()) + { + if ($this->resume_data === null) + { + $this->resume_data = $this->reparser_manager->get_resume_data($this->reparser_name); + } + + /** + * @var \phpbb\textreparser\reparser_interface $reparser + */ + $reparser = $this->reparsers[$this->reparser_name]; + + $min = !empty($this->resume_data['range-min']) ? $this->resume_data['range-min'] : self::MIN; + $current = !empty($this->resume_data['range-max']) ? $this->resume_data['range-max'] : $reparser->get_max_id(); + $size = !empty($this->resume_data['range-size']) ? $this->resume_data['range-size'] : self::SIZE; + + if ($current >= $min) + { + $start = max($min, $current + 1 - $size); + $end = max($min, $current); + + $reparser->reparse_range($start, $end); + + $this->reparser_manager->update_resume_data($this->reparser_name, $min, $start - 1, $size); + } + + $this->config->set($this->reparser_name . '_last_cron', time()); + $this->reparse_lock->release(); + } + } +} diff --git a/phpBB/phpbb/db/driver/driver.php b/phpBB/phpbb/db/driver/driver.php index 7e2d7a5ea6..5f06cb08fc 100644 --- a/phpBB/phpbb/db/driver/driver.php +++ b/phpBB/phpbb/db/driver/driver.php @@ -66,6 +66,15 @@ abstract class driver implements driver_interface */ var $sql_server_version = false; + const LOGICAL_OP = 0; + const STATEMENTS = 1; + const LEFT_STMT = 0; + const COMPARE_OP = 1; + const RIGHT_STMT = 2; + const SUBQUERY_OP = 3; + const SUBQUERY_SELECT_TYPE = 4; + const SUBQUERY_BUILD = 5; + /** * Constructor */ @@ -809,21 +818,21 @@ abstract class driver implements driver_interface { // In cases where an array exists but there is no head condition, // it should be because there's only 1 WHERE clause. This seems the best way to deal with it. - if ($operations_ary[0] !== 'AND' && - $operations_ary[0] !== 'OR') + if ($operations_ary[self::LOGICAL_OP] !== 'AND' && + $operations_ary[self::LOGICAL_OP] !== 'OR') { - $operations_ary = array('AND', $operations_ary); + $operations_ary = array('AND', array($operations_ary)); } return $this->_process_boolean_tree($operations_ary) . "\n"; } protected function _process_boolean_tree($operations_ary) { - $operation = array_shift($operations_ary); + $operation = $operations_ary[self::LOGICAL_OP]; - foreach ($operations_ary as &$condition) + foreach ($operations_ary[self::STATEMENTS] as &$condition) { - switch ($condition[0]) + switch ($condition[self::LOGICAL_OP]) { case 'AND': case 'OR': @@ -844,40 +853,40 @@ abstract class driver implements driver_interface case 3: // Typical 3 element clause with {left hand} {operator} {right hand} - switch ($condition[1]) + switch ($condition[self::COMPARE_OP]) { case 'IN': case 'NOT_IN': // As this is used with an IN, assume it is a set of elements for sql_in_set() - $condition = $this->sql_in_set($condition[0], $condition[2], $condition[1] === 'NOT_IN', true); + $condition = $this->sql_in_set($condition[self::LEFT_STMT], $condition[self::RIGHT_STMT], $condition[self::COMPARE_OP] === 'NOT_IN', true); break; case 'LIKE': - $condition = $condition[0] . ' ' . $this->sql_like_expression($condition[2]) . ' '; + $condition = $condition[self::LEFT_STMT] . ' ' . $this->sql_like_expression($condition[self::RIGHT_STMT]) . ' '; break; case 'NOT_LIKE': - $condition = $condition[0] . ' ' . $this->sql_not_like_expression($condition[2]) . ' '; + $condition = $condition[self::LEFT_STMT] . ' ' . $this->sql_not_like_expression($condition[self::RIGHT_STMT]) . ' '; break; case 'IS_NOT': - $condition[1] = 'IS NOT'; + $condition[self::COMPARE_OP] = 'IS NOT'; // no break case 'IS': // If the value is NULL, the string of it is the empty string ('') which is not the intended result. // this should solve that - if ($condition[2] === null) + if ($condition[self::RIGHT_STMT] === null) { - $condition[2] = 'NULL'; + $condition[self::RIGHT_STMT] = 'NULL'; } $condition = implode(' ', $condition); @@ -897,8 +906,8 @@ abstract class driver implements driver_interface // Subquery with {left hand} {operator} {compare kind} {SELECT Kind } {Sub Query} - $condition = $condition[0] . ' ' . $condition[1] . ' ' . $condition[2] . ' ( '; - $condition .= $this->sql_build_query($condition[3], $condition[4]); + $condition = $condition[self::LEFT_STMT] . ' ' . $condition[self::COMPARE_OP] . ' ' . $condition[self::SUBQUERY_OP] . ' ( '; + $condition .= $this->sql_build_query($condition[self::SUBQUERY_SELECT_TYPE], $condition[self::SUBQUERY_BUILD]); $condition .= ' )'; break; @@ -917,11 +926,11 @@ abstract class driver implements driver_interface if ($operation === 'NOT') { - $operations_ary = implode("", $operations_ary); + $operations_ary = implode("", $operations_ary[self::STATEMENTS]); } else { - $operations_ary = implode(" \n $operation ", $operations_ary); + $operations_ary = implode(" \n $operation ", $operations_ary[self::STATEMENTS]); } return $operations_ary; diff --git a/phpBB/phpbb/db/migration/data/v320/text_reparser.php b/phpBB/phpbb/db/migration/data/v320/text_reparser.php new file mode 100644 index 0000000000..1d73b74a76 --- /dev/null +++ b/phpBB/phpbb/db/migration/data/v320/text_reparser.php @@ -0,0 +1,101 @@ +<?php +/** + * + * This file is part of the phpBB Forum Software package. + * + * @copyright (c) phpBB Limited <https://www.phpbb.com> + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbb\db\migration\data\v320; + +class text_reparser extends \phpbb\db\migration\container_aware_migration +{ + static public function depends_on() + { + return array('\phpbb\db\migration\data\v310\contact_admin_form'); + } + + public function effectively_installed() + { + return isset($this->config['reparse_lock']); + } + + public function update_data() + { + return array( + array('config.add', array('reparse_lock', 0, true)), + array('config.add', array('text_reparser.pm_text_cron_interval', 10)), + array('config.add', array('text_reparser.pm_text_last_cron', 0)), + array('config.add', array('text_reparser.poll_option_cron_interval', 10)), + array('config.add', array('text_reparser.poll_option_last_cron', 0)), + array('config.add', array('text_reparser.poll_title_cron_interval', 10)), + array('config.add', array('text_reparser.poll_title_last_cron', 0)), + array('config.add', array('text_reparser.post_text_cron_interval', 10)), + array('config.add', array('text_reparser.post_text_last_cron', 0)), + array('config.add', array('text_reparser.user_signature_cron_interval', 10)), + array('config.add', array('text_reparser.user_signature_last_cron', 0)), + array('custom', array(array($this, 'reparse'))), + ); + } + + public function reparse($resume_data) + { + // Somtimes a cron job is too much + $limit = 100; + $fast_reparsers = array( + 'text_reparser.contact_admin_info', + 'text_reparser.forum_description', + 'text_reparser.forum_rules', + 'text_reparser.group_description', + ); + + if (!is_array($resume_data)) + { + $resume_data = array( + 'reparser' => 0, + 'current' => $this->container->get($fast_reparsers[0])->get_max_id(), + ); + } + + $fast_reparsers_size = sizeof($fast_reparsers); + $processed_records = 0; + while ($processed_records < $limit && $resume_data['reparser'] < $fast_reparsers_size) + { + $reparser = $this->container->get($fast_reparsers[$resume_data['reparser']]); + + // New reparser + if ($resume_data['current'] === 0) + { + $resume_data['current'] = $reparser->get_max_id(); + } + + $start = max(1, $resume_data['current'] + 1 - ($limit - $processed_records)); + $end = max(1, $resume_data['current']); + $reparser->reparse_range($start, $end); + + $processed_records = $end - $start + 1; + $resume_data['current'] = $start - 1; + + if ($start === 1) + { + // Prevent CLI command from running these reparsers again + $reparser_manager = $this->container->get('text_reparser.manager'); + $reparser_manager->update_resume_data($fast_reparsers[$resume_data['reparser']], 1, 0, $limit); + + $resume_data['reparser']++; + } + } + + if ($resume_data['reparser'] === $fast_reparsers_size) + { + return true; + } + + return $resume_data; + } +} diff --git a/phpBB/phpbb/install/controller/helper.php b/phpBB/phpbb/install/controller/helper.php index fdb07d9c4a..ed817f7396 100644 --- a/phpBB/phpbb/install/controller/helper.php +++ b/phpBB/phpbb/install/controller/helper.php @@ -197,7 +197,7 @@ class helper $this->language_cookie = $lang; } - $lang = (!empty($lang) && strpos($lang, '/')) ? $lang : null; + $lang = (!empty($lang) && strpos($lang, '/') === false) ? $lang : null; $this->render_language_select($lang); diff --git a/phpBB/phpbb/install/module/update_filesystem/task/show_file_status.php b/phpBB/phpbb/install/module/update_filesystem/task/show_file_status.php index 1c6b9aa058..e712b8ad6a 100644 --- a/phpBB/phpbb/install/module/update_filesystem/task/show_file_status.php +++ b/phpBB/phpbb/install/module/update_filesystem/task/show_file_status.php @@ -100,7 +100,7 @@ class show_file_status extends task_base { $this->file_updater->create_new_file( $filename, - $this->cache->get('_file_' . md5($filename)), + base64_decode($this->cache->get('_file_' . md5($filename))), true ); } diff --git a/phpBB/phpbb/install/module/update_filesystem/task/update_files.php b/phpBB/phpbb/install/module/update_filesystem/task/update_files.php index 747a86281b..fbb465cc66 100644 --- a/phpBB/phpbb/install/module/update_filesystem/task/update_files.php +++ b/phpBB/phpbb/install/module/update_filesystem/task/update_files.php @@ -164,20 +164,20 @@ class update_files extends task_base { case 'delete': $this->file_updater->delete_file($path); - break; + break; case 'new': $this->file_updater->create_new_file($path, $new_path . $path); - break; + break; case 'update_without_diff': $this->file_updater->update_file($path, $new_path . $path); - break; + break; case 'update_with_diff': $this->file_updater->update_file( $path, - $this->cache->get('_file_' . md5($path)), + base64_decode($this->cache->get('_file_' . md5($path))), true ); - break; + break; } // Save progress diff --git a/phpBB/phpbb/textreparser/manager.php b/phpBB/phpbb/textreparser/manager.php new file mode 100644 index 0000000000..fddd867923 --- /dev/null +++ b/phpBB/phpbb/textreparser/manager.php @@ -0,0 +1,128 @@ +<?php +/** + * + * This file is part of the phpBB Forum Software package. + * + * @copyright (c) phpBB Limited <https://www.phpbb.com> + * @license GNU General Public License, version 2 (GPL-2.0) + * + * For full copyright and license information, please see + * the docs/CREDITS.txt file. + * + */ + +namespace phpbb\textreparser; + +class manager +{ + /** + * @var \phpbb\config\config + */ + protected $config; + + /** + * @var \phpbb\config\db_text + */ + protected $config_text; + + /** + * @var \phpbb\di\service_collection + */ + protected $reparsers; + + /** + * @var array + */ + protected $resume_data; + + /** + * Constructor + * + * @param \phpbb\config\config $config + * @param \phpbb\config\db_text $config_text + * @param \phpbb\di\service_collection $reparsers + */ + public function __construct(\phpbb\config\config $config, \phpbb\config\db_text $config_text, \phpbb\di\service_collection $reparsers) + { + $this->config = $config; + $this->config_text = $config_text; + $this->reparsers = $reparsers; + } + + /** + * Loads resume data from the database + * + * @param string $name Name of the reparser to which the resume data belongs + * + * @return array + */ + public function get_resume_data($name) + { + if ($this->resume_data === null) + { + $resume_data = $this->config_text->get('reparser_resume'); + $this->resume_data = !empty($resume_data) ? unserialize($resume_data) : array(); + } + + return isset($this->resume_data[$name]) ? $this->resume_data[$name] : array(); + } + + /** + * Updates the resume data in the database + * + * @param string $name Name of the reparser to which the resume data belongs + * @param int $min Lowest record ID + * @param int $current Current record ID + * @param int $size Number of records to process at a time + * @param bool $update_db True if the resume data should be written to the database, false if not. (default: true) + */ + public function update_resume_data($name, $min, $current, $size, $update_db = true) + { + // Prevent overwriting the old, stored array + if ($this->resume_data === null) + { + $this->get_resume_data(''); + } + + $this->resume_data[$name] = array( + 'range-min' => $min, + 'range-max' => $current, + 'range-size' => $size, + ); + + if ($update_db) + { + $this->config_text->set('reparser_resume', serialize($this->resume_data)); + } + } + + /** + * Sets the interval for a text_reparser cron task + * + * @param string $name Name of the reparser to schedule + * @param int $interval Interval in seconds, 0 to disable the cron task + */ + public function schedule($name, $interval) + { + if (isset($this->reparsers[$name]) && isset($this->config[$name . '_cron_interval'])) + { + $this->config->set($name . '_cron_interval', $interval); + } + } + + /** + * Sets the interval for all text_reparser cron tasks + * + * @param int $interval Interval in seconds, 0 to disable the cron task + */ + public function schedule_all($interval) + { + // This way we don't construct every registered reparser + $reparser_array = array_keys($this->reparsers->getArrayCopy()); + + foreach ($reparser_array as $reparser) + { + $this->schedule($reparser, $interval); + } + } +} |