From ee89ede59faab3809cc61b7773a99f3e24376fb8 Mon Sep 17 00:00:00 2001 From: Nathan Date: Wed, 2 Oct 2013 14:23:01 -0500 Subject: [ticket/11881] Timezone migration can take a long time PHPBB3-11881 --- phpBB/phpbb/db/migration/data/v310/timezone.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index dd0c6a2093..8443f7b492 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -39,16 +39,22 @@ class timezone extends \phpbb\db\migration\migration ); } - public function update_timezones() + public function update_timezones($start) { + $start = (int) $start; + $limit = 1; + $converted_timezones = 0; + // Update user timezones $sql = 'SELECT user_dst, user_timezone FROM ' . $this->table_prefix . 'users GROUP BY user_timezone, user_dst'; - $result = $this->db->sql_query($sql); + $result = $this->db->sql_query_limit($sql, $limit, $start); while ($row = $this->db->sql_fetchrow($result)) { + $converted_timezones++; + $sql = 'UPDATE ' . $this->table_prefix . "users SET user_timezone = '" . $this->db->sql_escape($this->convert_phpbb30_timezone($row['user_timezone'], $row['user_dst'])) . "' WHERE user_timezone = '" . $this->db->sql_escape($row['user_timezone']) . "' @@ -57,6 +63,12 @@ class timezone extends \phpbb\db\migration\migration } $this->db->sql_freeresult($result); + if ($converted_timezones == $limit) + { + // There are still more to convert + return $start + $limit; + } + // Update board default timezone $sql = 'UPDATE ' . $this->table_prefix . "config SET config_value = '" . $this->convert_phpbb30_timezone($this->config['board_timezone'], $this->config['board_dst']) . "' -- cgit v1.2.1 From 9653276ebcd1a5c24302cb8d06f905f90e158e45 Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Fri, 4 Oct 2013 12:10:27 -0500 Subject: [ticket/11881] Better split the timezone conversion into chunks; add test PHPBB3-11881 --- phpBB/phpbb/db/migration/data/v310/timezone.php | 32 +++++---- tests/dbal/convert_timezones.php | 94 +++++++++++++++++++++++++ tests/dbal/fixtures/convert_timezones.xml | 80 +++++++++++++++++++++ 3 files changed, 194 insertions(+), 12 deletions(-) create mode 100644 tests/dbal/convert_timezones.php create mode 100644 tests/dbal/fixtures/convert_timezones.xml diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index 8443f7b492..13e2a5ceb6 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -42,28 +42,36 @@ class timezone extends \phpbb\db\migration\migration public function update_timezones($start) { $start = (int) $start; - $limit = 1; - $converted_timezones = 0; + $limit = 5000; + $converted = 0; - // Update user timezones - $sql = 'SELECT user_dst, user_timezone + $update_blocks = array(); + + $sql = 'SELECT user_id, user_timezone, user_dst FROM ' . $this->table_prefix . 'users - GROUP BY user_timezone, user_dst'; + ORDER BY user_id ASC'; $result = $this->db->sql_query_limit($sql, $limit, $start); - while ($row = $this->db->sql_fetchrow($result)) { - $converted_timezones++; + $converted++; + + $update_blocks[$row['user_timezone'] . ':' . $row['user_dst']][] = (int) $row['user_id']; + } + $this->db->sql_freeresult($result); + + // Update blocks of users who share the same timezone/dst + foreach ($update_blocks as $timezone => $user_ids) + { + $timezone = explode(':', $timezone); + $converted_timezone = $this->convert_phpbb30_timezone($timezone[0], $timezone[1]); $sql = 'UPDATE ' . $this->table_prefix . "users - SET user_timezone = '" . $this->db->sql_escape($this->convert_phpbb30_timezone($row['user_timezone'], $row['user_dst'])) . "' - WHERE user_timezone = '" . $this->db->sql_escape($row['user_timezone']) . "' - AND user_dst = " . (int) $row['user_dst']; + SET user_timezone = '" . $this->db->sql_escape($converted_timezone) . "' + WHERE " . $this->db->sql_in_set('user_id', $user_ids); $this->sql_query($sql); } - $this->db->sql_freeresult($result); - if ($converted_timezones == $limit) + if ($converted == $limit) { // There are still more to convert return $start + $limit; diff --git a/tests/dbal/convert_timezones.php b/tests/dbal/convert_timezones.php new file mode 100644 index 0000000000..718b28b9ff --- /dev/null +++ b/tests/dbal/convert_timezones.php @@ -0,0 +1,94 @@ +db = $this->new_dbal(); + $db_tools = new \phpbb\db\tools($this->db); + + // user_dst doesn't exist anymore, must re-add it to test this + $db_tools->sql_column_add('phpbb_users', 'user_dst', array('BOOL', 1)); + + return $this->createXMLDataSet(dirname(__FILE__) . '/fixtures/convert_timezones.xml'); + } + + public function revert_schema() + { + return array( + 'drop_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst', + ), + ), + ); + } + + public function update_schema() + { + return array( + 'add_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst' => array('BOOL', 0), + ), + ), + ); + } + + protected function setUp() + { + parent::setUp(); + + global $phpbb_root_path, $phpEx; + + $this->db = $this->new_dbal(); + + $this->migration = new \phpbb\db\migration\data\v310\timezone( + new \phpbb\config\config(array()), + $this->db, + new \phpbb\db\tools($this->db), + $phpbb_root_path, + $phpEx, + 'phpbb_' + ); + } + + protected $expected_results = array( + //user_id => user_timezone + 1 => 'Etc/GMT+12', + 2 => 'Etc/GMT+11', + 3 => 'Etc/GMT-3', + 4 => 'Etc/GMT-4', + 5 => 'America/St_Johns', + 6 => 'Australia/Eucla', + ); + + public function test_convert() + { + $this->migration->update_timezones(0); + + $sql = 'SELECT user_id, user_timezone + FROM phpbb_users + ORDER BY user_id ASC'; + $result = $this->db->sql_query($sql); + while ($row = $this->db->sql_fetchrow($result)) + { + $this->assertEquals($this->expected_results[$row['user_id']], $row['user_timezone']); + } + $this->db->sql_freeresult($result); + + $db_tools = new \phpbb\db\tools($this->db); + + // Remove the user_dst field again + $db_tools->sql_column_remove('phpbb_users', 'user_dst'); + } +} diff --git a/tests/dbal/fixtures/convert_timezones.xml b/tests/dbal/fixtures/convert_timezones.xml new file mode 100644 index 0000000000..ce941d8b74 --- /dev/null +++ b/tests/dbal/fixtures/convert_timezones.xml @@ -0,0 +1,80 @@ + + + + user_id + username + username_clean + user_permissions + user_sig + user_occ + user_interests + user_timezone + user_dst + + 1 + 1 + 1 + + + + + -12 + 0 + + + 2 + 2 + 2 + + + + + -12 + 1 + + + 3 + 3 + 3 + + + + + 3 + 0 + + + 4 + 4 + 4 + + + + + 3 + 1 + + + 5 + 5 + 5 + + + + + -3.5 + 0 + + + 6 + 6 + 6 + + + + + 8.75 + 0 + +
+ -- cgit v1.2.1 From cd7d29b90edcf292464beaf3ab814750703f3f17 Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Fri, 4 Oct 2013 15:04:08 -0500 Subject: [ticket/11881] Make sure user_timezone isn't converted twice PHPBB3-11881 --- phpBB/phpbb/db/migration/data/v310/timezone.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index 13e2a5ceb6..6106e179cb 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -55,7 +55,12 @@ class timezone extends \phpbb\db\migration\migration { $converted++; - $update_blocks[$row['user_timezone'] . ':' . $row['user_dst']][] = (int) $row['user_id']; + // In case this is somehow run twice on a row. + // Otherwise it would just end up as UTC on the second run + if (is_numeric($row['user_timezone'])) + { + $update_blocks[$row['user_timezone'] . ':' . $row['user_dst']][] = (int) $row['user_id']; + } } $this->db->sql_freeresult($result); -- cgit v1.2.1 From efe130bcfc84577294069641621f93f0d9801d99 Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Fri, 4 Oct 2013 15:04:30 -0500 Subject: [ticket/11881] Limit to 500 PHPBB3-11881 --- phpBB/phpbb/db/migration/data/v310/timezone.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpBB/phpbb/db/migration/data/v310/timezone.php b/phpBB/phpbb/db/migration/data/v310/timezone.php index 6106e179cb..61f8dc488e 100644 --- a/phpBB/phpbb/db/migration/data/v310/timezone.php +++ b/phpBB/phpbb/db/migration/data/v310/timezone.php @@ -42,7 +42,7 @@ class timezone extends \phpbb\db\migration\migration public function update_timezones($start) { $start = (int) $start; - $limit = 5000; + $limit = 500; $converted = 0; $update_blocks = array(); -- cgit v1.2.1 From c6491c9078c4dac365227550c7daa5f8eab01aad Mon Sep 17 00:00:00 2001 From: Nathan Guse Date: Mon, 30 Dec 2013 14:27:13 -0600 Subject: [ticket/11881] Fix test filename PHPBB3-11881 --- tests/dbal/convert_timezones.php | 94 ----------------------------------- tests/dbal/convert_timezones_test.php | 94 +++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 94 deletions(-) delete mode 100644 tests/dbal/convert_timezones.php create mode 100644 tests/dbal/convert_timezones_test.php diff --git a/tests/dbal/convert_timezones.php b/tests/dbal/convert_timezones.php deleted file mode 100644 index 718b28b9ff..0000000000 --- a/tests/dbal/convert_timezones.php +++ /dev/null @@ -1,94 +0,0 @@ -db = $this->new_dbal(); - $db_tools = new \phpbb\db\tools($this->db); - - // user_dst doesn't exist anymore, must re-add it to test this - $db_tools->sql_column_add('phpbb_users', 'user_dst', array('BOOL', 1)); - - return $this->createXMLDataSet(dirname(__FILE__) . '/fixtures/convert_timezones.xml'); - } - - public function revert_schema() - { - return array( - 'drop_columns' => array( - $this->table_prefix . 'users' => array( - 'user_dst', - ), - ), - ); - } - - public function update_schema() - { - return array( - 'add_columns' => array( - $this->table_prefix . 'users' => array( - 'user_dst' => array('BOOL', 0), - ), - ), - ); - } - - protected function setUp() - { - parent::setUp(); - - global $phpbb_root_path, $phpEx; - - $this->db = $this->new_dbal(); - - $this->migration = new \phpbb\db\migration\data\v310\timezone( - new \phpbb\config\config(array()), - $this->db, - new \phpbb\db\tools($this->db), - $phpbb_root_path, - $phpEx, - 'phpbb_' - ); - } - - protected $expected_results = array( - //user_id => user_timezone - 1 => 'Etc/GMT+12', - 2 => 'Etc/GMT+11', - 3 => 'Etc/GMT-3', - 4 => 'Etc/GMT-4', - 5 => 'America/St_Johns', - 6 => 'Australia/Eucla', - ); - - public function test_convert() - { - $this->migration->update_timezones(0); - - $sql = 'SELECT user_id, user_timezone - FROM phpbb_users - ORDER BY user_id ASC'; - $result = $this->db->sql_query($sql); - while ($row = $this->db->sql_fetchrow($result)) - { - $this->assertEquals($this->expected_results[$row['user_id']], $row['user_timezone']); - } - $this->db->sql_freeresult($result); - - $db_tools = new \phpbb\db\tools($this->db); - - // Remove the user_dst field again - $db_tools->sql_column_remove('phpbb_users', 'user_dst'); - } -} diff --git a/tests/dbal/convert_timezones_test.php b/tests/dbal/convert_timezones_test.php new file mode 100644 index 0000000000..718b28b9ff --- /dev/null +++ b/tests/dbal/convert_timezones_test.php @@ -0,0 +1,94 @@ +db = $this->new_dbal(); + $db_tools = new \phpbb\db\tools($this->db); + + // user_dst doesn't exist anymore, must re-add it to test this + $db_tools->sql_column_add('phpbb_users', 'user_dst', array('BOOL', 1)); + + return $this->createXMLDataSet(dirname(__FILE__) . '/fixtures/convert_timezones.xml'); + } + + public function revert_schema() + { + return array( + 'drop_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst', + ), + ), + ); + } + + public function update_schema() + { + return array( + 'add_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst' => array('BOOL', 0), + ), + ), + ); + } + + protected function setUp() + { + parent::setUp(); + + global $phpbb_root_path, $phpEx; + + $this->db = $this->new_dbal(); + + $this->migration = new \phpbb\db\migration\data\v310\timezone( + new \phpbb\config\config(array()), + $this->db, + new \phpbb\db\tools($this->db), + $phpbb_root_path, + $phpEx, + 'phpbb_' + ); + } + + protected $expected_results = array( + //user_id => user_timezone + 1 => 'Etc/GMT+12', + 2 => 'Etc/GMT+11', + 3 => 'Etc/GMT-3', + 4 => 'Etc/GMT-4', + 5 => 'America/St_Johns', + 6 => 'Australia/Eucla', + ); + + public function test_convert() + { + $this->migration->update_timezones(0); + + $sql = 'SELECT user_id, user_timezone + FROM phpbb_users + ORDER BY user_id ASC'; + $result = $this->db->sql_query($sql); + while ($row = $this->db->sql_fetchrow($result)) + { + $this->assertEquals($this->expected_results[$row['user_id']], $row['user_timezone']); + } + $this->db->sql_freeresult($result); + + $db_tools = new \phpbb\db\tools($this->db); + + // Remove the user_dst field again + $db_tools->sql_column_remove('phpbb_users', 'user_dst'); + } +} -- cgit v1.2.1 From 46c5ab64d4bafb04fe062cca89b2d872567f53c1 Mon Sep 17 00:00:00 2001 From: Nils Adermann Date: Fri, 2 May 2014 14:37:54 +0200 Subject: [ticket/11881] Move convert_timezones_test to migrator PHPBB3-11881 --- tests/dbal/convert_timezones_test.php | 94 --------------------------- tests/dbal/fixtures/convert_timezones.xml | 80 ----------------------- tests/migrator/convert_timezones_test.php | 94 +++++++++++++++++++++++++++ tests/migrator/fixtures/convert_timezones.xml | 80 +++++++++++++++++++++++ 4 files changed, 174 insertions(+), 174 deletions(-) delete mode 100644 tests/dbal/convert_timezones_test.php delete mode 100644 tests/dbal/fixtures/convert_timezones.xml create mode 100644 tests/migrator/convert_timezones_test.php create mode 100644 tests/migrator/fixtures/convert_timezones.xml diff --git a/tests/dbal/convert_timezones_test.php b/tests/dbal/convert_timezones_test.php deleted file mode 100644 index 718b28b9ff..0000000000 --- a/tests/dbal/convert_timezones_test.php +++ /dev/null @@ -1,94 +0,0 @@ -db = $this->new_dbal(); - $db_tools = new \phpbb\db\tools($this->db); - - // user_dst doesn't exist anymore, must re-add it to test this - $db_tools->sql_column_add('phpbb_users', 'user_dst', array('BOOL', 1)); - - return $this->createXMLDataSet(dirname(__FILE__) . '/fixtures/convert_timezones.xml'); - } - - public function revert_schema() - { - return array( - 'drop_columns' => array( - $this->table_prefix . 'users' => array( - 'user_dst', - ), - ), - ); - } - - public function update_schema() - { - return array( - 'add_columns' => array( - $this->table_prefix . 'users' => array( - 'user_dst' => array('BOOL', 0), - ), - ), - ); - } - - protected function setUp() - { - parent::setUp(); - - global $phpbb_root_path, $phpEx; - - $this->db = $this->new_dbal(); - - $this->migration = new \phpbb\db\migration\data\v310\timezone( - new \phpbb\config\config(array()), - $this->db, - new \phpbb\db\tools($this->db), - $phpbb_root_path, - $phpEx, - 'phpbb_' - ); - } - - protected $expected_results = array( - //user_id => user_timezone - 1 => 'Etc/GMT+12', - 2 => 'Etc/GMT+11', - 3 => 'Etc/GMT-3', - 4 => 'Etc/GMT-4', - 5 => 'America/St_Johns', - 6 => 'Australia/Eucla', - ); - - public function test_convert() - { - $this->migration->update_timezones(0); - - $sql = 'SELECT user_id, user_timezone - FROM phpbb_users - ORDER BY user_id ASC'; - $result = $this->db->sql_query($sql); - while ($row = $this->db->sql_fetchrow($result)) - { - $this->assertEquals($this->expected_results[$row['user_id']], $row['user_timezone']); - } - $this->db->sql_freeresult($result); - - $db_tools = new \phpbb\db\tools($this->db); - - // Remove the user_dst field again - $db_tools->sql_column_remove('phpbb_users', 'user_dst'); - } -} diff --git a/tests/dbal/fixtures/convert_timezones.xml b/tests/dbal/fixtures/convert_timezones.xml deleted file mode 100644 index ce941d8b74..0000000000 --- a/tests/dbal/fixtures/convert_timezones.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - user_id - username - username_clean - user_permissions - user_sig - user_occ - user_interests - user_timezone - user_dst - - 1 - 1 - 1 - - - - - -12 - 0 - - - 2 - 2 - 2 - - - - - -12 - 1 - - - 3 - 3 - 3 - - - - - 3 - 0 - - - 4 - 4 - 4 - - - - - 3 - 1 - - - 5 - 5 - 5 - - - - - -3.5 - 0 - - - 6 - 6 - 6 - - - - - 8.75 - 0 - -
- diff --git a/tests/migrator/convert_timezones_test.php b/tests/migrator/convert_timezones_test.php new file mode 100644 index 0000000000..a1eed1dbdf --- /dev/null +++ b/tests/migrator/convert_timezones_test.php @@ -0,0 +1,94 @@ +db = $this->new_dbal(); + $db_tools = new \phpbb\db\tools($this->db); + + // user_dst doesn't exist anymore, must re-add it to test this + $db_tools->sql_column_add('phpbb_users', 'user_dst', array('BOOL', 1)); + + return $this->createXMLDataSet(dirname(__FILE__) . '/fixtures/convert_timezones.xml'); + } + + public function revert_schema() + { + return array( + 'drop_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst', + ), + ), + ); + } + + public function update_schema() + { + return array( + 'add_columns' => array( + $this->table_prefix . 'users' => array( + 'user_dst' => array('BOOL', 0), + ), + ), + ); + } + + protected function setUp() + { + parent::setUp(); + + global $phpbb_root_path, $phpEx; + + $this->db = $this->new_dbal(); + + $this->migration = new \phpbb\db\migration\data\v310\timezone( + new \phpbb\config\config(array()), + $this->db, + new \phpbb\db\tools($this->db), + $phpbb_root_path, + $phpEx, + 'phpbb_' + ); + } + + protected $expected_results = array( + //user_id => user_timezone + 1 => 'Etc/GMT+12', + 2 => 'Etc/GMT+11', + 3 => 'Etc/GMT-3', + 4 => 'Etc/GMT-4', + 5 => 'America/St_Johns', + 6 => 'Australia/Eucla', + ); + + public function test_convert() + { + $this->migration->update_timezones(0); + + $sql = 'SELECT user_id, user_timezone + FROM phpbb_users + ORDER BY user_id ASC'; + $result = $this->db->sql_query($sql); + while ($row = $this->db->sql_fetchrow($result)) + { + $this->assertEquals($this->expected_results[$row['user_id']], $row['user_timezone']); + } + $this->db->sql_freeresult($result); + + $db_tools = new \phpbb\db\tools($this->db); + + // Remove the user_dst field again + $db_tools->sql_column_remove('phpbb_users', 'user_dst'); + } +} diff --git a/tests/migrator/fixtures/convert_timezones.xml b/tests/migrator/fixtures/convert_timezones.xml new file mode 100644 index 0000000000..ce941d8b74 --- /dev/null +++ b/tests/migrator/fixtures/convert_timezones.xml @@ -0,0 +1,80 @@ + + + + user_id + username + username_clean + user_permissions + user_sig + user_occ + user_interests + user_timezone + user_dst + + 1 + 1 + 1 + + + + + -12 + 0 + + + 2 + 2 + 2 + + + + + -12 + 1 + + + 3 + 3 + 3 + + + + + 3 + 0 + + + 4 + 4 + 4 + + + + + 3 + 1 + + + 5 + 5 + 5 + + + + + -3.5 + 0 + + + 6 + 6 + 6 + + + + + 8.75 + 0 + +
+ -- cgit v1.2.1