From a7af736b9776d724fd2ca77b84b6623eba3edaef Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Sun, 14 Jul 2013 10:35:19 -0400 Subject: [ticket/11691] Stagger the convertion of soft delete updates PHPBB3-11691 --- .../phpbb/db/migration/data/v310/softdelete_p1.php | 26 +++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php index 0418d5cc2b..27bf5b6a57 100644 --- a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php +++ b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php @@ -101,7 +101,8 @@ class softdelete_p1 extends \phpbb\db\migration\migration return array( array('custom', array(array($this, 'update_post_visibility'))), array('custom', array(array($this, 'update_topic_visibility'))), - array('custom', array(array($this, 'update_topic_forum_counts'))), + array('custom', array(array($this, 'update_topics_post_counts'))), + array('custom', array(array($this, 'update_forums_topic_and_post_counts'))), array('permission.add', array('f_softdelete', false)), array('permission.add', array('m_softdelete', false)), @@ -122,7 +123,7 @@ class softdelete_p1 extends \phpbb\db\migration\migration $this->sql_query($sql); } - public function update_topic_forum_counts() + public function update_topics_post_counts() { $sql = 'UPDATE ' . $this->table_prefix . 'topics SET topic_posts_approved = topic_replies + 1, @@ -135,15 +136,24 @@ class softdelete_p1 extends \phpbb\db\migration\migration topic_posts_unapproved = (topic_replies_real - topic_replies) + 1 WHERE topic_visibility = ' . ITEM_UNAPPROVED; $this->sql_query($sql); + } + + public function update_forums_topic_and_post_counts($start) + { + $start = (int) $start; + $limit = 2; + $i = 0; $sql = 'SELECT forum_id, topic_visibility, COUNT(topic_id) AS sum_topics, SUM(topic_posts_approved) AS sum_posts_approved, SUM(topic_posts_unapproved) AS sum_posts_unapproved FROM ' . $this->table_prefix . 'topics GROUP BY forum_id, topic_visibility'; - $result = $this->db->sql_query($sql); + $result = $this->db->sql_query_limit($sql, $start, $limit); $update_forums = array(); while ($row = $this->db->sql_fetchrow($result)) { + $i++; + $forum_id = (int) $row['forum_id']; if (!isset($update_forums[$forum_id])) { @@ -169,5 +179,15 @@ class softdelete_p1 extends \phpbb\db\migration\migration WHERE forum_id = ' . $forum_id; $this->sql_query($sql); } + + + if ($i < $limit) + { + // There are no more topics, we are done + return; + } + + // There are still more topics to query, return the next start value + return $start + $limit; } } -- cgit v1.2.1 From 29c84be40a8c60fe98ba6bd2e817fcef14b491ba Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Sun, 4 Aug 2013 23:44:41 +0200 Subject: [ticket/11691] Stagger user notification reconversion even more PHPBB3-11691 --- .../data/v310/notification_options_reconvert.php | 31 +++++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php b/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php index bd7614e1c2..16509d7833 100644 --- a/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php +++ b/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php @@ -23,12 +23,19 @@ class notification_options_reconvert extends \phpbb\db\migration\migration ); } - public function convert_notifications() + public function convert_notifications($start) { $insert_table = $this->table_prefix . 'user_notifications'; $insert_buffer = new \phpbb\db\sql_insert_buffer($this->db, $insert_table); - $this->perform_conversion($insert_buffer, $insert_table); + $start = (int) $start; + if ($start == 0) + { + $sql = 'DELETE FROM ' . $insert_table; + $this->db->sql_query($sql); + } + + return $this->perform_conversion($insert_buffer, $insert_table, $start); } /** @@ -37,17 +44,19 @@ class notification_options_reconvert extends \phpbb\db\migration\migration * @param \phpbb\db\sql_insert_buffer $insert_buffer * @param string $insert_table */ - public function perform_conversion(\phpbb\db\sql_insert_buffer $insert_buffer, $insert_table) + public function perform_conversion(\phpbb\db\sql_insert_buffer $insert_buffer, $insert_table, $start) { - $sql = 'DELETE FROM ' . $insert_table; - $this->db->sql_query($sql); + $limit = 250; + $converted_users = 0; $sql = 'SELECT user_id, user_notify_type, user_notify_pm - FROM ' . USERS_TABLE; - $result = $this->db->sql_query($sql); + FROM ' . USERS_TABLE . ' + ORDER BY user_id'; + $result = $this->db->sql_query_limit($sql, $limit, $start); while ($row = $this->db->sql_fetchrow($result)) { + $converted_users++; $notification_methods = array(); // In-board notification @@ -91,6 +100,14 @@ class notification_options_reconvert extends \phpbb\db\migration\migration $this->db->sql_freeresult($result); $insert_buffer->flush(); + + if ($converted_users < $limit) + { + // No more users left, we are done... + return; + } + + return $start + $limit; } /** -- cgit v1.2.1 From a7cf62decf4e4e4d2c73ad7b4f91d40c681784fb Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 16 Aug 2013 17:07:31 +0200 Subject: [ticket/11691] Fix some more problems with softdelete update * wrong order of arguments on sql_query_limit * avoid "BIGINT UNSIGNED value is out of range" errors * increase limit for the loop PHPBB3-11691 --- phpBB/phpbb/db/migration/data/v310/softdelete_p1.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php index 27bf5b6a57..cdf61569b8 100644 --- a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php +++ b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php @@ -125,15 +125,20 @@ class softdelete_p1 extends \phpbb\db\migration\migration public function update_topics_post_counts() { + /* + * Using sql_case here, to avoid "BIGINT UNSIGNED value is out of range" errors. + * As we update all topics in 2 queries, one broken topic would stop the conversion + * for all topics and the surpressed error will cause the admin to not even notice it. + */ $sql = 'UPDATE ' . $this->table_prefix . 'topics SET topic_posts_approved = topic_replies + 1, - topic_posts_unapproved = topic_replies_real - topic_replies + topic_posts_unapproved = ' . $this->db->sql_case('topic_replies_real > topic_replies', 'topic_replies_real - topic_replies', '0') . ' WHERE topic_visibility = ' . ITEM_APPROVED; $this->sql_query($sql); $sql = 'UPDATE ' . $this->table_prefix . 'topics SET topic_posts_approved = 0, - topic_posts_unapproved = (topic_replies_real - topic_replies) + 1 + topic_posts_unapproved = (' . $this->db->sql_case('topic_replies_real > topic_replies', 'topic_replies_real - topic_replies', '0') . ') + 1 WHERE topic_visibility = ' . ITEM_UNAPPROVED; $this->sql_query($sql); } @@ -141,13 +146,13 @@ class softdelete_p1 extends \phpbb\db\migration\migration public function update_forums_topic_and_post_counts($start) { $start = (int) $start; - $limit = 2; + $limit = 10; $i = 0; $sql = 'SELECT forum_id, topic_visibility, COUNT(topic_id) AS sum_topics, SUM(topic_posts_approved) AS sum_posts_approved, SUM(topic_posts_unapproved) AS sum_posts_unapproved FROM ' . $this->table_prefix . 'topics GROUP BY forum_id, topic_visibility'; - $result = $this->db->sql_query_limit($sql, $start, $limit); + $result = $this->db->sql_query_limit($sql, $limit, $start); $update_forums = array(); while ($row = $this->db->sql_fetchrow($result)) -- cgit v1.2.1 From e7c43dbb6796d75445b50b0062d6e32cf650b3cd Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 16 Aug 2013 18:53:05 +0200 Subject: [ticket/11691] Fix some minor comments PHPBB3-11691 --- phpBB/phpbb/db/migration/data/v310/softdelete_p1.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php index cdf61569b8..a93171d931 100644 --- a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php +++ b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php @@ -147,7 +147,7 @@ class softdelete_p1 extends \phpbb\db\migration\migration { $start = (int) $start; $limit = 10; - $i = 0; + $converted_forums = 0; $sql = 'SELECT forum_id, topic_visibility, COUNT(topic_id) AS sum_topics, SUM(topic_posts_approved) AS sum_posts_approved, SUM(topic_posts_unapproved) AS sum_posts_unapproved FROM ' . $this->table_prefix . 'topics @@ -157,7 +157,7 @@ class softdelete_p1 extends \phpbb\db\migration\migration $update_forums = array(); while ($row = $this->db->sql_fetchrow($result)) { - $i++; + $converted_forums++; $forum_id = (int) $row['forum_id']; if (!isset($update_forums[$forum_id])) @@ -185,8 +185,7 @@ class softdelete_p1 extends \phpbb\db\migration\migration $this->sql_query($sql); } - - if ($i < $limit) + if ($converted_forums < $limit) { // There are no more topics, we are done return; -- cgit v1.2.1 From eace91af203e21fa82d054d42fc2117fa1230763 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 19 Aug 2013 10:30:23 +0200 Subject: [ticket/11691] Add order by to the query PHPBB3-11691 --- phpBB/phpbb/db/migration/data/v310/softdelete_p1.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php index a93171d931..14bd5f52fb 100644 --- a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php +++ b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php @@ -151,7 +151,8 @@ class softdelete_p1 extends \phpbb\db\migration\migration $sql = 'SELECT forum_id, topic_visibility, COUNT(topic_id) AS sum_topics, SUM(topic_posts_approved) AS sum_posts_approved, SUM(topic_posts_unapproved) AS sum_posts_unapproved FROM ' . $this->table_prefix . 'topics - GROUP BY forum_id, topic_visibility'; + GROUP BY forum_id, topic_visibility + ORDER BY forum_id, topic_visibility'; $result = $this->db->sql_query_limit($sql, $limit, $start); $update_forums = array(); -- cgit v1.2.1 From 3b7abb98e1178733054d91d260e8a6329ddaecca Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 19 Aug 2013 10:31:05 +0200 Subject: [ticket/11691] Fix typo in comment PHPBB3-11691 --- phpBB/phpbb/db/migration/data/v310/softdelete_p1.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php index 14bd5f52fb..f080c78c50 100644 --- a/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php +++ b/phpBB/phpbb/db/migration/data/v310/softdelete_p1.php @@ -126,7 +126,7 @@ class softdelete_p1 extends \phpbb\db\migration\migration public function update_topics_post_counts() { /* - * Using sql_case here, to avoid "BIGINT UNSIGNED value is out of range" errors. + * Using sql_case here to avoid "BIGINT UNSIGNED value is out of range" errors. * As we update all topics in 2 queries, one broken topic would stop the conversion * for all topics and the surpressed error will cause the admin to not even notice it. */ -- cgit v1.2.1 From 8aa9f1d3abe4ec2ad7d7692d70d19c4accb6ec43 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Mon, 19 Aug 2013 10:34:26 +0200 Subject: [ticket/11691] Move purge code into new function PHPBB3-11691 --- .../migration/data/v310/notification_options_reconvert.php | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php b/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php index 16509d7833..a4962b177d 100644 --- a/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php +++ b/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php @@ -19,22 +19,22 @@ class notification_options_reconvert extends \phpbb\db\migration\migration public function update_data() { return array( + array('custom', array(array($this, 'purge_notifications'))), array('custom', array(array($this, 'convert_notifications'))), ); } + public function purge_notifications() + { + $sql = 'DELETE FROM ' . $insert_table; + $this->sql_query($sql); + } + public function convert_notifications($start) { $insert_table = $this->table_prefix . 'user_notifications'; $insert_buffer = new \phpbb\db\sql_insert_buffer($this->db, $insert_table); - $start = (int) $start; - if ($start == 0) - { - $sql = 'DELETE FROM ' . $insert_table; - $this->db->sql_query($sql); - } - return $this->perform_conversion($insert_buffer, $insert_table, $start); } -- cgit v1.2.1 From 07186656c8ea73a3e303998d465f08cc9f4598c4 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Tue, 20 Aug 2013 00:35:34 +0200 Subject: [ticket/11691] Fix table names and arguments/docs PHPBB3-11691 --- .../data/v310/notification_options_reconvert.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'phpBB/phpbb/db/migration') diff --git a/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php b/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php index a4962b177d..4195623618 100644 --- a/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php +++ b/phpBB/phpbb/db/migration/data/v310/notification_options_reconvert.php @@ -26,31 +26,31 @@ class notification_options_reconvert extends \phpbb\db\migration\migration public function purge_notifications() { - $sql = 'DELETE FROM ' . $insert_table; + $sql = 'DELETE FROM ' . $this->table_prefix . 'user_notifications'; $this->sql_query($sql); } public function convert_notifications($start) { - $insert_table = $this->table_prefix . 'user_notifications'; - $insert_buffer = new \phpbb\db\sql_insert_buffer($this->db, $insert_table); + $insert_buffer = new \phpbb\db\sql_insert_buffer($this->db, $this->table_prefix . 'user_notifications'); - return $this->perform_conversion($insert_buffer, $insert_table, $start); + return $this->perform_conversion($insert_buffer, $start); } /** * Perform the conversion (separate for testability) * - * @param \phpbb\db\sql_insert_buffer $insert_buffer - * @param string $insert_table + * @param \phpbb\db\sql_insert_buffer $insert_buffer + * @param int $start Start of staggering step + * @return mixed int start of the next step, null if the end was reached */ - public function perform_conversion(\phpbb\db\sql_insert_buffer $insert_buffer, $insert_table, $start) + public function perform_conversion(\phpbb\db\sql_insert_buffer $insert_buffer, $start) { $limit = 250; $converted_users = 0; $sql = 'SELECT user_id, user_notify_type, user_notify_pm - FROM ' . USERS_TABLE . ' + FROM ' . $this->table_prefix . 'users ORDER BY user_id'; $result = $this->db->sql_query_limit($sql, $limit, $start); -- cgit v1.2.1