Browse Source

! backup updated to support bigger backups
+ Added warnings to the backup page depending on the forum size

emanuele 12 years ago
parent
commit
b758984492

+ 46 - 34
Sources/DbExtra-mysql.php

@@ -252,72 +252,81 @@ function smf_db_list_tables($db = false, $filter = false)
  * @return string, the query to insert the data back in, or an empty
  *  string if the table was empty.
  */
-function smf_db_insert_sql($tableName)
+function smf_db_insert_sql($tableName, $new_table = false)
 {
-	global $smcFunc, $db_prefix;
+	global $smcFunc, $db_prefix, $detected_id;
+	static $start = 0, $num_rows, $fields, $limit, $last_id;
+
+	if ($new_table)
+	{
+		$limit = strstr($tableName, 'log_') !== false ? 500 : 250;
+		$start = 0;
+		$last_id = 0;
+	}
 
+	$data = '';
 	$tableName = str_replace('{db_prefix}', $db_prefix, $tableName);
 
 	// This will be handy...
 	$crlf = "\r\n";
 
-	// Get everything from the table.
+	// This is done this way because retrieve data only with LIMIT will become slower after each query
+	// and for long tables (e.g. {db_prefix}messages) it could be a pain...
+	// Instead using WHERE speeds up thing *a lot* (especially after the first 50'000 records)
 	$result = $smcFunc['db_query']('', '
 		SELECT /*!40001 SQL_NO_CACHE */ *
-		FROM `{raw:table}`',
+		FROM `' . $tableName . '`' .
+		(!empty($last_id) && !empty($detected_id) ? '
+		WHERE ' . $detected_id . ' > ' . $last_id : '') . '
+		LIMIT ' . (empty($last_id) ? $start . ', ' : '') . $limit,
 		array(
-			'table' => $tableName,
+			'security_override' => true,
 		)
 	);
 
 	// The number of rows, just for record keeping and breaking INSERTs up.
 	$num_rows = $smcFunc['db_num_rows']($result);
-	$current_row = 0;
 
 	if ($num_rows == 0)
-		return '';
+		return false;
 
-	$fields = array_keys($smcFunc['db_fetch_assoc']($result));
-	$smcFunc['db_data_seek']($result, 0);
+	if ($new_table)
+	{
+		$fields = array_keys($smcFunc['db_fetch_assoc']($result));
+		$smcFunc['db_data_seek']($result, 0);
 
-	// Start it off with the basic INSERT INTO.
-	$data = 'INSERT INTO `' . $tableName . '`' . $crlf . "\t" . '(`' . implode('`, `', $fields) . '`)' . $crlf . 'VALUES ';
+		// Start it off with the basic INSERT INTO.
+		$data = 'INSERT INTO `' . $tableName . '`' . $crlf . "\t" . '(`' . implode('`, `', $fields) . '`)' . $crlf . 'VALUES ';
+	}
 
 	// Loop through each row.
-	while ($row = $smcFunc['db_fetch_row']($result))
+	while ($row = $smcFunc['db_fetch_assoc']($result))
 	{
-		$current_row++;
-
 		// Get the fields in this row...
 		$field_list = array();
-		for ($j = 0; $j < $smcFunc['db_num_fields']($result); $j++)
+
+		foreach ($row as $key => $item)
 		{
 			// Try to figure out the type of each field. (NULL, number, or 'string'.)
-			if (!isset($row[$j]))
+			if (!isset($item))
 				$field_list[] = 'NULL';
-			elseif (is_numeric($row[$j]) && (int) $row[$j] == $row[$j])
-				$field_list[] = $row[$j];
+			elseif (is_numeric($item) && (int) $item == $item)
+				$field_list[] = $item;
 			else
-				$field_list[] = '\'' . $smcFunc['db_escape_string']($row[$j]) . '\'';
+				$field_list[] = '\'' . $smcFunc['db_escape_string']($item) . '\'';
 		}
+		if (!empty($detected_id) && isset($row[$detected_id]))
+			$last_id = $row[$detected_id];
 
-		// 'Insert' the data.
-		$data .= '(' . implode(', ', $field_list) . ')';
-
-		// All done!
-		if ($current_row == $num_rows)
-			$data .= ';' . $crlf;
-		// Start a new INSERT statement after every 250....
-		elseif ($current_row > 249 && $current_row % 250 == 0)
-			$data .= ';' . $crlf . 'INSERT INTO `' . $tableName . '`' . $crlf . "\t" . '(`' . implode('`, `', $fields) . '`)' . $crlf . 'VALUES ';
-		// Otherwise, go to the next line.
-		else
-			$data .= ',' . $crlf . "\t";
+		$data .= '(' . implode(', ', $field_list) . ')' . ',' . $crlf . "\t";
 	}
+
 	$smcFunc['db_free_result']($result);
+	$data .= ';' . $crlf;
 
-	// Return an empty string if there were no rows.
-	return $num_rows == 0 ? '' : $data;
+	$start += $limit;
+
+	return $data;
 }
 
 /**
@@ -328,9 +337,10 @@ function smf_db_insert_sql($tableName)
  */
 function smf_db_table_sql($tableName)
 {
-	global $smcFunc, $db_prefix;
+	global $smcFunc, $db_prefix, $detected_id;
 
 	$tableName = str_replace('{db_prefix}', $db_prefix, $tableName);
+	$detected_id = '';
 
 	// This will be needed...
 	$crlf = "\r\n";
@@ -373,6 +383,8 @@ function smf_db_table_sql($tableName)
 
 		// And now any extra information. (such as auto_increment.)
 		$schema_create .= ($row['Extra'] != '' ? ' ' . $row['Extra'] : '') . ',' . $crlf;
+		if ($row['Extra'] == 'auto_increment')
+			$detected_id = $row['Field'];
 	}
 	$smcFunc['db_free_result']($result);
 

+ 93 - 44
Sources/DumpDatabase.php

@@ -27,12 +27,16 @@ if (!defined('SMF'))
  */
 function DumpDatabase2()
 {
-	global $db_name, $scripturl, $context, $modSettings, $crlf, $smcFunc, $db_prefix;
+	global $db_name, $scripturl, $context, $modSettings, $crlf, $smcFunc, $db_prefix, $db_show_debug;
 
 	// Administrators only!
 	if (!allowedTo('admin_forum'))
 		fatal_lang_error('no_dump_database', 'critical');
 
+	// We don't need debug when dumping the database
+	$modSettings['disableQueryCheck'] = true;
+	$db_show_debug = false;
+
 	// You can't dump nothing!
 	if (!isset($_REQUEST['struct']) && !isset($_REQUEST['data']))
 		$_REQUEST['data'] = true;
@@ -44,22 +48,22 @@ function DumpDatabase2()
 
 	// Attempt to stop from dying...
 	@set_time_limit(600);
+	$time_limit = ini_get('max_execution_time');
+	$start_time = time();
 	
 	// @todo ... fail on not getting the requested memory?
 	setMemoryLimit('256M');
+	$memory_limit = memoryReturnBytes(ini_get('memory_limit')) / 4;
+	$current_used_memory = 0;
+	$db_backup = '';
+	$output_function = 'un_compressed';
+
+	@ob_end_clean();
 
 	// Start saving the output... (don't do it otherwise for memory reasons.)
 	if (isset($_REQUEST['compress']) && function_exists('gzencode'))
 	{
-		// Make sure we're gzipping output, but then say we're not in the header ^_^.
-		if (empty($modSettings['enableCompressedOutput']))
-			@ob_start('ob_gzhandler', 65536);
-		// Try to clean any data already outputted.
-		elseif (ob_get_length() != 0)
-		{
-			ob_end_clean();
-			@ob_start('ob_gzhandler', 65536);
-		}
+		$output_function = 'gzencode';
 
 		// Send faked headers so it will just save the compressed output as a gzip.
 		header('Content-Type: application/x-gzip');
@@ -109,13 +113,13 @@ function DumpDatabase2()
 	$crlf = "\r\n";
 
 	// SQL Dump Header.
-	echo
-		'-- ==========================================================', $crlf,
-		'--', $crlf,
-		'-- Database dump of tables in `', $db_name, '`', $crlf,
-		'-- ', timeformat(time(), false), $crlf,
-		'--', $crlf,
-		'-- ==========================================================', $crlf,
+	$db_chunks = 
+		'-- ==========================================================' . $crlf .
+		'--' . $crlf .
+		'-- Database dump of tables in `' . $db_name . '`' . $crlf .
+		'-- ' . timeformat(time(), false) . $crlf .
+		'--' . $crlf .
+		'-- ==========================================================' . $crlf .
 		$crlf;
 
 	// Get all tables in the database....
@@ -134,49 +138,94 @@ function DumpDatabase2()
 	$tables = $smcFunc['db_list_tables'](false, $db_prefix . '%');
 	foreach ($tables as $tableName)
 	{
-		if (function_exists('apache_reset_timeout'))
-			@apache_reset_timeout();
-
 		// Are we dumping the structures?
 		if (isset($_REQUEST['struct']))
 		{
-			echo
-				$crlf,
-				'--', $crlf,
-				'-- Table structure for table `', $tableName, '`', $crlf,
-				'--', $crlf,
-				$crlf,
-				$smcFunc['db_table_sql']($tableName), ';', $crlf;
+			$db_chunks .= 
+				$crlf .
+				'--' . $crlf .
+				'-- Table structure for table `' . $tableName . '`' . $crlf .
+				'--' . $crlf .
+				$crlf .
+				$smcFunc['db_table_sql']($tableName) . ';' . $crlf;
 		}
+		else
+			// This is needed to speedup things later
+			$smcFunc['db_table_sql']($tableName);
 
 		// How about the data?
 		if (!isset($_REQUEST['data']) || substr($tableName, -10) == 'log_errors')
 			continue;
 
+		$first_round = true;
+		$close_table = false;
+
 		// Are there any rows in this table?
-		$get_rows = $smcFunc['db_insert_sql']($tableName);
+		while ($get_rows = $smcFunc['db_insert_sql']($tableName, $first_round))
+		{
+			if (empty($get_rows))
+				break;
+
+			// Time is what we need here!
+			if (function_exists('apache_reset_timeout'))
+				@apache_reset_timeout();
+			elseif (!empty($time_limit) && ($start_time + $time_limit - 20 > time()))
+			{
+				$start_time = time();
+				@set_time_limit(150);
+			}
+
+			if ($first_round)
+			{
+				$db_chunks .= 
+					$crlf .
+					'--' . $crlf .
+					'-- Dumping data in `' . $tableName . '`' . $crlf .
+					'--' . $crlf .
+					$crlf;
+				$first_round = false;
+			}
+			$db_chunks .= 
+				$get_rows;
+			$current_used_memory += $smcFunc['strlen']($db_chunks);
+
+			$db_backup .= $db_chunks;
+			unset($db_chunks);
+			$db_chunks = '';
+			if ($current_used_memory > $memory_limit)
+			{
+				echo $output_function($db_backup);
+				$current_used_memory = 0;
+				// This is probably redundant
+				unset($db_backup);
+				$db_backup = '';
+			}
+			$close_table = true;
+		}
 
 		// No rows to get - skip it.
-		if (empty($get_rows))
-			continue;
-
-		echo
-			$crlf,
-			'--', $crlf,
-			'-- Dumping data in `', $tableName, '`', $crlf,
-			'--', $crlf,
-			$crlf,
-			$get_rows,
-			'-- --------------------------------------------------------', $crlf;
-
-		unset($get_rows);
+		if ($close_table)
+			$db_backup .= 
+			'-- --------------------------------------------------------' . $crlf;
 	}
 
-	echo
-		$crlf,
-		'-- Done', $crlf;
+	$db_backup .= 
+		$crlf .
+		'-- Done' . $crlf;
+
+	echo $output_function($db_backup);
 
 	exit;
 }
 
+/**
+ * Dummy/helper function, it simply returns the string passed as argument
+ * @param $string, a string
+ * @return the string passed
+ */
+function un_compressed($string = '')
+{
+	return $string;
+}
+
 ?>

+ 45 - 1
Sources/ManageMaintenance.php

@@ -126,12 +126,56 @@ function ManageMaintenance()
  */
 function MaintainDatabase()
 {
-	global $context, $db_type, $db_character_set, $modSettings, $smcFunc, $txt;
+	global $context, $db_type, $db_character_set, $modSettings, $smcFunc, $txt, $maintenance;
 
 	// Show some conversion options?
 	$context['convert_utf8'] = $db_type == 'mysql' && (!isset($db_character_set) || $db_character_set !== 'utf8' || empty($modSettings['global_character_set']) || $modSettings['global_character_set'] !== 'UTF-8') && version_compare('4.1.2', preg_replace('~\-.+?$~', '', $smcFunc['db_server_info']()), '<=');
 	$context['convert_entities'] = $db_type == 'mysql' && isset($db_character_set, $modSettings['global_character_set']) && $db_character_set === 'utf8' && $modSettings['global_character_set'] === 'UTF-8';
 
+	// Check few things to give advices before make a backup
+	// If safe mod is enable the external tool is *always* the best (and probably the only) solution
+	$context['safe_mode_enable'] = @ini_get('safe_mode');
+	// This is just a...guess
+	$result = $smcFunc['db_query']('', '
+		SELECT COUNT(*)
+		FROM {db_prefix}messages',
+		array()
+	);
+	list($messages) = $smcFunc['db_fetch_row']($result);
+	$smcFunc['db_free_result']($result);
+
+	// 256 is what we use in the backup script
+	setMemoryLimit('256M');
+	$memory_limit = memoryReturnBytes(ini_get('memory_limit')) / (1024 * 1024);
+	// Zip limit is set to more or less 1/4th the size of the available memory * 1500
+	// 1500 is an estimate of the number of messages that generates a database of 1 MB (yeah I know IT'S AN ESTIMATION!!!)
+	// Why that? Because the only reliable zip package is the one sent out the first time, 
+	// so when the backup takes 1/5th (just to stay on the safe side) of the memory available
+	$zip_limit = $memory_limit * 1500 / 5;
+	// Here is more tricky: it depends on many factors, but the main idea is that
+	// if it takes "too long" the backup is not reliable. So, I know that on my computer it take
+	// 20 minutes to backup 2.5 GB, of course my computer is not representative, so I'll multiply by 4 the time.
+	// I would consider "too long" 5 minutes (I know it can be a long time, but let's start with that:
+	// 80 minutes for a 2.5 GB and a 5 minutes limit means 160 MB approx
+	$plain_limit = 240000;
+
+	$context['use_maintenance'] = 0;
+
+	if ($context['safe_mode_enable'])
+		$context['suggested_method'] = 'use_exernal_tool';
+	elseif ($zip_limit < $plain_limit && $messages < $zip_limit)
+		$context['suggested_method'] = 'zipped_file';
+	elseif ($zip_limit > $plain_limit || ($zip_limit < $plain_limit && $plain_limit < $messages))
+	{
+		$context['suggested_method'] = 'use_exernal_tool';
+		$context['use_maintenance'] = empty($maintenance) ? 2 : 0;
+	}
+	else
+	{
+		$context['use_maintenance'] = 1;
+		$context['suggested_method'] = 'plain_text';
+	}
+
 	if (isset($_GET['done']) && $_GET['done'] == 'convertutf8')
 		$context['maintenance_finished'] = $txt['utf8_title'];
 	if (isset($_GET['done']) && $_GET['done'] == 'convertentities')

+ 14 - 2
Themes/default/ManageMaintenance.template.php

@@ -60,13 +60,25 @@ function template_maintain_database()
 					<p><input type="submit" value="', $txt['maintain_backup_save'], '" id="submitDump" class="button_submit" />
 					<br class="clear_right" /></p>';
 	else
+	{
+		if ($context['safe_mode_enable'])
+			echo '
+					<div class="errorbox">', $txt['safe_mode_enabled'], '</div>';
+		else
+			echo '
+					<div class="', $context['suggested_method'] == 'use_exernal_tool' || $context['use_maintenance'] != 0 ? 'errorbox' : 'noticebox', '">
+						', $txt[$context['suggested_method']],
+						$context['use_maintenance'] != 0 ? '<br />' . $txt['enable_maintenance' . $context['use_maintenance']] : '',
+					'</div>';
+
 		echo '
 					<p><label for="struct"><input type="checkbox" name="struct" id="struct" onclick="document.getElementById(\'submitDump\').disabled = !document.getElementById(\'struct\').checked &amp;&amp; !document.getElementById(\'data\').checked;" class="input_check" checked="checked" /> ', $txt['maintain_backup_struct'], '</label><br />
 					<label for="data"><input type="checkbox" name="data" id="data" onclick="document.getElementById(\'submitDump\').disabled = !document.getElementById(\'struct\').checked &amp;&amp; !document.getElementById(\'data\').checked;" checked="checked" class="input_check" /> ', $txt['maintain_backup_data'], '</label><br />
-					<label for="compress"><input type="checkbox" name="compress" id="compress" value="gzip" checked="checked" class="input_check" /> ', $txt['maintain_backup_gz'], '</label></p>
+					<label for="compress"><input type="checkbox" name="compress" id="compress" value="gzip"', $context['suggested_method'] == 'zipped_file' ? ' checked="checked"' : '', ' class="input_check" /> ', $txt['maintain_backup_gz'], '</label></p>
 					<hr class="hrcolor" />
-					<p><input type="submit" value="', $txt['maintain_backup_save'], '" id="submitDump" onclick="return document.getElementById(\'struct\').checked || document.getElementById(\'data\').checked;" class="button_submit" />
+					<p><input ', $context['use_maintenance'] == 2 ? 'disabled="disabled" ' : '', 'type="submit" value="', $txt['maintain_backup_save'], '" id="submitDump" onclick="return document.getElementById(\'struct\').checked || document.getElementById(\'data\').checked;" class="button_submit" />
 					<br class="clear_right" /></p>';
+	}
 
 	echo '
 					<input type="hidden" name="', $context['session_var'], '" value="', $context['session_id'], '" />

+ 7 - 0
Themes/default/languages/ManageMaintenance.english.php

@@ -216,4 +216,11 @@ $txt['reattribute_cannot_find_member'] = 'Could not find member to attribute pos
 $txt['maintain_recountposts'] = 'Recount User Posts';
 $txt['maintain_recountposts_info'] = 'Run this maintenance task to update your users total post count.  It will recount all (countable) posts made by each user and then update their profile post count totals';
 
+$txt['safe_mode_enabled'] = '<a href="http://php.net/manual/en/features.safe-mode.php">safe_mode</a> is enabled on your server!<br />The backup done with this tool cannot be considered reliable!';
+$txt['use_exernal_tool'] = 'Please take in consideration to use an external tool to backup your database, any backup done with this tool cannot be considered 100% reliable.';
+$txt['zipped_file'] = 'If you want you can create a zipped backup.';
+$txt['plain_text'] = 'The more appropriate method to backup your database is to create a plain text file, a compressed package could not be completely reliable.';
+$txt['enable_maintenance1'] = 'Due to the size of your forum, is reccomended to enable the maintenance mode before start the backup.';
+$txt['enable_maintenance2'] = 'If you want to procede, due to the size of your forum, before start the backup please enable the maintenance mode.';
+
 ?>