<?php
/**
* @brief Conversion module
* @author <a href='https://www.invisioncommunity.com'>Invision Power Services, Inc.</a>
* @copyright (c) Invision Power Services, Inc.
* @license https://www.invisioncommunity.com/legal/standards/
* @package IPS Tools
* @since 4 Sept 2013
*/
namespace IPSUtf8;
/**
* Conversion class
*/
class Convert
{
const VERSION_ID = '1.1.20';
/**
* @brief Data Store
*/
protected $_data = array();
/**
* @brief Instance
*/
protected static $instance = NULL;
/**
* @brief Convertable table fields
*/
protected static $convertCols = array( 'text', 'mediumtext', 'longtext', 'varchar', 'char', 'tinytext' );
/**
* @brief Numeric table fields
*/
protected static $numericCols = array( 'integer' => 11, 'int' => 11, 'smallint' => 6, 'tinyint' => 4, 'mediumint' => 8, 'bigint' => 20, 'decimal' => null, 'numeric' => null, 'float' => null, 'double' => null );
/**
* @brief Tables we need to convert
*/
protected static $nonUtf8Tables = NULL;
/**
* @brief Tables we need to convert
*/
protected static $nonUtf8Collations = NULL;
/**
* @brief Number of rows we need to convert
*/
protected static $rowsToConvert = NULL;
/**
* @brief Native Database object
*/
protected static $db = NULL;
/**
* @brief UTF-8 Database object
*/
protected static $utf = NULL;
/**
* @brief UTF-8 table engine
*/
protected static $utfTableEngine = NULL;
/**
* @brief Create don't populate
*/
protected static $createOnly = array( 'content_cache_posts', 'content_cache_sigs', 'sessions', 'topic_views', 'search_keywords' );
/**
* @brief Problem Tables that can cause an error due to serialization while using "fast" mode
*/
protected static $problemTables = array( 'cache_store' );
/**
* @brief MySQL supported character sets
*/
protected static $mysqlCharSets = NULL;
/**
* Get instance
*
* @return Output
*/
public static function i()
{
if ( self::$instance === NULL )
{
static::$db = \IPSUtf8\Db::i();
if ( defined( 'SOURCE_DB_CHARSET' ) AND SOURCE_DB_CHARSET !== NULL )
{
static::$db->set_charset( SOURCE_DB_CHARSET );
}
static::$utf = \IPSUtf8\Db::i('utf8');
self::$instance = new self();
self::$instance->init();
}
return self::$instance;
}
/**
* Get value from data store
*
* @param mixed $key Key
* @return mixed Value from the datastore
*/
public function __get( $key )
{
if( isset( $this->_data[ $key ] ) )
{
return $this->_data[ $key ];
}
return NULL;
}
/**
* Set value in data store
*
* @param mixed $key Key
* @param mixed $value Value
* @return void
*/
public function __set( $key, $value )
{
if( array_key_exists( $key, $this->_data ) )
{
$this->_data[ $key ] = $value;
}
}
/**
* Process a batch
*
* @param int|null Number of rows to limit per batch
* @return boolean True (I converted some) False (I didn't)
*/
public function process( $limit=null, $fromError=false )
{
/* MB4? */
$sessionData = \IPSUtf8\Session::i()->json;
if ( isset( $sessionData['use_utf8mb4'] ) AND $sessionData['use_utf8mb4'] )
{
static::$utf->set_charset( 'utf8mb4' );
}
if ( \IPSUtf8\Session::i()->current_table === NULL AND \IPSUtf8\Session::i()->is_ipb )
{
/* Lock tasks until the end of time so they do not attempt to add / remove things during conversion */
if( static::$db->checkForTable('task_manager') )
{
static::$db->update( "task_manager", array( 'task_locked' => 2147483647 ) );
}
else
{
static::$db->update( "core_tasks", array( 'next_run' => 2147483647 ) );
}
}
$convertCount = 0;
$table = $this->getTable();
$convertCols = static::getConvertableColumns( $table['name'] );
$numericCols = static::getNumericColumns( $table['name'] );
$tableCharSet = \IPSUtf8\Convert::i()->database_charset;
$currentCharSet = \IPSUtf8\Session::i()->current_charset;
if ( ! empty( $table['charset'] ) )
{
$tableCharSet = $table['charset'];
/* Latin1 is actually cp1252, not ISO-8859-1 @link http://dev.mysql.com/doc/refman/5.0/en/charset-we-sets.html */
/* But only if our document character set is ISO-8859-1 or UTF-8. Otherwise, we're using the slow method which needs to real charset despite what the table is set too */
if ( $tableCharSet === 'latin1' AND in_array( mb_strtolower( $currentCharSet ), array( 'iso-8859-1', 'utf-8' ) ) )
{
$currentCharSet = 'windows-1252';
}
if ( defined( 'FORCE_CONVERT_CHARSET' ) AND FORCE_CONVERT_CHARSET !== NULL )
{
$currentCharSet = FORCE_CONVERT_CHARSET;
}
}
if ( !defined( 'SOURCE_DB_CHARSET' ) OR SOURCE_DB_CHARSET === NULL )
{
static::$db->set_charset( $tableCharSet );
}
/* Create table only */
if ( IPB_LOCK and in_array( $table['name'], static::$createOnly ) )
{
static::log( 'No content conversion of ' . $table['name'] . ' required' );
$this->getNextTable( $table['name'] );
\IPSUtf8\Session::i()->status = 'processing';
\IPSUtf8\Session::i()->json = $sessionData;
\IPSUtf8\Session::i()->save();
return true;
}
while( $convertCount < $limit )
{
$sessionData = \IPSUtf8\Session::i()->json;
if ( $table === null )
{
if ( count( array_keys( \IPSUtf8\Session::i()->tables ) ) == count( array_keys( \IPSUtf8\Session::i()->completed_json ) ) )
{
\IPSUtf8\Session::i()->status = 'completed';
\IPSUtf8\Session::i()->save();
return false;
}
return false;
}
/* Got any fields to convert? */
if ( count( $convertCols ) === 0 )
{
/* No data to convert, lets repopulate the easy way! */
$this->preInsert( $table );
static::$utf->delete( $table['name'] );
static::$utf->insert( $table['name'], static::$db->select( '*', $table['name'] ) );
$this->postInsert( $table );
/* Update counts */
$sessionData['convertedCount'] += $table['count'];
$convertCount += $table['count'];
static::log( 'No columns to convert in ' . $table['name'] . ' INSERT INTO FROM SELECT used' );
$table = $this->getNextTable( $table['name'] );
$convertCols = static::getConvertableColumns( $table['name'] );
$numericCols = static::getNumericColumns( $table['name'] );
if ( $table === null or $convertCount >= $limit )
{
\IPSUtf8\Session::i()->status = 'processing';
\IPSUtf8\Session::i()->json = $sessionData;
\IPSUtf8\Session::i()->save();
return true;
}
}
/* If the previous table had no columns to convert, then the table charset may be lost - reset it if it does not match */
if ( $table !== NULL AND $tableCharSet !== $table['charset'] )
{
$tableCharSet = $table['charset'];
}
/* Are we reeeeeeeeally sure we know the charset? Let's check one more time */
if ( ! $tableCharSet )
{
$createTable = static::$db->query( "SHOW CREATE TABLE `" . static::$db->prefix . $table['name'] . "`" )->fetch_assoc();
if ( preg_match( '#\scharset=([a-z0-9]+?)(\s|$)#i', $createTable['Create Table'], $matches ) )
{
$tableCharSet = mb_strtolower( $matches[1] );
}
}
/* Latin1 or ISO-8559-1? */
if ( ! in_array( $table['name'], static::$problemTables ) AND ( in_array( $tableCharSet, $this->getMysqlCharSets() ) ) and ( \IPSUtf8\Session::i()->current_charset == 'utf-8' or \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' ) AND ( !isset( $sessionData['force_conversion'] ) OR $sessionData['force_conversion'] !== 1 ) )
{
/* Convert the fast way! */
$count = static::$utf->select( 'COUNT(*)', $table['name'] )->first();
if ( $count > 0 )
{
/* TRUNCATE can hang so only do it if the table isn't empty (which it will be 99% of the time) */
/* @see http://dba.stackexchange.com/questions/28055/truncate-table-statement-sometimes-hangs */
static::log("TRUNCATE TABLE `" . static::$utf->prefix . $table['name'] . "`");
static::$utf->query("TRUNCATE TABLE `" . static::$utf->prefix . $table['name'] . "`" );
}
/* Update counts */
$sessionData['convertedCount'] += $table['count'];
$convertCount += $table['count'];
$select = array();
$convertCols = static::getConvertableColumns( $table['name'] );
$schematic = static::getTableSchematic( $table['name'] );
foreach( array_keys( $table['definition']['columns'] ) as $col )
{
if ( in_array( $col, $convertCols ) )
{
/* Even if the current charset is set to UTF-8, if the Table Charset is latin1, we need to use this method */
/* ... but only if the actual column isn't using utf8 */
$columnCollation = $schematic['definition']['columns'][$col]['collation'];
if ( UTF8_INSERT_ONLY === TRUE AND $tableCharSet == 'utf8' )
{
$select[] = 'CAST( `' . $col . '` AS BINARY )';
}
else if ( FORCE_MULTI_CONVERT === FALSE AND ( \IPSUtf8\Session::i()->current_charset != 'utf-8' OR ( $tableCharSet != 'utf8' AND ! in_array( $columnCollation, array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) ) OR ( $tableCharSet == 'utf8' AND mb_substr( $columnCollation, 0, 5 ) == 'latin' ) ) )
{
if ( \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' AND mb_substr( $columnCollation, 0, 5 ) == 'latin' )
{
/* Casting as binary truncates non-latin characters, so if we are using real Latin1, just convert */
$select[] = 'CONVERT( `' . $col . '` USING utf8 )';
}
else
{
/* Otherwise cast to binary */
$select[] = 'CONVERT( CAST(`' . $col . '` AS BINARY) USING utf8 )';
}
}
else
{
/**
This looks a little weird but bare with me. Semi-old databases (post-2.x but pre-4.x) can have UTF8 Tables and Columns, but contain Latin1 data.
We need to try and sniff this out and apply a different type of conversion, because in this instance you cannot just extract
as BINARY and then insert into the source. Instead, you need to convert to latin1, then to binary, then to utf8. But you can't
do that on actual UTF8 data because it will then truncate multibyte characters so we need to compare the conversion against what
is stored.
Mind = blown.
*/
$collation = 'utf8_unicode_ci';
$using = 'utf8';
if ( $columnCollation == 'utf8mb4_unicode_ci' )
{
$collation = 'utf8mb4_unicode_ci';
$using = 'utf8mb4';
}
$source_charset = 'latin1';
if ( defined( 'SOURCE_DB_CHARSET' ) AND SOURCE_DB_CHARSET !== NULL )
{
$source_charset = SOURCE_DB_CHARSET;
}
$select[] = 'CASE WHEN STRCMP(CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . '), `' . $col . '` COLLATE ' . $collation . ') = 0 THEN CONVERT(`' . $col . '` USING BINARY) ELSE CONVERT(CONVERT(CONVERT(`' . $col . '` USING ' . $source_charset . ') USING BINARY) USING ' . $using . ') END';
}
}
else
{
$select[] = '`' . $col . '`';
}
}
$this->preInsert( $table );
$sql = "INSERT IGNORE INTO `" . static::$utf->prefix . $table['name'] . "` SELECT " . implode( ',', $select ) . " FROM `" . static::$db->prefix . $table['name'] . "`";
static::log( $sql );
static::$utf->query( $sql );
$this->postInsert( $table );
$table = $this->getNextTable( $table['name'] );
\IPSUtf8\Session::i()->status = 'processing';
\IPSUtf8\Session::i()->json = $sessionData;
\IPSUtf8\Session::i()->save();
/* Always reset so we can test for no convertable columns above */
return true;
}
/* If we have a work table, do we have a PKEY and a current row? */
if ( \IPSUtf8\Session::i()->current_pkey )
{
$start = static::$utf->select('MAX(`' . \IPSUtf8\Session::i()->current_pkey . '`) as max', \IPSUtf8\Session::i()->current_table )->setKeyField('max')->first();
$rows = static::$db->select(
'*',
$table['name'],
array( \IPSUtf8\Session::i()->current_pkey . ' > ?', intval( $start ) ),
\IPSUtf8\Session::i()->current_pkey . ' ASC',
( $limit ? array( 0, ( $limit - $convertCount ) ) : null )
);
}
else
{
$start = static::$utf->select('COUNT(*) as count', \IPSUtf8\Session::i()->current_table )->setKeyField('count')->first();
/* Fetch via offset */
$rows = static::$db->select(
'*',
$table['name'],
null,
null,
( $limit ? array( $start, ( $limit - $convertCount ) ) : array( $start, 18446744073709551615 ) ) # No really, this is what MySQL recommends for offset, no limit (http://dev.mysql.com/doc/refman/5.1/en/select.html#id4651990)
);
}
if ( !defined( 'FORCE_CONVERT_METHOD' ) OR FORCE_CONVERT_METHOD === NULL )
{
\IPSUtf8\Text\Charset::$method = 'internal';
/* Optimise for latin character sets */
if ( function_exists( 'mb_convert_encoding' ) )
{
if ( in_array( strtolower( $currentCharSet ), array_map( 'strtolower', mb_list_encodings() ) ) )
{
\IPSUtf8\Text\Charset::$method = 'mb';
}
}
}
$rowCount = 0;
$gotRows = count( $rows );
$batch = array();
$batchBytes = 0;
$max = ( $limit ) ?: 250;
/* If we have a text column then the data set gets large */
if ( $limit > 50 AND static::hasTextColumn( $table['name'] ) )
{
$max = 50;
}
/* Throttle inserts for other reasons? */
$throttle = static::throttleInserts( $table['name'] );
if ( $throttle !== false )
{
$max = ( $throttle < $limit ) ? $throttle : $limit;
}
/* Anything fetched? */
if ( $gotRows === 0 )
{
try
{
static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` ENABLE KEYS;");
static::log( "{$table['name']} keys enabled." );
}
catch( \IPSUtf8\Db\Exception $e )
{
static::log( static::$utf->prefix . "{$table['name']} enable keys exception caught." );
}
$table = $this->getNextTable( $table['name'] );
$convertCols = static::getConvertableColumns( $table['name'] );
$numericCols = static::getNumericColumns( $table['name'] );
continue;
}
foreach( $rows as $row )
{
foreach( $convertCols as $col )
{
if ( isset( $row[ $col ] ) AND ! empty( $row[ $col ] ) )
{
$currentCharSet = ( defined( 'FORCE_CONVERT_CHARSET' ) AND FORCE_CONVERT_CHARSET != NULL ) ? FORCE_CONVERT_CHARSET : $currentCharSet;
if ( static::isSerialized( $row[ $col ] ) AND \IPSUtf8\Text\Charset::i()->needsConverting( $row[ $col ], $currentCharSet, 'UTF-8' ) )
{
/* Store a copy in case it doesn't work */
$original = $row[ $col ];
$work = unserialize( $row[ $col ] );
array_walk_recursive( $work, function( &$input, $key ) use ($currentCharSet)
{
$input = \IPSUtf8\Text\Charset::i()->convert( $input, $currentCharSet, 'UTF-8' );
} );
$row[ $col ] = serialize( $work );
/* Did it work? */
if ( ! @unserialize( $row[ $col ] ) )
{
/* Something went wrong... maybe we can figure it out */
$work = unserialize( $original );
array_walk_recursive( $work, function ( &$input, $key ) use ( $currentCharSet, $tableCharSet ) {
if ( function_exists( 'mb_detect_encoding' ) )
{
$encoding = mb_detect_encoding( $input );
$input = \IPSUtf8\Text\Charset::i()->convert( $input, $encoding, 'UTF-8' );
}
else
{
/* @todo expand */
if ( $tableCharSet == 'latin1' )
{
$encoding = 'ISO-8859-1';
}
$input = \IPSUtf8\Text\Charset::i()->convert( $input, $encoding, 'UTF-8' );
}
} );
$row[ $col ] = serialize( $work );
}
}
else
{
$row[ $col ] = \IPSUtf8\Text\Charset::i()->convert( $row[ $col ], $currentCharSet, 'UTF-8' );
}
}
}
/* Numeric columns */
foreach( $numericCols as $col )
{
if ( isset( $row[ $col ] ) AND ! empty( $row[ $col ] ) )
{
if ( $row[ $col ] === null OR ! is_numeric( $row[ $col ] ) )
{
$row[ $col ] = 0;
}
}
}
/* Roughly add the size of this insert in bytes */
$tmp = $row; // Use a copy so the reference in array_walk doesn't overwrite the row to be inserted
/* Make sure it's an associative array as topic_views only has 1 column so DB driver returns an indexed array only */
if ( is_array( $tmp ) )
{
array_walk_recursive( $tmp, function( &$input, $key )
{
$input = htmlentities( (string) $input, ENT_QUOTES | ENT_IGNORE, 'utf-8', false );
} );
}
$batchBytes += mb_strlen( @json_encode( $tmp ), '8bit');
unset( $tmp );
$batch[] = $row;
$nextBatch = null;
/* Give us an error margin to account for MySQL syntax */
if ( $batchBytes > ( $this->_data['max_allowed_packet'] - ( ( $this->_data['max_allowed_packet'] / 100 ) * ( count( $batch ) * 0.3 ) ) ) )
{
/* Remove last row */
$nextBatch = array_pop( $batch );
static::log( "Max packet hit with " . $batchBytes . 'b with ' . count( $batch ) . ' rows' );
}
/* If we have moved the only row from $batch to $nextbatch, then process that now */
if ( count( $nextBatch ) and ! count( $batch ) )
{
$batch = $nextBatch;
$nextBatch = array();
}
/* Got a batch to write? */
if ( $nextBatch !== null OR ( count( $batch ) === $max ) OR ( $gotRows === count( $batch ) ) )
{
try
{
/* Optimise */
$this->preInsert( $table );
$insertId = static::$utf->insert( $table['name'], $batch, true, true, $table );
if ( ! empty( static::$utf->error ) )
{
throw new \RuntimeException( static::$utf->error );
}
$this->postInsert( $table );
/* Update counts */
$sessionData['convertedCount'] += count( $batch );
$convertCount += count( $batch );
if ( \IPSUtf8\Session::i()->current_pkey )
{
\IPSUtf8\Session::i()->current_row = $insertId;
}
else
{
\IPSUtf8\Session::i()->current_row += count( $batch );
}
\IPSUtf8\Session::i()->status = 'processing';
\IPSUtf8\Session::i()->json = $sessionData;
\IPSUtf8\Session::i()->save();
static::log( count( $batch ) . " rows batch inserted using conversion method: " . \IPSUtf8\Text\Charset::$method . ', last insert ID ' . $insertId );
$batch = ( $nextBatch !== null ) ? array( $nextBatch ) : array();
$batchBytes = 0;
}
catch( \IPSUtf8\Db\Exception $e )
{
$msg = $e->getMessage();
\IPSUtf8\Session::i()->save();
/* Trying to insert duplicate data, just let it go to the next in case there is a small overlap */
if ( mb_stristr( $msg, 'duplicate entry' ) )
{
static::log( "Duplicate key failure on insert\n{$msg}\n" . var_export( $batch, true ) );
return $this->heal( $limit, $convertCount );
}
else
{
/* Anything else we should probably halt on */
static::log( "Exception Thrown:\n{$msg}\n" . var_export( $e, true ) );
throw new \RuntimeException( $msg );
}
}
}
}
/* Anything fetched? */
if ( $gotRows === 0 )
{
try
{
static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` ENABLE KEYS;");
static::log( "{$table['name']} keys enabled." );
}
catch( \IPSUtf8\Db\Exception $e )
{
static::log( static::$utf->prefix . "{$table['name']} enable keys exception caught." );
}
$table = $this->getNextTable( $table['name'] );
$convertCols = static::getConvertableColumns( $table['name'] );
}
}
return true;
}
/**
* Pre insert
*
* @param array $table Table definition data
* @return void
*/
public function preInsert( $table )
{
$row = static::$utf->query( "SHOW TABLE STATUS LIKE '" . static::$utf->prefix . $table['name'] . "'" )->fetch_assoc();
static::$utfTableEngine = $row['Engine'];
if ( empty( static::$utfTableEngine ) )
{
static::$utfTableEngine = static::$utf->defaultEngine();
}
if ( \strtolower( static::$utfTableEngine ) === 'myisam' )
{
static::log( "Pre inserts for MyISAM table " . $table['name'] );
try
{
//static::$utf->query("ALTER TABLE " . static::$utf->prefix . "{$table['name']} DISABLE KEYS;");
}
catch( \IPSUtf8\Db\Exception $e )
{
static::log( static::$utf->prefix . "{$table['name']} disable keys exception caught." );
}
}
else if ( \strtolower( static::$utfTableEngine ) === 'innodb' )
{
static::log( "Pre inserts for InnoDB table " . $table['name'] );
/* This doesn't seem necessary
try
{
static::$utf->query("SET autocommit=0");
static::$utf->query("SET unique_checks=0");
static::$utf->query("SET foreign_key_checks=0");
}
catch ( Exception $e ) { }*/
}
}
/**
* Post insert
*
* @param array $table Table definition data
* @return void
*/
public function postInsert( $table )
{
if ( \strtolower( static::$utfTableEngine ) === 'myisam' )
{
static::log( "Post inserts for MyISAM table " . $table['name'] );
try
{
//static::$utf->query("ALTER TABLE " . static::$utf->prefix . "{$table['name']} ENABLE KEYS;");
}
catch( \IPSUtf8\Db\Exception $e )
{
static::log( static::$utf->prefix . "{$table['name']} enable keys exception caught." );
}
}
else if ( \strtolower( static::$utfTableEngine ) === 'innodb' )
{
static::log( "Post inserts for InnoDB table " . $table['name'] );
/* This doesn't seem necessary
try
{
static::$utf->query("SET autocommit=1");
static::$utf->query("SET unique_checks=1");
static::$utf->query("SET foreign_key_checks=1");
}
catch ( Exception $e ) { }*/
}
}
/**
* Finish the conversion
*
* @return null
*/
public function finish()
{
$charset = 'utf8';
$collation = 'utf8_unicode_ci';
/* MB4? */
$sessionData = \IPSUtf8\Session::i()->json;
if ( isset( $sessionData['use_utf8mb4'] ) AND ! empty( $sessionData['use_utf8mb4'] ) )
{
static::$utf->set_charset( 'utf8mb4' );
$charset = 'utf8mb4';
$collation = 'utf8mb4_unicode_ci';
}
/* IPB 3? */
if ( isset( \IPSUtf8\Session::i()->tables['core_sys_conf_settings'] ) AND isset( \IPSUtf8\Session::i()->tables['cache_store'] ) )
{
if ( static::$utf->checkForTable('core_sys_conf_settings' ) )
{
static::$utf->update( 'core_sys_conf_settings', array( 'conf_value' => '' ), array( 'conf_key=?', 'gb_char_set' ) );
$rows = static::$utf->select( '*', 'core_sys_conf_settings', array( 'conf_add_cache=?', 1 ) );
$settings = array();
foreach( $rows as $row ) #row your boat
{
$value = $row['conf_value'] != "" ? $row['conf_value'] : $row['conf_default'];
if ( $value == '{blank}' )
{
$value = '';
}
$settings[ $row['conf_key'] ] = $value;
}
static::$utf->update( 'cache_store', array( 'cs_value' => serialize( $settings ) ), array( 'cs_key=?', 'settings' ) );
}
/* Update Language Locales */
if ( static::$utf->checkForTable('core_sys_lang') )
{
/* Store current */
$currentLocale = setlocale( LC_ALL, '0' );
/* Loop through languages */
$languages = static::$utf->select( '*', 'core_sys_lang' );
foreach( $languages AS $language )
{
$locale = explode( '.', $language['lang_short'] ); # We want to update even if a charset is already set.
foreach( array( "{$locale[0]}.UTF8", "{$locale[0]}.UTF-8", "{$locale[0]}.utf8" ) AS $test )
{
$verify = setlocale( LC_ALL, $test );
if ( $verify !== FALSE )
{
static::$utf->update( 'core_sys_lang', array( 'lang_short' => $test ), array( 'lang_id=?', $language['lang_id'] ) );
break;
}
}
}
foreach( explode( ";", $currentLocale ) as $locale )
{
$parts = explode( "=", $locale );
if( in_array( $parts[0], array( 'LC_ALL', 'LC_COLLATE', 'LC_CTYPE', 'LC_MONETARY', 'LC_NUMERIC', 'LC_TIME' ) ) )
{
setlocale( constant( $parts[0] ), $parts[1] );
}
}
/* Clear Cache so it regenerates */
static::$utf->update( 'cache_store', array( 'cs_value' => '' ), array( 'cs_key=?', 'lang_data' ) );
}
/* Unlock Tasks */
if( static::$utf->checkForTable('task_manager') )
{
static::$utf->update( "task_manager", array( 'task_locked' => 0 ) );
}
else if( static::$utf->checkForTable('core_tasks') )
{
static::$utf->update( "core_tasks", array( 'next_run' => time() ) );
}
}
/* Change database collation */
require( ROOT_PATH . '/conf_global.php' );
static::$utf->query( "ALTER DATABASE `" . $INFO['sql_database'] . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
}
/**
* Restore the original tables.
*
* @return null
*/
public function restoreOriginalTables()
{
/* Grab all tables to convert */
$stmt = static::$db->prepare( "SHOW TABLES" );
$stmt->execute();
$stmt->bind_result( $name );
$tables = array();
while ( $stmt->fetch() === true )
{
$tables[] = $name;
}
foreach( $tables as $name )
{
if ( mb_substr( $name, 0, 5 ) === 'orig_' )
{
$plainName = mb_substr( $name, 5 );
static::$db->query( "DROP TABLE IF EXISTS `" . $plainName . "`" );
$rename = "RENAME TABLE `" . $name . '` TO `' . $plainName . '`';
static::$db->query( $rename );
static::log( $rename );
}
}
}
/**
* Delete the original tables.
*
* @return null
*/
public function deleteOriginalTables()
{
/* Grab all tables to convert */
$stmt = static::$utf->prepare( "SHOW TABLES" );
$stmt->execute();
$stmt->bind_result( $name );
$tables = array();
while ( $stmt->fetch() === true )
{
$tables[] = $name;
}
foreach( $tables as $name )
{
if ( mb_substr( $name, 0, 5 ) === 'orig_' )
{
static::$db->query( "DROP TABLE IF EXISTS `" . $name . "`" );
static::log( "DROP TABLE IF EXISTS `" . $name . "`" );
}
}
}
/**
* Rename the tables
*
* @return null
*/
public function renameTables()
{
/* Grab all tables to convert */
$stmt = static::$db->prepare( "SHOW TABLES" );
$stmt->execute();
$stmt->bind_result( $name );
$tables = array();
$origPrefix = static::$db->prefix;
if ( mb_substr( $origPrefix, 0, 6 ) === 'x_utf_' )
{
$origPrefix = mb_substr( $origPrefix, 6 );
}
while ( $stmt->fetch() === true )
{
$tables[] = $name;
}
foreach( $tables as $name )
{
$tableNameNoPrefix = $name;
$isConvertedTable = ( mb_substr( $name, 0, 6 ) === 'x_utf_' );
$isOrigTable = ( mb_substr( $name, 0, 5 ) === 'orig_' );
if ( $isOrigTable )
{
continue;
}
if ( ! $isConvertedTable and $origPrefix )
{
$tableNameNoPrefix = mb_substr( $name, mb_strlen( $origPrefix ) );
}
/* Rename original tables */
if ( ( ! $isConvertedTable ) and mb_substr( $name, 0, mb_strlen( $origPrefix ) ) === $origPrefix )
{
static::$db->query( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );
static::log( "RENAME TABLE `" . $origPrefix . $tableNameNoPrefix . '` TO `orig_' . $origPrefix . $tableNameNoPrefix . '`' );
}
/* Grab x_utf_ prefixed tables */
if ( $isConvertedTable )
{
$tableName = mb_substr( $name, mb_strlen( static::$utf->prefix ) );
/* Rename the new one */
if ( $tableName !== 'convert_session' AND $tableName !== 'convert_session_tables' )
{
static::$utf->query( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );
static::log( "RENAME TABLE `" . $name . '` TO `' . $origPrefix . $tableName . '`' );
}
}
}
/* Update table data */
\IPSUtf8\Session::i()->updateTableData();
}
/**
* Go through the DB and fix the collation of UTF8 tables
*
* @return null
*/
public function fixCollation()
{
$charset = 'utf8';
$collation = 'utf8_unicode_ci';
/* MB4? */
$sessionData = \IPSUtf8\Session::i()->json;
if ( isset( $sessionData['use_utf8mb4'] ) AND ! empty( $sessionData['use_utf8mb4'] ) )
{
static::$utf->set_charset( 'utf8mb4' );
$charset = 'utf8mb4';
$collation = 'utf8mb4_unicode_ci';
}
/* Change database collation */
require( ROOT_PATH . '/conf_global.php' );
static::$utf->query( "ALTER DATABASE `" . $INFO['sql_database'] . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
foreach( \IPSUtf8\Session::i()->tables as $name => $data )
{
if ( in_array( $name, $this->getNonUtf8CollationTables() ) )
{
/* Skip if it's an _old table */
if( mb_substr( $name, -4 ) == '_old' )
{
continue;
}
$cols = static::getConvertableColumns( $data['name'] );
$tableData = \IPSUtf8\Session::i()->tables[ $data['name'] ];
if ( count( $cols ) )
{
$dropIndexes = array();
$addIndexes = array();
$modify = array();
$hasUnique = FALSE;
/* Changing collation can cause issues for duplicate entries in unique keys */
if ( isset( $tableData['definition']['indexes'] ) )
{
foreach( $tableData['definition']['indexes'] as $key => $index )
{
if ( $index['type'] === 'unique' OR $index['type'] === 'primary' )
{
/* Make sure none of the columns in this index are auto_increment */
foreach( $index['columns'] AS $k => $idxcol )
{
if ( $tableData['definition']['columns'][ $idxcol ]['auto_increment'] === TRUE )
{
continue 2;
}
}
if ( static::$db->checkForIndex( $name, $key ) )
{
$hasUnique = TRUE;
break;
}
}
}
}
}
/* We don't need to drop any indexes - just create a copy of the table and insert ignore */
if ( $hasUnique === TRUE )
{
/* Let's try an alter first - below may not be necessary */
try
{
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
}
catch( \IPSUtf8\Db\Exception $e )
{
/* Did not work - try the long way */
static::$utf->query( "RENAME TABLE `" . static::$db->prefix . $name . "` TO `" . static::$db->prefix . $name . "_temp`" );
static::log( $name . " has Unique Index - temp table created from main." );
/* Previously we used a raw create table query to create a copy, however this does not check index lengths. Leaving this for reference. */
//static::$utf->query( "CREATE TABLE `" . static::$db->prefix . $name . "` LIKE `" . static::$db->prefix . $name . "_temp`" );
static::$db->createTable( $this->checkTable( $tableData['definition'] ) );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
static::log( $name . " created from temp table." );
static::$utf->query( "REPLACE INTO `" . static::$db->prefix . $name . "` SELECT * FROM `" . static::$db->prefix . $name . "_temp`" );
static::log( $name . " data copied to new table." );
static::$utf->query( "DROP TABLE `" . static::$db->prefix . $name . "_temp`" );
static::log( $name . "_temp dropped" );
}
}
else
{
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` CONVERT TO CHARACTER SET " . $charset . " COLLATE " . $collation );
static::$utf->query( "ALTER TABLE `" . static::$db->prefix . $name . "` DEFAULT CHARACTER SET " . $charset . " COLLATE " . $collation );
}
if ( count( $cols ) )
{
foreach( $cols as $col )
{
if ( isset( $tableData['definition']['columns'][ $col ] ) )
{
$colData = $tableData['definition']['columns'][ $col ];
$modify[] = " MODIFY `" . $col . "` " . $colData['type'] . ( ( is_numeric( $colData['length'] ) AND $colData['length'] > 0 ) ? "(" . $colData['length'] . ")" : '' ) . " CHARACTER SET " . $charset . " COLLATE " . $collation;
}
}
}
if ( count( $cols ) )
{
if ( count( $modify ) )
{
$query = "ALTER TABLE `" . static::$db->prefix . $name . "` " . implode( ',', $modify );
static::log( $query );
static::$utf->query( $query );
}
if ( /*count( $modify ) and*/ count( $addIndexes ) )
{
$query = "ALTER TABLE `" . static::$db->prefix . $name . "` " . implode( ',', $addIndexes );
static::log( $query );
static::$utf->query( $query );
}
}
}
}
/* Update table data */
\IPSUtf8\Session::i()->updateTableData();
/* Update character set if we're in IP.Board */
if ( static::$utf->checkForTable('core_sys_conf_settings' ) )
{
static::$utf->update( 'core_sys_conf_settings', array( 'conf_value' => '' ), array( 'conf_key=?', 'gb_char_set' ) );
}
}
/**
* Something went wrong, so try and heal to restart progress
*
* @param int $limit Process() method limit
* @param int $cycleCount Items processed if this was in the middle of process()
* @return bool|null
*/
public function heal( $limit=250, $cycleCount=0 )
{
if ( \IPSUtf8\Session::i()->current_table )
{
$table = \IPSUtf8\Session::i()->tables[ \IPSUtf8\Session::i()->current_table ];
/* Table doesn't exist, so rewind back to the start of this table */
if ( ! static::$utf->checkForTable( $table['name'] ) )
{
if ( static::$utf->createTable( $table['definition'] ) === false )
{
throw new \RuntimeException( static::$utf->error . "\n" . var_export( $table['definition'], true ) );
}
$completed = \IPSUtf8\Session::i()->completed_json;
if ( isset( $completed[ \IPSUtf8\Session::i()->current_table ] ) )
{
unset( $completed[ \IPSUtf8\Session::i()->current_table ] );
}
/* Got a primary key so we can use a WHERE N > X query rather than a limit for efficiency? */
if ( isset( $table['definition']['indexes']['PRIMARY'] ) )
{
$pkey = $table['definition']['indexes']['PRIMARY']['columns'][0];
/* Is it numeric? */
if ( mb_stristr( $table['definition']['columns'][ $pkey ]['type'], 'int' ) )
{
\IPSUtf8\Session::i()->current_pkey = $pkey;
}
}
\IPSUtf8\Session::i()->status = 'processing';
\IPSUtf8\Session::i()->completed_json = $completed;
\IPSUtf8\Session::i()->current_row = 0;
\IPSUtf8\Session::i()->save();
/* Run again */
return $this->process( ( $limit - $cycleCount ) );
}
if ( \IPSUtf8\Session::i()->current_pkey )
{
/* Get the latest row */
$max = static::$utf->select('MAX(`' . \IPSUtf8\Session::i()->current_pkey . '`) as max', \IPSUtf8\Session::i()->current_table )->setKeyField('max')->first();
}
else
{
/* Get the count */
$max = static::$utf->select('COUNT(*) as count', \IPSUtf8\Session::i()->current_table )->setKeyField('count')->first();
}
\IPSUtf8\Session::i()->status = 'processing';
\IPSUtf8\Session::i()->current_row = $max;
\IPSUtf8\Session::i()->save();
/* Run again */
return $this->process( ( $limit - $cycleCount ), TRUE );
}
return null;
}
/**
* Set the current work table
*
* @return void
*/
public function getTable()
{
$table = null;
if ( \IPSUtf8\Session::i()->current_table AND isset( \IPSUtf8\Session::i()->tables[ \IPSUtf8\Session::i()->current_table ] ) )
{
/* Table selected and conversion in progress */
$table = \IPSUtf8\Session::i()->tables[ \IPSUtf8\Session::i()->current_table ];
static::log( "Continuing with " . $table['name'] . ' (PKEY: ' . \IPSUtf8\Session::i()->current_pkey . ')' );
if ( ! static::$utf->checkForTable( $table['name'] ) )
{
if ( static::$utf->createTable( $this->checkTable( $table['definition'] ) ) === false )
{
throw new \RuntimeException( static::$utf->error . "\n" . var_export( $table['definition'], true ) );
}
try
{
static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` DISABLE KEYS;");
static::log( "{$table['name']} keys disabled." );
}
catch( \IPSUtf8\Db\Exception $e )
{
static::log( static::$utf->prefix . "{$table['name']} disable keys exception caught." );
}
}
return $table;
}
else
{
/* need to resolve an issue where if you only need to convert a few tables, these few will be named x_utf_ while others won't,
so for now, just convert all */
$nonUtf8Tables = array_keys( \IPSUtf8\Session::i()->tables );//$this->getNonUtf8Tables();
/* No table selected */
if ( is_array( $nonUtf8Tables ) and count( $nonUtf8Tables ) )
{
if ( is_array( \IPSUtf8\Session::i()->completed_json ) )
{
$diff = array_diff( $nonUtf8Tables, array_keys( \IPSUtf8\Session::i()->completed_json ) );
if ( count( $diff ) )
{
$table = \IPSUtf8\Session::i()->tables[ array_shift( $diff ) ];
}
else
{
return null;
}
}
else
{
$table = array_shift( $nonUtf8Tables );
}
if ( isset( $table['name'] ) )
{
\IPSUtf8\Session::i()->current_table = $table['name'];
/* Got a primary key so we can use a WHERE N > X query rather than a limit for efficiency? */
if ( isset( $table['definition']['indexes']['PRIMARY'] ) )
{
$pkey = $table['definition']['indexes']['PRIMARY']['columns'][0];
/* Is it numeric? */
if ( mb_stristr( $table['definition']['columns'][ $pkey ]['type'], 'int' ) )
{
\IPSUtf8\Session::i()->current_pkey = $pkey;
}
}
\IPSUtf8\Session::i()->save();
/* Create the table for UTF8 goodness */
static::$utf->dropTable( $table['name'], true );
if ( static::$utf->createTable( $this->checkTable( $table['definition'] ) ) === false )
{
throw new \RuntimeException( static::$utf->error . "\n" . var_export( $table['definition'], true ) );
}
static::log( "Created UTF8 table " . $table['name'] . ' (PKEY: ' . \IPSUtf8\Session::i()->current_pkey . ')' );
try
{
static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$table['name']}` DISABLE KEYS;");
static::log( "{$table['name']} keys disabled." );
}
catch( \IPSUtf8\Db\Exception $e )
{
static::log( static::$utf->prefix . "{$table['name']} disable keys exception caught." );
}
return $table;
}
}
}
return null;
}
/**
* Attempt to fix issues with keys longer than 1000bytes
*
* @param array $definition Table definition
* @return array
*/
public function checkTable( $definition )
{
/* MB4? */
$sessionData = \IPSUtf8\Session::i()->json;
$length = 0;
$multiplier = 4;
$needsFixing = array();
$maxLen = 1000;
if ( \mb_strtolower( $definition['engine'] ) === 'innodb' )
{
$maxLen = 767;
}
if ( isset( $definition['indexes'] ) )
{
foreach( $definition['indexes'] as $key => $index )
{
$thisLength = null;
$hasText = false;
foreach( $index['columns'] as $i => $column )
{
$thisLength = ( isset( $index['length'][ $i ] ) ) ? $index['length'][ $i ] : ( ( (int) $definition['columns'][ $column ]['length'] or empty( $definition['columns'][ $column ]['length'] ) ) ? $definition['columns'][ $column ]['length'] : 250 );
$isText = in_array( mb_strtolower( $definition['columns'][ $column ]['type'] ), array( 'mediumtext', 'text' ) );
if ( $hasText === false and $isText === true )
{
$hasText = true;
}
if ( isset( $definition['columns'][ $column ] ) and ( ( ! empty( $thisLength ) or $isText ) ) )
{
$length += $thisLength;
}
}
if ( ( $length * $multiplier > $maxLen ) or $hasText )
{
foreach( $index['columns'] as $i => $column )
{
$thisLength = ( isset( $index['length'][ $i ] ) ) ? $index['length'][ $i ] : ( (int) $definition['columns'][ $column ]['length'] ? $definition['columns'][ $column ]['length'] : 250 );
if ( isset( $definition['columns'][ $column ] ) and ( ( ! empty( $thisLength ) or in_array( mb_strtolower( $definition['columns'][ $column ]['type'] ), array( 'mediumtext', 'text' ) ) ) ) )
{
/* Column name, column length, column type */
$needsFixing[ $key ][ $i ] = array( $column, $thisLength, $definition['columns'][ $column ]['type'] );
}
}
}
$length = 0;
}
}
if ( count( $needsFixing ) )
{
foreach( $needsFixing as $key => $i )
{
$totalLength = 0;
$maxChars = $maxLen / $multiplier;
foreach( $i as $vals )
{
$totalLength += $vals[1];
}
if ( $totalLength > $maxChars )
{
/* Check each column can be reduced by the amount we need reducing */
$debt = 0;
$reduceEachBy = ( ( 100 / $totalLength ) * $maxChars) / 100;
/* Apply debt if we have any. We do not reduce integers */
foreach( $i as $x => $vals )
{
if ( in_array( mb_strtoupper( $vals[2] ), array_keys( static::$numericCols ) ) )
{
$debt += $vals[1];
}
}
/* Recalculate value to multiply index sub lengths with (subtracting debt) */
if ( $debt < $totalLength )
{
$reduceEachBy = ( ( 100 / ($totalLength - $debt) ) * ( $maxChars - $debt ) ) / 100;
}
foreach( $i as $x => $vals )
{
/* No length? */
if ( empty( $vals[1] ) )
{
$vals[1] = 250;
}
if ( in_array( mb_strtoupper( $vals[2] ), array_keys( static::$numericCols ) ) )
{
/* Preserve col len where possible but if the column length is greater than subpart allowed, NULL the length
otherwise MySQL will complain as you cannot use subpart on non-string column. */
if ( $vals[1] > floor( $maxLen / $multiplier ) )
{
$vals[1] = NULL;
$i[ $x ] = $vals;
}
continue;
}
$vals[1] = floor( $vals[1] * $reduceEachBy );
$i[ $x ] = $vals;
}
}
foreach( $i as $x => $vals )
{
if ( in_array( mb_strtolower( $vals[2] ), static::$convertCols ) AND $definition['columns'][ $definition['indexes'][ $key ]['columns'][ $x ] ]['length'] != $vals[1] )
{
$definition['indexes'][ $key ]['length'][ $x ] = intval( $vals[1] );
}
else
{
$definition['indexes'][ $key ]['length'][ $x ] = NULL;
}
}
}
}
return $definition;
}
/**
* Fetches the next table to process or false if nothing left to process
*
* @param string $currentTable Name of table just processed
* @return array Array of table data
*/
public function getNextTable( $currentTable )
{
try
{
static::$utf->query("ALTER TABLE `" . static::$utf->prefix . "{$currentTable}` ENABLE KEYS;");
static::log( "{$table['name']} keys enabled." );
}
catch( \IPSUtf8\Db\Exception $e )
{
static::log( static::$utf->prefix . "{$currentTable} enable keys exception caught." );
}
$completed = \IPSUtf8\Session::i()->completed_json;
$completed[ $currentTable ] = $currentTable;
\IPSUtf8\Session::i()->completed_json = $completed;
\IPSUtf8\Session::i()->current_pkey = null;
\IPSUtf8\Session::i()->current_table = null;
\IPSUtf8\Session::i()->current_row = 0;
\IPSUtf8\Session::i()->save();
return $this->getTable();
}
/**
* Init this class
*/
public function init()
{
/* Grab all tables to convert */
$stmt = static::$db->query( "show variables" );
$tables = array();
while ( $row = $stmt->fetch_array( MYSQLI_ASSOC ) )
{
$key = $row['Variable_name'];
$value = $row['Value'];
if ( $key == 'character_set_database' )
{
$this->_data['database_charset'] = strtolower( $value );
}
if ( $key == 'bulk_insert_buffer_size' )
{
$this->_data['bulk_insert_buffer_size'] = $value;
}
if ( $key == 'max_allowed_packet' )
{
$this->_data['max_allowed_packet'] = $value;
}
}
if ( ! empty( $this->_data['max_allowed_packet'] ) AND ! empty( $this->_data['bulk_insert_buffer_size'] ) AND ( $this->_data['bulk_insert_buffer_size'] < $this->_data['max_allowed_packet'] ) )
{
$this->_data['max_allowed_packet'] = $this->_data['bulk_insert_buffer_size'];
}
if ( empty( $this->_data['max_allowed_packet'] ) AND empty( $this->_data['bulk_insert_buffer_size'] ) )
{
/* No value so use MySQL default of 1MB */
$this->_data['max_allowed_packet'] = 1048576;
}
}
/**
* Returns true if the table has a text or medium text column
*
* @param string $name Name of table
* @return boolean
*/
public static function hasTextColumn( $name )
{
$table = \IPSUtf8\Session::i()->tables[ $name ];
foreach( $table['definition']['columns'] as $col => $val )
{
if ( in_array( mb_strtolower( $val['type'] ), array( 'mediumtext', 'text' ) ) )
{
return true;
}
}
return false;
}
/**
* Tables with lots of dynamic columns can be slow
*
* @param string $name Name of table
* @return boolean
*/
public static function throttleInserts( $name )
{
$table = \IPSUtf8\Session::i()->tables[ $name ];
$total = 0;
$var = 0;
foreach( $table['definition']['columns'] as $col => $val )
{
$total++;
if ( in_array( mb_strtolower( $val['type'] ), array( 'varchar' ) ) )
{
$var++;
}
}
if ( $total > 70 )
{
return 50;
}
else if ( $total > 30 AND $var > 0 )
{
if ( ( $var / $total ) * 100 > 25 )
{
return 100;
}
}
return false;
}
/**
* Returns whether the database is Invision Community already
*
* @return boolean
*/
public function databaseIsIPS4()
{
return (boolean) static::$db->checkForTable( 'core_members' );
}
/**
* Returns whether the DB is UTF8 already
*
* @return bool
*/
public function databaseIsUtf8()
{
$yeahProbably = false;
if ( \IPSUtf8\Convert::i()->database_charset == 'utf8mb4' OR \IPSUtf8\Convert::i()->database_charset == 'utf8' OR \IPSUtf8\Session::i()->current_charset == 'utf-8' )
{
$yeahProbably = true;
}
/* Another check */
if ( $yeahProbably === true )
{
/* Best check the tables, then */
$json = \IPSUtf8\Session::i()->json;
if ( count( $this->getNonUtf8Tables() ) )
{
$yeahProbably = false;
}
}
return $yeahProbably;
}
/**
* Grab the character sets from MySQL
*
* @return array
*/
public function getMysqlCharSets()
{
if ( static::$mysqlCharSets === NULL )
{
static::$mysqlCharSets = array( 'latin1', 'utf8' );
$stmt = static::$db->query( "show character set" );
while ( $row = $stmt->fetch_array( MYSQLI_ASSOC ) )
{
static::$mysqlCharSets[] = $row['Charset'];
}
}
return static::$mysqlCharSets;
}
/**
* Return tables that need converting
*
* @param boolean $force Force a recount
* @return array
*/
public function getNonUtf8Tables( $force=false )
{
if ( static::$nonUtf8Tables === NULL or $force === TRUE )
{
static::$nonUtf8Tables = array();
/* Best check the tables, then */
$json = \IPSUtf8\Session::i()->json;
foreach( $json['charSets'] as $charSet => $tablesArray )
{
if ( ! empty( $json['force_conversion'] ) or ( $charSet != 'utf8' and $charSet != 'utf8mb4' ) )
{
static::$nonUtf8Tables = array_merge( static::$nonUtf8Tables, $tablesArray );
}
}
}
return static::$nonUtf8Tables;
}
/**
* Returns total rows to convert
*
* @return array
*/
public function getTotalRowsToConvert()
{
if ( static::$rowsToConvert === NULL )
{
static::$rowsToConvert = 0;
foreach( array_keys( \IPSUtf8\Session::i()->tables ) as $table )
{
static::$rowsToConvert += intval( \IPSUtf8\Session::i()->tables[ $table ]['count'] );
}
}
return static::$rowsToConvert;
}
/**
* Returns whether the DB has the correct collation
*
* @return bool
*/
public function getNonUtf8CollationTables()
{
if ( static::$nonUtf8Collations === null )
{
static::$nonUtf8Collations = array();
/* Best check the tables, then */
foreach( \IPSUtf8\Session::i()->tables as $table => $definition )
{
/* If the table itself is not utf8_unicode_ci, don't bother check columns */
if ( $definition['definition']['collation'] AND !in_array( $definition['definition']['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
{
static::$nonUtf8Collations[] = $table;
continue;
}
foreach( $definition['definition']['columns'] as $name => $column )
{
if ( $column['collation'] and ! in_array( $column['collation'], array( 'utf8_unicode_ci', 'utf8mb4_unicode_ci' ) ) )
{
static::$nonUtf8Collations[] = $table;
break;
}
}
}
}
return static::$nonUtf8Collations;
}
/**
* Fetch debug information
*
* @return array
*/
public function getDebugString()
{
$data = array();
$tableCharSet = \IPSUtf8\Convert::i()->database_charset;
$currentCharSet = \IPSUtf8\Session::i()->current_charset;
$json = \IPSUtf8\Session::i()->json;
$data[] = "IP.Board Character Set: " . $currentCharSet;
$data[] = "Database Character Set: " . $tableCharSet;
$data[] = "Original table prefix: " . static::$db->prefix;
$data[] = "Converted table prefix: " . static::$utf->prefix;
foreach( $json['charSets'] as $charSet => $tablesArray )
{
$data[] = count( $tablesArray ) . ' tables are ' . $charSet;
}
$data[] = count( $this->getNonUtf8CollationTables() ) . " tables have incorrect collations";
$data[] = "Can use 'dump' method: " . var_export( $this->canDump(), true );
return $data;
}
/**
* Detect whether this is a serialized string or not
*
* @param string $string The actual string
* @return boolean
*/
public static function isSerialized( $string )
{
/* If it looks nothing like a serialized string, then return it */
if ( ! preg_match( '#^a:\d+:\{#', $string ) )
{
return false;
}
/* Now make sure it's actually a serialized string */
return ( boolean ) @unserialize( $string );
}
/**
* Log data
*/
public static function log( $message )
{
if ( LOG )
{
$file = THIS_PATH . '/tmp/log_' . date('Y-m-d') . '.cgi';
$isNew = false;
if ( ! is_file( $file ) )
{
$isNew = true;
}
@file_put_contents( $file, "\n" . str_repeat( '-', 48 ) . "\n" . date('r') . "\n" . $message, FILE_APPEND );
if ( $isNew )
{
@chmod( $file, 0777 );
}
}
}
/**
* Return a list of numeric columns that need checking
*
* @param string $name Table
* @return array Array of columns that need checking as they can contain INT
*/
public static function getNumericColumns( $name )
{
if ( ! is_string( $name ) )
{
return false;
}
$table = \IPSUtf8\Session::i()->tables[ $name ];
$return = array();
foreach( $table['definition']['columns'] as $col => $val )
{
if ( in_array( mb_strtolower( $val['type'] ), static::$numericCols ) )
{
$return[] = $val['name'];
}
}
return $return;
}
/**
* Return a list of columns that need converting
*
* @param string $name Table
* @return array Array of columns that need converting as they can contain text
*/
public static function getConvertableColumns( $name )
{
if ( ! is_string( $name ) )
{
return false;
}
$table = \IPSUtf8\Session::i()->tables[ $name ];
$return = array();
foreach( $table['definition']['columns'] as $col => $val )
{
if ( in_array( mb_strtolower( $val['type'] ), static::$convertCols ) )
{
$return[] = $val['name'];
}
}
return $return;
}
/**
* Can we use the fast dump method?
*
* @return boolean
*/
public static function canDump()
{
/* Best check the tables, then */
$json = \IPSUtf8\Session::i()->json;
$tablesOk = true;
foreach( $json['charSets'] as $charSet => $tablesArray )
{
if ( $charSet != 'utf8' and $charSet != 'utf8mb4' and mb_substr( $charSet, 0, 5) != 'latin' and $charSet != 'windows-1252' )
{
$tablesOk = false;
break;
}
}
if (
( $tablesOk ) AND
( \IPSUtf8\Session::i()->current_charset == 'utf-8' or \IPSUtf8\Session::i()->current_charset == 'iso-8859-1' ) AND
( ( is_callable( 'exec' ) AND false === stripos( ini_get( 'disable_functions' ), 'exec' ) ) )
)
{
try
{
@exec("iconv -l", $output );
}
catch( \ErrorException $e )
{
return false;
}
if ( ! array( $output ) or ! count( $output ) or ( ! stristr( implode( "\n", $output ), 'latin' ) ) )
{
return false;
}
return true;
}
else
{
return false;
}
}
/**
* Get Table Schematic
*
* @return array
*/
protected static function getTableSchematic( $table )
{
return json_decode( \IPSUtf8\Db::i('utf8')->select( 'table_schema', 'convert_session_tables', array( "table_name=?", $table ) )->first(), TRUE );
}
}