perl-install/unused

<?php
/** 
*
* @package phpBB3
* @version $Id$
* @copyright (c) 2005 phpBB Group 
* @license http://opensource.org/licenses/gpl-license.php GNU Public License 
*
*/

if (php_sapi_name() != 'cli')
{
	die("This program must be run from the command line.\n");
}

set_time_limit(0);

define('IN_PHPBB', true);
$phpbb_root_path = '../';
$phpEx = substr(strrchr(__FILE__, '.'), 1);

echo "Checking for required files\n";
download('http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt');
download('http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt');
download('http://www.unicode.org/Public/UNIDATA/UnicodeData.txt');
echo "\n";

require_once($phpbb_root_path . 'includes/utf/utf_normalizer.' . $phpEx);
$file_contents = array();

/**
* Generate some Hangul/Jamo stuff
*/
echo "\nGenerating Hangul and Jamo tables\n";
for ($i = 0; $i < UNICODE_HANGUL_LCOUNT; ++$i)
{
	$utf_char = cp_to_utf(UNICODE_HANGUL_LBASE + $i);
	$file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_VCOUNT * UNICODE_HANGUL_TCOUNT + UNICODE_HANGUL_SBASE;
	$file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_L;
}

for ($i = 0; $i < UNICODE_HANGUL_VCOUNT; ++$i)
{
	$utf_char = cp_to_utf(UNICODE_HANGUL_VBASE + $i);
	$file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i * UNICODE_HANGUL_TCOUNT;
	$file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_V;
}

for ($i = 0; $i < UNICODE_HANGUL_TCOUNT; ++$i)
{
	$utf_char = cp_to_utf(UNICODE_HANGUL_TBASE + $i);
	$file_contents['utf_normalizer_common']['utf_jamo_index'][$utf_char] = $i;
	$file_contents['utf_normalizer_common']['utf_jamo_type'][$utf_char] = UNICODE_JAMO_T;
}

/**
* Load the CompositionExclusions table
*/
echo "Loading CompositionExclusion\n";
$fp = fopen('CompositionExclusions.txt', 'rt');

$exclude = array();
while (!feof($fp))
{
	$line = fgets($fp, 1024);

	if (!strpos(' 0123456789ABCDEFabcdef', $line[0]))
	{
		continue;
	}

	$cp = strtok($line, ' ');

	if ($pos = strpos($cp, '..'))
	{
		$start = hexdec(substr($cp, 0, $pos));
		$end = hexdec(substr($cp, $pos + 2));

		for ($i = $start; $i < $end; ++$i)
		{
			$exclude[$i] = 1;
		}
	}
	else
	{
		$exclude[hexdec($cp)] = 1;
	}
}
fclose($fp);

/**
* Load QuickCheck tables
*/
echo "Generating QuickCheck tables\n";
$fp = fopen('DerivedNormalizationProps.txt', 'rt');

while (!feof($fp))
{
	$line = fgets($fp, 1024);

	if (!strpos(' 0123456789ABCDEFabcdef', $line[0]))
	{
		continue;
	}

	$p = array_map('trim', explode(';', strtok($line, '#')));

	/**
	* Capture only NFC_QC, NFKC_QC
	*/
	if (!preg_match('#^NFK?C_QC$#', $p[1]))
	{
		continue;
	}

	if ($pos = strpos($p[0], '..'))
	{
		$start = hexdec(substr($p[0], 0, $pos));
		$end = hexdec(substr($p[0], $pos + 2));
	}
	else
	{
		$start = $end = hexdec($p[0]);
	}

	if ($start >= UTF8_HANGUL_FIRST && $end <= UTF8_HANGUL_LAST)
	{
		/**
		* We do not store Hangul syllables in the array
		*/
		continue;
	}

	if ($p[2] == 'M')
	{
		$val = UNICODE_QC_MAYBE;
	}
	else
	{
		$val = UNICODE_QC_NO;
	}

	if ($p[1] == 'NFKC_QC')
	{
		$file = 'utf_nfkc_qc';
	}
	else
	{
		$file = 'utf_nfc_qc';
	}

	for ($i = $start; $i <= $end; ++$i)
	{
		/**
		* The vars have the same name as the file: $utf_nfc_qc is in utf_nfc_qc.php
		*/
		$file_contents[$file][$file][cp_to_utf($i)] = $val;
	}
}
fclose($fp);

/**
* Do mappings
*/
echo "Loading Unicode decomposition mappings\n";
$fp = fopen($phpbb_root_path . 'develop/UnicodeData.txt', 'rt');

$map = array();
while (!feof($fp))
{
	$p = explode(';', fgets($fp, 1024));
	$cp = hexdec($p[0]);

	if (!empty($p[3]))
	{
		/**
		* Store combining class > 0
		*/
		$file_contents['utf_normalizer_common']['utf_combining_class'][cp_to_utf($cp)] = (int) $p[3];
	}

	if (!isset($p[5]) || !preg_match_all('#[0-9A-F]+#', strip_tags($p[5]), $m))
	{
		continue;
	}

	if (strpos($p[5], '>'))
	{
		$map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0]));
	}
	else
	{
		$map['NFD'][$cp] = $map['NFKD'][$cp] = implode(' ', array_map('hexdec', $m[0]));
	}
}
fclose($fp);

/**
* Build the canonical composition table
*/
echo "Generating the Canonical Composition table\n";
foreach ($map['NFD'] as $cp => $decomp_seq)
{
	if (!strpos($decomp_seq, ' ') || isset($exclude[$cp]))
	{
		/**
		* Singletons are excluded from canonical composition
		*/
		continue;
	}

	$utf_seq = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq)));

	if (!isset($file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq]))
	{
		$file_contents['utf_canonical_comp']['utf_canonical_comp'][$utf_seq] = cp_to_utf($cp);
	}
}

/**
* Decompose the NF[K]D mappings recursively and prepare the file contents
*/
echo "Generating the Canonical and Compatibility Decomposition tables\n\n";
foreach ($map as $type => $decomp_map)
{
	foreach ($decomp_map as $cp => $decomp_seq)
	{
		$decomp_map[$cp] = decompose($decomp_map, $decomp_seq);
	}
	unset($decomp_seq);

	if ($type == 'NFKD')
	{
		$file = 'utf_compatibility_decomp';
		$var = 'utf_compatibility_decomp';
	}
	else
	{
		$file = 'utf_canonical_decomp';
		$var = 'utf_canonical_decomp';
	}

	/**
	* Generate the corresponding file
	*/
	foreach ($decomp_map as $cp => $decomp_seq)
	{
		$file_contents[$file][$var][cp_to_utf($cp)] = implode('', array_map('cp_to_utf', explode(' ', $decomp_seq)));
	}
}

/**
* Generate and/or alter the files
*/
foreach ($file_contents as $file => $contents)
{
	/**
	* Generate a new file
	*/
	echo "Writing to $file.$phpEx\n";

	if (!$fp = fopen($phpbb_root_path . 'includes/utf/data/' . $file . '.' . $phpEx, 'wb'))
	{
		trigger_error('Cannot open ' . $file . ' for write');
	}

	fwrite($fp, '<?php');
	foreach ($contents as $var => $val)
	{
		fwrite($fp, "\n\$GLOBALS[" . my_var_export($var) . ']=' . my_var_export($val) . ";");
	}
	fclose($fp);
}

echo "\n*** UTF-8 normalization tables done\n\n";

/**
* Now we'll generate the files needed by the search indexer
*/
echo "Generating search indexer tables\n";

$fp = fopen($phpbb_root_path . 'develop/UnicodeData.txt', 'rt');

$map = array();
while ($line = fgets($fp, 1024))
{
	/**
	* The current line is split, $m[0] hold the codepoint in hexadecimal and
	* all other fields numbered as in http://www.unicode.org/Public/UNIDATA/UCD.html#UnicodeData.txt
	*/
	$m = explode(';', $line);

	/**
	* @var	integer	$cp			Current char codepoint
	* @var	string	$utf_char	UTF-8 representation of current char
	*/
	$cp = hexdec($m[0]);
	$utf_char = cp_to_utf($cp);

	/**
	* $m[2] holds the "General Category" of the character
	* @link http://www.unicode.org/Public/UNIDATA/UCD.html#General_Category_Values
	*/
	switch ($m[2][0])
	{
		case 'L':
			/**
			* We allow all letters and map them to their lowercased counterpart on the fly
			*/
			$map_to_hex = (isset($m[13][0])) ? $m[13] : $m[0];

			if (preg_match('#^LATIN.*(?:LETTER|LIGATURE) ([A-Z]{2}(?![A-Z]))$#', $m[1], $capture))
			{
				/**
				* Special hack for some latin ligatures. Using the name of a character
				* is bad practice, but for now it works well enough.
				*
				* @todo Note that ligatures with combining marks such as U+01E2 are
				* not supported at this time
				*/
				$map[$cp] = strtolower($capture[1]);
			}
			else if (isset($m[13][0]))
			{
				/**
				* If the letter has a lowercased form, use it
				*/
				$map[$cp] = hex_to_utf($m[13]);
			}
			else
			{
				/**
				* In all other cases, map the letter to itself
				*/
				$map[$cp] = $utf_char;
			}
			break;

		case 'M':
			/**
			* We allow all marks, they are mapped to themselves
			*/
			$map[$cp] = $utf_char;
			break;

		case 'N':
			/**
			* We allow all numbers, but we map them to their numeric value whenever
			* possible. The numeric value (field #8) is in ASCII already
			*
			* @todo Note that fractions such as U+00BD will be converted to something
			* like "1/2", with a slash. However, "1/2" entered in ASCII is converted
			* to "1 2". This will have to be fixed.
			*/
			$map[$cp] = (isset($m[8][0])) ? $m[8] : $utf_char;
			break;

		default:
			/**
			* Everything else is ignored, skip to the next line
			*/
			continue 2;
	}
}
fclose($fp);

/**
* Add some cheating
*/
$cheats = array(
	'00DF'	=>	'ss',		#	German sharp S
	'00D6'	=>	'oe',		#	Capital O with diaeresis
	'00F6'	=>	'oe',		#	Small O with diaeresis
);

/**
* Add our "cheat replacements" to the map
*/
foreach ($cheats as $hex => $map_to)
{
	$map[hexdec($hex)] = $map_to;
}

/**
* Split the map into smaller blocks
*/
$file_contents = array();
foreach ($map as $cp => $map_to)
{
	$file_contents[$cp >> 11][cp_to_utf($cp)] = $map_to;
}
unset($map);

foreach ($file_contents as $idx => $contents)
{
	echo "Writing to search_indexer_$idx.$phpEx\n";
	$fp = fopen($phpbb_root_path . 'includes/utf/data/search_indexer_' . $idx . '.' . $phpEx, 'wb');
	fwrite($fp, '<?php return ' . my_var_export($contents) . ';');
	fclose($fp);
}
echo "\n*** Search indexer tables done\n\n";


die("\nAll done!\n");
Mode	Name	Size
-rw-r--r--	.cvsignore	2	log stats plain
-rw-r--r--	cdrom.pm	710	log stats plain
-rw-r--r--	dns.pm	1538	log stats plain
-rw-r--r--	otherinsmod.pm	488	log stats plain
-rw-r--r--	scsi.pm	3229	log stats plain