#!/usr/bin/perl
#
# BNCweb: A user-friendly interface to the British National Corpus - Version 4.3
# Copyright (C) 2005-10 Hoffmann/Evert/Lehmann
#
# Updates and information available at http://www.bncweb.info
#
# This file is part of BNCweb.
# 
# BNCweb is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
# 
# BNCweb is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

#
# based on BNCweb Version 2.0:
# © 1996/2004 Lehmann/Hoffmann/Schneider
#

#Module Declarations
package bncLibCQPXML;
require Exporter;
@ISA = qw(Exporter);
@EXPORT = qw(createSaradSock CQPconnect printHeader disconnect printFooter getParams setParam %params %paramsEnc getURLparams
             bark barkFull freqCalculate freqCalculate2 output2 makeDecimals createTitle subCorporaList %typeValues %catValues 
             $bwCGIalias $bwHTMLalias $bwServer $bwWebServer $bwSavePath $bwCorpusPath $bwMYSQLhost $bwMYSQLusertable $bwMYSQLspeakerIDstable 
             $bwMYSQLtable $null $bwMysqlPort $bwConcurrentProcesses $bwSuperuser $bwTempPath $bwCopyright 
             $bwDefaultCollocation $bwDefaultDist $bwDefaultSort $bwHeaderLength 
             $CQP CQPdisconnect CQPconnect undump_file getSpokenPositions hexURL $bwMYSQLcategorizetable $bwMysqlUser $bwMysqlPwd deleteCachedFreqlists
             deleteCachedQueries deleteCachedMySQLTables
             $C $Word $POS $U_WHO $TEXT_ID $S_N $S cqp_error_handler cqp_error_handler_full $bwMySQLTempPath $bwFileSize $bwMaxMySQLSize $bwCorpus $bwMYSQLfrequency 
             $bwMaxFreqlistMySQL subcorpusCheck compileFlist linefeed reverseFilemapping Filemapping %spoken class2cpos getTableSize checkSuperUser checkSuperUser2 encURL decURL encodeForm
             checkAuthLevel
);

use bncConfigXML;

# The length of the longest header in the BNC (for sucking in header with seek/read) - DON'T CHANGE THIS
$bwHeaderLength=100000;

# copyright notice for display
$bwCopyright = "<SMALL><FONT COLOR=\"#AAAAAA\">BNCweb (CQP-edition) &#169; 1996-2010</FONT></SMALL>";
$bwCopyright = "<table width=\"100%\"><tr><TD>$bwCopyright</TD><TD ALIGN=RIGHT><SMALL><FONT COLOR=\"#AAAAAA\">You are logged in as user \"$ENV{'REMOTE_USER'}\"</FONT></SMALL></TD></tr></TABLE>";

# connect to mysql
use DBI;
$bncHandle = DBI->connect("DBI:mysql:$bwMYSQLtable:$bwMYSQLhost:$bwMysqlPort", "$bwMysqlUser", "$bwMysqlPwd");
$userHandle = DBI->connect("DBI:mysql:$bwMYSQLusertable:$bwMYSQLhost:$bwMysqlPort", "$bwMysqlUser", "$bwMysqlPwd") or bark("Can't connect to mysql on $bwMYSQLhost at port: $bwMysqlPort <BR>" . $userHandle::errstr);

# Functions
sub getParams {
	my $theQuery=$_[0];
	my (@inputArray, $param, $paramName, $paramValue);
	@inputArray=split(/\&/, $theQuery);
	foreach $param (@inputArray) {
		($paramName, $paramValue) = split(/=/, $param,2);
		
		unless ($paramValue eq '') {
			$paramsEnc{$paramName}=$paramValue;
			$paramValue=~s/\+/ /g;
			$paramValue=~s/%([\dA-Fa-f][\dA-Fa-f])/pack ("C", hex ($1))/eg;
			$params{$paramName}=$paramValue;
		}
	}
	if ($params{urlTest} eq 'yes') {
		return 1;
	} else {
		return 0;
	}
}

sub setParam {
	my($pName, $value)=@_;
	my $valueEnc= encURL($value);
	$params{$pName}=$value;
	$paramsEnc{$pName}=$valueEnc;

}

sub getURLparams {
	my ($pName, $out);
	foreach $pName (keys %paramsEnc) {
		$out.= "&$pName=$paramsEnc{$pName}" unless $pName eq 'urlTest';
	}
	#$out.='&urlTest=yes';
	$out=~s/\A\&//;
	return $out;
}

sub encURL {
	my $url=$_[0];
	$url=~s/([^\w ])/&hexURL($1)/eg;
	$url=~s/ /\+/g;
	return($url);
}

sub decURL {
	my $Value=$_[0];
	$Value=~s/\+/ /g;
	$Value=~s/%([\dA-Fa-f][\dA-Fa-f])/pack ("C", hex ($1))/eg;
	return($Value);
}

sub hexURL {
	my $char=$_[0];
	$char=sprintf("%X", ord($char));
	$char= '%' . $char;
	return($char);
} 

sub encodeForm {
	my $chunk=$_[0];
	$chunk=~s/</\&lt\;/g;
	$chunk=~s/>/\&gt\;/g;
	$chunk=~s/"/\&quot\;/g;
	return($chunk);
}


####################################
# CQP stuff
####################################

use CWB::CQP;
use CWB::CL;

sub CQPconnect {
	my $theCorpus = "$bwCorpus";
	my $barkFull = $_[0];

 	$CQP = new CWB::CQP;
 	if ($barkFull) {
 		$CQP->set_error_handler(\&cqp_error_handler_full);
 	} else {
 		$CQP->set_error_handler(\&cqp_error_handler);
 	}
 	$CQP->exec("$theCorpus");

	$C = new CWB::CL::Corpus "$theCorpus"			# corpus handle
	  or die "Can't access corpus BNC\n";
	$Word = $C->attribute("word", "p")		# surface forms of tokens
	  or die "Can't access attribute BNC.word\n";
	$POS = $C->attribute("pos", "p")		# part-of-speech tags
	  or die "Can't access attribute BNC.pos\n";
	$U_WHO = $C->attribute("u_who", "s")		# this is important: speaker information ...
	  or die "Can't access attribute BNC.u_who\n";
	$TEXT_ID = $C->attribute("text_id", "s")	# ... document ID ...
	  or die "Can't access attribute BNC.text_id\n";
	$S_N = $C->attribute("s_n", "s")		# ... and sentence number (within document)
	  or die "Can't access attribute BNC.s_n\n";
	$S = $C->attribute("s", "s")	   # sentence regions without annotations (could also use $S_N below)
	  or die "Can't access attribute BNC.s\n";
	
	#return ("OK");	
}

sub CQPdisconnect {
 	die "CQPdisconnect: not connected to CQP process."
 	 	unless $CQP;
 	undef $CQP;
}

sub cqp_error_handler {
	print <<END;
	<TABLE BORDER="0" WIDTH="100%" CELLPADDING=3 CELLSPACING=3>
	<TR>
		<TD HEIGHT="27" COLSPAN="2" BGCOLOR="#FF6666"><FONT SIZE="5"><B>Error message</B></FONT></TD>
	</TR>
	<TR>
		<TD BGCOLOR="#DDDDDD" COLSPAN="2" HEIGHT="37">
END
	  foreach (@_) {
		print "\t$_<BR>";
	  }
	print <<END;

		</TD>
	</TR>
	</TABLE>
END
	&printFooter();
	&CQPdisconnect();

    exit 1;
}

sub cqp_error_handler_full {
  
  printHeader("Error");


	print <<END;
	<TABLE BORDER="0" WIDTH="100%" CELLPADDING=3 CELLSPACING=3>
	<TR>
		<TD HEIGHT="27" COLSPAN="2" BGCOLOR="#FF6666"><FONT SIZE="5"><B>Error message</B></FONT></TD>
	</TR>
	<TR>
		<TD BGCOLOR="#DDDDDD" COLSPAN="2" HEIGHT="37">
END
	  foreach (@_) {
		print "\t$_<BR>";
	  }
	print <<END;

		</TD>
	</TR>
	</TABLE>
END
	&printFooter();
	&CQPdisconnect();

    exit 1;
}

# load named query from TAB-delimited text file (without target or keyword anchors)
#   undump_file($query_name, $file_name [, $num_lines]);
# if $num_lines is not specified, runs wc -l on $file_name to determine number of matches in named query
# (note: this function trusts you to have called CQPconnect() and activated the right corpus)
sub undump_file {
	my ($nqr, $file, $lines) = @_;
	unless ($lines) {
		chomp($lines = `wc -l $file`);
	}
	my $temp = new CWB::TempFile;
	$temp->write("$lines\n");
	$temp->finish;
	my $tempfile = $temp->name;
	system("cat $file >> $tempfile");
	$CQP->exec("undump $nqr < '$tempfile'");
	$temp->close;
}

# HTML stuff
sub printHeader {
	$title=shift(@_);
	print "Content-type: text/html\n\n";
	print "<HEAD>\n<TITLE>" , $title , "<\/TITLE>";
	print "<META HTTP-EQUIV=\"Content-Type\" CONTENT=\"text/html; CHARSET=iso-8859-1\">";
	print "</HEAD>\n";
	print "<BODY>\n";
}

sub printFooter {
	print <<END;
		<script language="JavaScript" type="text/javascript" src="/$bwHTMLalias/wz_tooltip.js"></script>
		<HR>
		
		$bwCopyright

		<\/BODY><\/HTML>

END
}


sub linefeed {
	# checking for the user's platform to set newline scheme
	my $browserProg = $ENV{'HTTP_USER_AGENT'};
	my $bncHandle = DBI->connect("DBI:mysql:$bwMYSQLtable:$bwMYSQLhost:$bwMysqlPort", "$bwMysqlUser", "$bwMysqlPwd");
	my $username = $ENV{'REMOTE_USER'};
	
	$sqlQuery = "SELECT crlf FROM settings where user='$username'";
	$sth= $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
	$sth->execute or bark("Can't execute $sqlQuery: " . $sth::errstr);
	$rows = $sth->rows;
	if ($rows) {
		@rowArray=$sth->fetchrow_array;
		$setVal=$rowArray[0];
		$rows=0 if ($setVal == 0);
	}
	if (!$rows) {	
		if ($browserProg=~m/Windows/) {
			$crlf = "\015\012";
		} elsif ($browserProg=~m/(Macintosh|Mac_PowerPC)/) {
			if ($browserProg=~m/OS X/) {
				$crlf = "\012";
			} else {
				$crlf = "\015";
			}
		} else {
			$crlf = "\012";
		}
	} else {
		if ($setVal == 1) {
			$crlf = "\015";
		} elsif ($setVal == 2) {
			$crlf = "\015\012";
		} elsif ($setVal == 3) {
			$crlf = "\012";
		} 
	}
	$sth->finish();
	$bncHandle->disconnect;
	
	return($crlf);
}

sub bark {
    warn "@_\n";
    
    print <<END;
	
	<table width="100%">
		<tr>
			<TD bgcolor="#FF3333">
				<H2>Error</H2>
			<TD>
		</tr>
		<tr>
			<TD>
				<BR><B>Type of error: @_</B><BR><BR>
			</TD>
		</tr>
	</table>
	<HR>$bwCopyright

	</BODY>
	</HTML>
END

	exit(0);
}

sub barkFull {
    &printHeader("BNC Error Alert");
    
    print <<END;
	
	<table width="100%">
		<tr>
			<TD bgcolor="#FF3333">
				<H2>Error</H2>
			<TD>
		</tr>
		<tr>
			<TD>
				<BR><B>Type of error: @_</B><BR><BR>
			</TD>
		</tr>
	</table>
	<HR>$bwCopyright

	</BODY>
	</HTML>
END
	exit(0);
}

sub createTitle {

	local($thMode)=$_[0];
	local($theDataL)=$_[1];
	local($numOfSolutions)=$_[2];
	local($numOfFiles)=$_[3];
	local($listFiles)=$_[4];
	local($numTotalWords)=$_[5];
	local($numTotalWordsOrig)=$_[6];
	local($subcorpus)=$_[7];
	local($restrictions)=$_[8];
	local($timeTaken)=$_[9];
	local($inCache)=$_[10];
	local($numOfSpeakers)=$_[11];
		
	local($check, $titleBar, $history, $newRes, $cat, $type, $thMod, $item, $item2, $tag, $letter, $tagName, $order, $perc, $showWords, $numFilesBlurb, $sortpos, $numOfSolutionsOrig);
	my $username = $ENV{'REMOTE_USER'};
			
	$sqlQuery = "SELECT historyView FROM settings where user='$username'";
	$sth= $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
	$sth->execute or bark("Can't execute $sqlQuery: " . $sth::errstr);
	$rows = $sth->rows;
	if ($rows) {
		@rowArray=$sth->fetchrow_array;
		$historyView=$rowArray[0];
	} else {
		$historyView="simple";
	}

	#get simple query out of savedQueries table
	if ($historyView eq "simple") {
		$sqlQuery = "SELECT simpleQuery FROM savedQueries where queryID='$params{queryID}'";		
		$sth99 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
		$sth99->execute or bark("Can't execute $sqlQuery: ". $sth99::errstr);
		@rowArray=$sth99->fetchrow_array;
		$theDataL=$rowArray[0] if ($rowArray[0]);
		$theDataL =~s/\%D\%A//g;
	}
	
  ## ***DEL***
  # $theDataL=~s/\A###/\(\(longest\)\) /;
	
	
	unless ($theDataL=~m/\A\".*\"\Z/) {
		$theDataL=~s/\</&lt;/g;
		$theDataL=~s/\>/&gt;/g;
		
		$theDataL = "&quot;" . $theDataL . "&quot;";
	}
	
	if ($listFiles) {
		$numTotalWords = &makeDecimals($numTotalWords);
		if ($restrictions=~m/(age|sex|class|firstLanguage|dialect|education|role)=/) {
			$titleBar = "<B>Your query returned $numOfSolutions speakers. They produce a total of $numTotalWords words.";
		} else {
			$titleBar = "<B>Your query returned $numOfSolutions files with a total of $numTotalWords words.";
		}
	} else {
		
		$check = $thMode=~m/\AM(\d+)#(\d+)#([^#]+)/;
		if ($check) {
			$numOfSolutions = $1;
			$numOfFiles = $2;
			$subcorpus = $3;
			$theRest = $';
			
		}
		
		if ($numOfFiles) {
			if ($subcorpus) {
				$subcorpusQuote=$bncHandle->quote($subcorpus);
				$sqlQuery = "SELECT metatextual, textOrSpeaker FROM subcorpora WHERE username='$username' and subcorpus = $subcorpusQuote";

				$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
				$sth1->execute or bark("Can't execute $sqlQuery: $sth::errstr");
				$res = $sth1->rows;
				@rowArray=$sth1->fetchrow_array;
				if ($res) {
					$subcorpus=$rowArray[0] if ($subcorpus=~m/last_(spoken|written)/);
					$textOrSpeaker=$rowArray[1];
				}
			}
			if (($subcorpus=~m/(age|sex|class|firstLanguage|dialect|education|role)=/) || ($textOrSpeaker eq "speaker")) {
				if ($numOfSpeakers > 1) {
					$numFilesBlurb = " by $numOfSpeakers different speakers";
				} else {
					$numFilesBlurb = " by 1 speaker";
				}
			} else {
				if ($numOfFiles > 1) {
					$numFilesBlurb = " in $numOfFiles different texts";
				} else {
					$numFilesBlurb = " in 1 text";
				}
			}
		} else {
			$numFilesBlurb="";
		}


		if ($theRest=~m/\A#+I/) {
			$theRest = $';
			$check = $theRest=~m/\A(.*?)#/;
			if ($check) {
				$theSavedName = $1;
			} else {
				$theSavedName = $theRest;
			}
			
			if ($numOfSolutions > 1) {
				$titleBar="<B>Your saved query '$theSavedName' contains $numOfSolutions instances${numFilesBlurb}";
			} else {
				$titleBar="<B>Your saved query '$theSavedName' contains $numOfSolutions instance${numFilesBlurb}";
			}
			
			
		} else {
		
			
			$subcorpBlurb = "";
			$numPossibleFiles=4048;
			unless (($subcorpus eq "no_subcorpus") || (!$subcorpus)) {
				if ($subcorpus eq "(spowri=1)") {
					$subcorpBlurb = " in written texts ";
					$numPossibleFiles=3140;
				} elsif ($subcorpus eq "(spowri=2)") {
					$subcorpBlurb = " in spoken texts ";
					$numPossibleFiles=908;
				} elsif ($subcorpus=~m/\Alast_(written|spoken)\Z/) {
					$sqlQuery = "SELECT metatextual, numfiles FROM subcorpora WHERE username='$username' and subcorpus = '$subcorpus'";
					$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
					$sth1->execute or bark("Can't execute $sqlQuery: $sth::errstr");
					@rowArray=$sth1->fetchrow_array;
					$subcorpus=$rowArray[0];
					$numPossibleFiles=$rowArray[1];
				} else {
					$subcorpusQuote=$bncHandle->quote($subcorpus);
					$sqlQuery = "SELECT metatextual, numfiles FROM subcorpora WHERE username='$username' and subcorpus = $subcorpusQuote";
					$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
					$sth1->execute or bark("Can't execute $sqlQuery: $sth::errstr");
					$res = $sth1->rows;		
					if ($res) {
						$subcorpBlurb = " in the subcorpus \"$subcorpus\" ";
						@rowArray=$sth1->fetchrow_array;
						$subcorpus=$rowArray[0];
						$numPossibleFiles=$rowArray[1];
					} else {
						
						while ($subcorpus=~m/(\w+)=\'?\d+/g) {
							$theCat = $1;
							if ($theCat=~m/\A(age|sex|class|firstLanguage|dialect|education|role)/) {
								$personCheckCount++;
							} else {
								$catRefCheckCount++;
							}						
						}
						if (($catRefCheckCount) && (!$personCheckCount)) {
							$sqlQuery = "SELECT count(fname) FROM headerInfo WHERE " . $subcorpus;
						} elsif ((!$catRefCheckCount) && ($personCheckCount)) {
							$sqlQuery = "SELECT count(spid) FROM speakers WHERE " . $subcorpus;
						} elsif (($catRefCheckCount) && ($personCheckCount)){
							$sqlQuery = "SELECT count(spid) FROM headerInfo, speakers WHERE fname = filename AND" . $subcorpus;
						}						
						
						$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
						$sth1->execute or bark("Can't execute $sqlQuery: $sth::errstr");
						@rowArray=$sth1->fetchrow_array;
						$numPossibleFiles=$rowArray[0];
					}
				}
			}
			unless (($subcorpBlurb) || ($subcorpus eq "no_subcorpus")) {
				
				
				if (($thMode eq "INIT") && ($restrictions)) {
					$subcorpus = $restrictions;
				}
				
				$subcorpus=~s/##\Z//;
				$subcorpus=~s/\A##//;
				$subcorpus =~s/\'//g;
				$subcorpus =~s/ AND / \& /g;
				
				if ($subcorpus=~m/\&/) {
					@subcorpStuff = split(/ \& /, $subcorpus);
				} else {
					$subcorpStuff[0] = $subcorpus;
				}
				
				foreach $restrictionItem (@subcorpStuff) {
					$itemList = "";
					$check = $restrictionItem=~m/\(([^\=]+)\=/;
					if ($check) {
						$cat = $1;
						$type = $typeValues{$cat};
					} else {
						warn "problem with matching category in $restrictionItem!!!"
					}
					
					$restrictionItem=~s/\A\(//;
					$restrictionItem=~s/\)\Z//;
					if ($restrictionItem=~m/ or /) {
						@restrictionItems = split(/ or /, $restrictionItem);
					} else {
						$restrictionItems[0] = $restrictionItem;
					}
					
					foreach $feature (@restrictionItems) {
						$itemList .= " or <I>$catValues{$feature}</I>";
					}
					$itemList =~s/\A or //;
					
					$itemList = "<I>$type: </I>" . $itemList;
					
					$subcorpBlurb.= " and  $itemList";
				}
				$subcorpBlurb =~s/\A and //;
				$subcorpBlurb =~s/\A //;
				$subcorpBlurb= " restricted to \"$subcorpBlurb\" " if ($subcorpBlurb);
			}
	
	
			if ($numOfSolutions > 1) {
				$titleBar="<B>Your query $theDataL $subcorpBlurb returned $numOfSolutions hits${numFilesBlurb}";
			} else {
				$titleBar="<B>Your query $theDataL $subcorpBlurb returned $numOfSolutions hit${numFilesBlurb}";
			}
	
			unless ($theDataL =~m/bncDoc id/)  {
				$hit=$thMode=~m/\AM?(\d+)/;
				if ($hit) {
					$numOfSolutionsOrig = $1;
				} else {
					$numOfSolutionsOrig = $numOfSolutions;
				}
				if ($numTotalWordsOrig) {
					$perc = int((($numOfSolutionsOrig / ($numTotalWordsOrig / 1000000)) * 100) + 0.5)/ 100;
					$numTotalWordsD = &makeDecimals($numTotalWordsOrig);
					$numPossibleFilesD = &makeDecimals($numPossibleFiles);
					if ($subcorpus=~m/(age|sex|class|firstLanguage|dialect|education|role)=/) {
						$showWords =  "$numTotalWordsD words [$numPossibleFilesD speakers]; " if ($numTotalWordsOrig < 100000000);
					} else {
						$showWords =  "$numTotalWordsD words [$numPossibleFilesD texts]; " if ($numTotalWordsOrig < 100000000);
					}
					$titleBar .= " \(${showWords}frequency: $perc instances per million words\)";
				}
			}
		}
	}
	
	

	$hit=$thMode=~m/\AM(\d+)#\d+#[^#]+#+T\d#(\d+)/;
	if ($hit) {
		$full=$1;
		$thinned=$2;
		$thinning=($thinned / $full);
	} else {
		$thinning=1;
	}
	
	$thinningFactor = int(((1 / $thinning) * 100) + 0.5) / 100;
	
	# this does not take care of metatextual restrictions!!!!
	$check = $thMode=~m/\AM(\d+)#(\d+)#([^#]+)/;
	$postProcessInfo = $' if ($check);
	
	
	@history=split(/##/, $postProcessInfo);
	# add something that makes sure of singular if there is only one hit
	foreach $history (@history) {
		next unless ($history=~m/\S/);
		if ($history=~m/F(\d+)#(\d+)/g) {
			$thinning = $thinning * ($2 / $1);
			$thinningFactor = int(((1 / $thinning) * 100) + 0.5) / 100;
		}		

		# ABCDEFGLMPRSUXYZ
		if ($history=~m/\AT(\d)#(\d+)/) {
			$thMod = $1;
			$newRes = $2;
			$thMod= "first n hits" if $thMod eq '0';
			$thMod= "random selection" if $thMod eq '1';
			$thMod= "random selection (not reproducible)" if $thMod eq '2';
			$titleBar .= ", thinned with method <I>$thMod</I> to $newRes hits";
		} elsif ($history=~m/\AF(\d+)#(\d+)/) {
			$oldRes = $1;
			$newRes = $2;
			$titleBar .= ", thinned with method <I>random selection</I> to $newRes hits";
		} elsif ($history=~m/\ADC([^#]+)#([^#]+)#(\d+)/) {
			$item = $1;
			$item = $catValues{$item};
			$item2 = $2;
			$item2 = $catValues{$item2};
			$newRes = $3;
			$titleBar .= ", distribution over categories <I>$item</I> and <I>$item2</I> ($newRes hits)";
		} elsif ($history=~m/\ADF([^#]+)#(\d+)/) {
			$item=$1;
			$newRes = $2;
			$titleBar .= ", occurrences in text <I>$item</I> ($newRes hits)";
		}  elsif ($history=~m/\ADS([^#]+)#([^#]+)#(\d+)/) {
			$item=$1;
			$text = $2;
			$newRes = $3;
			$titleBar .= ", uttered by speaker <I>$item</I> in text <I>$text</I> ($newRes hits)";
		} elsif ($history=~m/\AG([^#]+)#(\d+)/) {
			$item=$1;
			$newRes = $2;
			if (($thMode=~m/##G[^#]+\Z/) && ($numTotalWords)) {
				$perc = int((($newRes / ($numTotalWords / 1000000) ) * 100) + 0.5)/ 100;
				$perc2 = int((($newRes / ($numTotalWords / 1000000 * $thinning) ) * 100) + 0.5)/ 100;

				$numTotalWordsD = &makeDecimals($numTotalWords);
				$showWords =  "in $numTotalWordsD words; " if ($numTotalWords < 100000000);
				if ($thinning == 1) {
					$titleBar .= ", restricted to subcorpus <I>$item</I> ($newRes hits ${showWords}freq: $perc instances per million words\)";
				} else {
					$titleBar .= ", restricted to subcorpus <I>$item</I> ($newRes hits ${showWords}freq: $perc instances per million words - extrapolated frequency based on thinning factor $thinningFactor: $perc2 instances per million words\)";
				}
			} else {
				$titleBar .= ", restricted to subcorpus <I>$item</I> ($newRes hits)";
			}
		} elsif ($history=~m/\ADO([^#]+)#(\d+)/) {
			$item=$1;
			$newRes = $2;
			$item=~m/\A(\w+)/;
			$cat=$1;
			$type = $typeValues{$cat};
			$item = $catValues{$item};
			$titleBar .= ", distribution over <I>$type: $item</I> ($newRes hits)";
		} elsif ($history=~m/\AC([^#]+)#(\d+)#(\S+)/) {
			$item=$1;
			$newRes = $2;
			$tag=$3;
			$tagName=tagStuff($tag);
			$titleBar .= ", collocating with <I>$item</I> $tagName ($newRes hits)";
		} elsif ($history=~m/\AR([^#]+)#(\d+)#(-?\d+)#(\S+)/) {
			$item=$1;
			$newRes = $2;
			$range=$3;
			$tag=$4;
			$tagName=tagStuff($tag);
			$titleBar .= ", collocating with <I>$item</I> $tagName at position $range ($newRes hits)";
		} elsif ($history=~m/\AB([^#]+)#(\d+)#(-?\d+)/) {
			$item=$1;
			$newRes = $2;
			$range=$3;
			$titleBar .= ", collocating with lemma <I>$item</I> at position $range ($newRes hits)";
		} elsif ($history=~m/\AY([^#]+)#(\d+)/) {
			$item=$1;
			$newRes = $2;
			$tag=$3;
			$titleBar .= ", collocating with words tagged as <I>$item</I> ($newRes hits)";
		} elsif ($history=~m/\AZ([^#]+)#(\d+)/) {
			$item=$1;
			$newRes = $2;
			$tag=$3;
			$titleBar .= ", collocating with lemma <I>$item</I> ($newRes hits)";
		} elsif ($history=~m/\AL([^#]+)#(\d+)#(-?\d+)/) {
			$item=$1;
			$newRes = $2;
			$range=$3;
			$titleBar .= ", collocating with words tagged as <I>$item</I> at position $range ($newRes hits)";
		} elsif ($history=~m/\AA(refnum#)?(before\d+|after\d+|node)#L(\((\w)\|\w\))?#T(E|N)(\S+)?#H(\d+)?/) {
			$textOrderView = $1;
			$order=$2;
			$letter=$4;
			$excluded = $5;
			$tag = $6;
			$hits = $7;
			if ($order=~m/before(\d+)/) {
				$order = "position \-$1";
			} elsif ($order=~m/after(\d+)/) {
				$order = "position \+$1";
			}
			
			$tagName=$tag;
			$tagName="any verb" if ($tag eq "VERB");
			$tagName="any noun" if ($tag eq "SUBST");
			$tagName="any adjective" if ($tag eq "ADJ");
			$tagName="any adverb" if ($tag eq "ADV");
			$tagName="any pronoun" if ($tag eq "PRON");
			$tagName="any preposition" if ($tag eq "PREP");
			$tagName="any conjunction" if ($tag eq "CONJ");
			$tagName="any article" if ($tag eq "ART");
			$tagName="any interjection" if ($tag eq "INTERJ");
			$tagName="any punctuation" if ($tag eq "STOP");
			$tagName="any unclear" if ($tag eq "UNC");
			
			if ($order eq "node") {
				$titleBar .= ", sorted on <I>$order</I> word";
			} else {
				$titleBar .= ", sorted on <I>$order</I>";
			}
			
			$titleBar .= " starting with letter <I>$letter</I>" if ($letter);
			if ($tag) {
				if ($excluded eq "N") {
					$titleBar .= " with tag-restriction <I>$tagName</I>";
				} else {
					$titleBar .= " with tag-restriction <I>exclude $tagName</I>";
				}
			}
			$titleBar .= " ($hits hits)" if ($hits);
			$titleBar .= ", displayed by text order" if ($textOrderView);
			
		} elsif ($history=~m/\AU([^#]+)#([^#]+)#(\w+)/) {
			$item=$1;
			$newRes = $2;
			$display = $3;
			if ($display eq "both") {
				$display = "word-tag combination";	
			}
			$titleBar .= ", reduced with frequency list ($display: <I>$item</I>) to $newRes hits";
		} elsif ($history=~m/\AQ([^#]+)#(\d+)/) {
			$catName=$1;
			$newRes = $2;
			$titleBar .= ", manually categorized as &quot;$catName&quot;($newRes hits)";

		} elsif ($history=~m/\AV#(\d+)/) {
			$titleBar .= " (displayed in random order)";

		}
	}
	$titleBar .= "</B>";
	
	if (($timeTaken) || ($timeTaken eq "0")) {
		unless ($timeTaken eq "cache") {
			$inCacheText = "";
			$inCacheText = " - retrieved from cache" if ($inCache);
			$titleBar .= "<font color=\"#AAAAAA\" size=\"1\"> [$timeTaken seconds$inCacheText]</font>" unless ($timeTaken == 0);
		} else {
			$titleBar .= "<font color=\"#AAAAAA\" size=\"1\"> [data retrieved from cache]</font>";
		}
	}
	return($titleBar);
}

sub tagStuff {
	
	local($tagClean)=$_[0];
	local($tagName);
	
	unless ($tagClean eq "all") {

		$tagName=" with tag restriction <I>any verb</I>" if ($tag eq "VERB");
		$tagName=" with tag restriction <I>any noun</I>" if ($tag eq "SUBST");
		$tagName=" with tag restriction <I>any adjective</I>" if ($tag eq "ADJ");
		$tagName=" with tag restriction <I>any adverb</I>" if ($tag eq "ADV");
		$tagName=" with tag restriction <I>any pronoun</I>" if ($tag eq "PRON");
		$tagName=" with tag restriction <I>any preposition</I>" if ($tag eq "PREP");
		$tagName=" with tag restriction <I>any conjunction</I>" if ($tag eq "CONJ");
		$tagName=" with tag restriction <I>any article</I>" if ($tag eq "ART");
		$tagName=" with tag restriction <I>any interjection</I>" if ($tag eq "INTERJ");
		$tagName=" with tag restriction <I>any punctuation</I>" if ($tag eq "STOP");
		$tagName=" with tag restriction <I>any unclear" if ($tag eq "UNC");

	} else {
		$tagName="";
	}			

	return ($tagName);
}

sub Filemapping {
	local($fname)=$_[0];
	local($fnum);
	
	$sqlQuery = "SELECT fnum FROM headerInfo where fname = '$fname'";
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: $bncHandle::errstr");
	$sth->execute or bark("Can't execute $sqlQuery: $sth::errstr");

	if ($sth->rows) {
		@rowArray=$sth->fetchrow_array;
		$fnum=$rowArray[0];
	}
	return ($fnum);
	
}

sub reverseFilemapping {
	local($fnum)=$_[0];
	local($fname);
	
	$sqlQuery = "SELECT fname FROM headerInfo where fnum = $fnum";
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: $bncHandle::errstr");
	$sth->execute or bark("Can't execute $sqlQuery: $sth::errstr");

	if ($sth->rows) {
		@rowArray=$sth->fetchrow_array;
		$fname=$rowArray[0];
	}
	return ($fname);
	
}

sub subCorporaList {
	local($username)=$_[0];
	local($subcorpus)=$_[1];
	
	my $subcorpusQuote = $bncHandle->quote($subcorpus);
	
	
	$sqlQuery = "SELECT fnums, textOrSpeaker FROM subcorpora where username = '$username' and subcorpus = $subcorpusQuote";
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: $bncHandle::errstr");
	$sth->execute or bark("Can't execute $sqlQuery: $sth::errstr");

	if ($sth->rows) {
		@rowArray=$sth->fetchrow_array;
		$fnums=$rowArray[0];
		$fnums=~s/\A·//;
		
		$textOrSpeaker=$rowArray[1];
		@fnumArray = split(/·/, $fnums);
		if ($textOrSpeaker eq "text") {
			$fnumList = "(" . join(" OR ", map { "fnum = $_" } @fnumArray) . ")";
		} else {
			$fnumList = "(" . join(" OR ", map { "snum = $_" } @fnumArray) . ")";
		}
				
		
	} else {
		$fnumList = "";
	}
	
	return($fnumList, $textOrSpeaker);
}

sub subcorpusCheck {
	local($thMode)=$_[0];
	local($subcorpus);
	local($username) = $ENV{'REMOTE_USER'};
	$check = $thMode=~m/\AM(\d+)#(\d+)#([^#]+)/;

	if ($check) {
		$subcorpus = $3;
		$postprocessing = $';
		unless (($subcorpus eq "no_subcorpus") || (!$subcorpus)) {
			if ($subcorpus eq "(spowri=1)") {
				#$subcorpBlurb = " in written texts ";
			} elsif ($subcorpus eq "(spowri=2)") {
				#$subcorpBlurb = " in spoken texts ";
			} else {
				$subcorpusQuote=$bncHandle->quote($subcorpus);
				$sqlQuery = "SELECT numfiles FROM subcorpora WHERE username='$username' and subcorpus = $subcorpusQuote";
				$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
				$sth1->execute or bark("Can't execute $sqlQuery: $sth::errstr");
				$res = $sth1->rows;		
				if ($res) {
					($subcorpus, $textOrSpeaker) = &subCorporaList($username, $subcorpus);
				} else {
					# keep as is
				}
			}
		}
		$subcorpus = "" if ($subcorpus eq "no_subcorpus");
		
		@history=split(/##/, $postprocessing);
		foreach $history (@history) {
			next unless ($history=~m/\S/);
			if ($history=~m/\ADC([^#]+)#([^#]+)#(\d+)/) {
				$item1 = $1;
				$item2 = $2;
				$subcorpus .= " & ($item1) & ($item2)";
			} elsif ($history=~m/\ADO([^#]+)#(\d+)/) {
				$item1=$1;
				$subcorpus .= " & ($item1)";
			}
		}
		
		$subcorpus=~s/AND/&/g;
		$subcorpus=~s/\A *&//;
		$subcorpus=~s/\A //;
		
		$subcorpus=~s/\'//g;
		
	} else {
		warn "no subcorpus matched!!!";
	}	
	
	$subcorpus = "" if ($subcorpus eq "no_subcorpus");

	return ($subcorpus);
}


sub freqCalculate2 {
	local($theData) = $_[0];
	local($subcorpus) = $_[1];
	local($username) = $_[2];
	local($source) = $_[3];
	local($catRefCount, $catRefString, $element, $category, $number, $elementCount, $personCount);
	
	%catMapping = (
		"soc" => "class",
		"flang" => "firstLanguage",
		"educ" => "education",
		"role" => "role"
	);
	
	
	if ($subcorpus eq "no_subcorpus") {
		$hit=$params{thMode}=~m/M(\d+)#\d+\#([^#]+)#/;
		if ($hit) {
			$subcorpus = $2;
		}
	}
	
	$subcorpus = "" if ($subcorpus eq "no_subcorpus");
		
	$catRefCount = 0;
	$catRefString = "";
	
	unless (!$subcorpus) {
		my ($fnumList, $textOrSpeaker) = subCorporaList($username, $subcorpus);	
		if ($fnumList) {
			$subcorpus = $fnumList;
			if ($textOrSpeaker eq "text") {
				$catRefCount++;
			} else {
				$personCount++;
			}
		} else {			
			if ($subcorpus=~m/(age|sex|class|firstLanguage|dialect|education|role)=\'/) {
				$personCount++;
			}
			if ($subcorpus=~m/(spowri|derivedtexttype|allava|alltyp|alltim|scgdom|sdeage|sdecla|sdesex|spolog|sporeg|wriaag|wriad|wriase|wriaty|wriaud|wridom|wrilev|wrimed|wripp|wrisam|wrista|writas|genre)=/i) {
				$catRefCount++;
			}
			
		}
	
	}
		
	$distributionRestrictions = "";
	if ($source eq "dist") {
		while ($params{thMode}=~m/##DO([^#]+)#(\d+)/g) {
			$item=$1;
			$item=~s/¨/=/;
			if ($item=~m/\A(age|sex|class|firstLanguage|dialect|education|role)/) {
				$personCount++;
			} else {
				$catRefCount++;
			}
			$distributionRestrictions = "AND (" . $item . ")";
		}
		while ($params{thMode}=~m/##DC([^#]+)#([^#]+)#(\d+)/g) {
			$item=$1;
			$item=~s/¨/=/;
			if ($item=~m/\A(age|sex|class|firstLanguage|dialect|education|role)/) {
				$personCount++;
			} else {
				$catRefCount++;
			}
			$item2=$2;
			$item2=~s/¨/=/;
			
			$distributionRestrictions = "AND (" . $item . ") AND (" . $item2 . ")";
		}

		$distributionRestrictions =~s/\A\s*AND// if ($subcorpus eq "no_subcorpus");
		
	}
	if (($catRefCount) && (!$personCount)) {
		$catRefStringOrig = "SELECT sum(numwords) FROM headerInfo WHERE " . $subcorpus;
		$catRefString = "SELECT sum(numwords) FROM headerInfo WHERE " . $subcorpus . $distributionRestrictions;
	} elsif ((!$catRefCount) && ($personCount)) {
		$catRefStringOrig = "SELECT sum(speakerWords) FROM speakers WHERE " . $subcorpus;
		$catRefString = "SELECT sum(speakerWords) FROM speakers WHERE " . $subcorpus . $distributionRestrictions;
	} elsif (($catRefCount) && ($personCount)){
		$catRefStringOrig = "SELECT sum(speakerWords) FROM headerInfo, speakers WHERE fname = filename AND" . $subcorpus;
		$catRefString = "SELECT sum(speakerWords) FROM headerInfo, speakers WHERE fname = filename AND" . $subcorpus . $distributionRestrictions;
	} else {
		$catRefString = "SELECT sum(numwords) FROM headerInfo" unless ($source eq "dist");
		$catRefStringOrig = "SELECT sum(numwords) FROM headerInfo";
		unless ((!$subcorpus) && (!$distributionRestrictions)) {
			$catRefString = "SELECT sum(numwords) FROM headerInfo WHERE " . $fnames . $distributionRestrictions;
			
		}
	}
	# an ugly fudge!
	$catRefString=~s/WHERE\s+AND/WHERE /;
	return ($catRefString, $catRefStringOrig);

}

sub getSpokenPositions {
	local($snumList)=$_[0];
	local($tempfile)=$_[1];
	local($written)=$_[2];
	local($count);
	if ($written) {
		$sqlQuery = "select beginPositions, endPositions from headerInfo, speakers where headerInfo.fname = speakers.filename AND $snumList";
	} else {
		$sqlQuery = "select beginPositions, endPositions from speakers where $snumList";
	}
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
	$sth->execute or bark("Can't execute $sqlQuery: ". $sth::errstr);
	$resArrayResult=$sth->fetchall_arrayref;
	for $k ( 0 .. ($#{$resArrayResult}) ) {
		$beginPositions=$resArrayResult->[$k][0];
		$endPositions=$resArrayResult->[$k][1];
		
		next unless $beginPositions;
		

		push @begin, split / /, $beginPositions;
		push @end, split / /, $endPositions;
	}
	
	$count = @begin;
	# improve error message...
	die "oopsie!" unless @begin == @end;

		
	open (OUT,">$tempfile") || die "can't open control";
	print OUT "$count\n";
	
	for $i (0 .. $count-1) {
		print OUT "$begin[$i]\t$end[$i]\n";
	}
	close(OUT);
	return($count);
}


sub makeDecimals {
	local($newCount, $figure, $result, $blurb, $length);
	$result = shift (@_);
	$length=length($result);
	$figure=$length % 3;
	$figure=3 if (!$figure);
	$newCount="";
	for ($j = 0; $j < ($length + 1); $j++) {
		$figure--;
		$blurb=substr($result, $j, 1);
		$newCount .= $blurb;
		if (!$figure) {
			$newCount .= "\,";
			$figure=3;
		}
	}
	chop ($newCount);
	return ($newCount);
}



sub compileFlist {

	my $subcorpus = $_[0];
	my $tempname = $_[1];
	my $username = $_[2];
	
		
	$frequencyHandle = DBI->connect("DBI:mysql:$bwMYSQLfrequency:$bwMYSQLhost:$bwMysqlPort", "$bwMysqlUser", "$bwMysqlPwd") or bark("Can't connect to mysql on $bwMYSQLhost at port: $bwMysqlPort <BR>" . $frequencyHandle::errstr);

	$spokenRes++ if ($subcorpus=~m/\((age|sex|class|firstLanguage|education|role|dialect)\=/);
	$spokenRes++ if ($subcorpus=~m/snum/);
	$writtenRes++ if ($subcorpus=~m/\((spowri|all(tim|typ)|wri(med|dom|ad|aag|ase|aty|aud|lev|tas|sta|sam)|sde(age|cla|sex)|spo(log|reg)|scgdom)|genre|derivedtexttype\=/);
	$writtenRes++ if ($subcorpus=~m/fnum/);
	
	print "<TR><TD>Compiling data for base frequency list... <BR></TD></TR>";

	if (($writtenRes) && (!$spokenRes)) {
		$text_or_speakers_corpus    = "$bwCorpusText";
		$query = "SELECT `start`, `end` FROM bncFreqTextIndex AS t1, `headerInfo` AS t2 WHERE t1.fname = t2.fname AND $subcorpus ORDER BY start asc";
	}  elsif ((!$writtenRes) && ($spokenRes)) {
		$text_or_speakers_corpus    = "$bwCorpusSpeaker"; 
		$query = "SELECT `start`, `end` FROM bncFreqSpeakerIndex AS t1, `speakers` AS t2 WHERE t1.spid = t2.spid AND $subcorpus ORDER BY start asc";
	} else {
		$text_or_speakers_corpus    = "$bwCorpusSpeaker";
		$query = "SELECT `start`, `end` FROM bncFreqSpeakerIndex AS t1, `speakers` AS t2, headerInfo WHERE headerInfo.fname = t2.filename and t1.spid = t2.spid AND $subcorpus ORDER BY start asc";
	}
	my $registry_flag = ($Registry) ? "-r $Registry" : "";
		
	## execute query
	my $regions   = $bncHandle->selectall_arrayref($query);
	my $n_regions = @$regions;
	
	## disconnect from this database
	$bncHandle->disconnect;
	
	## set up temporary table for subcorpus frequencies
	my $sql = "CREATE TABLE `$tempname` (
	   `freq` int(11) NOT NULL default 0,
	   `word` char(50) NOT NULL default '',
	   `tag` char(7) NOT NULL default '',
	   `lemma` char(50) NOT NULL default '',
	   `lemma_tag` char(7) NOT NULL default '',
	   key (`word`, `tag`)
	   ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
	$frequencyHandle->do($sql);
	
	## store regions to be scanned in temporary file
	my $tf = new CWB::TempFile;
	foreach my $region (@$regions) {
		$tf->write(join("\t", @$region), "\n");
	}
	$tf->finish;
	my $tf_name = $tf->name;
	
	#warn "$CWB::ScanCorpus $registry_flag -F f -R $tf_name $text_or_speakers_corpus word+0 pos+0 lemma+0 class+0 > /$bwTempPath/$tempname";
	
	qx{$CWB::ScanCorpus $registry_flag -F f -R $tf_name $text_or_speakers_corpus word+0 pos+0 lemma+0 class+0> /$bwTempPath/$tempname};
	
	print "<TR><TD>Importing base frequency list... <BR></TD></TR>";
	# import into master table
	$sqlQuery = "Alter table $tempname disable keys";
	$frequencyHandle->do($sqlQuery);
	$sqlQuery = "LOAD DATA INFILE '/$bwMySQLTempPath/$tempname' into table $tempname fields escaped by ''";
	$frequencyHandle->do($sqlQuery);
	$sqlQuery = "Alter table $tempname enable keys";
	$frequencyHandle->do($sqlQuery);
	
	unlink("/$bwTempPath/$tempname") if -f "/$bwTempPath/$tempname";
	
	print "<TR><TD>Creating separate frequency lists for words, POS-tags and lemmas... <BR></TD></TR>";
	
	# create separate tables from master table
	
	$sqlQuery = "CREATE TABLE `allWords_$tempname` (
	  `word` char(50) NOT NULL default '',
	  `freq` mediumint(8) unsigned default NULL,
	  PRIMARY KEY  (`word`)
	) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
	$frequencyHandle->do($sqlQuery) or bark("Can't do $sqlQuery");
	
	$sqlQuery = "insert into allWords_$tempname select word, sum(freq) from $tempname group by word";
	$frequencyHandle->do($sqlQuery);
		
	$sqlQuery = "CREATE TABLE `allWordsT_$tempname` (
	  `word` char(58) NOT NULL default '',
	  `freq` mediumint(8) unsigned default NULL,
	  PRIMARY KEY  (`word`)
	) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
	$frequencyHandle->do($sqlQuery) or bark("Can't do $sqlQuery");
	
	$sqlQuery = "insert into allWordsT_$tempname select concat(word, '_', tag) as bla, sum(freq) from $tempname group by bla";
	$frequencyHandle->do($sqlQuery);
		
	
	$sqlQuery = "CREATE TABLE `allWordsG_$tempname` (
	  `word` char(52) NOT NULL default '',
	  `freq` mediumint(8) unsigned default NULL,
	  PRIMARY KEY  (`word`)
	) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
	$frequencyHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

	$sqlQuery = "insert into allWordsG_$tempname select concat(word, '_', lemma_tag), sum(freq) as bla from $tempname group by concat(word, '_', lemma_tag)" or bark("Can't do $sqlQuery");
 	$frequencyHandle->do($sqlQuery);
		
	$sqlQuery = "CREATE TABLE `allLemma_$tempname` (
	  `lemma` char(50) NOT NULL default '',
	  `freq` mediumint(8) unsigned default NULL,
	  PRIMARY KEY  (`lemma`)
	) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
	$frequencyHandle->do($sqlQuery) or bark("Can't do $sqlQuery");
	
	$sqlQuery = "insert into allLemma_$tempname select lemma, sum(freq) as bla from $tempname group by lemma";
	$frequencyHandle->do($sqlQuery);
		
	$sqlQuery = "CREATE TABLE `tag_$tempname` (
	  `tag` char(7) NOT NULL default '',
	  `freq` int(11) unsigned default NULL,
	  PRIMARY KEY  (`tag`)
	) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
	$frequencyHandle->do($sqlQuery) or bark("Can't do $sqlQuery");
		
	$sqlQuery = "insert into tag_$tempname select tag, sum(freq) from $tempname group by tag";
	$frequencyHandle->do($sqlQuery);
		
	$sqlQuery = "DROP TABLE `$tempname`";
	$frequencyHandle->do($sqlQuery) or bark("Can't do $sqlQuery");
	
	$theSize = 0;
	$theSize += getTablesSizeFreqlist("allWords_$tempname");
	$theSize += getTablesSizeFreqlist("allWordsT_$tempname");
	$theSize += getTablesSizeFreqlist("allWordsG_$tempname");
	$theSize += getTablesSizeFreqlist("allLemma_$tempname");
	$theSize += getTablesSizeFreqlist("tag_$tempname");

	$frequencyHandle->disconnect;
	
	$time = int(time);
	# save information in cache table
	$sqlQuery = "insert into frequencyLists values (
		'$tempname',
		'$username',
		$time,
		$theSize,
		'$subcorpus'
	)";
	$bncHandle->do($sqlQuery);
}

sub getTablesSizeFreqlist {
	local($theSize);
	my $dbname = $_[0];
	my $sqlQuery = "SHOW TABLE STATUS LIKE '$dbname'";
	$sth1 = $frequencyHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $frequencyHandle::errstr);
	$sth1->execute or bark("Can't execute $sqlQuery: ". $sth::errstr);
	$tableInfo = $sth1->fetchrow_hashref();
	$theSize = $tableInfo->{'Data_length'} + $tableInfo->{'Index_length'};
	return($theSize);
	
}

sub getTableSize {
	local($theSize);
	my $dbname = $_[0];
	my $sqlQuery = "SHOW TABLE STATUS LIKE '$dbname'";
	$sth1 = $userHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $userHandle::errstr);
	$sth1->execute or bark("Can't execute $sqlQuery: ". $sth::errstr);
	$tableInfo = $sth1->fetchrow_hashref();
	$theSize = $tableInfo->{'Data_length'} + $tableInfo->{'Index_length'};
	return($theSize);
	
}

sub deleteCachedFreqlists {

	# delete cached frequency lists if the (configurable) limit has been reached
	# to change the limit, modify variable $bwMaxFreqlistMySQL in bncConfig.pm

	my $theSize = $_[0];
	
	$frequencyHandle = DBI->connect("DBI:mysql:$bwMYSQLfrequency:$bwMYSQLhost:$bwMysqlPort", "$bwMysqlUser", "$bwMysqlPwd") or bark("Can't connect to mysql on $bwMYSQLhost at port: $bwMysqlPort <BR>" . $frequencyHandle::errstr);

	my @dbnames = ("allWords_", "allWordsT_", "allWordsG_", "allLemma_", "tag_");
	
	$sqlQuery = "select sum(numRows) from frequencyLists";
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
	$sth->execute or bark("Can't execute $sqlQuery: ". $sth::errstr);
	@rowArray=$sth->fetchrow_array;
	$sumNumrows=$rowArray[0];
	
	if ($sumNumrows > $theSize) {
		
		$toDeleteSize = $sumNumrows - $theSize;
		$sqlQuery = "select theName, numRows from frequencyLists order by time_of_creation asc";
		$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
		$res = $sth1->execute or bark("Can't execute $sqlQuery: " . $sth1::errstr);
		$delStuff=$sth1->fetchall_arrayref;
		
		@del = ();
		deleteLoop: for $k ( 0 .. ($#{$delStuff}) ) {
			$toDelete=$delStuff->[$k][0];
			$deleteSize = $delStuff->[$k][1];
			$deleteSize = 0 if (!$deleteSize);
			
			
			foreach $dName (@dbnames) {
				$sqlQuery = "DROP TABLE if exists ${dName}$toDelete";
				$frequencyHandle->do($sqlQuery) or bark("Can't delete: $sqlQuery: " . $frequencyHandle::errstr);
			}
			
			$toDeleteSize = $toDeleteSize - $deleteSize;
			$del[$k] = "$toDelete";
			last deleteLoop if ($toDeleteSize < 0);
		}
		
		foreach $toDelete (@del) {
			$sqlQuery = "DELETE FROM frequencyLists where theName = '$toDelete'";
			$bncHandle->do($sqlQuery) or bark("Can't delete: $sqlQuery: " . $bncHandle::errstr);
		}
	
	}

}

sub deleteCachedQueries {

	# delete cached queries if the (configurable) limit has been reached
	# to change the limit, modify variable $bwFileSize in bncConfigXML.pm

	my $theSize = $_[0];
	
	# delete temporary CQP files if the (configurable) limit has been reached
	# to change the limit, modify variable $bwFileSize in bncConfig.pm
	$sqlQuery = "select sum(fileSize) from savedQueries where saved = 0";
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
	$sth->execute or bark("Can't execute $sqlQuery: ". $sth::errstr);
	@rowArray=$sth->fetchrow_array;
	$sumFileSize=$rowArray[0];
	
	if ($sumFileSize > $theSize) {
		
		$toDeleteSize = $sumFileSize - $theSize;
		$sqlQuery = "select queryID, fileSize, execute_time from savedQueries where saved = 0 order by time_of_query asc";
		$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
		$res = $sth1->execute or bark("Can't execute $sqlQuery: " . $sth1::errstr);
		$delStuff=$sth1->fetchall_arrayref;
		
		@del = ();
		deleteLoop: for $k ( 0 .. ($#{$delStuff}) ) {
			$toDelete=$delStuff->[$k][0];
			$deleteSize = $delStuff->[$k][1];
			$execute_time = $delStuff->[$k][2];
			$deleteSize = 0 if (!$deleteSize);
			$execute_time = 0 if (!$execute_time);
			
			# Add algorithm to keep cached queries with small file sizes but long execute times. 
			# But don't keep them forever...
			unlink("/$bwTempPath/$bwCorpus:$toDelete") if -f "/$bwTempPath/$bwCorpus:$toDelete";
			$toDeleteSize = $toDeleteSize - $deleteSize;
			$del[$k] = "$toDelete";
			last deleteLoop if ($toDeleteSize < 0);
		}
		
		foreach $toDelete (@del) {
			$sqlQuery = "DELETE FROM savedQueries where queryID = '$toDelete'";
			$bncHandle->do($sqlQuery) or bark("Can't delete: $sqlQuery: " . $bncHandle::errstr);
		}
	
	}
	

}

sub deleteCachedMySQLTables {

	# delete temporary MySQL tables if the (configurable) limit has been reached
	# to change the limit, modify variable $bwMaxRowsMySQL in bncConfig.pm
	
	my $theSize = $_[0];
	
	$sqlQuery = "select sum(numRows) from tableLog";
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
	$sth->execute or bark("Can't execute $sqlQuery: ". $sth::errstr);
	@rowArray=$sth->fetchrow_array;
	$sumNumrows=$rowArray[0];

	if ($sumNumrows > $theSize) {
		
		$toDeleteSize = $sumNumrows - $theSize;
		$sqlQuery = "select db_name, numRows from tableLog order by crTime asc";
		$sth1 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
		$res = $sth1->execute or bark("Can't execute $sqlQuery: " . $sth1::errstr);
		$delStuff=$sth1->fetchall_arrayref;
		
		@del = ();
		deleteLoop: for $k ( 0 .. ($#{$delStuff}) ) {
			$toDelete=$delStuff->[$k][0];
			$deleteSize = $delStuff->[$k][1];
			$deleteSize = 0 if (!$deleteSize);

			$sqlQuery = "DROP TABLE IF EXISTS $toDelete";
			$userHandle->do($sqlQuery) or bark("Can't delete: $sqlQuery: " . $userHandle::errstr);
			
			$toDeleteSize = $toDeleteSize - $deleteSize;
			$del[$k] = "$toDelete";
			last deleteLoop if ($toDeleteSize < 0);
		}
		
		foreach $toDelete (@del) {
			$sqlQuery = "DELETE FROM tableLog where db_name = '$toDelete'";
			$bncHandle->do($sqlQuery) or bark("Can't delete: $sqlQuery: " . $bncHandle::errstr);
		}
	
	}

}


sub checkSuperUser {
	my $username = $ENV{'REMOTE_USER'};
	unless (($username=~m/\A($bwSuperuser)\Z/) || (&checkSuperUser2())) {

		print <<END;
		<TABLE BORDER="0" CELLPADDING="0" CELLSPACING="6" WIDTH="100%">
			<TR>
				<TD WIDTH="100%" HEIGHT="22" BGCOLOR="#FFCC99">
					<H3 ALIGN="CENTER"><B>You are not authorised to access this page! Please don't fiddle with the URL... ;-)</B>
				</TD>
			</TR>
		</TABLE>
END
		print "<HR>$bwCopyright</TD></TR></TABLE></TD></TR></TABLE>";
		exit(0);	
	}

}

sub checkSuperUser2 {
	my $username = $ENV{'REMOTE_USER'};
	
	my $theUserQuote = $bncHandle->quote($username);
	$sqlQuery = "select * from adminStatus where user = $theUserQuote";
	$sth11 = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $bncHandle::errstr);
	$sth11->execute or bark("Can't execute $sqlQuery: " . $sth11::errstr);
	$res = $sth11->rows;
	if ($res) {
		return(1);
	} else {
		return(0);
	}

}

sub checkAuthLevel {
	my $username = $ENV{'REMOTE_USER'};
	
	my $theUserQuote = $bncHandle->quote($username);
	$sqlQuery = "select license from deadlock_users where username = $theUserQuote";
	$sth = $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: $bncHandle::errstr");
	$sth->execute or bark("Can't execute $sqlQuery: $sth::errstr");

	if ($sth->rows) {
		@rowArray=$sth->fetchrow_array;
		$authLevel=$rowArray[0];
	} else {
		$authLevel = 0;
	}
	#$authLevel = 0 if ($username eq "sebhoff");
	return($authLevel);
}


sub class2cpos ( $ ) {
  my $class = shift;
  return "^(AJ.|CRD|DT0|ORD)" if $class eq "ADJ";
  return "^(AV.|XX0)" if $class eq "ADV";
  return "^AT0" if $class eq "ART";
  return "^(CJ.)" if $class eq "CONJ";
  return "^(ITJ)" if $class eq "INTERJ";
  return "^(PR.|TO0)" if $class eq "PREP";
  return "^(DPS|DT[SQ]|EX0|PN.)" if $class eq "PRON";
  return "^(POS|PU.)" if $class eq "STOP";
  return "^(N[NP].|ZZ0|NN1-NP0|NP0-NN1)" if $class eq "SUBST";
  return "^(V..|VVD-VVN|VVN-VVD)" if $class eq "VERB";
  return "^UNC"; # everything else is UNC
}


%catValues = (
	"spowri=1" => "Written",
	"spowri=2" => "Spoken",
	"derivedtexttype=0" => "Academic prose",
	"derivedtexttype=1" => "Fiction and verse",
	"derivedtexttype=2" => "Non-academic prose and biography",
	"derivedtexttype=3" => "Newspapers",
	"derivedtexttype=4" => "Other published written material",
	"derivedtexttype=5" => "Unpublished written material",
	"derivedtexttype=6" => "Spoken conversation",
	"derivedtexttype=7" => "Other spoken material",
	"derivedtexttype=8" => "n/a",
	"alltim=0" => "Unknown",
	"alltim=1" => "1960-1974",
	"alltim=2" => "1975-1984",
	"alltim=3" => "1985-1993",
	"alltim=4" => "n/a",
	"alltyp=1" => "Demographically sampled",
	"alltyp=2" => "Context-governed",
	"alltyp=3" => "Written books and periodicals",
	"alltyp=4" => "Written-to-be-spoken",
	"alltyp=5" => "Written miscellaneous",
	"alltyp=6" => "n/a",
	"wripp=0" => "unknown",
	"wripp=1" => "UK (unspecific)",
	"wripp=2" => "Ireland",
	"wripp=3" => "UK: North (north of Mersey-Humber line)",
	"wripp=4" => "UK: Midlands (north of Bristol Channel-Wash line)",
	"wripp=5" => "UK: South (south of Bristol Channel-Wash line)",
	"wripp=6" => "United States",
	"wripp=7" => "n/a",
	"wrimed=1" => "Book",
	"wrimed=2" => "Periodical",
	"wrimed=3" => "Miscellaneous published",
	"wrimed=4" => "Miscellaneous unpublished",
	"wrimed=5" => "To-be-spoken",
	"wrimed=6" => "n/a",
	"wrisam=0" => "unknown",
	"wrisam=1" => "Whole text",
	"wrisam=2" => "Beginning sample",
	"wrisam=3" => "Middle sample",
	"wrisam=4" => "End sample",
	"wrisam=5" => "Composite",
	"wrisam=6" => "n/a",
	"wridom=1" => "Imaginative prose",
	"wridom=2" => "Informative: Natural and pure sciences",
	"wridom=3" => "Informative: Applied science",
	"wridom=4" => "Informative: Social science",
	"wridom=5" => "Informative: World affairs",
	"wridom=6" => "Informative: Commerce and finance",
	"wridom=7" => "Informative: Arts",
	"wridom=8" => "Informative: Belief and thought",
	"wridom=9" => "Informative: Leisure",
	"wridom=10" => "n/a",
	"wriad=0" => "unknown",
	"wriad=1" => "UK and Ireland",
	"wriad=2" => "Commonwealth",
	"wriad=3" => "Continental Europe",
	"wriad=4" => "USA",
	"wriad=5" => "Elsewhere",
	"wriad=6" => "n/a",
	"wriaag=0" => "unknown",
	"wriaag=1" => "0-14",
	"wriaag=2" => "15-24",
	"wriaag=3" => "25-34",
	"wriaag=4" => "35-44",
	"wriaag=5" => "45-59",
	"wriaag=6" => "60+",
	"wriaag=7" => "n/a",
	"wriase=0" => "unknown",
	"wriase=1" => "Male",
	"wriase=2" => "Female",
	"wriase=3" => "Mixed",
	"wriase=4" => "n/a",
	"wriaty=1" => "Corporate",
	"wriaty=2" => "Multiple",
	"wriaty=3" => "Sole",
	"wriaty=4" => "n/a",
	"wriaud=1" => "Child",
	"wriaud=2" => "Teenager",
	"wriaud=3" => "Adult",
	"wriaud=4" => "Any",
	"wriaud=5" => "n/a",
	"wrilev=1" => "Low",
	"wrilev=2" => "Medium",
	"wrilev=3" => "High",
	"wrilev=4" => "n/a",
	"writas=0" => "Unknown",
	"writas=1" => "Male",
	"writas=2" => "Female",
	"writas=3" => "Mixed",
	"writas=4" => "n/a",
	"wrista=0" => "Unknown",
	"wrista=1" => "Low",
	"wrista=2" => "Medium",
	"wrista=3" => "High",
	"wrista=4" => "n/a",
	"sdeage=1" => "0-14",
	"sdeage=2" => "15-24",
	"sdeage=3" => "25-34",
	"sdeage=4" => "35-44",
	"sdeage=5" => "45-59",
	"sdeage=6" => "60+",
	"sdeage=7" => "n/a",
	"sdecla=0" => "Unknown",
	"sdecla=1" => "AB (top or middle management, administrative or professional)",
	"sdecla=2" => "C1 (junior management, supervisory or clerical)",
	"sdecla=3" => "C2 (skilled manual)",
	"sdecla=4" => "DE (semi-skilled or unskilled)",
	"sdecla=5" => "n/a",
	"sdesex=0" => "unknown",
	"sdesex=1" => "Male",
	"sdesex=2" => "Female",
	"sdesex=3" => "n/a",
	"spolog=1" => "Monologue",
	"spolog=2" => "Dialogue",
	"spolog=3" => "n/a",
	"sporeg=0" => "Unknown",
	"sporeg=1" => "South",
	"sporeg=2" => "Midlands",
	"sporeg=3" => "North",
	"sporeg=4" => "n/a",
	"scgdom=1" => "Educational/Informative",
	"scgdom=2" => "Business",
	"scgdom=3" => "Public/Institutional",
	"scgdom=4" => "Leisure",
	"scgdom=5" => "n/a",
	"age=0" => "Unknown",
	"age=1" => "0-14",
	"age=2" => "15-24",
	"age=3" => "25-34",
	"age=4" => "35-44",
	"age=5" => "45-59",
	"age=6" => "60+",
	"age=7" => "n/a",
	"sex=0" => "Unknown",
	"sex=1" => "Male",
	"sex=2" => "Female",
	"sex=3" => "n/a",
	"class=0" => "Unknown",
	"class=1" => "AB",
	"class=2" => "C1",
	"class=3" => "C2",
	"class=4" => "DE",
	"class=5" => "n/a",
	"firstLanguage=0" => "Unknown",
	"firstLanguage=1" => "German",
	"firstLanguage=2" => "French",
	"firstLanguage=3" => "British English",
	"firstLanguage=4" => "North American English",
	"firstLanguage=5" => "Unknown Indian language",
	"firstLanguage=6" => "n/a",
	"education=0" => "Unknown",
	"education=1" => "Still in education",
	"education=2" => "Left school 14 or under",
	"education=3" => "Left school 15/16",
	"education=4" => "Left school 17/18",
	"education=5" => "Education continued until 19 or over",
	"education=6" => "n/a",
	"role=1" => "recruited",
	"role=2" => "not recruited",
	"role=3" => "unknown",
	"dialect=0" => "No accent recorded",
	"dialect=1" => "Canadian",
	"dialect=2" => "German",
	"dialect=3" => "East Anglian",
	"dialect=4" => "French",
	"dialect=5" => "Home Counties",
	"dialect=6" => "Humberside",
	"dialect=7" => "Irish",
	"dialect=8" => "Indian subcontinent",
	"dialect=9" => "Lancashire",
	"dialect=10" => "London",
	"dialect=11" => "Central Midlands",
	"dialect=12" => "Merseyside",
	"dialect=13" => "North-east Midlands",
	"dialect=14" => "Midlands",
	"dialect=15" => "South Midlands",
	"dialect=16" => "North-west Midlands",
	"dialect=17" => "Central Northern England",
	"dialect=18" => "North-east England",
	"dialect=19" => "Northern England",
	"dialect=20" => "Other or unidentifiable",
	"dialect=21" => "Scottish",
	"dialect=22" => "Lower south-west England",
	"dialect=23" => "Central south-west England",
	"dialect=24" => "Upper south-west England",
	"dialect=25" => "European",
	"dialect=26" => "American (US)",
	"dialect=27" => "Welsh",
	"dialect=28" => "West Indian",
	"dialect=29" => "N/A",
	"genre=1" => "S:brdcast:discussn",
	"genre=2" => "S:brdcast:documentary",
	"genre=3" => "S:brdcast:news",
	"genre=4" => "S:classroom",
	"genre=5" => "S:consult",
	"genre=6" => "S:conv",
	"genre=7" => "S:courtroom",
	"genre=8" => "S:demonstratn",
	"genre=9" => "S:interview",
	"genre=10" => "S:interview:oral_history",
	"genre=11" => "S:lect:commerce",
	"genre=12" => "S:lect:humanities_arts",
	"genre=13" => "S:lect:nat_science",
	"genre=14" => "S:lect:polit_law_edu",
	"genre=15" => "S:lect:soc_science",
	"genre=16" => "S:meeting",
	"genre=17" => "S:parliament",
	"genre=18" => "S:pub_debate",
	"genre=19" => "S:sermon",
	"genre=20" => "S:speech:scripted",
	"genre=21" => "S:speech:unscripted",
	"genre=22" => "S:sportslive",
	"genre=23" => "S:tutorial",
	"genre=24" => "S:unclassified",
	"genre=25" => "W:ac:humanities_arts",
	"genre=26" => "W:ac:medicine",
	"genre=27" => "W:ac:nat_science",
	"genre=28" => "W:ac:polit_law_edu",
	"genre=29" => "W:ac:soc_science",
	"genre=30" => "W:ac:tech_engin",
	"genre=31" => "W:admin",
	"genre=32" => "W:advert",
	"genre=33" => "W:biography",
	"genre=34" => "W:commerce",
	"genre=35" => "W:email",
	"genre=36" => "W:essay:school",
	"genre=37" => "W:essay:univ",
	"genre=38" => "W:fict:drama",
	"genre=39" => "W:fict:poetry",
	"genre=40" => "W:fict:prose",
	"genre=41" => "W:hansard",
	"genre=42" => "W:institut_doc",
	"genre=43" => "W:instructional",
	"genre=44" => "W:letters:personal",
	"genre=45" => "W:letters:prof",
	"genre=46" => "W:misc",
	"genre=47" => "W:news_script",
	"genre=48" => "W:newsp:brdsht_nat:arts",
	"genre=49" => "W:newsp:brdsht_nat:commerce",
	"genre=50" => "W:newsp:brdsht_nat:editorial",
	"genre=51" => "W:newsp:brdsht_nat:misc",
	"genre=52" => "W:newsp:brdsht_nat:report",
	"genre=53" => "W:newsp:brdsht_nat:science",
	"genre=54" => "W:newsp:brdsht_nat:social",
	"genre=55" => "W:newsp:brdsht_nat:sports",
	"genre=56" => "W:newsp:other:arts",
	"genre=57" => "W:newsp:other:commerce",
	"genre=58" => "W:newsp:other:report",
	"genre=59" => "W:newsp:other:science",
	"genre=60" => "W:newsp:other:social",
	"genre=61" => "W:newsp:other:sports",
	"genre=62" => "W:newsp:tabloid",
	"genre=63" => "W:non_ac:humanities_arts",
	"genre=64" => "W:non_ac:medicine",
	"genre=65" => "W:non_ac:nat_science",
	"genre=66" => "W:non_ac:polit_law_edu",
	"genre=67" => "W:non_ac:soc_science",
	"genre=68" => "W:non_ac:tech_engin",
	"genre=69" => "W:pop_lore",
	"genre=70" => "W:religion",
	"genre=71" => "N/A"
);
%typeValues = (
	"spowri" => "Spoken or Written",
	"derivedtexttype" => "Derived text type",
	"allava" => "Text availability",
	"alltyp" => "Text type",
	"alltim" => "Publication date",
	"wrimed" => "Medium of Text",
	"wripp" => "Place of publication",
	"wrisam" => "Text Sample",
	"wridom" => "Text Domain",
	"wriad" => "Domicile of Author",
	"wriaag" => "Age of Author",
	"wriase" => "Sex of Author",
	"wriaty" => "Type of Author",
	"wriaud" => "Age of Audience",
	"wrilev" => "Perceived level of difficulty",
	"writas" => "Target audience sex",
	"wripp" => "Place of publication",
	"wrista" => "Estimated circulation size",
	"sdeage" => "Age of Respondent",
	"sdecla" => "Social Class of Respondent",
	"sdesex" => "Gender of Respondent",
	"spolog" => "Type of Interaction",
	"sporeg" => "Region where spoken text was captured",
	"scgdom" => "Domain (spoken context governed texts)",
	"age" => "Age",
	"sex" => "Sex",
	"class" => "Social Class",
	"firstLanguage" => "First Language",
	"education" => "Education",
	"role" => "Role",
	"dialect" => "Dialect/Accent",
	"numwords" => "Number of words",
	"genre" => "Genre"
);

sub output2 {
	
	local($chunk) = shift(@_);
	local($thing);

	$chunk=~s/&bquo(\;)?/\"/g;
	$chunk=~s/&equo(\;)?/\"/g;
	$chunk=~s/&quot(\;)?/\"/g;
	$chunk=~s/&hellip(\;)?/ \.\.\. /g;
	$chunk=~s/&mdash(\;)?/ \-\- /g;
	$chunk=~s/&ndash(\;)?/ \- /g;
	$chunk=~s/&lsqb(\;)?/\[/g;
	$chunk=~s/&rsqb(\;)?/\]/g;
	$chunk=~s/&rehy(\;)?/-/g;
	$thing='&#149'; # (2150)
	$chunk=~s/&bull(\;)?/$thing\;/g;
	$thing='&#176'; # (4068)
	$chunk=~s/&deg(\;)?/$thing\;/g;
	$thing='&#185'; # 
	$chunk=~s/&(amp;?)?sup1\;?/$thing/g;
	$thing='&#178'; # 
	$chunk=~s/&(amp;?)?sup2\;?/$thing/g;
	$thing='&#179'; # 
	$chunk=~s/&(amp;?)?sup3\;?/$thing/g;
	$chunk=~s/&rarr(\;)?/\-&gt\;/g; # (182)
	$chunk=~s/&larr(\;)/&lt\-\;/g; # (1)
	$thing='&#181'; # (1487)
	$chunk=~s/&micro(\;)?/$thing\;/g; 
	$thing='&#042'; # (919)
	$chunk=~s/&ast(\;)?/$thing\;/g; 
	$thing='&#061'; # (3)
	$chunk=~s/&equals(\;)?/$thing\;/g; 
	$chunk=~s/&percnt(\;)?/\% /g; # (144)
	$thing='&#043'; # (198)
	$chunk=~s/&plus(\;)?/$thing\;/g; 
	$chunk=~s/&sol(\;)?/\//g; # (356)
	$chunk=~s/&bsol(\;)?/\\/g; # (226)
	$chunk=~s/&amp(\;)?/&amp\;/g; # (19006)
	$thing='&#162'; # (3)
	$chunk=~s/&cent(\;)?/$thing\;/g;
	$thing='&#064'; # (190)
	$chunk=~s/&commat(\;)?/$thing\;/g;
	$chunk=~s/&copy(\;)?/&copy\;/g;
	$thing='&#036'; # (24584)
	$chunk=~s/&dollar(\;)?/$thing\;/g;
	$chunk=~s/&ge(\;)?/&gt\;=/g; # (18)
	$chunk=~s/&ins(\;)?/in/g; # (2308)
	$thing='&#191'; # (11)
	$chunk=~s/&iquest(\;)?/$thing\;/g;
	$thing='&#123'; # (345)
	$chunk=~s/&lcub(\;)?/$thing\;/g;
	$thing='&#125'; # (343)
	$chunk=~s/&rcub(\;)?/$thing\;/g;
	$chunk=~s/&num(\;)?/No\./g; # (138)
	$thing='&#188'; # (37) only Mac?
	$chunk=~s/&pgr(\;)?/$thing\;/g;
	$thing='&#189'; # (95) only Mac?
	$chunk=~s/&Pgr(\;)?/$thing\;/g;
	$thing='&#177'; # (122)
	$chunk=~s/&plusmn(\;)?/$thing\;/g;
	$thing='&#163'; # (71753)
	$chunk=~s/&pound(\;)?/$thing\;/g;
	$chunk=~s/&shilling(\;)?/s\./g;
	$thing='&#174'; # (11)
	$chunk=~s/&reg(\;)?/$thing\;/g;
	$thing='&#215'; # (2301)
	$chunk=~s/&times(\;)?/$thing\;/g;
	$thing='&#153'; # (12)
	$chunk=~s/&trade(\;)?/$thing\;/g;
	$thing='&#124'; # (269)
	$chunk=~s/&verbar(\;)?/$thing\;/g;
	$thing='&#165'; # (120)
	$chunk=~s/&yen(\;)?/$thing\;/g;
	$thing='&#914'; # 
	$chunk=~s/&Bgr(\;)?/$thing\;/g;
	$thing='&#946'; # 
	$chunk=~s/&bgr(\;)?/$thing\;/g;
	$thing='&#915'; # 
	$chunk=~s/&Ggr(\;)?/$thing\;/g;
	$thing='&#947'; # 
	$chunk=~s/&ggr(\;)?/$thing\;/g;
	$thing='&#916'; # 
	$chunk=~s/&Dgr(\;)?/$thing\;/g;
	$thing='&#948'; # 
	$chunk=~s/&dgr(\;)?/$thing\;/g;
	$thing='&#918'; # 
	$chunk=~s/&Zgr(\;)?/$thing\;/g;
	$thing='&#950'; # 
	$chunk=~s/&zgr(\;)?/$thing\;/g;
	$thing='&#920'; # 
	$chunk=~s/&THgr(\;)?/$thing\;/g;
	$thing='&#952'; # 
	$chunk=~s/&thgr(\;)?/$thing\;/g;
	$thing='&#954'; # 
	$chunk=~s/&kgr(\;)?/$thing\;/g;
	$thing='&#957'; # 
	$chunk=~s/&ngr(\;)?/$thing\;/g;
	$thing='&#935'; # 
	$chunk=~s/&PHgr(\;)?/$thing\;/g;
	$thing='&#966'; # 
	$chunk=~s/&phgr(\;)?/$thing\;/g;
	$thing='&#961'; # 
	$chunk=~s/&rgr(\;)?/$thing\;/g;
	$thing='&#945'; # 
	$chunk=~s/&agr(\;)?/$thing\;/g;
	$thing='&#951'; # 
	$chunk=~s/&eegr(\;)?/$thing\;/g;
	$thing='&#917'; # 
	$chunk=~s/&Egr(\;)?/$thing\;/g;
	$thing='&#959'; # 
	$chunk=~s/&egr(\;)?/$thing\;/g;
	$thing='&#953'; # 
	$chunk=~s/&igr(\;)?/$thing\;/g;
	$thing='&#967'; # 
	$chunk=~s/&khgr(\;)?/$thing\;/g;
	$thing='&#955'; # 
	$chunk=~s/&lgr(\;)?/$thing\;/g;
	$thing='&#924'; # 
	$chunk=~s/&Mgr(\;)?/$thing\;/g;
	$thing='&#956'; # 
	$chunk=~s/&mgr(\;)?/$thing\;/g;
	$thing='&#926'; # 
	$chunk=~s/&Ogr(\;)?/$thing\;/g;
	$thing='&#958'; # 
	$chunk=~s/&ogr(\;)?/$thing\;/g;
	$thing='&#937'; # 
	$chunk=~s/&OHgr(\;)?/$thing\;/g;
	$thing='&#969'; # 
	$chunk=~s/&ohgr(\;)?/$thing\;/g;
	$thing='&#936'; # 
	$chunk=~s/&PSgr(\;)?/$thing\;/g;
	$thing='&#968'; # 
	$chunk=~s/&psgr(\;)?/$thing\;/g;
	$thing='&#964'; # 
	$chunk=~s/&tgr(\;)?/$thing\;/g;
	$thing='&#933'; # 
	$chunk=~s/&Ugr(\;)?/$thing\;/g;
	$thing='&#965'; # 
	$chunk=~s/&ugr(\;)?/$thing\;/g;
	$thing='&#958'; # 
	$chunk=~s/&xgr(\;)?/$thing\;/g;	
	$chunk=~s/&formula(\;)?/&lt;formula&gt;/g;	
	
	return($chunk);
}

%spoken = (
"KS0" => 1, "KS1" => 1, "KS2" => 1, "KS3" => 1, "KS4" => 1, "KS5" => 1, "KS6" => 1, "KS7" => 1, "KSN" => 1, "KSP" => 1, "KSR" => 1, "KSS" => 1, 
"KST" => 1, "KSU" => 1, "KSV" => 1, "KSW" => 1, "KR0" => 1, "KR1" => 1, "KR2" => 1, "KRE" => 1, "KRF" => 1, "KRG" => 1, "KRH" => 1, "KRJ" => 1, 
"KRK" => 1, "KRL" => 1, "KRM" => 1, "KRN" => 1, "KRP" => 1, "KRR" => 1, "KRS" => 1, "KRT" => 1, "KRU" => 1, "KRV" => 1, "KRW" => 1, "KRX" => 1, 
"KRY" => 1, "KP0" => 1, "KP1" => 1, "KP2" => 1, "KP3" => 1, "KP4" => 1, "KP5" => 1, "KP6" => 1, "KP7" => 1, "KP8" => 1, "KP9" => 1, "KPA" => 1, 
"KPB" => 1, "KPC" => 1, "KPD" => 1, "KPE" => 1, "KPF" => 1, "KPG" => 1, "KPH" => 1, "KPJ" => 1, "KPK" => 1, "KPL" => 1, "KPM" => 1, "KPN" => 1, 
"KPP" => 1, "KPR" => 1, "KPS" => 1, "KPT" => 1, "KPU" => 1, "KPV" => 1, "KPW" => 1, "KPX" => 1, "KPY" => 1, "KN1" => 1, "KN2" => 1, "KN3" => 1, 
"KN6" => 1, "KN7" => 1, "KN8" => 1, "KN9" => 1, "KNA" => 1, "KNB" => 1, "KNC" => 1, "KND" => 1, "KNF" => 1, "KNG" => 1, "KNH" => 1, "KNJ" => 1, 
"KNK" => 1, "KNR" => 1, "KNS" => 1, "KNT" => 1, "KNU" => 1, "KNV" => 1, "KNW" => 1, "KNY" => 1, "KM0" => 1, "KM1" => 1, "KM2" => 1, "KM3" => 1, 
"KM4" => 1, "KM5" => 1, "KM6" => 1, "KM7" => 1, "KM8" => 1, "KLE" => 1, "KLF" => 1, "KLG" => 1, "KLH" => 1, "KLS" => 1, "KLT" => 1, "KLV" => 1, 
"KLW" => 1, "KLX" => 1, "KLY" => 1, "KJS" => 1, "KJT" => 1, "KJU" => 1, "KJV" => 1, "KGH" => 1, "KGK" => 1, "KGL" => 1, "KGM" => 1, "KGN" => 1, 
"KGP" => 1, "KGR" => 1, "KGS" => 1, "KGT" => 1, "KGU" => 1, "KGV" => 1, "KGW" => 1, "KGX" => 1, "KE0" => 1, "KE1" => 1, "KE2" => 1, "KE3" => 1, 
"KE4" => 1, "KE5" => 1, "KE6" => 1, "KEB" => 1, "KD0" => 1, "KD1" => 1, "KD2" => 1, "KD3" => 1, "KD4" => 1, "KD5" => 1, "KD6" => 1, "KD7" => 1, 
"KD8" => 1, "KD9" => 1, "KDA" => 1, "KDB" => 1, "KDC" => 1, "KDD" => 1, "KDE" => 1, "KDF" => 1, "KDG" => 1, "KDH" => 1, "KDJ" => 1, "KDK" => 1, 
"KDL" => 1, "KDM" => 1, "KDN" => 1, "KDP" => 1, "KDR" => 1, "KDS" => 1, "KDT" => 1, "KDU" => 1, "KDV" => 1, "KDW" => 1, "KDX" => 1, "KDY" => 1, 
"KC0" => 1, "KC1" => 1, "KC2" => 1, "KC3" => 1, "KC4" => 1, "KC5" => 1, "KC6" => 1, "KC7" => 1, "KC8" => 1, "KC9" => 1, "KCA" => 1, "KCB" => 1, 
"KCC" => 1, "KCD" => 1, "KCE" => 1, "KCF" => 1, "KCG" => 1, "KCH" => 1, "KCJ" => 1, "KCK" => 1, "KCL" => 1, "KCM" => 1, "KCN" => 1, "KCP" => 1, 
"KCR" => 1, "KCS" => 1, "KCT" => 1, "KCU" => 1, "KCV" => 1, "KCW" => 1, "KCX" => 1, "KCY" => 1, "KB0" => 1, "KB1" => 1, "KB2" => 1, "KB3" => 1, 
"KB4" => 1, "KB5" => 1, "KB6" => 1, "KB7" => 1, "KB8" => 1, "KB9" => 1, "KBA" => 1, "KBB" => 1, "KBC" => 1, "KBD" => 1, "KBE" => 1, "KBF" => 1, 
"KBG" => 1, "KBH" => 1, "KBJ" => 1, "KBK" => 1, "KBL" => 1, "KBM" => 1, "KBN" => 1, "KBP" => 1, "KBR" => 1, "KBS" => 1, "KBT" => 1, "KBU" => 1, 
"KBV" => 1, "KBW" => 1, "KBX" => 1, "KBY" => 1, "K70" => 1, "K71" => 1, "K73" => 1, "K74" => 1, "K75" => 1, "K76" => 1, "K77" => 1, "K78" => 1, 
"K79" => 1, "K7D" => 1, "K7E" => 1, "K7F" => 1, "K7G" => 1, "K60" => 1, "K61" => 1, "K62" => 1, "K63" => 1, "K64" => 1, "K65" => 1, "K66" => 1, 
"K67" => 1, "K68" => 1, "K69" => 1, "K6A" => 1, "K6B" => 1, "K6C" => 1, "K6D" => 1, "K6E" => 1, "K6F" => 1, "K6G" => 1, "K6H" => 1, "K6J" => 1, 
"K6K" => 1, "K6L" => 1, "K6M" => 1, "K6N" => 1, "K6P" => 1, "K6R" => 1, "K6S" => 1, "K6T" => 1, "K6U" => 1, "K6V" => 1, "K6W" => 1, "K6X" => 1, 
"K6Y" => 1, "JYL" => 1, "JYM" => 1, "JYN" => 1, "JWA" => 1, "JT0" => 1, "JT1" => 1, "JT2" => 1, "JT3" => 1, "JT4" => 1, "JT5" => 1, "JT7" => 1, 
"JT8" => 1, "JT9" => 1, "JTA" => 1, "JTB" => 1, "JTC" => 1, "JTD" => 1, "JTE" => 1, "JTF" => 1, "JS7" => 1, "JS8" => 1, "JS9" => 1, "JSA" => 1, 
"JSC" => 1, "JSD" => 1, "JSE" => 1, "JSF" => 1, "JSG" => 1, "JSH" => 1, "JSJ" => 1, "JSK" => 1, "JSL" => 1, "JSM" => 1, "JSN" => 1, "JST" => 1, 
"JSU" => 1, "JSV" => 1, "JSY" => 1, "JP0" => 1, "JP1" => 1, "JP2" => 1, "JP3" => 1, "JP4" => 1, "JP5" => 1, "JP6" => 1, "JP7" => 1, "JP8" => 1, 
"JN6" => 1, "JN7" => 1, "JN8" => 1, "JNB" => 1, "JND" => 1, "JNE" => 1, "JNF" => 1, "JNG" => 1, "JNH" => 1, "JNJ" => 1, "JNK" => 1, "JNL" => 1, 
"JNM" => 1, "JNN" => 1, "JNP" => 1, "JNR" => 1, "JNS" => 1, "JNT" => 1, "JNU" => 1, "JNV" => 1, "JNW" => 1, "JNX" => 1, "JNY" => 1, "JK0" => 1, 
"JK1" => 1, "JK2" => 1, "JK5" => 1, "JK6" => 1, "JK7" => 1, "JK8" => 1, "JK9" => 1, "JJ6" => 1, "JJ7" => 1, "JJ8" => 1, "JJ9" => 1, "JJA" => 1, 
"JJC" => 1, "JJD" => 1, "JJE" => 1, "JJF" => 1, "JJG" => 1, "JJH" => 1, "JJJ" => 1, "JJL" => 1, "JJN" => 1, "JJP" => 1, "JJR" => 1, "JJS" => 1, 
"JJT" => 1, "JJU" => 1, "JJV" => 1, "JJW" => 1, "JJX" => 1, "JJY" => 1, "JA0" => 1, "JA1" => 1, "JA2" => 1, "JA3" => 1, "JA4" => 1, "JA5" => 1, 
"JA6" => 1, "JA8" => 1, "JA9" => 1, "JAA" => 1, "JAB" => 1, "JAC" => 1, "JAD" => 1, "JAE" => 1, "J90" => 1, "J91" => 1, "J92" => 1, "J94" => 1, 
"J95" => 1, "J97" => 1, "J98" => 1, "J99" => 1, "J9A" => 1, "J9B" => 1, "J9C" => 1, "J9D" => 1, "J9E" => 1, "J9F" => 1, "J9G" => 1, "J9H" => 1, 
"J9J" => 1, "J9K" => 1, "J9L" => 1, "J9M" => 1, "J9N" => 1, "J9P" => 1, "J9R" => 1, "J9S" => 1, "J9T" => 1, "J9U" => 1, "J9V" => 1, "J9X" => 1, 
"J9Y" => 1, "J8B" => 1, "J8D" => 1, "J8F" => 1, "J8G" => 1, "J8J" => 1, "J8K" => 1, "J8M" => 1, "J8Y" => 1, "J40" => 1, "J41" => 1, "J42" => 1, 
"J43" => 1, "J44" => 1, "J45" => 1, "J3M" => 1, "J3N" => 1, "J3P" => 1, "J3R" => 1, "J3S" => 1, "J3T" => 1, "J3U" => 1, "J3V" => 1, "J3W" => 1, 
"J3X" => 1, "J3Y" => 1, "HYC" => 1, "HYD" => 1, "HYE" => 1, "HYF" => 1, "HYG" => 1, "HYH" => 1, "HYJ" => 1, "HYK" => 1, "HYL" => 1, "HYM" => 1, 
"HYN" => 1, "HYP" => 1, "HYR" => 1, "HYS" => 1, "HYT" => 1, "HYU" => 1, "HYV" => 1, "HYW" => 1, "HYX" => 1, "HYY" => 1, "HV0" => 1, "HV1" => 1, 
"HV2" => 1, "HV3" => 1, "HV4" => 1, "HV5" => 1, "HV6" => 1, "HV7" => 1, "HV8" => 1, "HV9" => 1, "HVA" => 1, "HVB" => 1, "HVC" => 1, "HVD" => 1, 
"HVE" => 1, "HVF" => 1, "HVG" => 1, "HVH" => 1, "HVJ" => 1, "HVK" => 1, "HUC" => 1, "HUD" => 1, "HUE" => 1, "HUF" => 1, "HUG" => 1, "HUH" => 1, 
"HUJ" => 1, "HUK" => 1, "HUL" => 1, "HUM" => 1, "HUN" => 1, "HUP" => 1, "HUR" => 1, "HUS" => 1, "HUT" => 1, "HUU" => 1, "HUV" => 1, "HUW" => 1, 
"HUX" => 1, "HUY" => 1, "HM2" => 1, "HM4" => 1, "HM5" => 1, "HM6" => 1, "HM7" => 1, "HMA" => 1, "HMD" => 1, "HMG" => 1, "HMH" => 1, "HMJ" => 1, 
"HMK" => 1, "HML" => 1, "HMM" => 1, "HMN" => 1, "HMP" => 1, "HLU" => 1, "HLW" => 1, "HLX" => 1, "HLY" => 1, "HF0" => 1, "HF1" => 1, "HF2" => 1, 
"HF3" => 1, "HE0" => 1, "HE1" => 1, "HE2" => 1, "HE3" => 1, "HE4" => 1, "HE5" => 1, "HE6" => 1, "HE7" => 1, "HE8" => 1, "HE9" => 1, "HEA" => 1, 
"HEC" => 1, "HED" => 1, "HEE" => 1, "HEF" => 1, "HEG" => 1, "HEH" => 1, "HEJ" => 1, "HEK" => 1, "HEL" => 1, "HEM" => 1, "HEN" => 1, "HEP" => 1, 
"HER" => 1, "HES" => 1, "HET" => 1, "HEU" => 1, "HEV" => 1, "HEW" => 1, "HEX" => 1, "HEY" => 1, "HDD" => 1, "HDE" => 1, "HDH" => 1, "HDJ" => 1, 
"HDK" => 1, "HDL" => 1, "HDM" => 1, "HDN" => 1, "HDP" => 1, "HDR" => 1, "HDS" => 1, "HDT" => 1, "HDU" => 1, "HDV" => 1, "HDW" => 1, "HDX" => 1, 
"HDY" => 1, "H60" => 1, "H61" => 1, "H50" => 1, "H51" => 1, "H52" => 1, "H53" => 1, "H54" => 1, "H55" => 1, "H56" => 1, "H57" => 1, "H58" => 1, 
"H59" => 1, "H5A" => 1, "H5B" => 1, "H5C" => 1, "H5D" => 1, "H5E" => 1, "H5G" => 1, "H5H" => 1, "H5J" => 1, "H5K" => 1, "H5L" => 1, "H5M" => 1, 
"H5N" => 1, "H5P" => 1, "H5R" => 1, "H5S" => 1, "H5T" => 1, "H5U" => 1, "H5V" => 1, "H5W" => 1, "H5X" => 1, "H5Y" => 1, "H47" => 1, "H48" => 1, 
"H49" => 1, "H4A" => 1, "H4B" => 1, "H4C" => 1, "H4D" => 1, "H4E" => 1, "H4F" => 1, "H4G" => 1, "H4H" => 1, "H4J" => 1, "H4K" => 1, "H4L" => 1, 
"H4M" => 1, "H4N" => 1, "H4P" => 1, "H4R" => 1, "H4S" => 1, "H4T" => 1, "H4U" => 1, "H4V" => 1, "H4W" => 1, "H4X" => 1, "H4Y" => 1, "H00" => 1, 
"H01" => 1, "H02" => 1, "H03" => 1, "H04" => 1, "H05" => 1, "GY4" => 1, "GY5" => 1, "GY6" => 1, "GY7" => 1, "GY8" => 1, "GY9" => 1, "GYA" => 1, 
"GYB" => 1, "GYC" => 1, "GYD" => 1, "GYE" => 1, "GYF" => 1, "GYG" => 1, "GYH" => 1, "GYJ" => 1, "GYK" => 1, "GYL" => 1, "GYM" => 1, "GYN" => 1, 
"GYP" => 1, "GYR" => 1, "GYS" => 1, "GYT" => 1, "GYU" => 1, "GYV" => 1, "GYW" => 1, "GYX" => 1, "GYY" => 1, "G60" => 1, "G61" => 1, "G62" => 1, 
"G63" => 1, "G64" => 1, "G50" => 1, "G51" => 1, "G52" => 1, "G53" => 1, "G54" => 1, "G55" => 1, "G56" => 1, "G57" => 1, "G58" => 1, "G59" => 1, 
"G5A" => 1, "G5B" => 1, "G5C" => 1, "G5D" => 1, "G5E" => 1, "G5F" => 1, "G5G" => 1, "G5H" => 1, "G5J" => 1, "G5K" => 1, "G5L" => 1, "G5M" => 1, 
"G5N" => 1, "G5P" => 1, "G5R" => 1, "G5S" => 1, "G5T" => 1, "G5U" => 1, "G5V" => 1, "G5W" => 1, "G5X" => 1, "G5Y" => 1, "G42" => 1, "G43" => 1, 
"G44" => 1, "G45" => 1, "G46" => 1, "G47" => 1, "G48" => 1, "G49" => 1, "G4A" => 1, "G4B" => 1, "G4C" => 1, "G4D" => 1, "G4E" => 1, "G4F" => 1, 
"G4G" => 1, "G4H" => 1, "G4J" => 1, "G4K" => 1, "G4N" => 1, "G4P" => 1, "G4R" => 1, "G4S" => 1, "G4T" => 1, "G4U" => 1, "G4V" => 1, "G4W" => 1, 
"G4X" => 1, "G4Y" => 1, "G3U" => 1, "G3V" => 1, "G3W" => 1, "G3X" => 1, "G3Y" => 1, "FY0" => 1, "FY1" => 1, "FY2" => 1, "FY3" => 1, "FY4" => 1, 
"FY5" => 1, "FY6" => 1, "FY7" => 1, "FY8" => 1, "FY9" => 1, "FYA" => 1, "FYB" => 1, "FYD" => 1, "FYE" => 1, "FYF" => 1, "FYG" => 1, "FYH" => 1, 
"FYJ" => 1, "FYK" => 1, "FYL" => 1, "FYM" => 1, "FYP" => 1, "FX5" => 1, "FX6" => 1, "FX7" => 1, "FX8" => 1, "FX9" => 1, "FXB" => 1, "FXC" => 1, 
"FXD" => 1, "FXE" => 1, "FXF" => 1, "FXG" => 1, "FXH" => 1, "FXJ" => 1, "FXK" => 1, "FXL" => 1, "FXM" => 1, "FXN" => 1, "FXP" => 1, "FXR" => 1, 
"FXT" => 1, "FXU" => 1, "FXV" => 1, "FXW" => 1, "FXX" => 1, "FXY" => 1, "FUE" => 1, "FUF" => 1, "FUG" => 1, "FUH" => 1, "FUJ" => 1, "FUK" => 1, 
"FUL" => 1, "FUM" => 1, "FUN" => 1, "FUP" => 1, "FUR" => 1, "FUS" => 1, "FUT" => 1, "FUU" => 1, "FM0" => 1, "FM1" => 1, "FM2" => 1, "FM3" => 1, 
"FM4" => 1, "FM5" => 1, "FM7" => 1, "FM8" => 1, "FM9" => 1, "FMA" => 1, "FMB" => 1, "FMC" => 1, "FMD" => 1, "FME" => 1, "FMF" => 1, "FMG" => 1, 
"FMH" => 1, "FMJ" => 1, "FMK" => 1, "FML" => 1, "FMM" => 1, "FMN" => 1, "FMP" => 1, "FMR" => 1, "FMS" => 1, "FL4" => 1, "FL5" => 1, "FL6" => 1, 
"FL7" => 1, "FL8" => 1, "FL9" => 1, "FLA" => 1, "FLB" => 1, "FLC" => 1, "FLD" => 1, "FLE" => 1, "FLF" => 1, "FLG" => 1, "FLH" => 1, "FLK" => 1, 
"FLL" => 1, "FLM" => 1, "FLP" => 1, "FLR" => 1, "FLS" => 1, "FLU" => 1, "FLW" => 1, "FLX" => 1, "FLY" => 1, "F81" => 1, "F82" => 1, "F84" => 1, 
"F85" => 1, "F86" => 1, "F87" => 1, "F88" => 1, "F89" => 1, "F8A" => 1, "F8B" => 1, "F8C" => 1, "F8D" => 1, "F8E" => 1, "F8F" => 1, "F8G" => 1, 
"F8H" => 1, "F8J" => 1, "F8L" => 1, "F8M" => 1, "F8N" => 1, "F8P" => 1, "F8R" => 1, "F8S" => 1, "F8U" => 1, "F71" => 1, "F72" => 1, "F73" => 1, 
"F74" => 1, "F75" => 1, "F76" => 1, "F77" => 1, "F78" => 1, "F7A" => 1, "F7C" => 1, "F7E" => 1, "F7F" => 1, "F7G" => 1, "F7J" => 1, "F7K" => 1, 
"F7L" => 1, "F7M" => 1, "F7N" => 1, "F7R" => 1, "F7S" => 1, "F7T" => 1, "F7U" => 1, "F7V" => 1, "F7W" => 1, "F7X" => 1, "F7Y" => 1, "DCH" => 1, 
"DCJ" => 1, "DCK" => 1, "D90" => 1, "D91" => 1, "D92" => 1, "D94" => 1, "D95" => 1, "D96" => 1, "D97" => 1, "D8Y" => 1
);


1;
