#!/usr/bin/perl 
#
#
# BNCweb: A user-friendly interface to the British National Corpus - Version 4.3
# Copyright (C) 2005-10 Hoffmann/Evert/Lehmann
#
# Updates and further information available at http://www.bncweb.info
#
# This file is part of BNCweb.
# 
# BNCweb is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
# 
# BNCweb is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


use DBI;
use FindBin qw($Bin);
use lib "$Bin/../lib_files";

use bncConfigXML;

print STDOUT "\n\n";
print STDOUT "***************************\n";
print STDOUT "* MySQL set-up for BNCweb *\n";
print STDOUT "***************************\n\n";

$already = 1 if ($bncHandle = DBI->connect("DBI:mysql:$bwMYSQLtable:$bwMYSQLhost:$bwMysqlPort", "$bwMysqlUser", "$bwMysqlPwd"));

if ($already) {
	print <<END;
It looks as if the database '$bwMYSQLtable' already exists on your MySQL server. This may be because the databases were installed manually, or you may have previously run this installation script. However, it might also be the case that an older version of BNCweb is already installed. If you proceed, all data stored in the existing databases will be overwritten.
	
Please type 'yes' if you wish to proceed with running this script.
END
	$anyway = <STDIN>;
	chop($anyway);
	
	if ($anyway ne "yes") {
		print STDERR "\nGood bye!\n";
		exit(0);
	}
}

unless ($anyway eq "yes") {
	print <<END;
This script creates several MySQL databases that are required to run BNCweb, using the names defined in the Perl library bncConfigXML.pm (located in the folder \"lib_files\").
Please review the default settings for database names before running this script.

This script will alter the MySQL configuration to allow the user '$bwMysqlUser' (as defined in the variable \$bwMysqlUser in bncConfigXML.pm) general access to the newly created databases. It also gives the user \"file priviledges\" to three of the four databases to enable use of the commands 'SELECT ... INTO OUTFILE' and 'LOAD DATA INFILE'.
You may wish to make changes to these settings depending on your local setup and security requirements.

The following SQL-queries will be executed in administrator mode:
-----------------------------------------------------------------
DROP DATABASE if exists $bwMYSQLtable;
DROP DATABASE if exists $bwMYSQLusertable;
DROP DATABASE if exists $bwMYSQLcategorizetable;
DROP DATABASE if exists $bwMYSQLfrequency;

CREATE DATABASE $bwMYSQLtable CHARACTER SET latin1 COLLATE latin1_general_ci;
CREATE DATABASE $bwMYSQLusertable CHARACTER SET latin1 COLLATE latin1_general_ci;
CREATE DATABASE $bwMYSQLcategorizetable CHARACTER SET latin1 COLLATE latin1_general_ci;
CREATE DATABASE $bwMYSQLfrequency CHARACTER SET latin1 COLLATE latin1_general_ci;

GRANT ALL ON $bwMYSQLtable.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd';
GRANT ALL ON $bwMYSQLusertable.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd';
GRANT ALL ON $bwMYSQLcategorizetable.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd';
GRANT ALL ON $bwMYSQLfrequency.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd';
GRANT FILE ON *.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd';

Please enter a MySQL administrator username (e.g. root):
END
	
	$root = <STDIN>;
	chop($root);
	
	print STDOUT "Please enter the password for the MySQL user: ";
	
	$password = <STDIN>;
	chop($password);
	
	$rootHandle = DBI->connect("DBI:mysql:mysql:$bwMYSQLhost:$bwMysqlPort", "$root", "$password") or bark("Can't connect to mysql on $bwMYSQLhost at port: $bwMysqlPort <BR>$rootHandle->errstr\n");
	
	$sqlQuery = "DROP DATABASE if exists $bwMYSQLtable";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "DROP DATABASE if exists $bwMYSQLusertable";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "DROP DATABASE if exists $bwMYSQLcategorizetable";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "DROP DATABASE if exists $bwMYSQLfrequency";
	$rootHandle->do($sqlQuery);	
	
	$sqlQuery = "CREATE DATABASE $bwMYSQLtable CHARACTER SET latin1 COLLATE latin1_general_ci";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "CREATE DATABASE $bwMYSQLusertable CHARACTER SET latin1 COLLATE latin1_general_ci";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "CREATE DATABASE $bwMYSQLcategorizetable CHARACTER SET latin1 COLLATE latin1_general_ci";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "CREATE DATABASE $bwMYSQLfrequency CHARACTER SET latin1 COLLATE latin1_general_ci";
	$rootHandle->do($sqlQuery);	
	
	$sqlQuery = "GRANT ALL ON $bwMYSQLtable.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd'";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "GRANT ALL ON $bwMYSQLusertable.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd'";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "GRANT ALL ON $bwMYSQLcategorizetable.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd'";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "GRANT ALL ON $bwMYSQLfrequency.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd'";
	$rootHandle->do($sqlQuery);	
	$sqlQuery = "GRANT FILE ON *.* TO '$bwMysqlUser'\@'$bwServer' IDENTIFIED BY '$bwMysqlPwd'";
	$rootHandle->do($sqlQuery);	
	
	print STDOUT "Privileges successfully set for user '$bwMysqlUser'\n\n";

} else {
	$sqlQuery = "SHOW TABLES";
	$sth= $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $sth->errstr);
	$sth->execute or bark("Can't execute $sqlQuery: " . $sth->errstr);
	$resArrayResult=$sth->fetchall_arrayref or bark("Can't fetch $sqlQuery: " . $sth->errstr);
	$numTables = $sth->rows;
	print STDERR "\nDeleting $numTables existing tables in '$bwMYSQLtable'...\n" if ($numTables);
	for $k ( 0 .. ($#{$resArrayResult}) ) {
		$tableName=$resArrayResult->[$k][0];
		$sqlQuery = "DROP TABLE $tableName";
		$bncHandle->do($sqlQuery);	
	}
	$sqlQuery = "SHOW TABLES FROM $bwMYSQLusertable";
	$sth= $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $sth->errstr);
	$sth->execute or bark("Can't execute $sqlQuery: " . $sth->errstr);
	$resArrayResult=$sth->fetchall_arrayref or bark("Can't fetch $sqlQuery: " . $sth->errstr);
	$numTables = $sth->rows;
	print STDERR "\nDeleting $numTables existing tables in '$bwMYSQLusertable'...\n" if ($numTables);
	for $k ( 0 .. ($#{$resArrayResult}) ) {
		$tableName=$resArrayResult->[$k][0];
		$sqlQuery = "DROP TABLE $bwMYSQLusertable.$tableName";
		$bncHandle->do($sqlQuery);	
	}
	$sqlQuery = "SHOW TABLES FROM $bwMYSQLcategorizetable";
	$sth= $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $sth->errstr);
	$sth->execute or bark("Can't execute $sqlQuery: " . $sth->errstr);
	$resArrayResult=$sth->fetchall_arrayref or bark("Can't fetch $sqlQuery: " . $sth->errstr);
	$numTables = $sth->rows;
	print STDERR "\nDeleting $numTables existing tables in '$bwMYSQLcategorizetable'...\n" if ($numTables);
	for $k ( 0 .. ($#{$resArrayResult}) ) {
		$tableName=$resArrayResult->[$k][0];
		$sqlQuery = "DROP TABLE $bwMYSQLcategorizetable.$tableName";
		$bncHandle->do($sqlQuery);	
	}
	$sqlQuery = "SHOW TABLES FROM $bwMYSQLfrequency";
	$sth= $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $sth->errstr);
	$sth->execute or bark("Can't execute $sqlQuery: " . $sth->errstr);
	$resArrayResult=$sth->fetchall_arrayref or bark("Can't fetch $sqlQuery: " . $sth->errstr);
	$numTables = $sth->rows;
	print STDERR "\nDeleting $numTables existing tables in '$bwMYSQLfrequency'...\n" if ($numTables);
	for $k ( 0 .. ($#{$resArrayResult}) ) {
		$tableName=$resArrayResult->[$k][0];
		$sqlQuery = "DROP TABLE $bwMYSQLfrequency.$tableName";
		$bncHandle->do($sqlQuery);	
	}
	
}

print STDOUT "\nPlease indicate the directory and prefix where the scripts EncodeBNC.perl and MakeFreqTables.perl saved tables with text and speaker metadata summaries, category codes and other meta-information. Default: [tables/bnc]\n";

$tables = <STDIN>;
chop($tables);

unless ($tables=~/\S/) {
	$tables = "tables";
	$prefix = "bnc";
} else {
	$check = $tables=~m/(.*)\/([^\/]+)\Z/;
	if ($check) {
		$tables = $1;
		$prefix = $2;
		
		
	} else {
		print STDERR "There is something wrong with your input. Please make sure that you enter a directory name (or path to a directory), followed by the same prefix that you chose when compiling the index files (e.g. tables/bnc).\n";
		exit(0);
	}
}

print STDOUT "\nCopying tables to $bwTempPath/...\n";

0 == system "cp $tables/${prefix}_* $bwTempPath/" or die "***** Error: The script cannot find the required files in $tables/$prefix. Please check your input and try again.\n\n";


# connect to mysql
$bncHandle = DBI->connect("DBI:mysql:$bwMYSQLtable:$bwMYSQLhost:$bwMysqlPort", "$bwMysqlUser", "$bwMysqlPwd");


# first, create all tables

$sqlQuery = "DROP TABLE if exists headerInfo";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE headerInfo (
	fname CHAR(5) NOT NULL, 
	title CHAR(255) NOT NULL,
	num_orthogr_words MEDIUMINT UNSIGNED NOT NULL,
	numtokens MEDIUMINT UNSIGNED NOT NULL,
	numwords MEDIUMINT UNSIGNED NOT NULL,
	numPuns MEDIUMINT UNSIGNED NOT NULL,
	numSentences MEDIUMINT UNSIGNED NOT NULL,
	alltim TINYINT UNSIGNED NOT NULL default '0',
	alltyp TINYINT UNSIGNED NOT NULL default '0',
	scgdom TINYINT UNSIGNED NOT NULL default '0',
	sdeage TINYINT UNSIGNED NOT NULL default '0',
	sdecla TINYINT UNSIGNED NOT NULL default '0',
	sdesex TINYINT UNSIGNED NOT NULL default '0',
	spolog TINYINT UNSIGNED NOT NULL default '0',
	sporeg TINYINT UNSIGNED NOT NULL default '0',
	wriaag TINYINT UNSIGNED NOT NULL default '0',
	wriad  TINYINT UNSIGNED NOT NULL default '0',
	wriase TINYINT UNSIGNED NOT NULL default '0',
	wriaty TINYINT UNSIGNED NOT NULL default '0',
	wriaud TINYINT UNSIGNED NOT NULL default '0',
	wridom TINYINT UNSIGNED NOT NULL default '0',
	wrilev TINYINT UNSIGNED NOT NULL default '0',
	wrimed TINYINT UNSIGNED NOT NULL default '0',
	wripp  TINYINT UNSIGNED NOT NULL default '0',
	wrisam TINYINT UNSIGNED NOT NULL default '0',
	wrista TINYINT UNSIGNED NOT NULL default '0',
	writas TINYINT UNSIGNED NOT NULL default '0',
	allava TINYINT UNSIGNED NOT NULL default '0',
	spowri TINYINT UNSIGNED NOT NULL default '0',
	derivedTextType TINYINT UNSIGNED NOT NULL default '0',
	genre SMALLINT UNSIGNED NOT NULL default '0',
	beginPosition BIGINT UNSIGNED,
	endPosition BIGINT UNSIGNED,
	fnum MEDIUMINT UNSIGNED NOT NULL AUTO_INCREMENT,
	primary key(fnum),
	key(fname),
	key(title),
	key(spowri), 
	key(allava),
	key(alltyp), 
	key(alltim), 
	key(scgdom),
	key(sdeage),
	key(sdecla),
	key(sdesex),
	key(spolog),
	key(sporeg),
	key(wriaag), 
	key (wriad), 
	key(wriase), 
	key(wriaty),
	key(wriaud), 
	key(wridom), 
	key(wrilev), 
	key(wrimed),
	key(wripp),
	key(wrisam), 
	key(wrista),
	key(writas),
	key(derivedTextType),
	key(fnum)
	) CHARACTER SET latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't bloody create table $sqlQuery: " . $bncHandle->errstr);

$sqlQuery = "DROP TABLE if exists speakers";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE speakers (
	filename char(3) NOT NULL,
	spid char(20) NOT NULL,
	age TINYINT UNSIGNED NOT NULL,
	dialect TINYINT UNSIGNED NOT NULL,
	education TINYINT UNSIGNED NOT NULL,
	firstLanguage TINYINT UNSIGNED NOT NULL,
	sex TINYINT UNSIGNED NOT NULL,
	class TINYINT UNSIGNED NOT NULL,
	ageFull CHAR(100),
	nameFull CHAR(100),
	occupation CHAR(100),
	role CHAR(100),
	num_orthogr_words MEDIUMINT UNSIGNED NOT NULL,
	numtokens MEDIUMINT UNSIGNED NOT NULL,
	speakerWords MEDIUMINT UNSIGNED NOT NULL,
	speakerPuns MEDIUMINT UNSIGNED NOT NULL,
	speakerTurns MEDIUMINT UNSIGNED NOT NULL,
	speakerSunits MEDIUMINT UNSIGNED NOT NULL,
	`beginPositions` blob,
	`endPositions` blob,
	`snum` mediumint(9) NOT NULL AUTO_INCREMENT,
	primary key(spid),
	key(snum)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't bloody create table $sqlQuery: " . $bncHandle->errstr);


$sqlQuery = "DROP TABLE if exists allWordsMw";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsMw` (
  `freq` int(11) NOT NULL default '0',
  `word` char(50) NOT NULL default '',
  `tag` char(7) NOT NULL default '',
  `lemma_tag` char(7) NOT NULL default '',
  `sw` tinyint NOT NULL default '0',
  KEY `word` (`word`,`tag`,`sw`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists allLemmaMw";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allLemmaMw` (
  `freq` int(11) NOT NULL default '0',
  `lemma` char(50) NOT NULL default '',
  `lemma_tag` char(7) NOT NULL default '',
  `sw` tinyint NOT NULL default '0',
  KEY `word` (`lemma`,`lemma_tag`,`sw`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allLemma";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allLemma` (
  `lemma` char(50) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`lemma`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists allLemmaS";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allLemmaS` (
  `lemma` char(50) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`lemma`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allLemmaW";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allLemmaW` (
  `lemma` char(50) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`lemma`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWords";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWords` (
  `word` char(50) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsG";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsG` (
  `word` char(52) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsS";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsS` (
  `word` char(50) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsSG";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsSG` (
  `word` char(52) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsT";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsT` (
  `word` char(58) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsTS";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsTS` (
  `word` char(58) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsTW";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsTW` (
  `word` char(58) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsW";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsW` (
  `word` char(50) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists allWordsWG";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `allWordsWG` (
  `word` char(52) NOT NULL default '',
  `freq` mediumint(8) unsigned default NULL,
  PRIMARY KEY  (`word`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists tags_master";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `tags_master` (
   `freq` int(11) unsigned default NULL,
	`tag` char(7) NOT NULL default '',
	`sw` TINYINT unsigned default NULL,
  KEY  (`tag`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists tags_overall";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `tags_overall` (
  `tag` char(7) NOT NULL default '',
  `freq` int(11) unsigned default NULL,
  PRIMARY KEY  (`tag`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists tags_spoken";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `tags_spoken` (
  `tag` char(7) NOT NULL default '',
  `freq` int(11) unsigned default NULL,
  PRIMARY KEY  (`tag`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists tags_written";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `tags_written` (
  `tag` char(7) NOT NULL default '',
  `freq` int(11) unsigned default NULL,
  PRIMARY KEY  (`tag`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists categorize";
$bncHandle->do($sqlQuery);	

$sqlQuery = "CREATE TABLE `categorize` (
  `user` varchar(20) default NULL,
  `queryID` varchar(30) default NULL,
  `query` text,
  `thMode` text,
  `hits` mediumint(9) default NULL,
  `time_of_query` int(11) default NULL,
  `theName` varchar(50) default NULL,
  `link` text,
  `date_of_saving` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  `theValues` text,
  KEY `user` (`user`),
  KEY `time_of_query` (`time_of_query`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists COPAC_keywords";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `COPAC_keywords` (
  `filename` char(3) default NULL,
  `keyword` blob,
  `keyword_low` blob
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists keywords";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `keywords` (
  `filename` char(3) default NULL,
  `keyword` blob,
  `keyword_low` blob
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

#### user-defined CQP macros in BNCweb are currently on hold, but may be re-introduced in later release
# $sqlQuery = "DROP TABLE if exists macros";
# $bncHandle->do($sqlQuery);
# $sqlQuery = "CREATE TABLE `macros` (
#   `username` varchar(20) default NULL,
#   `theName` varchar(30) default NULL,
#   `theMacro` text,
#   `time_of_creation` int(11) default NULL,
#   `theStatus` tinyint default 1,
#   `numberOfArguments` tinyint default 0,
#   KEY `username` (`username`)
# ) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
# $bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists spoken_settings";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `spoken_settings` (
	`filename` char(3) default NULL,
	`setting_id` char(12) default NULL,
	`setting_number` INT,
	`beginPosition` BIGINT UNSIGNED,
	`endPosition` BIGINT UNSIGNED,
	`speaker_list` blob,
	`theDate` char(3) default NULL,
	`locale` blob,
	`activity` blob,
	`placeName` blob  
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists genres";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `genres` (
  `fname` char(5) NOT NULL default '',
  `fnum` smallint(5) unsigned NOT NULL default '0',
  `genre` char(20) default NULL,
  KEY `genre` (`genre`),
  KEY `fname` (`fname`),
  KEY `fnum` (`fnum`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists genres2";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `genres2` (
  `fname` char(5) NOT NULL default '',
  `fnum` smallint(5) unsigned NOT NULL default '0',
  `genre` char(40) default NULL,
  KEY `genre` (`genre`),
  KEY `fname` (`fname`),
  KEY `fnum` (`fnum`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists history";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `history` (
  `user` varchar(12) NOT NULL default '',
  `link` text NOT NULL,
  `query` text NOT NULL,
  `hits` int(11) default NULL,
  `time_of_query` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  `checked` varchar(30) default NULL,
  `subcorpus` blob,
  `typeOfRestriction` char(10) default NULL,
  `items` blob,
  `textOrSpeaker` char(10) default NULL,
  `simple_query` text,
  `queryMode` char(10) default NULL,
  KEY `user` (`user`),
  KEY `query` (`query`(256))
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists frequencyLists";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `frequencyLists` (
  `theName` varchar(30) default NULL,
  `username` varchar(20) default NULL,
  `time_of_creation` int(11) default NULL,
  `numRows` bigint UNSIGNED default NULL,
  `subcorpus` text,
  FULLTEXT KEY `subcorpus` (`subcorpus`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists mysqlLog";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `mysqlLog` (
  `db_name` char(60) default NULL,
  `feature` char(15) default NULL,
  `crTime` int(11) default NULL,
  `processID` mediumint(9) NOT NULL default '0'
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists savedQueries";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `savedQueries` (
  `user` varchar(20) default NULL,
  `queryID` varchar(80) default NULL,
  `query` text character set latin1 collate latin1_bin,
  `thMode` text,
  `hits` int(11) default NULL,
  `time_of_query` int(11) default NULL,
  `saved` tinyint(4) default NULL,
  `theName` varchar(50) default NULL,
  `link` text,
  `date_of_saving` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
  `simpleQuery` text character set latin1 collate latin1_bin,
  `initialQuery` tinyint(4) default NULL,
  `originalFile` char(30) default NULL,
  `theID` char(30) default NULL,
  `theRestrictions` text,
  `subcorpus` text,
  `fileSize` int(10) unsigned default NULL,
  `execute_time` float default NULL,
  KEY `time_of_query` (`time_of_query`),
  KEY `user` (`user`),
  FULLTEXT KEY `query` (`query`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists settings";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `settings` (
  `user` varchar(30) default NULL,
  `disp_tags` tinyint(3) unsigned default NULL,
  `disp_lines` tinyint(3) unsigned default NULL,
  `coll_warning` tinyint(4) default NULL,
  `database_col` tinyint(3) unsigned default NULL,
  `crlf` tinyint(4) unsigned default NULL,
  `statType` tinyint(4) default NULL,
  `delRare` mediumint(9) default NULL,
  `totBNC` mediumint(9) default NULL,
  `range` tinyint(4) default NULL,
  `minusrange` tinyint(4) default NULL,
  `disp_color` tinyint(4) default NULL,
  `show_value` varchar(15) default NULL,
  `view` varchar(10) default NULL,
  `new_features` tinyint(3) unsigned default NULL,
  `markup` tinyint(3) unsigned default NULL,
  `historyView` varchar(10) default NULL,
  `showRandom` varchar(3) collate latin1_general_ci default 'no'
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists subcorpora";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `subcorpora` (
  `subcorpus` varchar(200) default NULL,
  `username` varchar(30) default NULL,
  `fnums` mediumtext,
  `numfiles` mediumint(8) unsigned default NULL,
  `numwords` bigint(21) default NULL,
  `spowri` varchar(10) default NULL,
  `textOrspeaker` varchar(10) default NULL,
  `metatextual` blob
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists tableLog";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `tableLog` (
  `db_name` varchar(60) default NULL,
  `crTime` int(11) default NULL,
  `thMode` text,
  `query` text,
  `theRestrictions` text,
  `db_type` varchar(15) default NULL,
  `numRows` bigint UNSIGNED default NULL,
  `date_of_saving` timestamp(14) NOT NULL,
  `includeLemma` char(3),
  `sentBound` tinyint(3) unsigned default NULL,
  `totrange` tinyint(3) unsigned default NULL,
  `sortpos` tinyint(3) unsigned default NULL
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists userLimits";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `userLimits` (
  `user` char(50) default NULL,
  `collocationMax` int(10) unsigned default NULL,
  `sortMax` int(10) unsigned default NULL,
  `distMax` int(10) unsigned default NULL,
  KEY `user` (`user`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists `adminStatus`";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `adminStatus` (
  `user` char(50) default NULL,
  `adminStatus` char(3) default 'no',
  KEY `user` (`user`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists bncFreqSpeakerIndex";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `bncFreqSpeakerIndex` (
  `start` int(11) unsigned NOT NULL,
  `end` int(11) unsigned NOT NULL,
  `spid` char(20) NOT NULL default '',
  KEY `spid` (`spid`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");


$sqlQuery = "DROP TABLE if exists bncFreqTextIndex";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `bncFreqTextIndex` (
  `start` int(11) unsigned NOT NULL,
  `end` int(11) unsigned NOT NULL,
  `fname` char(3) NOT NULL default '',
  KEY `fname` (`fname`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

$sqlQuery = "DROP TABLE if exists bnc_s_unit_pos";
$bncHandle->do($sqlQuery);

$sqlQuery = "CREATE TABLE `bnc_s_unit_pos` (
	`filename` char(3) NOT NULL,
	`sunit` int(11) unsigned NOT NULL,
	`sunit_internal` int(11) unsigned NOT NULL,
	`beginPosition` BIGINT UNSIGNED,
	`endPosition` BIGINT UNSIGNED,
	KEY `mixed` (filename, sunit),
	KEY `sunit_internal` (`sunit_internal`)
	
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE latin1_general_ci";
$bncHandle->do($sqlQuery) or bark("Can't do $sqlQuery");

print STDERR "Importing into table 'bncFreqTextIndex'...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_freqdb_index_text.tbl' into table bncFreqTextIndex";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'bncFreqSpeakerIndex'...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_freqdb_index_spkr.tbl' into table bncFreqSpeakerIndex";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'speakers'...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_speaker.tbl' into table speakers";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'headerInfo'...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_metadata.tbl' into table headerInfo";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'spoken_settings'...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_settings.tbl' into table spoken_settings";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'bnc_s_unit_pos'...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_s_cpos.tbl' into table bnc_s_unit_pos";
$bncHandle->do($sqlQuery);


### THE PATH NEEDS TO BE MADE USER-DEFINABLE
open (OTHER, ">$bwTempPath/${prefix}_keywords1.tbl") || die "can't open: $!";	
open (COPAC, ">$bwTempPath/${prefix}_keywords2.tbl") || die "can't open: $!";	
open (H, "<$bwTempPath/${prefix}_keywords.tbl") || die "can't open: $!";	
while (<H>) {
	$line = $_;
	chomp($line);
	($fname, $type, $keyword) = split (/\t/, $line);
	$LCkeyword = lc($keyword);
	if ($type eq "other") {
		print OTHER "$fname\t$keyword\t$LCkeyword\n";
	} else {
		print COPAC "$fname\t$keyword\t$LCkeyword\n";
	}
}
close(H);
close(OTHER);
close(COPAC);

print STDERR "Importing into keywords tables...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_keywords1.tbl' into table keywords fields escaped by ''";
$bncHandle->do($sqlQuery);
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_keywords2.tbl' into table COPAC_keywords fields escaped by ''";
$bncHandle->do($sqlQuery);


open (OUT, ">$bwTempPath/${prefix}_genres.tbl") || die "can't open: $!";	
open (OUT2, ">$bwTempPath/${prefix}_genres2.tbl") || die "can't open: $!";	

$sqlQuery = "SELECT fname, fnum, genre from headerInfo ";
$sth= $bncHandle->prepare($sqlQuery) or bark("Can't prepare $sqlQuery: " . $sth->errstr);
$sth->execute or bark("Can't execute $sqlQuery: " . $sth->errstr);
$resArrayResult=$sth->fetchall_arrayref or bark("Can't fetch $sqlQuery: " . $sth->errstr);
for $k ( 0 .. ($#{$resArrayResult}) ) {
	
	$fname=$resArrayResult->[$k][0];
	$fnum=$resArrayResult->[$k][1];
	$genre=$resArrayResult->[$k][2];
	$string = "genre=$genre";
	$value = $catValues{$string};
	print OUT2 "$fname\t$fnum\t$value\n";
	
	@values = split(/:/, $value);
	foreach $value (@values) {
		print OUT "$fname\t$fnum\t$value\n";
	}
	
}
close(OUT);
close(OUT2);

print STDERR "Importing into genres table...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_genres.tbl' into table genres fields escaped by ''";
$bncHandle->do($sqlQuery);

print STDERR "Importing into genres2 table...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_genres2.tbl' into table genres2 fields escaped by ''";
$bncHandle->do($sqlQuery);

print STDERR "Importing into master tags table...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_freq_pos.tbl' into table tags_master fields escaped by ''";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'tags_overall'...\n";
$sqlQuery = 'insert into tags_overall select tag, sum(freq) as bla from tags_master group by tag';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'tags_written'...\n";
$sqlQuery = 'insert into tags_written select tag, sum(freq) as bla from tags_master where sw="1" group by tag';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'tags_spoken'...\n";
$sqlQuery = 'insert into tags_spoken select tag, sum(freq) as bla from tags_master where sw="2" group by tag';
$bncHandle->do($sqlQuery);


print STDERR "Importing into master word table...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_freq_word.tbl' into table allWordsMw fields escaped by ''";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'allWords'...\n";
$sqlQuery = 'insert into allWords select word, sum(freq) as bla from allWordsMw group by word';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allWordsW'...\n";
$sqlQuery = 'insert into allWordsW select word, sum(freq) as bla from allWordsMw where sw="1" group by word';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allWordsS'...\n";
$sqlQuery = 'insert into allWordsS select word, sum(freq) as bla from allWordsMw where sw="2" group by word';
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'allWordsT'...\n";
$sqlQuery = 'insert into allWordsT select concat(word, "_", tag), sum(freq) as bla from allWordsMw group by concat(word, "_", tag)';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allWordsTW'...\n";
$sqlQuery = 'insert into allWordsTW select concat(word, "_", tag), sum(freq) as bla from allWordsMw where sw="1" group by concat(word, "_", tag)';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allWordsTS'...\n";
$sqlQuery = 'insert into allWordsTS select concat(word, "_", tag), sum(freq) as bla from allWordsMw where sw="2" group by concat(word, "_", tag)';
$bncHandle->do($sqlQuery);


print STDERR "Importing into table 'allWordsG'...\n";
$sqlQuery = 'insert into allWordsG select concat(word, "_", lemma_tag), sum(freq) as bla from allWordsMw group by concat(word, "_", lemma_tag)';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allWordsWG'...\n";
$sqlQuery = 'insert into allWordsWG select concat(word, "_", lemma_tag), sum(freq) as bla from allWordsMw where sw="1" group by concat(word, "_", lemma_tag)';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allWordsSG'...\n";
$sqlQuery = 'insert into allWordsSG select concat(word, "_", lemma_tag), sum(freq) as bla from allWordsMw where sw="2" group by concat(word, "_", lemma_tag)';
$bncHandle->do($sqlQuery);


print STDERR "Importing into master lemma table...\n";
$sqlQuery = "load data infile '$bwMySQLTempPath/${prefix}_freq_lemma.tbl' into table allLemmaMw fields escaped by ''";
$bncHandle->do($sqlQuery);

print STDERR "Importing into table 'allLemma'...\n";
$sqlQuery = 'insert into allLemma select concat(lemma, "_", lemma_tag) , sum(freq) as bla from allLemmaMw group by concat(lemma, "_", lemma_tag)';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allLemmaW'...\n";
$sqlQuery = 'insert into allLemmaW select concat(lemma, "_", lemma_tag) , sum(freq) as bla from allLemmaMw where sw="1" group by concat(lemma, "_", lemma_tag)';
$bncHandle->do($sqlQuery);
print STDERR "Importing into table 'allLemmaS'...\n";
$sqlQuery = 'insert into allLemmaS select concat(lemma, "_", lemma_tag) , sum(freq) as bla from allLemmaMw where sw="2" group by concat(lemma, "_", lemma_tag)';
$bncHandle->do($sqlQuery);

print STDERR "Deleting master tables...\n";
$sqlQuery = 'Drop table allLemmaMw';
$bncHandle->do($sqlQuery);
$sqlQuery = 'Drop table allWordsMw';
$bncHandle->do($sqlQuery);
$sqlQuery = 'Drop table tags_master';
$bncHandle->do($sqlQuery);

print STDERR "Removing files from $bwTempPath/...\n";

qx{rm -f $bwTempPath/${prefix}_*};

print STDERR "Congratulations! If you have configured the settings for the web server, BNCweb should now be up and running!\n";

exit(0);



