Changed import method so that the importer loads into memory then dumps from memory to the drive to speed up importing of databases to 10,000 per second.

perl/database.sql

@@ -0,0 +1,60 @@
+SET NAMES latin1;
+SET FOREIGN_KEY_CHECKS = 0;
+
+-- ----------------------------
+--  Table structure for `email`
+-- ----------------------------
+DROP TABLE IF EXISTS `email`;
+CREATE TABLE `email` (
+  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `email` varchar(255) NOT NULL,
+  `password` blob NOT NULL,
+  `leak` varchar(255) NOT NULL DEFAULT '',
+  PRIMARY KEY (`id`),
+  KEY `email` (`email`) USING BTREE
+) ENGINE=Aria DEFAULT CHARSET=utf8 PAGE_CHECKSUM=1;
+
+-- ----------------------------
+--  Table structure for `emailload`
+-- ----------------------------
+DROP TABLE IF EXISTS `emailload`;
+CREATE TABLE `emailload` (
+  `email` varchar(255) NOT NULL,
+  `password` binary(255) NOT NULL,
+  `leak` varchar(255) NOT NULL DEFAULT '',
+  PRIMARY KEY (`email`,`password`)
+) ENGINE=MEMORY DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+--  Table structure for `hash`
+-- ----------------------------
+DROP TABLE IF EXISTS `hash`;
+CREATE TABLE `hash` (
+  `hash` varchar(255) NOT NULL,
+  `leak` varchar(255) NOT NULL,
+  PRIMARY KEY (`hash`)
+) ENGINE=MEMORY DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+--  Table structure for `settings`
+-- ----------------------------
+DROP TABLE IF EXISTS `settings`;
+CREATE TABLE `settings` (
+  `name` varchar(255) NOT NULL,
+  `value` blob NOT NULL,
+  PRIMARY KEY (`name`)
+) ENGINE=Aria DEFAULT CHARSET=utf8 PAGE_CHECKSUM=1;
+
+-- ----------------------------
+--  Table structure for `sha1`
+-- ----------------------------
+DROP TABLE IF EXISTS `sha1`;
+CREATE TABLE `sha1` (
+  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `hash` varchar(40) NOT NULL,
+  `leak` varchar(255) NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `hash` (`hash`) USING BTREE
+) ENGINE=Aria DEFAULT CHARSET=utf8 PAGE_CHECKSUM=1;
+
+SET FOREIGN_KEY_CHECKS = 1;

perl/hashPasswords.pl

@@ -57,6 +57,7 @@ $dbConnection = DBI->connect("DBI:mysql:$dbName;host=$dbHost", $dbUser, $dbPassw
 
 open(passwords, $file);
 my $i=0;
+my $count=0;
 while (<passwords>) {
 	chomp;
 	$i++;
@@ -70,10 +71,25 @@ while (<passwords>) {
 		next;
 	}
 	$result->finish();
-	my $result = $dbConnection->prepare("INSERT INTO `sha1` (`hash`,`leak`) VALUES (?,'Email Database')");
+	my $result = $dbConnection->prepare("INSERT INTO `hash` (`hash`,`leak`) VALUES (?,'Email Database')");
 	$result->execute($sha1);
 	$result->finish();
+	$count++;
+	if ($count%10000==0) {
+		my $result = $dbConnection->prepare("INSERT INTO `sha1` (`hash`,`leak`) SELECT `hash`,`leak` FROM `hash`");
+		$result->execute();
+		$result->finish();
+		my $result = $dbConnection->prepare("DELETE FROM `hash`");
+		$result->execute();
+		$result->finish();
+	}
 }
+my $result = $dbConnection->prepare("INSERT INTO `sha1` (`hash`,`leak`) SELECT `hash`,`leak` FROM `hash`");
+$result->execute();
+$result->finish();
+my $result = $dbConnection->prepare("DELETE FROM `hash`");
+$result->execute();
+$result->finish();
 close(passwords);
 
 $dbConnection->disconnect();

perl/loadPasswords.pl

@@ -55,6 +55,7 @@ $dbConnection = DBI->connect("DBI:mysql:$dbName;host=$dbHost", $dbUser, $dbPassw
 
 open(passwords, $file);
 my $i=0;
+my $count=0;
 while (<passwords>) {
 	chomp;
 	my $entry = $_;
@@ -66,7 +67,7 @@ while (<passwords>) {
 			next;
 		}
 		print "$i Email: $email Password: $password\n";
-		my $result = $dbConnection->prepare("SELECT * FROM `emailplaintext` WHERE `email`=? AND `password`=?");
+		my $result = $dbConnection->prepare("SELECT * FROM `email` WHERE `email`=? AND `password`=?");
 		$result->execute($email, $password);
 		my $exists = $result->fetchrow_hashref();
 		if ($exists!=undef) {
@@ -74,11 +75,26 @@ while (<passwords>) {
 			next;
 		}
 		$result->finish();
-		my $result = $dbConnection->prepare("INSERT INTO `emailplaintext` (`email`,`password`) VALUES (?,?)");
+		my $result = $dbConnection->prepare("INSERT INTO `emailload` (`email`,`password`,`leak`) VALUES (?,?,'Leak')");
 		$result->execute($email, $password);
 		$result->finish();
+		$count++;
+		if ($count%10000==0) {
+			my $result = $dbConnection->prepare("INSERT INTO `email` (`email`,`password`,`leak`) SELECT `email`,`password`,`leak` FROM `emailload`");
+			$result->execute();
+			$result->finish();
+			my $result = $dbConnection->prepare("DELETE FROM `emailload`");
+			$result->execute();
+			$result->finish();
+		}
 	}
 }
+my $result = $dbConnection->prepare("INSERT INTO `email` (`email`,`password`,`leak`) SELECT `email`,`password`,`leak` FROM `emailload`");
+$result->execute();
+$result->finish();
+my $result = $dbConnection->prepare("DELETE FROM `emailload`");
+$result->execute();
+$result->finish();
 close(passwords);
 
 $dbConnection->disconnect();

perl/loadSHA1.pl

@@ -47,7 +47,7 @@ $dbName = "passwords";
 $dbUser = "root";
 $dbPassword = "password";
 
-$file = "/Users/grmrgecko/Desktop/passwords/SHA1.txt";
+$file = "/Users/grmrgecko/Desktop/passwords/combo_not.txt";
 
 #print localtime(time).": Connecting to DataBase\n";
 
@@ -55,6 +55,7 @@ $dbConnection = DBI->connect("DBI:mysql:$dbName;host=$dbHost", $dbUser, $dbPassw
 
 open(passwords, $file);
 my $i=0;
+my $count=0;
 while (<passwords>) {
 	chomp;
 	$i++;
@@ -68,10 +69,25 @@ while (<passwords>) {
 		next;
 	}
 	$result->finish();
-	my $result = $dbConnection->prepare("INSERT INTO `sha1` (`hash`,`leak`) VALUES (?,'LinkedIn')");
+	my $result = $dbConnection->prepare("INSERT INTO `hash` (`hash`,`leak`) VALUES (?,'LinkedIn 6/6/12')");
 	$result->execute($sha1);
 	$result->finish();
+	$count++;
+	if ($count%10000==0) {
+		my $result = $dbConnection->prepare("INSERT INTO `sha1` (`hash`,`leak`) SELECT `hash`,`leak` FROM `hash`");
+		$result->execute();
+		$result->finish();
+		my $result = $dbConnection->prepare("DELETE FROM `hash`");
+		$result->execute();
+		$result->finish();
+	}
 }
+my $result = $dbConnection->prepare("INSERT INTO `sha1` (`hash`,`leak`) SELECT `hash`,`leak` FROM `hash`");
+$result->execute();
+$result->finish();
+my $result = $dbConnection->prepare("DELETE FROM `hash`");
+$result->execute();
+$result->finish();
 close(passwords);
 
 $dbConnection->disconnect();