loading xls but not skipping dupes

api/loadXLS.php

@@ -6,6 +6,7 @@ include 'db.inc.php';
 ini_set('memory_limit', '256M');
 
 
+//for reference!
 $columns_to_fields = array(
 	'Division' => 'division',
 	'Consignee' => 'consignee',
@@ -37,10 +38,12 @@ $columns_to_fields = array(
 $fields_by_col = array();
 
 
-processFile('../test.xls', $columns_to_fields);
+$id_company = 1;
 
+processFile('../test.xls', $id_company);
 
-function processFile($filename_report, $columns_to_fields)
+
+function processFile($filename_report, $id_company)
 {
 	global $dbh;
 	
@@ -48,7 +51,7 @@ function processFile($filename_report, $columns_to_fields)
 	//TODO: check if we already processed this file
 	
 	
-	$stmt = $dbh->prepare('INSERT INTO records (
+	$stmt = $dbh->prepare("INSERT INTO records (
 			id_company,
 			division, consignee, date_called_in, dispatch_number, shipper, date_ready,
 			address_1, address_2, city, state, zip, phone,
@@ -57,12 +60,12 @@ function processFile($filename_report, $columns_to_fields)
 			ctns, weight, cube
 		) VALUES (
 			:id_company,
-			:division, :consignee, :date_called_in, :dispatch_number, :shipper, :date_ready,
+			:division, :consignee, STR_TO_DATE(:date_called_in, '%m/%d/%Y'), :dispatch_number, :shipper, STR_TO_DATE(:date_ready, '%m/%d/%Y'),
 			:address_1, :address_2, :city, :state, :zip, :phone,
 			:time_ready, :time_close, :time_pickup, :time_depart, :bol_delivered,
 			:dispatch_reference,
 			:ctns, :weight, :cube
-		)');
+		)");
 		
 	if(!$stmt)
 	{
@@ -112,20 +115,6 @@ function processFile($filename_report, $columns_to_fields)
 		$columns_by_names = array();
 		
 		
-		//make sure all found columns are mapped
-		for($i = 0; $i < count($column_names[0]); $i++)
-		{
-			$name = trim($column_names[0][$i]);
-			
-			if(!$name && !strlen($name))
-				continue;
-			
-			if(!isset($columns_to_fields[$name]))
-				echo "column not mapped: $name\n";
-			
-			$fields_by_col[$i] = $columns_to_fields[$name];
-		}
-		
 		
 		//flip the column names to reflect the indexes
 		foreach($column_names[0] as $index => $name)
@@ -140,7 +129,7 @@ function processFile($filename_report, $columns_to_fields)
 			echo "processing row #$r...";
 			
 			// read row of data into array
-			$row_array = $sheet->rangeToArray($range, NULL, TRUE, FALSE);
+			$row_array = $sheet->rangeToArray($range, NULL, TRUE, TRUE);
 
 			if(!count($row_array))
 			{
@@ -148,40 +137,50 @@ function processFile($filename_report, $columns_to_fields)
 				continue;
 			}
 			
+			//pull the row out and format the nulls as needed
 			$row = $row_array[0];
-
+			for($c = 0; $c < count($row); $c++)
+				$row[$c] = $row[$c] && strlen(trim($row[$c])) ? trim($row[$c]) : null;
+			
+			//skip if no data for first three columns
+			if(!$row[0] && !$row[1] && !$row[2])
+			{
+				echo "SKIP\n";
+				continue;
+			}
+			
 			//bind appropriate values
-			$params = array(
-				':id_company' => 1
-			);
-
-			/*
-			$EXCEL_DATE = $row[2];
-			$UNIX_DATE = round(($EXCEL_DATE - 25569) * 86400);
-			echo gmdate("d-m-Y H:i:s", $UNIX_DATE) . "\n";
-			*/
+			$params = array();
 			
-			for($c = 0; $c < count($row); $c++)
-				if(isset($fields_by_col[$c]))
-				{
-					$cell = $sheet->getCellByColumnAndRow($c, $r);
-					
-					//if column is a date, then calculate the timestamp and pass that
-					if(PHPExcel_Shared_Date::isDateTime($cell))
-					{
-						$value = PHPExcel_Shared_Date::ExcelToPHP($cell->getValue());
-					}
-					else
-					{
-						$value = trim($cell->getValue());
-					
-						if(!strlen($value))
-							$value = null;
-					}
-					
-					$params[':' . $fields_by_col[$c]] = $value;
-				}
+			$params[':id_company'] = $id_company;
+			
+			$params[':division'] = $row[0];
+			$params[':consignee'] = $row[1];
+			$params[':date_called_in'] = $row[2];
+			$params[':dispatch_number'] = $row[3];
+			$params[':shipper'] = $row[4];
+			$params[':date_ready'] = $row[5];
+			
+			$params[':address_1'] = $row[6];
+			$params[':address_2'] = $row[7];
+			$params[':city'] = $row[8];
+			$params[':state'] = $row[9];
+			$params[':zip'] = $row[10];
+			$params[':phone'] = $row[11];
+			
+			$params[':time_ready'] = $row[14] ? DateTime::createFromFormat('H:i A', $row[14])->format('H:i:s') : null;
+			$params[':time_close'] = $row[15] ? DateTime::createFromFormat('H:i A', $row[15])->format('H:i:s') : null;
+			$params[':time_pickup'] = $row[16] ? DateTime::createFromFormat('H:i A', $row[16])->format('H:i:s') : null;
+			$params[':time_depart'] = $row[17] ? DateTime::createFromFormat('H:i A', $row[17])->format('H:i:s') : null;
+			$params[':bol_delivered'] = $row[18];
+			
+			$params[':dispatch_reference'] = $row[35];
+			
+			$params[':ctns'] = $row[44];
+			$params[':weight'] = $row[45];
+			$params[':cube'] = $row[46];
 
+				
 			try
 			{
 				$stmt->execute($params);

schema.sql

@@ -0,0 +1,113 @@
+/*
+SQLyog Ultimate v12.14 (64 bit)
+MySQL - 5.7.11 : Database - sdpickup
+*********************************************************************
+*/
+
+/*!40101 SET NAMES utf8 */;
+
+/*!40101 SET SQL_MODE=''*/;
+
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+/*Table structure for table `company` */
+
+DROP TABLE IF EXISTS `company`;
+
+CREATE TABLE `company` (
+  `id_company` smallint(5) unsigned NOT NULL AUTO_INCREMENT,
+  `ftp_directory` varchar(32) DEFAULT NULL COMMENT 'name of subdirectory for ftp scanning',
+  `name` varchar(64) NOT NULL,
+  `description` text,
+  PRIMARY KEY (`id_company`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1;
+
+/*Table structure for table `files_processed` */
+
+DROP TABLE IF EXISTS `files_processed`;
+
+CREATE TABLE `files_processed` (
+  `id_file_processed` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `id_company` smallint(5) unsigned NOT NULL,
+  `filename` varchar(64) DEFAULT NULL,
+  `size` bigint(20) unsigned DEFAULT NULL,
+  `num_rows` int(10) unsigned DEFAULT NULL,
+  `num_failed` int(10) unsigned DEFAULT NULL,
+  `date_processed` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id_file_processed`),
+  KEY `id_company` (`id_company`),
+  CONSTRAINT `files_processed_ibfk_1` FOREIGN KEY (`id_company`) REFERENCES `company` (`id_company`) ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+/*Table structure for table `records` */
+
+DROP TABLE IF EXISTS `records`;
+
+CREATE TABLE `records` (
+  `id_record` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  `id_company` smallint(6) unsigned NOT NULL,
+  `division` varchar(64) DEFAULT NULL COMMENT 'Dynamic Delivery Services',
+  `consignee` varchar(32) DEFAULT NULL COMMENT 'GW-PVH',
+  `date_called_in` date DEFAULT NULL COMMENT '05/04/2016',
+  `dispatch_number` varchar(32) DEFAULT NULL COMMENT '22940402-001',
+  `shipper` varchar(128) DEFAULT NULL COMMENT 'Joe Shipping Firm',
+  `date_ready` date DEFAULT NULL COMMENT '05/04/2016',
+  `address_1` varchar(128) DEFAULT NULL COMMENT '123 4th St',
+  `address_2` varchar(128) DEFAULT NULL COMMENT 'Apt 1',
+  `city` varchar(128) DEFAULT NULL COMMENT 'Beverly Hills',
+  `state` char(2) DEFAULT NULL COMMENT 'CA',
+  `zip` varchar(32) DEFAULT NULL COMMENT '90210',
+  `phone` varchar(32) DEFAULT NULL COMMENT '2015551234',
+  `time_ready` time DEFAULT NULL COMMENT '08:00 AM',
+  `time_close` time DEFAULT NULL COMMENT '05:00 PM',
+  `time_pickup` time DEFAULT NULL,
+  `time_depart` time DEFAULT NULL,
+  `bol_delivered` varchar(64) DEFAULT NULL,
+  `dispatch_reference` varchar(32) DEFAULT NULL,
+  `ctns` int(11) DEFAULT NULL COMMENT '20',
+  `weight` decimal(10,0) DEFAULT NULL COMMENT '500.00',
+  `cube` decimal(10,0) DEFAULT NULL COMMENT '42.00',
+  PRIMARY KEY (`id_record`),
+  KEY `records_ibfk_1` (`id_company`),
+  CONSTRAINT `records_ibfk_1` FOREIGN KEY (`id_company`) REFERENCES `company` (`id_company`) ON UPDATE CASCADE
+) ENGINE=InnoDB AUTO_INCREMENT=162 DEFAULT CHARSET=latin1;
+
+/*Table structure for table `update` */
+
+DROP TABLE IF EXISTS `update`;
+
+CREATE TABLE `update` (
+  `id_record` bigint(10) unsigned NOT NULL,
+  `id_user` int(10) unsigned NOT NULL,
+  `key` varchar(32) DEFAULT NULL,
+  `value` varchar(64) DEFAULT NULL,
+  `date_modified` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+  KEY `id_user` (`id_user`),
+  KEY `id_record` (`id_record`),
+  CONSTRAINT `update_ibfk_1` FOREIGN KEY (`id_user`) REFERENCES `user` (`id_user`) ON UPDATE CASCADE,
+  CONSTRAINT `update_ibfk_2` FOREIGN KEY (`id_record`) REFERENCES `records` (`id_record`) ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+/*Table structure for table `user` */
+
+DROP TABLE IF EXISTS `user`;
+
+CREATE TABLE `user` (
+  `id_user` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `id_company` smallint(5) unsigned NOT NULL,
+  `email` varchar(64) NOT NULL,
+  `password` varchar(128) DEFAULT NULL,
+  `date_created` timestamp NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id_user`),
+  UNIQUE KEY `email` (`email`),
+  KEY `id_user` (`id_user`),
+  KEY `id_company` (`id_company`),
+  CONSTRAINT `user_ibfk_1` FOREIGN KEY (`id_company`) REFERENCES `company` (`id_company`) ON UPDATE CASCADE
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;