<?php
/*
*    Copyright 2008 - 2011 Maarch
*
*   This file is part of Maarch Framework.
*
*   Maarch Framework is free software: you can redistribute it and/or modify
*   it under the terms of the GNU General Public License as published by
*   the Free Software Foundation, either version 3 of the License, or
*   (at your option) any later version.
*
*   Maarch Framework is distributed in the hope that it will be useful,
*   but WITHOUT ANY WARRANTY; without even the implied warranty of
*   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
*   GNU General Public License for more details.
*
*   You should have received a copy of the GNU General Public License
*   along with Maarch Framework.  If not, see <http://www.gnu.org/licenses/>.
*/

/**
* @defgroup dispatch
*/

/**
*
* Massive files import in Maarch DataBase (Entreprise > 1.1)
*
* This script must be launched in command line, it has many arguments :
*
* Usage: maarch_auto_import_new.php -c config -m mapping [-i] [-e] [-f] [-n]
*           [-w] [-d importDirectory] [-l dirlog] [-s sqldir] [-b] [-h]
*
* OPTIONS
* -c,--config <config>       Config file path is mandatory.
* -m,--mapping <mapping>     Mapping file path is mandatory.
* -i,--insertMode            Instead of a mass load in database, use of insert
*                            queries. This mode is slower than normal mode.
*                            (default: false)
* -e,--excludeExistingDocs   Make a check on the fingerprint to exclude already
*                            existing docs of the database load. This option
*                            slows the processus. (default: false)
* -f,--fileComplete          Check if the file is complete. (default: false)
* -n,--noBackup              Disabled the automatic backup of loaded files.
*                            (default: false)
* -w,--withoutXml            Do not use xml files to get indexes but file
*                            names. (default: false)
* -d,--importDirectory <importDirectory>   Directory where the batch find the
*                            documents to load (default: incoming)
* -l,--dirlog <dirlog>       Directory of the log file, logs directory if not
*                            specified (default: .)
* -s,--sqldir <sqldir>       Directory of the sql file, current directory if
*                            not specified (default: .)
* -b,--noDbLog               Disable the autoimport log in database.
*                            (default: false)
* -t,--letterbox             Activate letterbox compatibility, by default
*                            autoimport works with maarch_framework
*                            (default: false)
* -h,--help                  Display help and exits (default: false)
*
* Errors :
*  1  : Configuration file missing
*  2  : Mapping file missing
*  3  : Error with script arguments
*  4  : Log directory is missing
*  5  : Log directory does not exists or is not writable
*  6  : Import directory is missing
*  7  : Import directory does not exists or is not readable
*  8  : Configuration file does not exist
*  9  : Mapping file does not exist
*  10 : Error on loading config file
*  11 : Batch not complete
*  12 : Docserver path does not exists
*  13 : SQL Query error
*  14 : Mapping file : dateFormat tag is missing for one column
*  15 : Workbatch already exist! Unable to send file in the Docserver
*  16 : Unable to move file in putFileInAlreadyExistsDirectory function
*  17 : Not enough space in the docserver
*  18 : Tag format error
*  19 : Unable to move file in createLockFile function
*  20 : Move imported files in backup failed
*  21 : Impossible to create a directory (mkdir failed)
*  22 : Docserver is not enabled or is readonly
*  23 : Docserver identifier unknown
*  24 : Database error (connexion, constructor)
*  25 : Mapping File : Format tag is missing
*  26 : Sql directory is missing
*  27 : Sql directory does not exists or is not writable
*  28 : Can not write in sqlFile
*  29 : Can not open sql file
*  30 : Database type unknown
*  31 : Lock file is present
*
*  Warnings :
*  1  : Image not found
*  2  : File copy failed during process
*  3  : Xml file not found
*  4  : File not found
*  5  : Tag missing or empty tag in xml file
*  6  : Wrong format in tag
*  7  : Impossible to copy file in docserver
*
*
* @file
* @author Claire Figueras <dev@maarch.org>
* @author Loïc Vinet <dev@maarch.org>
* @author Laurent Giovanonni <dev@maarch.org>
* @date $date$
* @version $Revision$
* @ingroup dispatch
* @brief Massive files import in Maarch DataBase (Entreprise > 1.1)
*/

/********** START **********************/
date_default_timezone_set('Europe/Paris');
echo 'Start : '.date("Y-m-d H:i:s")."\n";

// Include CLITools files and class_db
try {
    include 'Maarch_CLITools/ArgsParser.php';
    include 'Maarch_CLITools/Logger.php';
    include 'Maarch_CLITools/FileHandler.php';
    include 'Maarch_CLITools/ConsoleHandler.php';
    require_once 'class_db.php';
}
catch(IncludeFileError $e) {
    echo "Maarch_CLITools and MIME_type required ! \n (pear.maarch.org)\n";
    exit();
}
//Define globals var

//Batch of folder to load in Maarch
$folderLoaded = array();

// Scan import directory
$scanImportDirectory = '';

// Work batch number of AutoImport
$wbai = 0;

//AutoImport directory
$autoImportDirectory = '';

// Configuration name
$configName = '';

// Lock file
$lockFile = '';

// Csv separator
$csvSeparator = ';';

//Name of the resource table
$tableName = '';

//Format of date fields of the config file
$dateTimeFormat = '';

//Total nb files processed
$totalNbfile = 0;

// Nb of errors
$nbErrors = 0;

// Infos of errors
$infoErrors = '';

main();
echo 'End : ' . date("Y-m-d H:i:s") . "\n";
exit(0);

/********** END script *************/

/**
 * Class to manage files includes errors
 */
class IncludeFileError extends Exception
{
    public function __construct($file)
    {
        $this->file = $file;
        parent::__construct("Include File \"$file\" is missing!", 1);
    }
}

/**
 * File include function which test file existence before inclusion to avoid
 * error and throw an IncludeFileError error in case file is missing
 *
 * @param $file String File path to include
 */
function myInclude($file)
{
    if (file_exists($file)) {
        include_once($file);
    } else {
        throw new IncludeFileError($file);
    }
}

/**
* Make the database query and logs it if necessary
*
* @param  $dbConn db_query Object
* @param  $queryTxt String Text of the query to launch
* @param  $logger Logger Object Use to log data if necessary
*/
function doQuery($dbConn, $queryTxt, $logger, $logDbError=true)
{
    $res = $dbConn->connect();
    if (!$res) {
        manageLog(
            $dbConn->getErrorMsg(), 'ERROR', $logger, 24
        );
    }
    $res = $dbConn->query($queryTxt, true);

    if (! $res ) {
        manageLog(
            'SQL Query error : ' . $queryTxt, 'ERROR', $logger, 13, $dbConn,
            true, '', $logDbError
        );
    }
    manageLog('SQL Query : ' . $queryTxt, 'DEBUG', $logger);
    return true;
}

/**
* Browse a directory and its first level subdirectories to see if there are
* files
*
* @param $directory Path of the directory to check
* @param $excludeFiles Array List of files and directories to exclude
*
* @return Bool True if there are files in the 2 levels , false otherwise
*/
function containFiles($directory, $excludeFiles=array())
{
    $foundDoc = false;
    $classScan = dir($directory);
    while (($fileScan = $classScan->read()) != false) {
        if ($foundDoc) {
            return $foundDoc;
        }
        if (in_array($fileScan, $excludeFiles)) {
            continue;
        } else if (is_dir($directory . DIRECTORY_SEPARATOR . $fileScan)) {
            //Looking in the sub directories
            $subClassScan = dir($directory . DIRECTORY_SEPARATOR . $fileScan);
            while (($subFileScan = $subClassScan->read()) != false) {
                if (in_array($subFileScan, $excludeFiles)) {
                    continue;
                } else {
                    $foundDoc = true;
                }
            }
        } else {
            $foundDoc = true;
        }
    }
    return $foundDoc;
}

/**
* Checks if all files from a directory are complete (files only in the specified
* directory and its first level subdirectories)
*
* @param $directory String Path to the directory to check
* @param  $logger Logger Object Use to log data if necessary
* @param $excludeFiles Array List of files and directories to exclude
*
* @return Bool True if all files to check are completed
*/
function isCompleteBatch($directory, $logger, $excludeFiles)
{
    $complete = true;
    $classScan = dir($directory);
    while (($fileScan = $classScan->read()) != false) {
        if (in_array($fileScan, $excludeFiles)) {
            continue;
        } else if (is_dir($directory . DIRECTORY_SEPARATOR . $fileScan)) {
            //Looking in the sub directories
            $subClassScan = dir($directory . DIRECTORY_SEPARATOR . $fileScan);
            manageLog(
                'Directory : ' . $directory . DIRECTORY_SEPARATOR . $fileScan,
                'INFO', $logger
            );
            while (($subFileScan = $subClassScan->read()) != false) {
                if (in_array($subFileScan, $excludeFiles)) {
                    continue;
                } else {
                    manageLog(
                        'Control of ' . $directory . $fileScan
                        . DIRECTORY_SEPARATOR . $subFileScan, 'INFO', $logger
                    );
                    $complete = isCompleteFile(
                        $directory . $fileScan . DIRECTORY_SEPARATOR
                        . $subFileScan
                    );
                }
            }
        } else {
            manageLog(
                'Control of ' . $directory . $fileScan, 'INFO', $logger
            );
            $complete = isCompleteFile(
                $directory . $fileScan . DIRECTORY_SEPARATOR. $subFileScan
            );
        }
    }
    return $complete;
}

/**
* Check if a file is complete (contains EOF)
*
* @param  $filename Path to the file to check
*
* @return Bool True if the file contains EOF, false otherwise
*/
function isCompleteFile($filename)
{
    $file = @fopen($filename, 'r');
    if (endOfFileReached($file, 500, 0)) {
        fclose($file);
        return true;
    } else {
        return false;
    }
}

/**
* Recursive function which return true when reach EOF or false if the
* filehandler is not set
*
* @param  $filehandler File handler
* @param  $delay Integer
* @param  $pointer Integer Pointer to browse the file
*
* @return Bool True if the file contains EOF, false otherwise
*/
function endOfFileReached($filehandler, $delay=500, $pointer=0)
{
    if ($filehandler == null) {
        return false;
    }
    fseek($filehandler, $pointer);
    $currentLine = fgets($filehandler);
    while (!feof($filehandler)) {
        $currentLine = fgets($filehandler);
    }
    $currentPos = ftell($filehandler);
    //Wait $delay ms
    usleep($delay * 1000);
    if ( $currentPos == $pointer ) {
        return true;
    } else {
        return endOfFileReached($filehandler, $delay, $currentPos);
    }
}

/**
* Get the workbatch identifier for the current batch
*
* @param  $logger Logger Object Use to log data if necessary
* @param  $dbconn Dbquery object
*
* @return Integer Workbatch Identifier
*/
function getWorkBatchAutoImport($logger, $dbConn)
{
    $queryTxt = "select param_value_int from parameters where id = "
              . "'work_batch_autoimport_id'";
    doQuery($dbConn, $queryTxt, $logger);
    while ($reqResult = $dbConn->fetch_array()) {
        $workBatchAutoImport = $reqResult[0] + 1;
    }
    return $workBatchAutoImport;
}

/**
* Update the database with the new workbatchbatch id
*
* @param  $workBatchAutoImport Integer Workbatch identifier
* @param  $dbconn Dbquery object
* @param  $logger Logger Object Use to log data if necessary
*/
function updateWorkBatchAutoImport($workBatchAutoImport, $dbConn, $logger)
{
    $queryTxt = "update parameters set param_value_int  = '"
               . $workBatchAutoImport
               . "' where id = 'work_batch_autoimport_id'";
    doQuery($dbConn, $queryTxt, $logger);
    return true;
}

/**
* Find if the docserver is available and not in read only mode
*
* @param  $docserverId String Docserver identifier
* @param  $dbconn Dbquery object
* @param  $logger Logger Object Use to log data if necessary
*
* @return Bool true if the docserver is available, false otherwise
*/
function isDocserverAvailable($docserverId, $dbConn, $logger)
{
    $queryTxt = "SELECT docserver_id FROM docservers WHERE docserver_id = '"
              . $docserverId . "' and enabled = 'Y' and is_readonly = 'N'";
    doQuery($dbConn, $queryTxt, $logger);
    if ($dbConn->nb_result() == 1) {
        return true;
    }
    return false;
}

/**
* Return in array the docserver data
*
* @param  $docserverId String Docserver identifier
* @param  $logger Logger Object Use to log data if necessary
* @param  $dbconn Dbquery object
* @param  $options Array contains possible options like letterbox
*
* @return Array (
*       'name'           =>  docserver identifier,
*       'path'           =>  docserver path template
*       'availableSpace' =>  Available space in the docserver
*       'hashMode'       =>  Fingerprint mode (md5, sha12, ...)
*       'sizeLimit'      =>  Allowed size of docserver (if 0, no limit defined)
*   )
*/
function getDocserverData($docserverId, $logger, $dbConn, $options)
{
    if ($options['letterbox']) {
        $queryTxt = 'SELECT d.size_limit, '
                  . 'd.actual_size, d.path_template '
                  . "FROM docservers  d WHERE docserver_id = '"
                  . $docserverId . "' ";
    } else {
        $queryTxt = 'SELECT d.docserver_type_id, d.size_limit_number, '
                  . 'd.actual_size_number, d.path_template, dt.fingerprint_mode '
                  . "FROM docservers  d, docserver_types dt WHERE docserver_id = '"
                  . $docserverId . "' and d.docserver_type_id = "
                  . "dt.docserver_type_id";
    }
    doQuery($dbConn, $queryTxt, $logger);
    if ($dbConn->nb_result() == 0) {
        manageLog(
            'Docserver identifier unknown', 'ERROR', $logger, 23, $dbConn
        );
    }
    $res = $dbConn->fetch_object();
    if ($options['letterbox']) {
        return array(
        	'name'           => $docserverId,
            'path'           => $res->path_template,
        	'availableSpace' => $res->size_limit - $res->actual_size,
        	'hashMode'       => 'md5',
            'sizeLimit'      => $res->size_limit,
        );
    }
    return array(
        'name'           => $docserverId,
        'path'           => $res->path_template,
        'availableSpace' => $res->size_limit_number - $res->actual_size_number,
        'hashMode'       => $res->fingerprint_mode,
        'sizeLimit'      => $res->size_limit_number
    );
}

/**
* Create subdirectories on the docserver : Year/Month/workbatch
*
* @param  $docserverPath String Docserver path
* @param  $logger Logger Object Use to log data if necessary
* @param  $dbconn Dbquery object
*
* @return String False if an error occur or the subdirectories relative path
*/
function createPathOnDocServer($docserverPath, $logger, $dbConn)
{
    $subdirectories = array(date('Y'), date('m'), $GLOBALS['wbai']);
    $result = buildDirectories($docserverPath, $subdirectories);

    if ($result['created'] == false && empty($result['error'])) {
        manageLog(
        	'Workbatch already exist! Unable to send file in the Docserver',
            'ERROR', $logger, 15, $dbConn
        );
        return false;
    } else if ($result['created'] == false && !empty($result['error'])) {
        manageLog($result['error'], 'ERROR', $logger, 21, $dbConn);
        return false;
    }
    return $result['path'];
}

/**
* Create subdirectories in a given directory
*
* @param $sourceDir String Directory path
* @param $subdirectories Array contains the name of the subdirectories to create
*
* @return Array (
*           'new'   => bool true if subdirectories are created, false otherwise
*           'path'  => absolute path of the subdirectories
*           'error' => error message
*       )
*/
function buildDirectories($sourceDir, $subdirectories)
{
    $tmp = '';
    $created = false;
    $error = '';
    for ($i = 0; $i < count($subdirectories); $i ++) {
        if (! is_dir(
            $sourceDir . $tmp . $subdirectories[$i] . DIRECTORY_SEPARATOR
        )
        ) {
            $res = mkdir(
                $sourceDir . $tmp . $subdirectories[$i] . DIRECTORY_SEPARATOR,
                0777
            );
            if (!$res) {
                $created = false;
                $error = 'Impossible to create directory mkdir failed : '
                       . $sourceDir . $tmp . $subdirectories[$i]
                       . DIRECTORY_SEPARATOR;
                break;
            }
            if ($i + 1 == count($subdirectories)) {
                $created = true;
            }
        }
        $tmp = $tmp . $subdirectories[$i] . DIRECTORY_SEPARATOR;
    }

    return array(
        'created' => $created,
        'path'    => $tmp,
        'error'   => $error,
    );
}

/**
* Check duplicate file if option excludeExistingDocs is set to true,
* returns the filename if the file is not xml or false otherwise
*
* @param $sourceDir String Directory path
* @param $targetDir String Target directory where to put duplicate file, only if
*   excludeExistingDocs = true
* @param $filename String File name
* @param $hashMode String Hash mode (md5, sha12, ...)
* @param  $logger Logger Object Use to log data if necessary
* @param  $options Array contains possible options like excludeExistingDocs
* @param  $dbconn Dbquery object
* @param  $subDir String if the file is in a subdirectory of the sourceDir it is
*       filled otherwise empty (empty by default)
*
* @return False if the file is xml or already in the database, the filename
*       otherwise
*/
function checkFile($sourceDir, $targetDir, $filename, $hashMode, $logger,
$options, $dbConn, $extensions, $subDir='') {
    $fullpath = $sourceDir . $subDir . DIRECTORY_SEPARATOR . $filename ;
    $ext = strtolower(extractFileExt($filename));
    if ( $ext <> 'xml' && ! in_array($ext, $extensions)) {
        manageLog(
            'File Extension ' . $ext . ' not allowed ('. $fullpath . ')',
            'INFO', $logger
        );
        return false;
    }

    if ($ext <> 'xml') {
        if (! $options['excludeExistingDocs']
            || ! controlExistingDocsInDb($fullpath, $hashMode, $dbConn, $logger)
        ) {
            if (!empty($subDir)) {
                return $subDir . DIRECTORY_SEPARATOR . $filename;
            }
            return $filename;
        } else {
            manageLog(
                'File already Exists in DB ' . $fullpath, 'INFO', $logger
            );
            putFileInAlreadyExistsDirectory(
                $sourceDir, $targetDir, $filename, $logger, $options, $dbConn,
                $subDir
            );
        }
    }
    return false;
}

/**
* Check in the database if the file already exists (using fingerprint)
*
* @param  $filename String Path to the file to check
* @param  $hashMode String Fingerprint type (md5, sha12, ...)
* @param  $dbconn Dbquery object
* @param  $logger Logger Object Use to log data if necessary
*
* @return String True if file already exists, false otherwise
*/
function controlExistingDocsInDb($filename, $hashMode, $dbConn, $logger)
{
    $fingerprint = doFingerprint($filename, $hashMode);
    $flagExists = false;

    if ($GLOBALS['dateTimeFormat'] == 'YYYY-MM-DD') {
        $myDate = date('Y/m/d', strtotime('-3 month'));
    } else if ($GLOBALS['dateTimeFormat'] == 'MM/DD/YYYY') {
        $myDate = date('m/d/Y', strtotime('-3 month'));
    } else {
        $myDate = date('d/m/Y', strtotime('-3 month'));
    }
    if ($GLOBALS['databasetype'] == 'ORACLE') {
        $queryTxt = 'select fingerprint from ' . $GLOBALS['tableName']
             . " where fingerprint = '" . $fingerprint
             . "' and creation_date > to_date('" . $myDate . "','"
             . $GLOBALS['dateTimeFormat'] . "')";
    } else {
        $queryTxt = "select fingerprint from " . $GLOBALS['tableName']
             . " where fingerprint = '" . $fingerprint
             . "' and creation_date > '" . $myDate . "'";
    }
    doQuery($dbConn, $queryTxt, $logger);
    while ($reqResult = $dbConn->fetch_array()) {
        if ($reqResult[0] <> '') {
            $flagExists = true;
            break;
        }
    }
    return $flagExists;
}

/**
* Calculate the fingerprint for a file
*
* @param  $path String Path to the file
* @param  $hashMode String Fingerprint type (md5, sha12, ...)
*
* @return String '0' or the fingerprint
*/
function doFingerprint($path, $hashMode)
{
    if ($hashMode == 'NONE' || $hashMode == '') {
        return '0';
    } else {
        return hash_file(strtolower($hashMode), $path);
    }
}

/**
* Get the file extension
*
* @param  $filepath String Path of the file
*
* @return String Extension of the file or empty string
*/
function extractFileExt($filepath)
{
    if (strpos($filepath, '.') == 0) {
        return '';
    }
    $extractFileExt = explode('.', $filepath);
    return $extractFileExt[count($extractFileExt) - 1];
}

/**
* Checks for a given file if there is a xml file with the same name
*
* @param  $filepath String Path of the file
* @param  $fileExt String File extension
*
* @return Bool True if xml file found, false otherwise
*/
function xmlExists($filepath, $fileExt)
{
    $found = false;
    $lowerXmlFile = preg_replace('/' . $fileExt . '$/', 'xml', $filepath);
    $upperXmlFile = preg_replace('/' . $fileExt . '$/', 'XML', $filepath);

    if (file_exists($lowerXmlFile) == true || file_exists($upperXmlFile)) {
        return true;
    } else {
        return false;
    }
}

/**
* Checks a date pattern
*
* @param  $datePattern String Pattern to check
*
* @return Array ( 'status' => true if check ok, false otherwise,
*                 'error'  => Error message (string)
*/
function checkFormatDate($datePattern)
{
    if (!isset($datePattern) || empty($datePattern)) {
        return array(
            'status' => false,
            'error'  => 'Empty tag date_format',
        );
    }
    if (! preg_match('/YYYY/', $datePattern)
        && !preg_match('/YY/', $datePattern)
    ) {
        return array(
            'status' => false,
            'error'  => 'date_format tag error : wrong description of the '
           	. 'date in the mapping file (year)',
        );
    }
    if (!preg_match('/MM/', $datePattern)) {
        return array(
            'status' => false,
            'error'  => 'date_format tag error : wrong description of the '
                        . 'date in the mapping file (month)'
        );
    }
    if (!preg_match('/DD/', $datePattern)) {
        return array(
            'status' => false,
            'error'  => 'date_format tag error : wrong description of the '
                        . 'date in the mapping file (day)'
        );
    }
    if ((preg_match('/H/', $datePattern) && ! preg_match('/HH/', $datePattern))
        || (preg_match('/HHH/', $datePattern))
    ) {// if One H or more than HH
        return array(
            'status' => false,
            'error'  => 'date_format tag error : wrong description of the '
                        . 'date in the mapping file (hour)'
        );
    }
    if ((preg_match('/I/', $datePattern) && ! preg_match('/II/', $datePattern))
        || (preg_match('/III/', $datePattern))
    ) {// if One I or more than two I
        return array(
            'status' => false,
            'error'  => 'date_format tag error : wrong description of the '
                        . 'date in the mapping file (minutes)'
        );
    }
    if ((preg_match('/S/', $datePattern) && !preg_match('/SS/', $datePattern))
        || (preg_match('/SSS/', $datePattern))
    ) {// if One S or more than two S
        return array(
            'status' => false,
            'error'  => 'date_format tag error : wrong description of the '
                        . 'date in the mapping file (seconds)'
        );
    }
    return array(
        'status' => true,
        'error'  => '',
    );
}

/**
* Browse each file and folder in the import directory
*
* @param  $sourceDir String Path to the directory to inspect
* @param  $docserverData Array contains docserver data
* @param  $logger Logger Object Use to log data if necessary
* @param  $excludeFiles Array contains names of files or directory to excludes
* @param  $options Array contains different options
* @param  $dbconn Dbquery object
*
* @return Array all filenames sorted by names
*/
function getFiles($sourceDir, $docserverData, $logger, $excludeFiles, $options,
$dbConn, $extensions)
{
    $classScan = dir($sourceDir);
    $createNewSubdir = true;
    $arrayFiles = array();
    while (($fileScan = $classScan->read()) != false) {
        $tmp = '';
        if (in_array($fileScan, $excludeFiles)) {
            continue;
        } else if (is_dir($sourceDir . DIRECTORY_SEPARATOR . $fileScan)) {
            //Looking in the sub directories
            array_push($GLOBALS['folderLoaded'], $fileScan);
            $subClassScan = dir($sourceDir . DIRECTORY_SEPARATOR . $fileScan);
            while (($subFileScan = $subClassScan->read()) != false) {
                if (in_array($subFileScan, $excludeFiles)) {
                    continue;
                } else {
                    $tmp = checkFile(
                        $sourceDir, $sourceDir, $subFileScan,
                        $docserverData['hashMode'], $logger, $options, $dbConn,
                        $extensions, $fileScan
                    );
                    if (isset($tmp) && $tmp <> '') {
                        array_push($arrayFiles, $tmp);
                    }
                }
            }
        } else {
            $tmp = checkFile(
                $sourceDir, $sourceDir, $fileScan, $docserverData['hashMode'],
                $logger, $options, $dbConn, $extensions
            );
            if (isset($tmp) && $tmp <> '') {
                array_push($arrayFiles, $tmp);
            }
        }
    }
    //Sorting tab for compatibilites with Unix system
    sort($arrayFiles);
    return $arrayFiles;
}

/**
* Process Files to import : calculate indexes for each file, copy file to
* docserver; if insert mode, insert indexes in database, otherwise create a csv
* file
*
* @param $sourceDir Path of the directory to import
* @param $arrFiles Array List of files to load (path)
* @param $knownvalueFields Array List of values set by default in mapping file
* @param $mappedFields Array List of indexes to set with values of the files
* @param $docserverData Array (
*       'name'           =>  docserver identifier,
*       'path'           =>  docserver path template
*       'availableSpace' =>  Available space in the docserver
*       'hashMode'       =>  Fingerprint mode (md5, sha12, ...)
*       'sizeLimit'      =>  Allowed size of docserver (if 0, no limit defined),
*       'subdocserver'   =>  Subdirectories in the docserver (YYYY/MM/wbai)
*       )
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
* @param $options Array  (
*       'insertMode'           => Insert mode (true/false),
*       'withoutXml'           => Use of xml or not (true/false),
*       'noDbLog'              => Logs won't be in the database (true/false),
*       'excludeExistingDocs'  => Make the checks if files are already in db,
*       'backupBatch'          => Backup all imported files in a backup dir,
*       'controlCompleteFiles' => Check if files in import directory are
*       						  complete
*       )
* @param $sqldir String Directory where to store the csv import file
*
* @return Array (
*           'status'    => True if process went ok, false otherwise
*           'sqlFile'   => path to the csv import ficle
*           'sqlFields' => List of the db fields (used to import csv)
*           'files'     => List of files to import
*           )
*/
function processFiles($sourceDir, $arrFiles, $knownvalueFields, $mappedFields,
$docserverData, $logger, $dbConn, $options, $sqldir = '')
{
    $totalNbfile = 0;
    $createNewSubdir = true;
    $currentSubDir = 0;
    $currentTransfertSize = 0;
    $niceSubDir = str_pad($currentSubDir, 4, '0', STR_PAD_LEFT);
    $processedFiles  = array();
    if (! $options['insertMode']) {
        $sqlFileName = $sqldir . DIRECTORY_SEPARATOR . 'sql_import_file_'
            . $GLOBALS['configName'] . '.sql';
        if (file_exists($sqlFileName)) {
            unlink($sqlFileName);
        }
        $sqlFile = fopen($sqlFileName, 'a');
        if ( ! $sqlFile) {
            manageLog(
                'Can not open sql file ' . $sqlFile, 'ERROR', $logger, 29,
                $dbConn
            );
        }
    }
    // Browsing files
    for ($l = 0; $l < count($arrFiles); $l++) {
        // Building new subdirectory in docserver every 1000 files
        if ($l % 1000 == 0) {
            $createNewSubdir = true;
        }
        if ($createNewSubdir == true) {
            $nbFiles = 0;
            $res = buildDirectories(
                $docserverData['path'] . $docserverData['subdocserver'],
                array($niceSubDir)
            );
            if (!empty($result['error'])) {
                manageLog($result['error'], 'ERROR', $logger, 21, $dbConn);
            }
            $createNewSubdir = false;
            $niceSubDir = str_pad($currentSubDir, 4, '0', STR_PAD_LEFT);
            $currentSubDir ++ ;
        }
        $fileExtension = extractFileExt($arrFiles[$l]);
        $foundXml = true;
        // Test xml file existence
        if (! $options['withoutXml']
            && ! xmlExists($sourceDir . $arrFiles[$l], $fileExtension)
        ) {
            manageLog(
                'Xml file not found for ' . $sourceDir . $arrFiles[$l],
                'WARNING', $logger, 3, $dbConn, true, $sourceDir . $arrFiles[$l]
            );
            continue;
        }
        if (file_exists($sourceDir . $arrFiles[$l])) {
            manageLog(
                'Scan in progress on : ' . $sourceDir . $arrFiles[$l], 'INFO',
                $logger
            );
            $tempFileSize = filesize($sourceDir . $arrFiles[$l]);
            $currentTransfertSize = $currentTransfertSize + $tempFileSize;

            // Checking size of the docserver
            if ($docserverData['sizeLimit'] <> 0) {
                if (($docserverData['availableSpace'] - $tempFileSize) > 0) {
                    manageLog(
                        'The free space available in the docserver is '
                        . 'sufficient to continue', 'DEBUG', $logger
                    );
                } else {
                    manageLog(
                        'Not enough space in the docserver (available : '
                        . $docserverData['availableSpace'] . ', data size : '
                        . $currentTransfertSize, 'ERROR', $logger, 17, $dbConn
                    );
                }
            }
            // Get indexes values
            $values = getFileData(
                $sourceDir, $arrFiles[$l], $knownvalueFields, $mappedFields,
                $docserverData, $logger, $dbConn, $options, $foundXml
            );
            if ($values != false) {
                // Copy file on docserver
                $res = copyFileOnDocserver(
                    $docserverData, $sourceDir, $niceSubDir, $arrFiles[$l],
                    $nbFiles, $logger
                );
                if ($res['status'] == false) {
                    manageLog(
                        'Impossible to copy file from ' . $sourceDir
                        . $arrFiles[$l] . ' to docserver' , 'WARNING', $logger,
                        7, $dbConn, true, $sourceDir . $arrFiles[$l]
                    );
                    continue;
                }

                // Adding filename to the values array
                $values[] = array(
                    'targetColumn' => 'filename',
                    'value'        => $res['filename'],
                    'format'       => 'string',
                );

                // Adding path to the values array
                $tmpPath = str_replace('/', '#', $res['subdir']);
                $tmpPath = str_replace('\\', '#', $tmpPath);
                $values[] = array(
                    'targetColumn' => 'path',
                    'value'        => $tmpPath,
                    'format'       => 'string',
                );

                if ($options['insertMode']) {
                    $line = doSqlInsert($values, $logger, $dbConn);
                } else {
                    writeCsvLine($values, $sqlFile, $logger, $dbConn);
                }

                $nbFiles++;
                $totalNbfile = $totalNbfile + 1;
                $processedFiles[] = $arrFiles[$l];
            }
        } else {
            manageLog(
                'File not found : ' . $sourceDir . $arrFiles[$l],
                'WARNING', $logger, 4, $dbConn, true, $sourceDir . $arrFiles[$l]
            );
        }
    }
    manageLog(
        'Files found : ' . count($arrFiles), 'INFO', $logger
    );

    manageLog(
        'Files imported to docserver : ' . $nbFiles, 'INFO', $logger
    );

    $GLOBALS['totalNbfile'] = $totalNbfile;
    if ($options['insertMode'] == true) {
        return array(
            'status'        => true,
            'transfertSize' => $tempFileSize,
            'files'         => $processedFiles,
        );
    } else {
        fclose($sqlFile);
        manageLog(
            'Creation of SQL command file in : ' . $sqlFileName, 'INFO', $logger
        );
        $fields = array();
        for ($i = 0; $i < count($values); $i ++) {
            $fields [] = $values[$i]['targetColumn'];
        }

        return array(
            'status'        => true,
            'sqlFile'       => $sqlFileName,
            'sqlFields'     => $fields,
            'transfertSize' => $tempFileSize,
            'files'         => $processedFiles,
        );
    }
}

/**
* Make insert sql query
*
* @param  $arrValues Array contains values of the fields to insert
*         (
*               'targetColumn' => Name of the column in database,
*               'value'        => Value to insert,
*               'format'       => type of the value (string, date, integer
*                                 or float)
*         )
* @param  $logger Logger Object Use to log data if necessary
* @param  $dbconn Dbquery object
*/
function doSqlInsert($arrValues, $logger, $dbConn)
{
    $fields = array();
    $values = array();
    $sqlColumns = '';
    $sqlValues = '';
    for ($i = 0; $i < count($arrValues); $i ++) {
        $fields[] = $arrValues[$i]['targetColumn'];
        if ($arrValues[$i]['format'] == 'string') {
            $values[] = "'" . protectStringDb($arrValues[$i]['value']) . "'";
        } else if ($arrValues[$i]['format'] == 'integer'
            || $arrValues[$i]['format'] == 'float'
        ) {
            $values[] = $arrValues[$i]['value'];
        } else if ($arrValues[$i]['format'] == 'date') {
            if ($arrValues[$i]['dateFormat'] == 'skip') {
                $format = 'skip';
            } else {
                $format = sliceDateFormat(
                    $arrValues[$i]['dateFormat'], $logger, $dbConn
                );
            }
            $tmp = dateTimeConvert(
                $arrValues[$i]['value'], $GLOBALS['dateTimeFormat'], $format
            );
            $values[]  = "'" . $tmp ."'";
        } else {
            manageLog(
                'Tag format unkwnown : ' . $arrValues[$i]['format'], 'ERROR',
                $logger , 18, $dbConn
            );
        }
    }
    $sqlColumns = implode(',', $fields);
    $sqlValues = implode(',', $values);
    $sqlInsertQuery = 'insert into ' . $GLOBALS['tableName']
                    . ' (' . $sqlColumns . ') values(' . $sqlValues . ')';
    doQuery($dbConn, $sqlInsertQuery, $logger);
    manageLog('Sql query : ' . $sqlInsertQuery, 'DEBUG', $logger);
    return true;
}

/**
* Write a csv line in the csv import file
*
* @param  $arrValues Array contains values of the fields to insert
*         (
*               'targetColumn' => Name of the column in database,
*               'value'        => Value to insert,
*               'format'       => type of the value (string, date, integer
*                                 or float)
*         )
* @param  $sqlFile File handler Handler of the csv import file
* @param  $logger Logger Object Use to log data if necessary
* @param  $dbconn Dbquery object
*/
function writeCsvLine($arrValues, $sqlFile, $logger, $dbConn)
{
    $values = array();
    for ($i = 0; $i < count($arrValues); $i ++) {
        if ($arrValues[$i]['format'] == 'string') {
            $values[] = protectStringDb($arrValues[$i]['value']);
        } else if ($arrValues[$i]['format'] == 'integer'
            || $arrValues[$i]['format'] == 'float'
        ) {
            $values[] = $arrValues[$i]['value'];
        } else if ($arrValues[$i]['format'] == 'date') {
            if ($arrValues[$i]['dateFormat'] == 'skip') {
                $format = 'skip';
            } else {
                $format = sliceDateFormat(
                    $arrValues[$i]['dateFormat'], $logger, $dbConn
                );
            }
            $values[]  = dateTimeConvert(
                $arrValues[$i]['value'], $GLOBALS['dateTimeFormat'], $format
            );
        } else {
            manageLog(
                'Tag format unkwnown : ' . $arrValues[$i]['format'], 'ERROR',
                $logger , 18, $dbConn
            );
        }
    }
    if ( fputcsv($sqlFile, $values, $GLOBALS['csvSeparator']) === false) {
        manageLog(
            'Can not write in ' . $sqlFile, 'ERROR', $logger, 28,
            $dbConn
        );
    }
    return true;
}

/**
* Calculate indexes to import in databse for a given file
*
* @param $sourceDir String Path of the import directory
* @param $filename String Name of the file to load
* @param $knownvalueFields Array List of values set by default in mapping file
* @param $mappedFields Array List of indexes to set with values of the files
* @param $docserverData Array (
*       'name'           =>  docserver identifier,
*       'path'           =>  docserver path template
*       'availableSpace' =>  Available space in the docserver
*       'hashMode'       =>  Fingerprint mode (md5, sha12, ...)
*       'sizeLimit'      =>  Allowed size of docserver (if 0, no limit defined),
*       'subdocserver'   =>  Subdirectories in the docserver (YYYY/MM/wbai)
*       )
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
* @param $options Array  (
*       'insertMode'           => Insert mode (true/false),
*       'withoutXml'           => Use of xml or not (true/false),
*       'noDbLog'              => Logs won't be in the database (true/false),
*       'excludeExistingDocs'  => Make the checks if files are already in db,
*       'backupBatch'          => Backup all imported files in a backup dir,
*       'controlCompleteFiles' => Check if files in import directory are
*       						  complete
*       )
* @param $foundXml Bool True if there is a matching xml file, false otherwise
*
* @return Array Contains data associated to the file
*               (
*                   'targetColumn' => Column in the database
*                   'value'        => Value for this column
*                   'format'       => column format (string, integer, date
*                                     or float)
*                   'dateFormat'   => Date format expected in xml file (only
*                                     present if format is date)
*               )
*/
function getFileData($sourceDir, $filename, $knownvalueFields, $mappedFields,
$docserverData, $logger, $dbConn, $options, $foundXml)
{
    $fileData = array();
    $fileExt = extractFileExt($filename);
    // If withoutXml = false, then we look for the matching xml file
    if (! $options['withoutXml'] && $foundXml) {
        // Xml extension can be in lowercase
        $xmlfile = preg_replace('/' . $fileExt . '$/', 'xml', $filename);
        // Or in uppercase
        if (! file_exists($sourceDir . $xmlfile)) {
            $xmlfile = preg_replace('/' . $fileExt . '$/', 'XML', $filename);
        }
        // Loading xml file
        $xml = simplexml_load_file($sourceDir . $xmlfile);
        // browsing the $mappedFields array to retrieve matching values in xml
        // file
        foreach (array_keys($mappedFields) as $tag) {
            $tagValue = (string) $xml->$tag;
            $targetColumn = $mappedFields[$tag]['targetColumn'];
            $format = strtolower($mappedFields[$tag]['format']);
            $dateFormat = null;
            $defaultValue = null;

            // If format = date then we must have a dateFormat description
            if ($format == 'date' || $format == 'DATE') {
                $dateFormat = $mappedFields[$tag]['dateFormat'];
            }
            // DefaultValue case
            if (isset($mappedFields[$tag]['defaultValue'])) {
                $defaultValue =
                    $mappedFields[$tag]['defaultValue'];
            }
            // if the tag is not found in xml file or value is empty
            if ( ! isset($tagValue) || $tagValue == '') {
                // if there is a defaultValue we take it
                if (isset($defaultValue)) {
                    $fileData[] = array(
                        'targetColumn' => $targetColumn,
                        'value'        => $defaultValue,
                        'format'       => $format,
                        'dateFormat'   => $dateFormat,
                    );
                    continue;
                } else { // Else throw warning and
                    manageLog(
                        'Tag missing or empty tag for ' . $targetColumn . ' in '
                        . $sourceDir . $filename, 'WARNING', $logger, 5,
                        $dbConn, true, $sourceDir . $filename
                    );
                    return false;
                }
            }
            // Format check
            $res = checkFormat($tagValue, $format, $dateFormat);
            if (! $res['status']) {
                manageLog(
                    'Tag ' . $tag .' in file ' . $sourceDir . $filename . ' : '
                    . $res['error'], 'WARNING', $logger, 6, $dbConn, true,
                    $sourceDir . $filename
                );
                return false;
            }
            $fileData[] = array(
                'targetColumn' => $targetColumn,
                'value'        => $tagValue,
                'format'       => $format,
                'dateFormat'   => $dateFormat,
            );
        }
    }
    $fileData = array_merge($fileData, $knownvalueFields);

    // Adding mandatory values
    // typist
    $fileData[] = array(
        'targetColumn' => 'typist',
        'value'        => 'AUTOIMPORT',
        'format'       => 'string',
    );
    // creation_date
    if ($GLOBALS['databasetype'] == 'ORACLE') {
        $date = 'Date(sysdate)';
    } else if ($GLOBALS['databasetype'] == 'SQLSERVER') {
        $date = date('Ymd') . ' ' . date('H:i:s') . '.000';
    } else {
        $date = date('Y-m-d') . '  ' . date('H:i:s');
    }
    $fileData[] = array(
        'targetColumn' => 'creation_date',
        'value'        => $date,
        'format'       => 'date',
        'dateFormat'   => 'skip',
    );
    // offset_doc
    $fileData[] = array(
        'targetColumn' => 'offset_doc',
        'value'        => '',
        'format'       => 'string',
    );
    // fingerprint
     $fileData[] = array(
         'targetColumn' => 'fingerprint',
         'value'        => doFingerprint(
             $sourceDir . $filename, $docserverData['hashMode']
         ),
         'format'       => 'string',
    );
    //  docserver_id
    $fileData[] = array(
        'targetColumn' => 'docserver_id',
        'value'        => $docserverData['name'],
        'format'       => 'string',
    );
    // format
    $fileData[] = array(
        'targetColumn' => 'format',
        'value'        => $fileExt,
        'format'       => 'string',
    );
    // filesize
    $fileData[] = array(
        'targetColumn' => 'filesize',
        'value'        => filesize($sourceDir . $filename),
        'format'       => 'integer',
    );
    // work_batch
    $fileData[] = array(
        'targetColumn' => 'work_batch',
        'value'        => $GLOBALS['wbai'],
        'format'       => 'integer',
    );

    return $fileData;
}

/**
* Check if the date match the expected format
*
* @param  $value String date to check
* @param  $dateFormat String Date format
*
* @return Array (
*           'status' => True if date match format, false otherwise
*           'error'  => Error message if date do not match format
*           )
*/
function checkMyDate($value, $dateFormat)
{
    if (empty($value)) {
        return array(
            'status' => false,
            'error'  => 'Empty date value',
        );
    }
    if ($dateFormat == 'skip') {
        return array(
            'status' => true,
            'error'  => '',
        );
    }
    $tmp = $dateFormat;
    $tmp = preg_replace('/YY/', '[0-9][0-9]', $tmp);
    $tmp = preg_replace('/YY/', '[0-9][0-9]', $tmp);
    $tmp = preg_replace('/MM/', '[0-9][0-9]', $tmp);
    $tmp = preg_replace('/DD/', '[0-9][0-9]', $tmp);
    $tmp = preg_replace('/HH/', '[0-9][0-9]', $tmp);
    $tmp = preg_replace('/II/', '[0-9][0-9]', $tmp);
    $tmp = preg_replace('/SS/', '[0-9][0-9]', $tmp);

    if (! preg_match('#^' . $tmp . '$#', trim($value))) {
        return array(
            'status' => false,
            'error'  => 'The date do not match the expected date pattern ('
                        . $dateFormat . ')',
        );
    }
    return array(
    	'status' => true,
        'error'  => '',
    );
}

/**
* Check that the given value match the given format
*
* @param  $value String Value to check
* @param  $format String Format (staring, integer, date or float)
* @param  $dateFormat String Date format
*
* @return Array (
*           'status' => True if value match format, false otherwise
*           'error'  => Error message if date do not match format
*           )
*/
function checkFormat($value, $format, $dateFormat='')
{
    // #TODO : permettre d'ajouter des vérifications spécifiques
    switch (strtolower($format)) {
        case 'date':
            return checkMyDate($value, $dateFormat);
        case 'string':
            break;
        case 'integer':
            if (! preg_match('/^[0-9]+$/', $value)) {
                return array(
                    'status' => false,
                    'error'  => 'Format error, ' . $value . ' is not integer',
                );
            }
            break;
        case 'float':
            if (! preg_match('/^[0-9]*[.,]?[0-9]*$/', $value)) {
                return array(
                    'status' => false,
                    'error'  => 'Format error, ' . $value . ' is not float',
                );
            }
            break;
        default:
            return array(
            	'status' => false,
                'error'  => 'Unknown format ' . $format,
            );
    }
    return array(
        'status' => true,
        'error'  => '',
    );
}

/**
* Extract a file name from a filepath
*
* @param  $sFullPath string File path
*
* @return String File name
*/
function extractFileName($sFullPath)
{
    if ($GLOBALS['osName'] == 'UNIX') {
        if (! preg_match('@' . DIRECTORY_SEPARATOR . '@', $sFullPath)) {
            return $sFullPath;
        } else {
            return substr(
                $sFullPath, strrpos($sFullPath, DIRECTORY_SEPARATOR) + 1
            );
        }
    } else {
        if (! preg_match('/\\' . DIRECTORY_SEPARATOR . '/', $sFullPath)) {
            return $sFullPath;
        } else {
            return substr(
                $sFullPath, strrpos($sFullPath, DIRECTORY_SEPARATOR) + 1
            );
        }
    }
}

/**
* Put an existing file in a 'duplicate' subdirectory of the target directory
* @param  $sourceDir String Source directory where the file is
* @param  $targetDir String Target directory where to put the file
* @param  $filename String  File name
* @param $logger Logger Object Use to log data if necessary
* @param $options Array  (
*       'insertMode'           => Insert mode (true/false),
*       'withoutXml'           => Use of xml or not (true/false),
*       'noDbLog'              => Logs won't be in the database (true/false),
*       'excludeExistingDocs'  => Make the checks if files are already in db,
*       'backupBatch'          => Backup all imported files in a backup dir,
*       'controlCompleteFiles' => Check if files in import directory are
*       						  complete
*       )
* @param $dbconn Dbquery object
* @param $additionnalPath Adding subdirectories in the duplicate directory
*   (empty by default)
*/
function putFileInAlreadyExistsDirectory($sourceDir, $targetDir, $filename,
$logger, $options, $dbConn, $additionnalPath = '')
{
    $subdirectories = array('duplicate', $GLOBALS['wbai']);
    if ($additionnalPath <> '' ) {
        $subdirectories [] = $additionnalPath;
    }
    $result = buildDirectories($sourceDir, $subdirectories);

    if (!empty($result['error'])) {
        manageLog($result['error'], 'ERROR', $logger, 21, $dbConn);
    }
    $targetPath = $sourceDir . $result['path'];
    if ($options['withoutXml'] == false) {
        $filenameMinusExt = extractFileExt($filename);
        if (file_exists(
            $sourceDir . str_replace($filenameMinusExt, '', $filename)
            . 'xml'
        )
        ) {
            $targetFile = $targetPath . str_replace(
                extractFileExt($filename), '', $filename
            ) . 'xml';
            if (! @rename(
                $sourceDir . str_replace(
                    extractFileExt($filename), '', $filename
                ) . 'xml', $targetFile
            )
            ) {
                manageLog(
                    'Unable to move file in putFileInAlreadyExistsDirectory'
                    .' function', 'ERROR', $logger, 16, $dbConn
                );
            }
        } else if (file_exists(
            $sourceDir . str_replace($filenameMinusExt, '', $filename)
            . 'XML'
        )
        ) {
            $targetFile = $targetPath . str_replace(
                extractFileExt($filename), '', $filename
            ) . 'XML';
            if (! @rename(
                $sourceDir . str_replace(
                    extractFileExt($filename), '', $filename
                ) . 'XML', $targetFile
            )
            ) {
                manageLog(
                	'Unable to move file in putFileInAlreadyExistsDirectory'
                    .' function', 'ERROR', $logger, 16, $dbConn
                );
            }
        }
    }

    if (! @rename($sourceDir . $filename, $targetPath . $filename)) {
        manageLog(
            'Unable to move file in putFileInAlreadyExistsDirectory function',
            'ERROR', $logger, 16, $dbConn
        );
    }
}

/**
* Copy the file in the DoServer
* @param $docserverData Array (
*       'name'           =>  docserver identifier,
*       'path'           =>  docserver path template
*       'availableSpace' =>  Available space in the docserver
*       'hashMode'       =>  Fingerprint mode (md5, sha12, ...)
*       'sizeLimit'      =>  Allowed size of docserver (if 0, no limit defined),
*       'subdocserver'   =>  Subdirectories in the docserver (YYYY/MM/wbai)
*       )
* @param  $sourceDir String Source directory where the file is
* @param  $subdir String Subdirectories in docserver where to copy file
* @param  $filename String File name
* @param  $nbFiles Integer Subdirectory in docserver (one every 1000 files)
* @param  $logger Logger Object Use to log data if necessary
*
* @return Array (
*           'status'   => True if copy ok, false otherwise
*           'filename' => File name in the docserver
*           'subdir'   => path of the file on the docserver
*       )
*/
function copyFileOnDocserver($docserverData, $sourceDir, $subdir, $filename,
$nbFiles, $logger)
{
    $ext = strtolower(extractFileExt($filename));
    $docName = str_pad($nbFiles , 4, '0', STR_PAD_LEFT). '.' . strtolower($ext);
    $pathOnDocserver = $docserverData['subdocserver'] . $subdir
                     . DIRECTORY_SEPARATOR;
    $newPath = $docserverData['path'] . $pathOnDocserver . $docName ;
    if (! @copy($sourceDir . $filename, $newPath)) {
        manageLog(
            'File (' . $sourceDir . $filename . ') copy to ' . $newPath
            . ' failed ', 'WARNING', $logger, 2, null, true,
            $sourceDir . $filename
        );
        return array(
            'status'   => false,
            'filename' => $docName,
            'subdir'   => $pathOnDocserver,
        );
    }
    manageLog('New file created : ' . $newPath, 'INFO', $logger);
    return array(
        'status'   => true,
        'filename' => $docName,
        'subdir'   => $pathOnDocserver,
    );
}

/**
* Protect string to insert in the database
*
* @param  $string string String to format
*
* @return Protected string
*/
function protectStringDb($string)
{
    if ($GLOBALS['databasetype'] == 'SQLSERVER') {
        $string = str_replace("'", "''", $string);
        $string = str_replace('\\', '', $string);
    } else if ($GLOBALS['databasetype'] == 'ORACLE') {
        $string = str_replace("'", "''", $string);
        $string = str_replace('\\', '', $string);
    } else if ($GLOBALS['databasetype'] == 'MYSQL'
        || $GLOBALS['databasetype'] == 'POSTGRESQL'
    ) {
        $string = addslashes($string);
        $string = str_replace('\\', '#', $string);
    }
    return $string;
}

/**
* Update the actual size of the DocServer
*
* @param $docserverId String Docserver identifier
* @param $theSize Long Actual size of the docserver
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
* @param $dbconn2 Dbquery object
*/
function docserverSizeUpdate($docserverId, $theSize, $logger, $dbConn, $dbConn2,
$options)
{
    if ( $options['letterbox']) {
        $queryTxt = "Select actual_size from docservers where docserver_id='"
                  . $docserverId . "'";
    } else {
        $queryTxt = "Select actual_size_number from docservers where"
                  . " docserver_id='" . $docserverId . "'";
    }
    doQuery($dbConn, $queryTxt, $logger);
    while ($result = $dbConn->fetch_array()) {
        $actualSize = $result[0];
        $size = $actualSize + $theSize;
        if ( $options['letterbox']) {
            $req = "update docservers SET actual_size = '" . $size
                  . "' where docserver_id = '" . $docserverId . "'";
        } else {
            $req = "update docservers SET actual_size_number = '" . $size
                  . "' where docserver_id = '" . $docserverId . "'";
        }
        doQuery($dbConn2, $req, $logger);
    }
}

/**
* Manage script logs, create lock file in case of serious error, store log in
* database if needed or put file in failed directory
*
* @param $logTxt String Log message
* @param $logLevel String Log level (ERROR, INFO, DEBUG, ...)
* @param $logger Logger Object Use to log data if necessary
* @param $errorNumber If not ERROR or WARNING, error number = 0
* @param $dbconn Dbquery object (null by default)
* @param $createLockFile Bool If true, allow to create the lock file
* @param $filePath String File path (empty by default)
*/
function manageLog($logTxt, $logLevel, $logger, $errorNumber=0,
$dbConn=null, $createLockFile=true, $filePath='', $logDbError=true)
{
    if ($errorNumber > 0) { // level = Warning or error
        $logger->write($logTxt, $logLevel, $errorNumber);
        if ($logLevel == 'ERROR') {
            if ($createLockFile) {
                createLockFile($logger);
            }
            if ((! isset($GLOBALS['noDbLog']) || ! $GLOBALS['noDbLog'] )
                && isset($dbConn) && $logDbError
            ) {
                logInDataBase(0, 1, $logTxt, $logger, $dbConn);
            }
            exit($errorNumber);
        } else if ($logLevel == 'WARNING') {
            $GLOBALS['nbErrors'] ++;
            $GLOBALS['infoErrors'] .= $logTxt;
            if ($filePath <> '') {
                putFileInError($filePath, $logger, $dbConn);
            }
        }
    } else {
        $logger->write($logTxt, $logLevel);
    }
}

/**
* Create a lock file
*
* @param $logger Logger Object Use to log data if necessary
*
* @return Bool True if file created, false otherwise
*/
function createLockFile($logger)
{
    $GLOBALS['lockFile'] = $GLOBALS['autoImportDirectory'] .
        $GLOBALS['configName'] . '.lck';
    if (file_exists($GLOBALS['lockFile'])) {
        unlink($GLOBALS['lockFile']);
    }
    $lockFileOpened = @fopen($GLOBALS['lockFile'], 'a');
    if ($lockFileOpened) {
        fwrite($lockFileOpened, '1');
        fclose($lockFileOpened);
        return true;
    }
    return false;
}

/**
* Insert in the database the report of AI
* @param $totalProcessed Integer
* @param $totalErrors Integer
* @param $info String
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
*/
function logInDataBase($totalProcessed=0, $totalErrors=0, $info='',
$logger, $dbConn)
{
    if ($GLOBALS['databasetype'] == 'SQLSERVER') {
        $dateNow = 'getdate()';
    } else if ($GLOBALS['databasetype'] == 'MYSQL'
        || $GLOBALS['databasetype'] == 'POSTGRESQL'
    ) {
        $dateNow = 'now()';
    } else if ($GLOBALS['databasetype'] == 'ORACLE') {
        $dateNow = "SYSDATE";
    }
    $req = 'insert into history_batch '
         . '(module_name, batch_id, event_date, total_processed, total_errors,'
         . " info) values('AutoImport', " . $GLOBALS['wbai'] . ", " . $dateNow
         . ", " . $totalProcessed . ", " . $totalErrors . ", '"
         . protectStringDb(substr($info, 0, 999)) . "')";
    doQuery($dbConn, $req, $logger, false);
}

/**
* Move a file in the failed directory
*
* @param  $path string File path of the file to move
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
*/
function putFileInError($path, $logger, $dbConn)
{
    $filename = extractFileName($path);
    $directory = $GLOBALS['scanImportDirectory'];
    $subdirectories = array('failed', $GLOBALS['wbai']);
    $result = buildDirectories($directory, $subdirectories);
    if (!empty($result['error'])) {
        manageLog($result['error'], 'ERROR', $logger, 21, $dbConn);
    }
    $failedDir = $directory . $result['path'];
    $newPath = $failedDir . $filename;
    if (! @rename($path, $newPath)) {
        manageLog(
            'Unable to move file ' . $path .' in failed directory ' . $directory
            . $result['path'], 'ERROR', $logger, 20, $dbConn
        );
    }
    # TODO : test xmlfile only if !withoutxml
    $xmlfile = getXmlFile($directory, $filename);
    if (! empty($xmlfile)) {
        if (! @rename(
            $directory . $xmlfile, $failedDir . $xmlfile
        )
        ) {
            manageLog(
                'Unable to move file ' . $directory . $xmlfile .' in failed '
                . 'directory ' . $directory . $result['path'], 'ERROR', $logger,
                20, $dbConn
            );
        }
    }
}

/**
* Delete imported files in the import directory and backup them if option set
*
* @param  $directory String Imlport Directory
* @param $options Array  (
*       'insertMode'           => Insert mode (true/false),
*       'withoutXml'           => Use of xml or not (true/false),
*       'noDbLog'              => Logs won't be in the database (true/false),
*       'excludeExistingDocs'  => Make the checks if files are already in db,
*       'backupBatch'          => Backup all imported files in a backup dir,
*       'controlCompleteFiles' => Check if files in import directory are
*       						  complete
*       )
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
*/
function moveImportedFiles($directory, $files, $options, $logger, $dbConn)
{
    if ($options['backupBatch'] == true) {
        $subdirectories = array('backup', date('Ymd'), $GLOBALS['wbai']);
        $result = buildDirectories($directory, $subdirectories);
        if (!empty($result['error'])) {
            manageLog($result['error'], 'ERROR', $logger, 21, $dbConn);
        }
        $backupDir = $directory . $result['path'];

        //Create the subfolders if subfolders exists in the scan folder
        if (!empty($GLOBALS['folderLoaded'])) {
            for ($fo = 0; $fo < count($GLOBALS['folderLoaded']); $fo++) {
                if (!is_dir(
                    $backupDir . $GLOBALS['folderLoaded'][$fo]
                    . DIRECTORY_SEPARATOR
                )
                ) {
                    mkdir(
                        $backupDir . $GLOBALS['folderLoaded'][$fo]
                        . DIRECTORY_SEPARATOR, 0777
                    );
                }
            }
        }
    }
    for ($i = 0; $i < count($files); $i++) {
        if ($options['backupBatch'] == true) {
            if (! @rename($directory . $files[$i], $backupDir . $files[$i])) {
                manageLog(
                    'Can not move ' . $directory . $files[$i]
                    . ' to failed directory', 'ERROR', $logger, 20, $dbConn
                );
            }
        } else {
            unlink($directory . $files[$i]);
        }
        if (! $options['withoutXml']) {
            $xmlfile = getXmlFile($directory, $files[$i]);
            if (! empty($xmlfile)) {
                if ($options['backupBatch'] == true) {
                    if (! @rename(
                        $directory . $xmlfile, $backupDir . $xmlfile
                    )
                    ) {
                        manageLog(
                            'Move imported files in backup failed', 'ERROR',
                            $logger, 20, $dbConn
                        );
                    }
                } else {
                    unlink($directory . $xmlfile);
                }
            }
        }
    }
    //To delete folder in scan folder
    for ($vo = 0; $vo < count($GLOBALS['folderLoaded']); $vo++) {
        $filesToMove = 0;
        $classMoveScan = dir(
            $directory . DIRECTORY_SEPARATOR . $GLOBALS['folderLoaded'][$vo]
            . DIRECTORY_SEPARATOR
        );
        while (($fileScanmove = $classMoveScan->read()) != false) {
            if ($fileScanmove == '.' || $fileScanmove == '..') {
                continue;
            } else {
                $filesToMove++;
            }
        }
        if ($filesToMove == 0) {
            rmdir($directory . $GLOBALS['folderLoaded'][$vo]);
        }
    }
}

/**
* Convert datetime in a specific format
*
* @param  $value String Date to convert
* @param  $format String Convert format
* @param  $valueDescArr Array or string
*
* @return String converted date or empty string
*/
function dateTimeConvert($value, $format, $valueDescArr)
{
    if ($value <> '' && $valueDescArr <> 'skip') {
        $year = '';
        $month = '';
        $day = '';
        $hour = '00';
        $minutes = '00';
        $seconds = '00';
        $year2 = '';
        $year4 = '';
        $tmp = (string) $value;
        for ($i = $valueDescArr['year']['startPos'];
             $i < $valueDescArr['year']['endPos'] + 1;
             $i ++
        ) {
            $year .= $tmp[$i];
        }
        // Microsoft excel 2001 rule : year with 2 digits > 30 = XX century
        // and year < 30 = XXI century
        if (strlen($year) == 2) {
            $year2 = $year;
            if ((integer) $year > 30) {
                $year4 = '19' . $year;
            } else {
                $year4 = '20' . $year;
            }
        } else {
            $year4 = $year;
            $year2 = $year[2] . $year[3];
        }
        $month = $tmp[$valueDescArr['month']['startPos']]
               . $tmp[$valueDescArr['month']['endPos']];
        $day = $tmp[$valueDescArr['day']['startPos']]
             . $tmp[$valueDescArr['day']['endPos']];
        if (isset($valueDescArr['hour']['startPos'])
            && isset($valueDescArr['hour']['endPos'])
        ) {
            $hour = $tmp[$valueDescArr['hour']['startPos']]
                  . $tmp[$valueDescArr['hour']['endPos']];
        }
        if (isset($valueDescArr['minutes']['startPos'])
            && isset($valueDescArr['minutes']['endPos'])
        ) {
            $minutes = $tmp[$valueDescArr['minutes']['startPos']]
                     . $tmp[$valueDescArr['minutes']['endPos']];
        }
        if (isset($valueDescArr['seconds']['startPos'])
            && isset($valueDescArr['seconds']['endPos'])
        ) {
            $seconds = $tmp[$valueDescArr['seconds']['startPos']]
                     . $tmp[$valueDescArr['seconds']['endPos']];
        }
        $res = (string) $format;
        if (preg_match('/YYYY/', $res)) {
            $res = preg_replace('/YYYY/', $year4, $res);
        } else {
            $res = preg_replace('/YY/', $year2, $res);
        }
        $res = preg_replace('/MM/', $month, $res);
        $res = preg_replace('/DD/', $day, $res);
        $res = preg_replace('/HH/', $hour, $res);
        $res = preg_replace('/II/', $minutes, $res);
        $res = preg_replace('/SS/', $seconds, $res);
        return $res;
    } else if ($valueDescArr == 'skip') {
        return $value;
    } else {
        return '';
    }
}

/**
* Slice a date format in an array
*
* @param  $dateFormat String Date format
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
*
* @return array (
*                   'dateFormat' => original date format
*                   'year' => array(
*                        'startPos' => Position of the first number of the year
*                        'endPos'   => Position of the last number of the year
*                               )
*                   'month' => array(
*                        'startPos' => Position of the first number of the month
*                        'endPos'   => Position of the last number of the month
*                               )
*                   'day' => array(
*                        'startPos' => Position of the first number of the day
*                        'endPos'   => Position of the last number of the day
*                               )
*                   'hour' => array(
*                        'startPos' => Position of the first number of the hour
*                        'endPos'   => Position of the last number of the hour
*                               )
*                   'minutes' => array(
*                      'startPos' => Position of the first number of the minutes
*                      'endPos'   => Position of the last number of the minutes
*                               )
*                   'seconds' => array(
*                      'startPos' => Position of the first number of the seconds
*                      'endPos'   => Position of the last number of the seconds
*                               )
*               )
*/
function sliceDateFormat($dateFormat, $logger, $dbConn)
{
    $arr = array();
    if (empty($dateFormat) || !isset($dateFormat)) {
        manageLog(
            'Empty DATE_FORMAT tag in the mapping file', 'ERROR', $logger, 18,
            $dbConn
        );
    }
    $arr['dateFormat'] = $dateFormat;
    //Year check
    $arr['year']['startPos'] = strpos($dateFormat, 'Y');
    $arr['year']['endPos'] = strrpos($dateFormat, 'Y');
    if (($arr['year']['startPos'] == false && $arr['year']['startPos'] <> 0)
        || $arr['year']['endPos'] == false
    ) {
        manageLog(
            'Error of the DATE_FORMAT tag in the mapping file (YEAR)',
            'ERROR', $logger, 18, $dbConn
        );
    }
    //Month check
    $arr['month']['startPos'] = strpos($dateFormat, 'M');
    $arr['month']['endPos'] = strrpos($dateFormat, 'M');
    if (($arr['month']['startPos'] == false && $arr['month']['startPos'] <> 0)
        || $arr['month']['endPos'] == false
    ) {
        manageLog(
            'Error of the DATE_FORMAT tag in the mapping file (MONTH)',
            'ERROR', $logger, 18, $dbConn
        );
    }
    //Day check
    $arr['day']['startPos'] = strpos($dateFormat, 'D');
    $arr['day']['endPos'] = strrpos($dateFormat, 'D');
    if (($arr['day']['startPos'] == false && $arr['day']['startPos'] <> 0)
        || $arr['day']['endPos'] == false
    ) {
        manageLog(
            'Error of the DATE_FORMAT tag in the mapping file (DAY)',
            'ERROR', $logger, 18, $dbConn
        );
    }
    // Hours check
    if (preg_match('/H/', $dateFormat)) {
        $arr['hour']['startPos'] = strpos($dateFormat, 'H');
        $arr['hour']['endPos'] = strrpos($dateFormat, 'H');
        if (($arr['hour']['startPos'] == false && $arr['hour']['startPos'] <> 0)
            || $arr['hour']['endPos'] == false
        ) {
            manageLog(
                'Error of the DATE_FORMAT tag in the mapping file (HOUR)',
                'ERROR', $logger, 18, $dbConn
            );
        }
    }
    // Minutes check
    if (preg_match('/I/', $dateFormat)) {
        $arr['minutes']['startPos'] = strpos($dateFormat, 'I');
        $arr['minutes']['endPos'] = strrpos($dateFormat, 'I');
        if (( $arr['minutes']['startPos'] == false
            && $arr['minutes']['startPos'] <> 0)
                || $arr['minutes']['endPos'] == false
        ) {
            manageLog(
                'Error of the DATE_FORMAT tag in the mapping file (MINUTES)',
                'ERROR', $logger, 18, $dbConn
            );
        }
    }
    // Seconds check
    if (preg_match('/S/', $dateFormat)) {
        $arr['seconds']['startPos'] = strpos($dateFormat, 'S');
        $arr['seconds']['endPos'] = strrpos($dateFormat, 'S');
        if (($arr['seconds']['startPos'] == false
            && $arr['seconds']['startPos'] <> 0)
                || $arr['seconds']['endPos'] == false
        ) {
            manageLog(
                'Error of the DATE_FORMAT tag in the mapping file (SECONDS)',
                'ERROR', $logger, 18, $dbConn
            );
        }
    }
    return $arr;
}

/**
* Parse the mapping file and return the expected values in an array
*
* @param $mappingFile String Path to to mapping file
* @param $logger Logger Object Use to log data if necessary
* @param $dbconn Dbquery object
*/
function parseMappingFile($mappingFile, $logger, $db)
{
    //Loading the mapping xml file
    manageLog('Load mapping file : ' .$mappingFile, 'INFO', $logger);
    $xmlmapping = simplexml_load_file($mappingFile);
    $mappedFields = array();
    $knownvalueFields = array();
    $xml = $xmlmapping->mappingFile;
    $allowedFormats = array('string', 'integer', 'date', 'float');
    foreach ($xml->element as $elem) {
        $value = (string) $elem->value;
        $column = (string) $elem->column;

        // format tag is mandatory
        if (!isset($elem->format)
            || ! in_array(strtolower($elem->format), $allowedFormats)
        ) {
            manageLog(
                'Mapping File : Format tag is missing or unknwon for column '
                . $column, 'ERROR', $logger, 25, $db
            );
        }
        $format = (string) $elem->format;
        // If format is date then dateFormat is mandatory
        if (strtolower($format) == 'date') {
           if ((! isset($elem->dateFormat) || empty($elem->dateFormat))) {
                manageLog(
                    'Mapping file : dateFormat tag is missing for column '
                    . $column, 'ERROR', $logger, 14, $db
                );
            }
            $res = checkFormatDate($elem->dateFormat);
            if ($res['status'] == false) {
                manageLog(
                    $res['error'] . ':'. $column, 'ERROR', $logger, 18, $db
                );
            }
        }

        // Value is between '' then value is already set
        if (preg_match("/^'.*'$/", $value)) {

            $tmp = array(
                'targetColumn' => $column,
                'value'        => substr($value, 1, strlen($value) - 2),
                'format'       => $format,
            );
            if (strtolower($elem->format) == 'date') {
                $tmp['dateFormat'] =
                    (string) $elem->dateFormat;
            }
            $knownvalueFields [] = $tmp;
        } else { // Value is a field in xml files

            $mappedFields[$value] = array(
                'targetColumn' => $column,
                'format'      => (string) $elem->format,
            );

            if (strtolower($elem->format) == 'date') {
                $mappedFields[$value]['dateFormat'] =
                    (string) $elem->dateFormat;
            }

    		// #TODO : faire une vérification sur la valeur par défaut sur le format
            if (isset($elem->defaultValue) && !empty($elem->defaultValue)) {
                $mappedFields[$value]['defaultValue'] =
                    (string) $elem->defaultValue;
            }
        }
    }
    return array($knownvalueFields, $mappedFields);
}


/**
* Get the xml file for a given file (if exists)
*
* @param $sourceDir String Import directory absolute path
* @param $filename String File name
*
* @return String Xml path or empty string
*/
function getXmlFile($sourceDir, $filename)
{
    $fileExt = extractFileExt($filename);
    $lXml = preg_replace('/' . $fileExt . '$/', 'xml', $filename);
    $uXml = preg_replace('/' . $fileExt . '$/', 'XML', $filename);
    if (file_exists($sourceDir . $lXml)) {
        return $lXml;
    }
    if (file_exists($sourceDir .  $uXml)) {
        return $uXml;
    }
    return '';
}

/**
 * Main function
 */
function main()
{
    // Define allowed arguments in the script
    $argsparser = new ArgsParser();
    $argsparser->add_arg(
        'config', array(
            'short'     => 'c',
            'long'      => 'config',
            'mandatory' => true,
            'help'      => 'Config file path is mandatory.',
        )
    );
    $argsparser->add_arg(
        'mapping', array(
            'short'     => 'm',
            'long'      => 'mapping',
            'mandatory' => true,
            'help'      => 'Mapping file path is mandatory.',
        )
    );
    $argsparser->add_switch(
        'insertMode', array(
            'short' => 'i',
            'long'  => 'insertMode',
            'help'  => 'Instead of a mass load in database, use of insert '
                       . 'queries. This mode is slower than normal mode.',
        )
    );
    $argsparser->add_switch(
        'excludeExistingDocs', array(
            'short' => 'e',
            'long'  => 'excludeExistingDocs',
            'help'  => 'Make a check on the fingerprint to exclude already '
                       . 'existing docs of the database load. This option slows'
                       . ' the processus.',
        )
    );
    $argsparser->add_switch(
        'fileComplete', array(
            'short' => 'f',
            'long'  => 'fileComplete',
            'help'  => 'Check if the file is complete.',
        )
    );
    $argsparser->add_switch(
        'noBackup', array(
            'short' => 'n',
            'long'  => 'noBackup',
            'help'  => 'Disabled the automatic backup of loaded files.',
        )
    );
    $argsparser->add_switch(
        'withoutXml', array(
            'short' => 'w',
            'long'  => 'withoutXml',
            'help'  => 'Do not use xml files to get indexes but file names.',
        )
    );
    $argsparser->add_arg(
        'importDirectory', array(
            'short'     => 'd',
            'long'      => 'importDirectory',
            'mandatory' => false,
            'default'   => 'incoming',
            'help'      => 'Directory where the batch find the documents to '
    					   . 'load',
        )
    );
    $argsparser->add_arg(
        'dirlog', array(
            'short'     => 'l',
            'long'      => 'dirlog',
            'mandatory' => false,
            'default'   => '.',
            'help'      => 'Directory of the log file, logs directory if not '
                           . 'specified',
        )
    );
    $argsparser->add_arg(
        'sqldir', array(
            'short'     => 's',
            'long'      => 'sqldir',
            'mandatory' => false,
            'default'   => '.',
            'help'      => 'Directory of the sql file, current directory if not'
                           . ' specified',
        )
    );
    $argsparser->add_switch(
        'noDbLog', array(
            'short' => 'b',
            'long'  => 'noDbLog',
            'help'  => 'Disable the autoimport log in database.',
        )
    );
    $argsparser->add_switch(
        'letterbox', array(
            'short' => 't',
            'long'  => 'letterbox',
            'help'  => 'Activate the letterbox mode',
        )
    );
    // Log management init
    $logger = new Logger();
    $logger->set_threshold_level('DEBUG');
    $console = new ConsoleHandler();
    $logger->add_handler($console);

    // Parse script options
    try {
        $options = $argsparser->parse_args($GLOBALS['argv']);
        // If option = help then options = false and the script continues ...
        if ($options == false) {
            exit(0);
        }
    }
    catch (MissingArgumentError $e) {
        if ($e->arg_name == 'config') {
            manageLog('Configuration file missing', 'ERROR', $logger, 1);
        } elseif ($e->arg_name == 'mapping') {
            manageLog('Mapping file missing', 'ERROR', $logger, 2);
        } else {
            manageLog(
                'Arguments Error, try -h to view help', 'ERROR', $logger, 3
            );
        }
    }

    // If no dirlog set, exit
    if (!isset($options['dirlog'])) {
        manageLog('Log directory is mandatory', 'ERROR', $logger, 4);
    }
     // If dirlog does not exist or script has no sufficient rights, exit
    if (! is_dir($options['dirlog']) || ! is_writable($options['dirlog'])) {
        manageLog(
            'Log directory (' . $options['dirlog'] . ') does not exists or '
            . 'is not writable', 'ERROR', $logger, 5
        );
    }

    // If no sqldir set, exit
    if (!isset($options['sqldir'])) {
        manageLog('Sql directory is mandatory', 'ERROR', $logger, 26);
    }
     // If sqldir does not exist or script has no sufficient rights, exit
    if (! is_dir($options['sqldir']) || ! is_writable($options['sqldir'])) {
        manageLog(
            'Sql directory (' . $options['sqldir'] . ') does not exists or '
            . 'is not writable', 'ERROR', $logger, 27
        );
    }
    // If no import directory set, exit
    if (! isset($options['importDirectory'])) {
        manageLog('Import directory is mandatory', 'ERROR', $logger, 6);
    }

    $GLOBALS['scanImportDirectory'] = $options['importDirectory'];
    if (! preg_match(
        '#'.DIRECTORY_SEPARATOR.'$#', $GLOBALS['scanImportDirectory']
    )
    ) {
        $GLOBALS['scanImportDirectory'] .= DIRECTORY_SEPARATOR;
    }

    // If import directory does not exist or script has no sufficient rights,
    // exit
    if (! is_dir($GLOBALS['scanImportDirectory'])
        || ! is_readable($GLOBALS['scanImportDirectory'])
    ) {
        manageLog(
            'Import directory (' . $GLOBALS['scanImportDirectory'] . ') does '
            . 'not exists or is not readable', 'ERROR', $logger, 7
        );
    }
    // Adding log file to the logger
    $file = new FileHandler(
        $options['dirlog'] . DIRECTORY_SEPARATOR . 'log.txt'
    );
    $logger->add_handler($file);

    $txt = 'Launching Autoimport : '.$_SERVER['SCRIPT_FILENAME'];
    manageLog($txt, 'INFO', $logger);

    $txt = '';
    foreach (array_keys($options) as $key) {
        if (isset($options[$key]) && $options[$key] == false) {
            $txt .= $key . '=false,';
        } else {
            $txt .= $key . '=' . $options[$key] . ',';
        }
    }
    manageLog($txt, 'DEBUG', $logger);

    // Define script Options
    $scriptOptions = array(
        'insertMode'           => $options['insertMode'],
        'withoutXml'           => $options['withoutXml'],
        'noDbLog'              => $options['noDbLog'],
        'excludeExistingDocs'  => $options['excludeExistingDocs'],
        'backupBatch'          => ! $options['noBackup'],
        'controlCompleteFiles' => $options['fileComplete'],
        'letterbox'            => $options['letterbox'],
    );

    // #TODO : rendre non global noDbLog et scanImportDirectory
    $GLOBALS['noDbLog'] = $options['noDbLog'];

    $configFile = $options['config'];
    // Test existence of config file
    if (!file_exists($configFile)) {
        manageLog(
            'Configuration file ' . $configFile . ' does not exist',
            'ERROR', $logger, 8
        );
    }
    // Test existence of mapping file
    if (!file_exists($options['mapping'])) {
        manageLog(
            'Mapping file ' . $options['mapping'] . ' does not exist', 'ERROR',
            $logger, 9
        );
    }
    // Load config file
    manageLog(
        'Load xml config file : ' . $configFile, 'INFO', $logger
    );
    $xmlconfig = simplexml_load_file($configFile);
    if ($xmlconfig == false) {
        manageLog(
            'Error on loading config file : ' . $configFile,
            'ERROR', $logger, 10
        );
    }
    $config = $xmlconfig->config;

    $GLOBALS['configName'] = $configName = (string) $config -> configName;
    $GLOBALS['autoImportDirectory']  = $autoImportDirectory
                                     = (string) $config -> autoimportDirectory;
    $GLOBALS['tableName'] = (string) $config -> tableName;
    $docserverId = (string) $config -> docserverId;
    $GLOBALS['dateTimeFormat'] = $config -> dateTimeFormat;
    $logLevel = (string) $config->logLevel;
    $displayedLogLevel = (string) $config->displayedLogLevel;
    if (isset($config->csvSeparator) && !empty($config->csvSeparator)) {
        $GLOBALS['csvSeparator'] = (string) $config->csvSeparator;
    }
    $logger->change_handler_log_level($file, $logLevel);
    $logger->change_handler_log_level($console, $displayedLogLevel);

    $dbData = array(
        'server'         => (string) $config->databaseserver,
        'port'           => (string) $config->databaseserverport,
        'databasetype'   => (string) $config->databasetype,
        'base'           => (string) $config->databasename,
        'user'           => (string) $config->databaseuser,
        'pass'           => (string) $config->databasepassword,
        'workspace'      => (string) $config->databaseworkspace,
    );
    // #TODO databasetype ne doit plus etre globale
    $GLOBALS['databasetype'] = $dbData['databasetype'];

    $db = new dbquery($dbData);
    $tmp = $db->getErrorMsg();
    if (isset($tmp) && !empty($tmp)) {
        manageLog($tmp, 'ERROR', $logger, 24);
    }

    if (file_exists($autoImportDirectory . $GLOBALS['configName'] . '.lck')) {
        manageLog(
            'Lock file is present, please correct problems and delete lock file'
            . ' before starting autoimport again', 'ERROR', $logger, 31, $db,
            false
        );
    }

    $db2 = new dbquery($dbData);
    $tmp = $db2->getErrorMsg();
    if (isset($tmp) && !empty($tmp)) {
        manageLog($tmp, 'ERROR', $logger, 24);
    }
    // Allowed extensions
    $allowedExt = array();
    $extensions = $xmlconfig->extensions;
    foreach ($extensions->ext as $ext) {
        array_push($allowedExt, strtolower((string) $ext));
    }
    // Files or directory excluded
    $excludeFiles = array('.', '..', 'backup', 'failed', '.svn');
    if (isset($scriptOptions['excludeExistingDocs'])
        && $scriptOptions['excludeExistingDocs']
    ) {
        array_push($excludeFiles, 'duplicate');
    }

    // Checks on the docserver
    if (! isDocserverAvailable($docserverId, $db, $logger)) {
        manageLog(
            'Docserver ' . $docserverId .' is not enabled or is readonly',
            'ERROR', $logger, 22, $db
        );
    }
    // Get Docserver data
    $docserverData = getDocserverData($docserverId, $logger, $db, $scriptOptions);
    manageLog(
        'Docserver identifier : ' . $docserverId, 'INFO', $logger
    );
    manageLog('Docserver path : ' . $docserverData['path'], 'INFO', $logger);

    // Check if docserver path exists
    if (!is_dir($docserverData['path'])) {
        manageLog(
            'Docserver path does not exists', 'ERROR', $logger, 12,
            $db
        );
    }
    manageLog('Database type : ' . $GLOBALS['databasetype'], 'INFO', $logger);
    if ($scriptOptions['insertMode'] == true) {
        manageLog('Insert SQL mode', 'INFO', $logger);
    } else {
        manageLog('Mass load mode (csv) ', 'INFO', $logger);
    }
    if (DIRECTORY_SEPARATOR == '/') {
        $GLOBALS['osName'] = 'UNIX';
    } else {
        $GLOBALS['osName'] = 'WINDOWS';
    }
    manageLog('Target OS : ' . $GLOBALS['osName'], 'INFO', $logger);
    // Create the lockfile
    createLockFile($logger);
    manageLog('Lock File created', 'DEBUG', $logger);

    manageLog(
        'Scan import directory ' . $GLOBALS['scanImportDirectory'], 'INFO',
        $logger
    );
    // Look if there are files to process
    if (containFiles($GLOBALS['scanImportDirectory'], $excludeFiles)) {
        // File complete check
        if ($scriptOptions['controlCompleteFiles'] == true
            && !isCompleteBatch(
                $GLOBALS['scanImportDirectory'], $logger, $excludeFiles
            )
        ) {
            manageLog('Batch not complete', 'ERROR', $logger, $db, 11);
        }
        manageLog('All files to import are completed', 'DEBUG', $logger);

        // Calculate the work batch autoimport number
        $wbai = getWorkBatchAutoImport($logger, $db);
        $GLOBALS['wbai'] = $wbai;
        $wbai = str_pad($GLOBALS['wbai'], 8, '0', STR_PAD_LEFT);
        manageLog(
            'AutoImport WorkBatch is : ' . $GLOBALS['wbai'], 'INFO', $logger
        );

        // Updating the work batch autoimport number
        updateWorkBatchAutoImport($wbai, $db, $logger);
        manageLog(
            'Parameters table updated with new workbatch', 'DEBUG', $logger
        );

        // Parsing mapping file
        manageLog('Begin mapping file parsing', 'INFO', $logger);
        $tmp = parseMappingFile($options['mapping'], $logger, $db);
        $knownvalueFields = $tmp[0];
        $mappedFields = $tmp[1];
        manageLog('Mapping file parsing done', 'INFO', $logger);

        // Create the subdirectories in docserver
        $subDocServer = createPathOnDocServer(
            $docserverData['path'], $logger, $db
        );
        if ($subDocServer == false) {
            // managed in manageLog => exit the script with error
        }
        manageLog(
            'Current target docserver location  : '
            . $subDocServer, 'INFO', $logger
        );
        manageLog(
            'Scanning import folder : ' . $GLOBALS['scanImportDirectory'],
            'INFO', $logger
        );
        $docserverData['subdocserver'] = $subDocServer;
        manageLog(
            'Begin parsing scanImportDirectory to find files to process',
            'DEBUG', $logger
        );
        // Get allowed files to import
        $files = getFiles(
            $GLOBALS['scanImportDirectory'], $docserverData, $logger,
            $excludeFiles, $scriptOptions, $db, $allowedExt
        );
        if (count($files) == 0) {
            manageLog('No file to process', 'INFO', $logger);
        } else {

            manageLog(
                'All found files are now in the $files array', 'DEBUG', $logger
            );

            $res = processFiles(
                $GLOBALS['scanImportDirectory'], $files, $knownvalueFields,
                $mappedFields, $docserverData, $logger, $db, $scriptOptions,
                $options['sqldir']
            );
            if ($res['status'] == true) {
                docserverSizeUpdate(
                    $docserverId, $res['transfertSize'], $logger, $db, $db2,
                    $scriptOptions
                );
                if (! $scriptOptions['insertMode']) {
                    $fields = implode(',', $res['sqlFields']);
                    $sqlfile = preg_replace(
                        '#^.'.DIRECTORY_SEPARATOR.'#', getcwd()
                        . DIRECTORY_SEPARATOR, $res['sqlFile']
                    );
                    if ($GLOBALS['databasetype'] == 'MYSQL') {
                        $queryTxt = "LOAD DATA LOCAL INFILE '" . $sqlfile
                                  . "' INTO TABLE " . $GLOBALS['tableName']
                                  . " FIELDS TERMINATED BY '"
                                  . $GLOBALS['csvSeparator']
                                  . "' ENCLOSED BY '\"'  LINES TERMINATED BY "
                                  . "'\\n' (" . $fields . ");";
                    } else if ($GLOBALS['databasetype'] == 'POSTGRESQL') {
                        $queryTxt = "COPY " . $GLOBALS['tableName'] . " ("
                                  . $fields. ") FROM '" . $sqlfile
                                  . "' WITH DELIMITER AS '"
                                  . $GLOBALS['csvSeparator']. "' CSV ;";
                    } else if ($GLOBALS['databasetype'] == 'SQLSERVER') {
                        $queryTxt = "bulk insert " . $GLOBALS['database']
                                  . ".dbo." . $GLOBALS['tableName'] . " from '"
                                  . $sqlfile . "' with( DATAFILETYPE = "
                                  ."'widechar', formatfile = '"
                                  . $GLOBALS['autoImportDirectory']
                                  . "maarch_autoimport" . DIRECTORY_SEPARATOR
                                  . $GLOBALS['tableName'] .".xml');";
                    } else if ($GLOBALS['databasetype'] == 'ORACLE') {
                        $queryTxt = "LOAD DATA INFILE '" . $GLOBALS['database']
                                  . "' APPEND INTO TABLE "
                                  . $GLOBALS['tableName'] . " FIELDS TERMINATED"
                                  . " BY '" . $GLOBALS['csvSeparator']. "' ("
                                  . $fields . ")";
                        $controlFile = $GLOBALS['configName'] . '_control.txt';
                        $inF = fopen($controlFile, 'w');
                        fwrite($inF, $queryTxt);
                        fclose($inF);
                        if (isset($GLOBALS['osName'])
                            && $GLOBALS['osName'] == 'UNIX'
                        ) {
                            $oracleQueryResult = exec(
                                'sqlldr userid=' . $GLOBALS['databaseuser']
                                . '/' . $GLOBALS['databasepwd'] .' control='
                                . $GLOBALS['configName'] .'_control.txt '
                                . 'log=log/' . $GLOBALS['configName']
                                . '_log_oracle.txt parallel=true  bad=log/'
                                . $GLOBALS['configName'] . '_bad_oracle.txt '
                                . 'discard=log/' . $GLOBALS['configName']
                                . '_discard_oracle.txt errors=0', $returnTab,
                                $oracleReturn
                            );
                        } else {
                            $oracleQueryResult = exec(
                                'sqlldr ' . $GLOBALS['databaseuser'] . '/'
                                . $GLOBALS['databasepwd'] . '@'
                                . $GLOBALS['database'] . ' control='
                                . $GLOBALS['configName'] . '_control.txt '
                                . 'log=log/' . $GLOBALS['configName']
                                . '_log_oracle.txt parallel=true  bad=log/'
                                . $GLOBALS['configName'] . '_bad_oracle.txt '
                                . 'discard=log/' . $GLOBALS['configName']
                                . '_discard_oracle.txt errors=0', $returnTab,
                                $oracleReturn
                            );
                        }
                        //print_r($oracleReturn);
                    } else {
                        manageLog(
                            'Database type unknown', 'ERROR', $logger, 30, $db
                        );
                    }
                    if ($GLOBALS['databasetype'] <> 'ORACLE') {
                        //Execute the SQL query.
                        doQuery($db, $queryTxt, $logger);
                    } else if ($oracleReturn <> 0) {
                        manageLog(
                            'Maarch Auto Import cannot execute query on '
                            . 'database ' . $db->error(true), 'ERROR', $logger,
                            13, $db
                        );
                    }
                }
                moveImportedFiles(
                    $GLOBALS['scanImportDirectory'], $res['files'],
                    $scriptOptions, $logger, $db
                );
            }
        }
    } else {
        manageLog('No file to process', 'INFO', $logger);
    }

    if (file_exists($GLOBALS['lockFile'])) {
        unlink($GLOBALS['lockFile']);
    }
    if (! isset($scriptOptions['noDbLog']) || ! $scriptOptions['noDbLog']) {
        logInDataBase(
            $GLOBALS['totalNbfile'], $GLOBALS['nbErrors'],
            $GLOBALS['infoErrors'], $logger, $db
        );
    }
    manageLog('End autoimport (everything went well)', 'INFO', $logger);
    return true;
}