From d8d8bb2f9a5a3a755c8d8efddcb38cc3f60ad59d Mon Sep 17 00:00:00 2001 From: Anne-Cath Date: Mon, 26 May 2025 19:09:20 +0200 Subject: [PATCH 1/4] =?UTF-8?q?N=C2=B08421=20-=20CSV=20Collector:=20Align?= =?UTF-8?q?=20headerless=20CSV=20file=20collector=20behavior=20with=20head?= =?UTF-8?q?er=20CSV=20file=20collector=20behavior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/csvcollector.class.inc.php | 39 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/core/csvcollector.class.inc.php b/core/csvcollector.class.inc.php index 82fdd34..4da472d 100644 --- a/core/csvcollector.class.inc.php +++ b/core/csvcollector.class.inc.php @@ -134,20 +134,22 @@ public function Prepare() return false; } - - array_multisort($aCurrentConfiguredHeaderColumns); - $this->aConfiguredHeaderColumns = array_keys($aCurrentConfiguredHeaderColumns); + $this->aConfiguredHeaderColumns = []; + if ($this->bHasHeader) { + array_multisort($aCurrentConfiguredHeaderColumns); + $this->aConfiguredHeaderColumns = array_keys($aCurrentConfiguredHeaderColumns); if ($this->bHasHeader) { - foreach ($aCurrentConfiguredHeaderColumns as $sSynchroField => $sCsvColumn) { - $this->aMappingCsvColumnNameToFields[$sCsvColumn][] = $sSynchroField; - $this->aMappedFields[$sSynchroField] = ''; - } - } + foreach ($aCurrentConfiguredHeaderColumns as $sSynchroField => $sCsvColumn) { + $this->aMappingCsvColumnNameToFields[$sCsvColumn][] = $sSynchroField; + $this->aMappedFields[$sSynchroField] = ''; + } + } else { + $this->aConfiguredHeaderColumns = $aCurrentConfiguredHeaderColumns; } - - } - } + } + } + } if ($sCsvFilePath === '') { // No query at all !! @@ -166,7 +168,7 @@ public function Prepare() Utils::Log(LOG_DEBUG, "[".get_class($this)."] Default values [".var_export($this->aSynchroFieldsToDefaultValues, true)."]"); if (!empty($this->sCsvCliCommand)) { - Utils::Exec($this->sCsvCliCommand); + utils::Exec($this->sCsvCliCommand); } try { @@ -314,13 +316,14 @@ protected function Configure($aCsvHeaderColumns) } } } else { - - foreach ($this->aConfiguredHeaderColumns as $sSynchroColumn) { - $this->aMappingCsvColumnIndexToFields[] = [$sSynchroColumn]; - $this->aMappedFields[$sSynchroColumn] = ''; + foreach ($this->aConfiguredHeaderColumns as $sSynchroField => $sCsvColumn) { + $this->aMappingCsvColumnIndexToFields[$sCsvColumn-1][] = $sSynchroField; + $this->aMappedFields[$sSynchroField] = ''; } - } - + foreach ( $this->aIgnoredCsvColumns as $sCsvColumn) { + $this->aMappingCsvColumnIndexToFields[$sCsvColumn-1] = ['ignored_attribute_'.$sCsvColumn]; + } + } foreach ($this->aIgnoredCsvColumns as $sIgnoredCsvColumn) { $this->aIgnoredSynchroFields = array_merge( $this->aIgnoredSynchroFields, ($this->bHasHeader) ? $this->aMappingCsvColumnNameToFields[$sIgnoredCsvColumn] : $this->aMappingCsvColumnIndexToFields[$sIgnoredCsvColumn - 1]); } From ee81aee64007b65afaa8afba2679cc4b9dba2d21 Mon Sep 17 00:00:00 2001 From: Anne-Cath Date: Tue, 27 May 2025 16:25:01 +0200 Subject: [PATCH 2/4] Add tests --- test/CsvCollectorTest.php | 3 +- .../expected_generated.csv | 2 ++ .../iTopPersonCsvCollector.class.inc.php | 9 +++++ .../iTopPersonCsvCollector.csv | 1 + .../params.distrib.xml | 34 +++++++++++++++++++ 5 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 test/single_csv/map_1_column_twice_without_header/expected_generated.csv create mode 100644 test/single_csv/map_1_column_twice_without_header/iTopPersonCsvCollector.class.inc.php create mode 100644 test/single_csv/map_1_column_twice_without_header/iTopPersonCsvCollector.csv create mode 100644 test/single_csv/map_1_column_twice_without_header/params.distrib.xml diff --git a/test/CsvCollectorTest.php b/test/CsvCollectorTest.php index 05405db..578830d 100644 --- a/test/CsvCollectorTest.php +++ b/test/CsvCollectorTest.php @@ -81,6 +81,7 @@ public function OrgCollectorProvider() "separator_incolumns" => ["separator_incolumns"], "mapping 1 column twice" => ["map_1_column_twice"], "mapping 1 column twice adding primary key" => ["map_1_column_twice_primary_key"], + "mapping 1 column twice without header" => ["map_1_column_twice_without_header"], "mapping 1 column 3 times" => ["map_1_column_3_times"], "mapping 2 columns twice" => ["map_2_columns_twice"], "mapping 2 columns 3 times" => ["map_2_columns_3_times"], @@ -179,7 +180,7 @@ public function ErrorFileProvider() ), "no email" => array( "no_email.csv", - "[iTopPersonCsvCollector] The field \"email\", used for reconciliation, has missing column(s) in the csv file.", + "[iTopPersonCsvCollector] The column \"email\", used for reconciliation, is missing in the csv file.", "iTopPersonCsvCollector::Collect() got an exception: Missing columns in the csv file.", ), "empty csv" => array( diff --git a/test/single_csv/map_1_column_twice_without_header/expected_generated.csv b/test/single_csv/map_1_column_twice_without_header/expected_generated.csv new file mode 100644 index 0000000..e13e504 --- /dev/null +++ b/test/single_csv/map_1_column_twice_without_header/expected_generated.csv @@ -0,0 +1,2 @@ +primary_key;first_name;name;org_id;employee_number;phone;email;function;status;mobile_phone +Isaac;Asimov;Demo;+33123456789;EMP001;EMP001;issac.asimov@function.io;writer;Active;+33601234567 diff --git a/test/single_csv/map_1_column_twice_without_header/iTopPersonCsvCollector.class.inc.php b/test/single_csv/map_1_column_twice_without_header/iTopPersonCsvCollector.class.inc.php new file mode 100644 index 0000000..b4da829 --- /dev/null +++ b/test/single_csv/map_1_column_twice_without_header/iTopPersonCsvCollector.class.inc.php @@ -0,0 +1,9 @@ + + + + 6 + + + no + collectors/iTopPersonCsvCollector.csv + + 1 + 2 + 3 + 5 + 7 + 6 + 8 + 5 + 4 + + + +33601234567 + + + + admin + + + + + + synchro_data_person_1 + production + + \ No newline at end of file From 450cef8a5448c4e2bcaffbf5f37f4779a994c3ba Mon Sep 17 00:00:00 2001 From: Anne-Cath Date: Mon, 13 Oct 2025 10:57:36 +0200 Subject: [PATCH 3/4] fix change after rebase --- test/CsvCollectorTest.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/CsvCollectorTest.php b/test/CsvCollectorTest.php index 578830d..9de8f71 100644 --- a/test/CsvCollectorTest.php +++ b/test/CsvCollectorTest.php @@ -178,11 +178,11 @@ public function ErrorFileProvider() "[iTopPersonCsvCollector] The mandatory column \"primary_key\" is missing in the csv file.", 'iTopPersonCsvCollector::Collect() got an exception: Missing columns in the csv file.', ), - "no email" => array( - "no_email.csv", - "[iTopPersonCsvCollector] The column \"email\", used for reconciliation, is missing in the csv file.", - "iTopPersonCsvCollector::Collect() got an exception: Missing columns in the csv file.", - ), + "no email" => array( + "no_email.csv", + "[iTopPersonCsvCollector] The field \"email\", used for reconciliation, has missing column(s) in the csv file.", + "iTopPersonCsvCollector::Collect() got an exception: Missing columns in the csv file.", + ), "empty csv" => array( "empty_file.csv", "[iTopPersonCsvCollector] CSV file is empty. Data collection stops here.", From 17f1474be189d52aadb5e41f587bcfb88bf50028 Mon Sep 17 00:00:00 2001 From: Anne-Cath Date: Mon, 13 Oct 2025 11:18:30 +0200 Subject: [PATCH 4/4] fix change after rebase --- core/csvcollector.class.inc.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/csvcollector.class.inc.php b/core/csvcollector.class.inc.php index 4da472d..fb05974 100644 --- a/core/csvcollector.class.inc.php +++ b/core/csvcollector.class.inc.php @@ -139,7 +139,6 @@ public function Prepare() array_multisort($aCurrentConfiguredHeaderColumns); $this->aConfiguredHeaderColumns = array_keys($aCurrentConfiguredHeaderColumns); - if ($this->bHasHeader) { foreach ($aCurrentConfiguredHeaderColumns as $sSynchroField => $sCsvColumn) { $this->aMappingCsvColumnNameToFields[$sCsvColumn][] = $sSynchroField; $this->aMappedFields[$sSynchroField] = ''; @@ -160,7 +159,7 @@ public function Prepare() } Utils::Log(LOG_INFO, "[".get_class($this)."] CSV file is [".$sCsvFilePath."]"); - Utils::Log(LOG_DEBUG, "[".get_class($this)."] Has cs header [".$this->bHasHeader."]"); + Utils::Log(LOG_DEBUG, "[".get_class($this)."] Has csv header [".($this->bHasHeader?"yes":"no")."]"); Utils::Log(LOG_DEBUG, "[".get_class($this)."] Separator used is [".$this->sCsvSeparator."]"); Utils::Log(LOG_DEBUG, "[".get_class($this)."] Encoding used is [".$this->sCsvEncoding."]"); Utils::Log(LOG_DEBUG, "[".get_class($this)."] Fields [".var_export($this->aConfiguredHeaderColumns, true)."]"); @@ -168,7 +167,7 @@ public function Prepare() Utils::Log(LOG_DEBUG, "[".get_class($this)."] Default values [".var_export($this->aSynchroFieldsToDefaultValues, true)."]"); if (!empty($this->sCsvCliCommand)) { - utils::Exec($this->sCsvCliCommand); + Utils::Exec($this->sCsvCliCommand); } try {