Skip to content

N°8421 - CSV Collector: Align headerless CSV file collector behavior … #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 13, 2025
Merged

N°8421 - CSV Collector: Align headerless CSV file collector behavior … #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d8d8bb2
N°8421 - CSV Collector: Align headerless CSV file collector behavior …
accognet May 26, 2025
ee81aee
Add tests
accognet May 27, 2025
450cef8
fix change after rebase
accognet Oct 13, 2025
17f1474
fix change after rebase
accognet Oct 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 23 additions & 21 deletions core/csvcollector.class.inc.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,20 +134,21 @@ public function Prepare()

return false;
}

array_multisort($aCurrentConfiguredHeaderColumns);
$this->aConfiguredHeaderColumns = array_keys($aCurrentConfiguredHeaderColumns);

if ($this->bHasHeader) {
foreach ($aCurrentConfiguredHeaderColumns as $sSynchroField => $sCsvColumn) {
$this->aMappingCsvColumnNameToFields[$sCsvColumn][] = $sSynchroField;
$this->aMappedFields[$sSynchroField] = '';
}
}
$this->aConfiguredHeaderColumns = [];
if ($this->bHasHeader) {
array_multisort($aCurrentConfiguredHeaderColumns);
$this->aConfiguredHeaderColumns = array_keys($aCurrentConfiguredHeaderColumns);

foreach ($aCurrentConfiguredHeaderColumns as $sSynchroField => $sCsvColumn) {
$this->aMappingCsvColumnNameToFields[$sCsvColumn][] = $sSynchroField;
$this->aMappedFields[$sSynchroField] = '';
}
} else {
$this->aConfiguredHeaderColumns = $aCurrentConfiguredHeaderColumns;
}

}
}
}
}
}

if ($sCsvFilePath === '') {
// No query at all !!
Expand All @@ -158,15 +159,15 @@ public function Prepare()
}

Utils::Log(LOG_INFO, "[".get_class($this)."] CSV file is [".$sCsvFilePath."]");
Utils::Log(LOG_DEBUG, "[".get_class($this)."] Has cs header [".$this->bHasHeader."]");
Utils::Log(LOG_DEBUG, "[".get_class($this)."] Has csv header [".($this->bHasHeader?"yes":"no")."]");
Utils::Log(LOG_DEBUG, "[".get_class($this)."] Separator used is [".$this->sCsvSeparator."]");
Utils::Log(LOG_DEBUG, "[".get_class($this)."] Encoding used is [".$this->sCsvEncoding."]");
Utils::Log(LOG_DEBUG, "[".get_class($this)."] Fields [".var_export($this->aConfiguredHeaderColumns, true)."]");
Utils::Log(LOG_DEBUG, "[".get_class($this)."] Ignored csv fields [".var_export($this->aIgnoredCsvColumns, true)."]");
Utils::Log(LOG_DEBUG, "[".get_class($this)."] Default values [".var_export($this->aSynchroFieldsToDefaultValues, true)."]");

if (!empty($this->sCsvCliCommand)) {
Utils::Exec($this->sCsvCliCommand);
Utils::Exec($this->sCsvCliCommand);
}

try {
Expand Down Expand Up @@ -314,13 +315,14 @@ protected function Configure($aCsvHeaderColumns)
}
}
} else {

foreach ($this->aConfiguredHeaderColumns as $sSynchroColumn) {
$this->aMappingCsvColumnIndexToFields[] = [$sSynchroColumn];
$this->aMappedFields[$sSynchroColumn] = '';
foreach ($this->aConfiguredHeaderColumns as $sSynchroField => $sCsvColumn) {
$this->aMappingCsvColumnIndexToFields[$sCsvColumn-1][] = $sSynchroField;
$this->aMappedFields[$sSynchroField] = '';
}
}

foreach ( $this->aIgnoredCsvColumns as $sCsvColumn) {
$this->aMappingCsvColumnIndexToFields[$sCsvColumn-1] = ['ignored_attribute_'.$sCsvColumn];
}
}
foreach ($this->aIgnoredCsvColumns as $sIgnoredCsvColumn) {
$this->aIgnoredSynchroFields = array_merge( $this->aIgnoredSynchroFields, ($this->bHasHeader) ? $this->aMappingCsvColumnNameToFields[$sIgnoredCsvColumn] : $this->aMappingCsvColumnIndexToFields[$sIgnoredCsvColumn - 1]);
}
Expand Down
11 changes: 6 additions & 5 deletions test/CsvCollectorTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ public function OrgCollectorProvider()
"separator_incolumns" => ["separator_incolumns"],
"mapping 1 column twice" => ["map_1_column_twice"],
"mapping 1 column twice adding primary key" => ["map_1_column_twice_primary_key"],
"mapping 1 column twice without header" => ["map_1_column_twice_without_header"],
"mapping 1 column 3 times" => ["map_1_column_3_times"],
"mapping 2 columns twice" => ["map_2_columns_twice"],
"mapping 2 columns 3 times" => ["map_2_columns_3_times"],
Expand Down Expand Up @@ -177,11 +178,11 @@ public function ErrorFileProvider()
"[iTopPersonCsvCollector] The mandatory column \"primary_key\" is missing in the csv file.",
'iTopPersonCsvCollector::Collect() got an exception: Missing columns in the csv file.',
),
"no email" => array(
"no_email.csv",
"[iTopPersonCsvCollector] The field \"email\", used for reconciliation, has missing column(s) in the csv file.",
"iTopPersonCsvCollector::Collect() got an exception: Missing columns in the csv file.",
),
"no email" => array(
"no_email.csv",
"[iTopPersonCsvCollector] The field \"email\", used for reconciliation, has missing column(s) in the csv file.",
"iTopPersonCsvCollector::Collect() got an exception: Missing columns in the csv file.",
),
"empty csv" => array(
"empty_file.csv",
"[iTopPersonCsvCollector] CSV file is empty. Data collection stops here.",
Expand Down
2 changes: 2 additions & 0 deletions test/single_csv/map_1_column_twice_without_header/expected_generated.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
primary_key;first_name;name;org_id;employee_number;phone;email;function;status;mobile_phone
Isaac;Asimov;Demo;+33123456789;EMP001;EMP001;[email protected];writer;Active;+33601234567
9 changes: 9 additions & 0 deletions test/single_csv/map_1_column_twice_without_header/iTopPersonCsvCollector.class.inc.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<?php

class iTopPersonCsvCollector extends CSVCollector
{
protected function MustProcessBeforeSynchro()
{
return true;
}
}
1 change: 1 addition & 0 deletions test/single_csv/map_1_column_twice_without_header/iTopPersonCsvCollector.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Isaac;Asimov;Demo;+33123456789;EMP001;[email protected];writer;Active
34 changes: 34 additions & 0 deletions test/single_csv/map_1_column_twice_without_header/params.distrib.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Default values for parameters. Do NOT alter this file, copy its content to conf/params.local.xml and edit it instead -->
<parameters>
<console_log_level>6</console_log_level>

<iTopPersonCsvCollector>
<has_header>no</has_header>
<csv_file>collectors/iTopPersonCsvCollector.csv</csv_file>
<fields>
<primary_key>1</primary_key>
<first_name>2</first_name>
<name>3</name>
<employee_number>5</employee_number>
<function>7</function>
<email>6</email>
<status>8</status>
<phone>5</phone>
<org_id>4</org_id>
</fields>
<defaults>
<mobile_phone>+33601234567</mobile_phone>
</defaults>
</iTopPersonCsvCollector>

<synchro_user>admin</synchro_user>
<contact_to_notify></contact_to_notify>
<full_load_interval></full_load_interval>

<json_placeholders>
<prefix></prefix>
<persons_data_table>synchro_data_person_1</persons_data_table>
<synchro_status>production</synchro_status>
</json_placeholders>
</parameters>