add deviation tolerancy

detector/Fixtures/test3.csv

@@ -0,0 +1,5 @@
+Email Address,,first name,last name,Gender,Date of birth,country,city,STREET,POSTAL CODE,INTEREST 1 - Producer,INTEREST 2 - 2-Dutch Recs,INTEREST 3 - SKINK,INTEREST 4 - Musical Madness,INTEREST 5  - Dutch Master Works,INTEREST 6 - 4-Dots,INTEREST 7 - Blue Forest,INTEREST 8 - Brooks,INTEREST 9 - PROMOTER GENERAL,INTEREST 10 - PROMOTER CLEAN
+[email protected],First,Last,,,,,,,,,,,,,,Brooks,,
+[email protected],First,Last,,,,,,,,,,,,,,Brooks,,
+[email protected],First,Last,,,,,,,,,,,,,,Brooks,,
+[email protected],First,Last,,,,,,,,,,,,,,Brooks,,

detector/detect.go

@@ -3,6 +3,7 @@ package detector
 import (
 	"bufio"
 	"io"
+	// "log"
 	"math"
 	"regexp"
 )
@@ -145,11 +146,22 @@ func (d *detector) analyze(ft frequencyTable, sampleLine int) []byte {
 	}
 
 	var candidates []byte
+	var minDeviation float64
+	var minDelimiter byte
 	for delimiter, frequencyOfLine := range ft {
-		if float64(0.0) == deviation(frequencyOfLine, sampleLine) {
+		dev := deviation(frequencyOfLine, sampleLine)
+		if float64(0.0) == dev {
 			candidates = append(candidates, delimiter)
+		} else if minDeviation > dev || minDeviation == 0 {
+			// find minimum deviation available
+			minDeviation = dev
+			minDelimiter = delimiter
 		}
 	}
+	// if zero deviation candidates are not found, pick the minimum one
+	if len(candidates) == 0 && minDeviation > 0 {
+		candidates = append(candidates, minDelimiter)
+	}
 
 	return candidates
 }

detector/detect_test.go

@@ -66,6 +66,19 @@ func TestDetectDelimiterComma(t *testing.T) {
 	assert.Equal(t, []string{","}, delimiters)
 }
 
+func TestDetectDelimiterComma2(t *testing.T) {
+	detector := New()
+	sampleLines := 4
+	detector.Configure(&sampleLines, nil)
+	file, err := os.OpenFile("./Fixtures/test3.csv", os.O_RDONLY, os.ModePerm)
+	assert.NoError(t, err)
+	defer file.Close()
+
+	delimiters := detector.DetectDelimiter(file, '"')
+
+	assert.Equal(t, []string{","}, delimiters)
+}
+
 func TestDetectDelimiterSemicolon(t *testing.T) {
 	detector := New()
 	sampleLines := 4