From 0fc1e78ec9e2947191c121b42d2672d1ce48d0c5 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Sun, 8 Jan 2017 02:16:33 -0500 Subject: [PATCH 01/14] add SeekLine function --- tail.go | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tail.go b/tail.go index c99cdaa2..eaad73c3 100644 --- a/tail.go +++ b/tail.go @@ -14,6 +14,7 @@ import ( "strings" "sync" "time" + "math" "github.com/hpcloud/tail/ratelimiter" "github.com/hpcloud/tail/util" @@ -223,6 +224,103 @@ func (tail *Tail) readLine() (string, error) { return line, err } +const BufferLength = 32 * 1024 + +// seeks through file by line +// 0 lines, 1 whence will return begining of current line +// 1 lines, 1 whence will return begining of next line +// 0 lines, 2 whence will return begining of last line +// negative lines will move backwards in file +func (tail *Tail) SeekLine(lines int64, whence int) (int64, error) { + BOF := errors.New("Begining of file") + + // return error on bad whence + if whence < 0 || whence > 2 { + return tail.file.Seek(0, whence) + } + + position, err := tail.file.Seek(0, whence) + + buf := make([]byte, BufferLength) + bufLen := 0 + lineSep := byte('\n') + seekBack := lines < 1 + lines = int64(math.Abs(float64(lines))) + matchCount := int64(0) + + // seekBack ignores first match + // allows 0 to go to begining of current line + if seekBack { + matchCount = -1 + } + + leftPosition := position + offset := int64(BufferLength * -1) + + for b := 1; ; b++ { + if err != nil { + break + } + + if seekBack { + + // on seekBack 2nd buffer onward needs to seek + // past what was just read plus another buffer size + if b == 2 { + offset *= 2 + } + + // if next seekBack will pass beginning of file + // buffer is 0 to unread position + if position+int64(offset) <= 0 { + buf = make([]byte, leftPosition) + position, err = tail.file.Seek(0, io.SeekStart) + leftPosition = 0 + } else { + position, err = tail.file.Seek(offset, io.SeekCurrent) + leftPosition = leftPosition - BufferLength + } + } + if err != nil { + break + } + + bufLen, err = tail.file.Read(buf) + if err != nil { + break + } else if leftPosition == 0 { + err = BOF + } + + for i := 0; i < bufLen; i++ { + iToCheck := i + if seekBack { + iToCheck = bufLen - i - 1 + } + byteToCheck := buf[iToCheck] + + if byteToCheck == lineSep { + matchCount++ + } + + if matchCount == lines { + if seekBack { + return tail.file.Seek(int64(i)*-1, io.SeekCurrent) + } + return tail.file.Seek(int64(bufLen*-1+i+1), io.SeekCurrent) + } + } + } + + if err == io.EOF { + position, _ = tail.file.Seek(0, io.SeekEnd) + } else if err == BOF { + position, _ = tail.file.Seek(0, io.SeekStart) + } + + return position, err +} + func (tail *Tail) tailFileSync() { defer tail.Done() defer tail.close() From d9579af6e609f4d7d1d1a9d1f2384dc8d1c71e92 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Sun, 8 Jan 2017 02:16:55 -0500 Subject: [PATCH 02/14] add LineLocation to config --- tail.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tail.go b/tail.go index eaad73c3..8f6260cf 100644 --- a/tail.go +++ b/tail.go @@ -58,12 +58,13 @@ type logger interface { // Config is used to specify how a file must be tailed. type Config struct { // File-specifc - Location *SeekInfo // Seek to this location before tailing - ReOpen bool // Reopen recreated files (tail -F) - MustExist bool // Fail early if the file does not exist - Poll bool // Poll for file changes instead of using inotify - Pipe bool // Is a named pipe (mkfifo) - RateLimiter *ratelimiter.LeakyBucket + Location *SeekInfo // Seek to this location before tailing + LineLocation *SeekInfo // Seek to this line number before tailing + ReOpen bool // Reopen recreated files (tail -F) + MustExist bool // Fail early if the file does not exist + Poll bool // Poll for file changes instead of using inotify + Pipe bool // Is a named pipe (mkfifo) + RateLimiter *ratelimiter.LeakyBucket // Generic IO Follow bool // Continue looking for new lines (tail -f) @@ -106,6 +107,10 @@ func TailFile(filename string, config Config) (*Tail, error) { util.Fatal("cannot set ReOpen without Follow.") } + if config.Location != nil && config.LineLocation != nil { + util.Fatal("Location and LineLocation cannot be set at the same time") + } + t := &Tail{ Filename: filename, Lines: make(chan *Line), @@ -344,6 +349,13 @@ func (tail *Tail) tailFileSync() { tail.Killf("Seek error on %s: %s", tail.Filename, err) return } + } else if tail.LineLocation != nil { + _, err := tail.SeekLine(tail.LineLocation.Offset, tail.LineLocation.Whence) + tail.Logger.Printf("Seeked Line %s - %+v\n", tail.Filename, tail.LineLocation) + if err != nil { + tail.Killf("Seek error on %s: %s", tail.Filename, err) + return + } } tail.openReader() From 444cf6249cce56f8b1fc97e1206e15bb8453440a Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Sun, 8 Jan 2017 08:36:49 -0500 Subject: [PATCH 03/14] check error on final seek --- tail.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tail.go b/tail.go index 8f6260cf..d7073301 100644 --- a/tail.go +++ b/tail.go @@ -318,11 +318,10 @@ func (tail *Tail) SeekLine(lines int64, whence int) (int64, error) { } if err == io.EOF { - position, _ = tail.file.Seek(0, io.SeekEnd) + position, err = tail.file.Seek(0, io.SeekEnd) } else if err == BOF { - position, _ = tail.file.Seek(0, io.SeekStart) + position, err = tail.file.Seek(0, io.SeekStart) } - return position, err } From 42c2e2a1f8ba81b59b3fd632c04e28beaa8c9102 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Sun, 22 Jan 2017 17:50:49 -0500 Subject: [PATCH 04/14] use file line seeker package --- tail.go | 101 ++------------------------------------------------------ 1 file changed, 3 insertions(+), 98 deletions(-) diff --git a/tail.go b/tail.go index d7073301..df7e1916 100644 --- a/tail.go +++ b/tail.go @@ -14,11 +14,11 @@ import ( "strings" "sync" "time" - "math" "github.com/hpcloud/tail/ratelimiter" "github.com/hpcloud/tail/util" "github.com/hpcloud/tail/watch" + "github.com/stoicperlman/fls" "gopkg.in/tomb.v1" ) @@ -229,102 +229,6 @@ func (tail *Tail) readLine() (string, error) { return line, err } -const BufferLength = 32 * 1024 - -// seeks through file by line -// 0 lines, 1 whence will return begining of current line -// 1 lines, 1 whence will return begining of next line -// 0 lines, 2 whence will return begining of last line -// negative lines will move backwards in file -func (tail *Tail) SeekLine(lines int64, whence int) (int64, error) { - BOF := errors.New("Begining of file") - - // return error on bad whence - if whence < 0 || whence > 2 { - return tail.file.Seek(0, whence) - } - - position, err := tail.file.Seek(0, whence) - - buf := make([]byte, BufferLength) - bufLen := 0 - lineSep := byte('\n') - seekBack := lines < 1 - lines = int64(math.Abs(float64(lines))) - matchCount := int64(0) - - // seekBack ignores first match - // allows 0 to go to begining of current line - if seekBack { - matchCount = -1 - } - - leftPosition := position - offset := int64(BufferLength * -1) - - for b := 1; ; b++ { - if err != nil { - break - } - - if seekBack { - - // on seekBack 2nd buffer onward needs to seek - // past what was just read plus another buffer size - if b == 2 { - offset *= 2 - } - - // if next seekBack will pass beginning of file - // buffer is 0 to unread position - if position+int64(offset) <= 0 { - buf = make([]byte, leftPosition) - position, err = tail.file.Seek(0, io.SeekStart) - leftPosition = 0 - } else { - position, err = tail.file.Seek(offset, io.SeekCurrent) - leftPosition = leftPosition - BufferLength - } - } - if err != nil { - break - } - - bufLen, err = tail.file.Read(buf) - if err != nil { - break - } else if leftPosition == 0 { - err = BOF - } - - for i := 0; i < bufLen; i++ { - iToCheck := i - if seekBack { - iToCheck = bufLen - i - 1 - } - byteToCheck := buf[iToCheck] - - if byteToCheck == lineSep { - matchCount++ - } - - if matchCount == lines { - if seekBack { - return tail.file.Seek(int64(i)*-1, io.SeekCurrent) - } - return tail.file.Seek(int64(bufLen*-1+i+1), io.SeekCurrent) - } - } - } - - if err == io.EOF { - position, err = tail.file.Seek(0, io.SeekEnd) - } else if err == BOF { - position, err = tail.file.Seek(0, io.SeekStart) - } - return position, err -} - func (tail *Tail) tailFileSync() { defer tail.Done() defer tail.close() @@ -349,7 +253,8 @@ func (tail *Tail) tailFileSync() { return } } else if tail.LineLocation != nil { - _, err := tail.SeekLine(tail.LineLocation.Offset, tail.LineLocation.Whence) + lineFile := fls.LineFile(tail.file) + _, err := lineFile.SeekLine(tail.LineLocation.Offset, tail.LineLocation.Whence) tail.Logger.Printf("Seeked Line %s - %+v\n", tail.Filename, tail.LineLocation) if err != nil { tail.Killf("Seek error on %s: %s", tail.Filename, err) From ddbcc23ffe831854995493eca78baef7de148cb5 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Sun, 22 Jan 2017 18:47:18 -0500 Subject: [PATCH 05/14] add dependency to .travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 183f1b7d..40e193c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,3 +17,4 @@ matrix: install: - go get gopkg.in/fsnotify.v1 - go get gopkg.in/tomb.v1 + - go get github.com/StoicPerlman/fls From 2a149d72bb2a505e39ffda226f2f9a9834bc7f52 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Sun, 22 Jan 2017 19:35:24 -0500 Subject: [PATCH 06/14] add line location tests --- tail_test.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tail_test.go b/tail_test.go index 38d6b84b..ca26ef7c 100644 --- a/tail_test.go +++ b/tail_test.go @@ -208,6 +208,66 @@ func TestLocationMiddle(t *testing.T) { tailTest.Cleanup(tail, true) } +func TestLineLocationFull(t *testing.T) { + tailTest := NewTailTest("location-full", t) + tailTest.CreateFile("test.txt", "hello\nworld\n") + tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: nil}) + go tailTest.VerifyTailOutput(tail, []string{"hello", "world"}, false) + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + tailTest.RemoveFile("test.txt") + tailTest.Cleanup(tail, true) +} + +func TestLineLocationFullDontFollow(t *testing.T) { + tailTest := NewTailTest("location-full-dontfollow", t) + tailTest.CreateFile("test.txt", "hello\nworld\n") + tail := tailTest.StartTail("test.txt", Config{Follow: false, LineLocation: nil}) + go tailTest.VerifyTailOutput(tail, []string{"hello", "world"}, false) + + // Add more data only after reasonable delay. + <-time.After(100 * time.Millisecond) + tailTest.AppendFile("test.txt", "more\ndata\n") + <-time.After(100 * time.Millisecond) + + tailTest.Cleanup(tail, true) +} + +func TestLineLocationEnd(t *testing.T) { + tailTest := NewTailTest("location-end", t) + tailTest.CreateFile("test.txt", "hello\nworld\n") + tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{0, os.SEEK_END}}) + go tailTest.VerifyTailOutput(tail, []string{"more", "data"}, false) + + <-time.After(100 * time.Millisecond) + tailTest.AppendFile("test.txt", "more\ndata\n") + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + tailTest.RemoveFile("test.txt") + tailTest.Cleanup(tail, true) +} + +func TestLineLocationMiddle(t *testing.T) { + // Test reading from middle. + tailTest := NewTailTest("location-middle", t) + tailTest.CreateFile("test.txt", "hello\nworld\n") + tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{-1, os.SEEK_END}}) + go tailTest.VerifyTailOutput(tail, []string{"world", "more", "data"}, false) + + <-time.After(100 * time.Millisecond) + tailTest.AppendFile("test.txt", "more\ndata\n") + + // Delete after a reasonable delay, to give tail sufficient time + // to read all lines. + <-time.After(100 * time.Millisecond) + tailTest.RemoveFile("test.txt") + tailTest.Cleanup(tail, true) +} + // The use of polling file watcher could affect file rotation // (detected via renames), so test these explicitly. From 66de0d779b8d367e529c9b4978723b7d965583c2 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 17:51:52 -0500 Subject: [PATCH 07/14] fix travis tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 40e193c2..f9fc5923 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,4 +17,4 @@ matrix: install: - go get gopkg.in/fsnotify.v1 - go get gopkg.in/tomb.v1 - - go get github.com/StoicPerlman/fls + - go get github.com/stoicperlman/fls From 93fd86f176ea418285505f88b0bf495f402a8c68 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 18:18:25 -0500 Subject: [PATCH 08/14] fix test names --- tail_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tail_test.go b/tail_test.go index ca26ef7c..46d823a3 100644 --- a/tail_test.go +++ b/tail_test.go @@ -209,7 +209,7 @@ func TestLocationMiddle(t *testing.T) { } func TestLineLocationFull(t *testing.T) { - tailTest := NewTailTest("location-full", t) + tailTest := NewTailTest("line-location-full", t) tailTest.CreateFile("test.txt", "hello\nworld\n") tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: nil}) go tailTest.VerifyTailOutput(tail, []string{"hello", "world"}, false) @@ -222,7 +222,7 @@ func TestLineLocationFull(t *testing.T) { } func TestLineLocationFullDontFollow(t *testing.T) { - tailTest := NewTailTest("location-full-dontfollow", t) + tailTest := NewTailTest("line-location-full-dontfollow", t) tailTest.CreateFile("test.txt", "hello\nworld\n") tail := tailTest.StartTail("test.txt", Config{Follow: false, LineLocation: nil}) go tailTest.VerifyTailOutput(tail, []string{"hello", "world"}, false) @@ -236,7 +236,7 @@ func TestLineLocationFullDontFollow(t *testing.T) { } func TestLineLocationEnd(t *testing.T) { - tailTest := NewTailTest("location-end", t) + tailTest := NewTailTest("line-location-end", t) tailTest.CreateFile("test.txt", "hello\nworld\n") tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{0, os.SEEK_END}}) go tailTest.VerifyTailOutput(tail, []string{"more", "data"}, false) @@ -253,7 +253,7 @@ func TestLineLocationEnd(t *testing.T) { func TestLineLocationMiddle(t *testing.T) { // Test reading from middle. - tailTest := NewTailTest("location-middle", t) + tailTest := NewTailTest("line-location-middle", t) tailTest.CreateFile("test.txt", "hello\nworld\n") tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{-1, os.SEEK_END}}) go tailTest.VerifyTailOutput(tail, []string{"world", "more", "data"}, false) From ca3f42d3b44e386693790d51a2752f1f668985c0 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 19:21:39 -0500 Subject: [PATCH 09/14] use io.Seek consts --- tail_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tail_test.go b/tail_test.go index 46d823a3..5d5af7ee 100644 --- a/tail_test.go +++ b/tail_test.go @@ -8,6 +8,7 @@ package tail import ( _ "fmt" + "io" "io/ioutil" "os" "strings" @@ -238,7 +239,7 @@ func TestLineLocationFullDontFollow(t *testing.T) { func TestLineLocationEnd(t *testing.T) { tailTest := NewTailTest("line-location-end", t) tailTest.CreateFile("test.txt", "hello\nworld\n") - tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{0, os.SEEK_END}}) + tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{0, io.SeekEnd}}) go tailTest.VerifyTailOutput(tail, []string{"more", "data"}, false) <-time.After(100 * time.Millisecond) @@ -255,7 +256,7 @@ func TestLineLocationMiddle(t *testing.T) { // Test reading from middle. tailTest := NewTailTest("line-location-middle", t) tailTest.CreateFile("test.txt", "hello\nworld\n") - tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{-1, os.SEEK_END}}) + tail := tailTest.StartTail("test.txt", Config{Follow: true, LineLocation: &SeekInfo{-1, io.SeekEnd}}) go tailTest.VerifyTailOutput(tail, []string{"world", "more", "data"}, false) <-time.After(100 * time.Millisecond) From 9d054ed4f9bf85738b7daf882d3bff4372f38d47 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 19:38:12 -0500 Subject: [PATCH 10/14] update cmd -n flag to use line seeker --- cmd/gotail/gotail.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/gotail/gotail.go b/cmd/gotail/gotail.go index 3da55f23..fdbb3dc7 100644 --- a/cmd/gotail/gotail.go +++ b/cmd/gotail/gotail.go @@ -6,6 +6,7 @@ package main import ( "flag" "fmt" + "io" "os" "github.com/hpcloud/tail" @@ -36,7 +37,7 @@ func main() { } if n != 0 { - config.Location = &tail.SeekInfo{-n, os.SEEK_END} + config.LineLocation = &tail.SeekInfo{-n, io.SeekEnd} } done := make(chan bool) From 2ffa3cf9d74b3b2a9d12a412e538346422b12093 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 19:42:31 -0500 Subject: [PATCH 11/14] remove log line --- tail.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tail.go b/tail.go index df7e1916..647c54fe 100644 --- a/tail.go +++ b/tail.go @@ -255,7 +255,6 @@ func (tail *Tail) tailFileSync() { } else if tail.LineLocation != nil { lineFile := fls.LineFile(tail.file) _, err := lineFile.SeekLine(tail.LineLocation.Offset, tail.LineLocation.Whence) - tail.Logger.Printf("Seeked Line %s - %+v\n", tail.Filename, tail.LineLocation) if err != nil { tail.Killf("Seek error on %s: %s", tail.Filename, err) return From 2a5af2ac88949457d2b2148bd71b2b61cd6c9730 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 19:52:11 -0500 Subject: [PATCH 12/14] fix off by 1 errror --- cmd/gotail/gotail.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gotail/gotail.go b/cmd/gotail/gotail.go index fdbb3dc7..16f81584 100644 --- a/cmd/gotail/gotail.go +++ b/cmd/gotail/gotail.go @@ -37,7 +37,7 @@ func main() { } if n != 0 { - config.LineLocation = &tail.SeekInfo{-n, io.SeekEnd} + config.LineLocation = &tail.SeekInfo{-n + 1, io.SeekEnd} } done := make(chan bool) From 5f0e9bcc6718cb1326e5921fa019aab8afc61c95 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 22:24:56 -0500 Subject: [PATCH 13/14] swallow EOF err on -n past size --- tail.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tail.go b/tail.go index 647c54fe..fb892787 100644 --- a/tail.go +++ b/tail.go @@ -255,7 +255,7 @@ func (tail *Tail) tailFileSync() { } else if tail.LineLocation != nil { lineFile := fls.LineFile(tail.file) _, err := lineFile.SeekLine(tail.LineLocation.Offset, tail.LineLocation.Whence) - if err != nil { + if err != nil && err != io.EOF { tail.Killf("Seek error on %s: %s", tail.Filename, err) return } From 4c42fd0ebb8c526b90b48ae1938f079b43a591c1 Mon Sep 17 00:00:00 2001 From: Sam Kleiner Date: Fri, 29 Dec 2017 23:01:50 -0500 Subject: [PATCH 14/14] match tail handling of newlines at EOF --- cmd/gotail/gotail.go | 2 +- tail.go | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cmd/gotail/gotail.go b/cmd/gotail/gotail.go index 16f81584..fdbb3dc7 100644 --- a/cmd/gotail/gotail.go +++ b/cmd/gotail/gotail.go @@ -37,7 +37,7 @@ func main() { } if n != 0 { - config.LineLocation = &tail.SeekInfo{-n + 1, io.SeekEnd} + config.LineLocation = &tail.SeekInfo{-n, io.SeekEnd} } done := make(chan bool) diff --git a/tail.go b/tail.go index fb892787..2964513d 100644 --- a/tail.go +++ b/tail.go @@ -254,7 +254,28 @@ func (tail *Tail) tailFileSync() { } } else if tail.LineLocation != nil { lineFile := fls.LineFile(tail.file) - _, err := lineFile.SeekLine(tail.LineLocation.Offset, tail.LineLocation.Whence) + buf := make([]byte, 1) + + _, err := lineFile.Seek(-1, io.SeekEnd) + if err != nil { + tail.Killf("Seek error on %s: %s", tail.Filename, err) + return + } + + _, err = lineFile.Read(buf) + if err != nil { + tail.Killf("Seek error on %s: %s", tail.Filename, err) + return + } + + // if file ends in newline don't count it in lines + // to read from end (mimics unix tail command) + correction := int64(1) + if string(buf) == "\n" { + correction = 0 + } + + _, err = lineFile.SeekLine(tail.LineLocation.Offset+correction, tail.LineLocation.Whence) if err != nil && err != io.EOF { tail.Killf("Seek error on %s: %s", tail.Filename, err) return