Skip to content

Commit 21138a3

Browse files
paskalpaskal
committed
Update timeout configuration to use time.Duration for consistency
1 parent 5e6de45 commit 21138a3

File tree

6 files changed

+66
-34
lines changed

6 files changed

+66
-34
lines changed

‎backend/extractor/pics.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ func (f *UReadability) extractPics(iselect *goquery.Selection, url string) (main
5959

6060
// getImageSize loads image to get size
6161
func (f *UReadability) getImageSize(url string) (size int) {
62-
httpClient := &http.Client{Timeout: time.Second * 30}
62+
httpClient := &http.Client{Timeout: 30 * time.Second}
6363
req, err := http.NewRequest("GET", url, nil)
6464
if err != nil {
6565
log.Printf("[WARN] can't create request to get pic from %s", url)

‎backend/extractor/pics_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"os"
1010
"strings"
1111
"testing"
12+
"time"
1213

1314
"github.com/PuerkitoBio/goquery"
1415
"github.com/stretchr/testify/assert"
@@ -27,7 +28,7 @@ func TestExtractPics(t *testing.T) {
2728
defer ts.Close()
2829

2930
t.Log("test main pic")
30-
lr := UReadability{TimeOut: 30, SnippetSize: 200}
31+
lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200}
3132
a, err := lr.Extract(context.Background(), ts.URL+"/2015/09/25/poiezdka-s-apple-maps/")
3233
require.NoError(t, err)
3334
allImages := []string{
@@ -41,7 +42,7 @@ func TestExtractPics(t *testing.T) {
4142

4243
func TestExtractPicsDirectly(t *testing.T) {
4344
t.Log("test pic directly")
44-
lr := UReadability{TimeOut: 30, SnippetSize: 200}
45+
lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200}
4546
t.Run("normal image retrieval", func(t *testing.T) {
4647
data := `<body>
4748
<img class="alignright size-full wp-image-944214 lazyloadableImage lazyLoad-fadeIn" alt="View Page Source" width="308" height="508" data-original="https://cdn1.tnwcdn.com/wp-content/blogs.dir/1/files/2016/01/page-source.jpg" src="https://cdn1.tnwcdn.com/wp-content/blogs.dir/1/files/2016/01/page-source.jpg"></body>`

‎backend/extractor/readability.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule
7373
log.Printf("[INFO] extract %s", reqURL)
7474
rb := &Response{}
7575

76-
httpClient := &http.Client{Timeout: time.Second * f.TimeOut}
76+
httpClient := &http.Client{Timeout: f.TimeOut}
7777
req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil)
7878
if err != nil {
7979
log.Printf("[WARN] failed to create request for %s, error=%v", reqURL, err)

‎backend/extractor/readability_test.go‎

Lines changed: 58 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -40,29 +40,60 @@ func TestExtractURL(t *testing.T) {
4040
}))
4141
defer ts.Close()
4242

43-
lr := UReadability{TimeOut: 30, SnippetSize: 200}
44-
t.Log("full url")
45-
rb, err := lr.Extract(context.Background(), ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/")
46-
assert.NoError(t, err)
47-
assert.Equal(t, ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", rb.URL, "not changed")
48-
assert.Equal(t, "Всем миром для общей пользы • Umputun тут был", rb.Title)
49-
assert.Equal(t, 9665, len(rb.Content))
43+
lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200}
44+
45+
tests := []struct {
46+
name string
47+
url string
48+
wantURL string
49+
wantTitle string
50+
wantContentLen int
51+
wantErr bool
52+
}{
53+
{
54+
name: "full url",
55+
url: ts.URL + "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/",
56+
wantURL: ts.URL + "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/",
57+
wantTitle: "Всем миром для общей пользы • Umputun тут был",
58+
wantContentLen: 9665,
59+
wantErr: false,
60+
},
61+
{
62+
name: "short url",
63+
url: ts.URL + "/IAvTHr",
64+
wantURL: ts.URL + "/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/",
65+
wantTitle: "Всем миром для общей пользы • Umputun тут был",
66+
wantContentLen: 9665,
67+
wantErr: false,
68+
},
69+
{
70+
name: "bad body",
71+
url: ts.URL + "/bad_body",
72+
wantErr: true,
73+
},
74+
{
75+
name: "bad url",
76+
url: "http://bad_url",
77+
wantErr: true,
78+
},
79+
}
80+
81+
for _, tt := range tests {
82+
t.Run(tt.name, func(t *testing.T) {
83+
rb, err := lr.Extract(context.Background(), tt.url)
84+
85+
if tt.wantErr {
86+
assert.Error(t, err)
87+
assert.Nil(t, rb)
88+
return
89+
}
5090

51-
t.Log("short url")
52-
rb, err = lr.Extract(context.Background(), ts.URL+"/IAvTHr")
53-
assert.NoError(t, err)
54-
assert.Equal(t, ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/", rb.URL, "full url")
55-
assert.Equal(t, 9665, len(rb.Content))
56-
57-
t.Log("bad body")
58-
rb, err = lr.Extract(context.Background(), ts.URL+"/bad_body")
59-
//assert.Error(t, err) // TODO: uncomment, wtf?! this should return error!
60-
assert.Nil(t, rb)
61-
62-
t.Log("bad url")
63-
rb, err = lr.Extract(context.Background(), "http://bad_url")
64-
assert.Error(t, err)
65-
assert.Nil(t, rb)
91+
assert.NoError(t, err)
92+
assert.Equal(t, tt.wantURL, rb.URL)
93+
assert.Equal(t, tt.wantTitle, rb.Title)
94+
assert.Equal(t, tt.wantContentLen, len(rb.Content))
95+
})
96+
}
6697
}
6798

6899
func TestExtractGeneral(t *testing.T) {
@@ -92,7 +123,7 @@ func TestExtractGeneral(t *testing.T) {
92123
}))
93124
defer ts.Close()
94125

95-
lr := UReadability{TimeOut: 30, SnippetSize: 200}
126+
lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200}
96127
a, err := lr.Extract(context.Background(), ts.URL+"/2015/11/26/vsiem-mirom-dlia-obshchiei-polzy/")
97128
assert.NoError(t, err)
98129
assert.Equal(t, "Всем миром для общей пользы • Umputun тут был", a.Title)
@@ -114,7 +145,7 @@ func TestExtractGeneral(t *testing.T) {
114145
}
115146

116147
func TestNormalizeLinks(t *testing.T) {
117-
lr := UReadability{TimeOut: 30, SnippetSize: 200}
148+
lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200}
118149
inp := `blah <img src="/aaa.png"/> sdfasd <a href="/blah2/aa.link">something</a> blah33 <img src="//aaa.com/xyz.jpg">xx</img>`
119150
u, _ := url.Parse("http://ukeeper.com/blah")
120151
out, links := lr.normalizeLinks(inp, &http.Request{URL: u})
@@ -130,7 +161,7 @@ func TestNormalizeLinks(t *testing.T) {
130161
}
131162

132163
func TestNormalizeLinksIssue(t *testing.T) {
133-
lr := UReadability{TimeOut: 30, SnippetSize: 200}
164+
lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200}
134165
_, err := lr.Extract(context.Background(), "https://git-scm.com/book/en/v2/Git-Tools-Submodules")
135166
assert.NoError(t, err)
136167
}
@@ -150,8 +181,8 @@ func (m RulesMock) Disable(_ context.Context, _ primitive.ObjectID) error { retu
150181
func (m RulesMock) All(_ context.Context) []datastore.Rule { return make([]datastore.Rule, 0) }
151182

152183
func TestGetContentCustom(t *testing.T) {
153-
lr := UReadability{TimeOut: 30, SnippetSize: 200, Rules: RulesMock{}}
154-
httpClient := &http.Client{Timeout: time.Second * 30}
184+
lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200, Rules: RulesMock{}}
185+
httpClient := &http.Client{Timeout: 30 * time.Second}
155186
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
156187
if r.URL.String() == "/2015/09/25/poiezdka-s-apple-maps/" {
157188
fh, err := os.Open("testdata/poiezdka-s-apple-maps.html")

‎backend/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func main() {
4141
log.Fatalf("[ERROR] can't connect to mongo %v", err)
4242
}
4343
srv := rest.Server{
44-
Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300, Rules: db.GetStores()},
44+
Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()},
4545
Token: opts.Token,
4646
Credentials: opts.Credentials,
4747
Version: revision,

‎backend/rest/server_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func TestServer_FileServer(t *testing.T) {
3636
require.NoError(t, err)
3737

3838
srv := Server{
39-
Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300},
39+
Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300},
4040
Credentials: map[string]string{"admin": "password"},
4141
}
4242
ts := httptest.NewServer(srv.routes(dir))
@@ -613,7 +613,7 @@ func startupT(t *testing.T) (*httptest.Server, *Server) {
613613
db, err := datastore.New("mongodb://localhost:27017/", "test_ureadability", 0)
614614
assert.NoError(t, err)
615615
srv := Server{
616-
Readability: extractor.UReadability{TimeOut: 30, SnippetSize: 300, Rules: db.GetStores()},
616+
Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()},
617617
Credentials: map[string]string{"admin": "password"},
618618
Version: "dev-test",
619619
}

0 commit comments

Comments
 (0)