Add -header flag

Author: Uğur Özyılmazel

README.md

@@ -62,6 +62,7 @@ usage: scraper-cli [flags...]
   -referer      HTTP referer header
   -selector     CSS style selector path such as: a.btn div li
   -version      display version information
+  -header       request header(s)
   -help, -h     display help
 
 
@@ -74,6 +75,8 @@ usage: scraper-cli [flags...]
 
   $ PROMPTAPI_TOKEN="your-api-key" scraper-cli -url "https://promptapi.com"
   $ scraper-cli -url "https://promptapi.com" -token "your-api-key"
+  $ scraper-cli -url "https://promptapi.com" -token "your-api-key" -header "X-Referer: https://www.google.com"
+  $ scraper-cli -url "https://promptapi.com" -token "your-api-key" -header "X-Referer: https://www.google.com" -header "X-Custom-Header: Hello"
 ```
 
 ---

go.mod

@@ -2,4 +2,4 @@ module github.com/promptapi/scraper-cli
 
 go 1.15
 
-require github.com/promptapi/scraper-go v0.1.2
+require github.com/promptapi/scraper-go v0.1.4

go.sum

@@ -1,2 +1,6 @@
 github.com/promptapi/scraper-go v0.1.2 h1:iYCVbrONu7hNkV2US1S7Qtn55w+wmDuZS17DPH/ifuE=
 github.com/promptapi/scraper-go v0.1.2/go.mod h1:ABmGIUogizw4JUmpR5gRXvT4KLgjgrJ8LFOnonjd2cw=
+github.com/promptapi/scraper-go v0.1.3 h1:wMgvae6Vj8asPN0YNBcNtYZjgAi8zhxLfI4cbroJ93I=
+github.com/promptapi/scraper-go v0.1.3/go.mod h1:ABmGIUogizw4JUmpR5gRXvT4KLgjgrJ8LFOnonjd2cw=
+github.com/promptapi/scraper-go v0.1.4 h1:x5ijz9D8K9VicMVakesyYzpWDu5tl/XYgMbVSHgLwRU=
+github.com/promptapi/scraper-go v0.1.4/go.mod h1:ABmGIUogizw4JUmpR5gRXvT4KLgjgrJ8LFOnonjd2cw=

pkg/app/app.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/url"
 	"os"
+	"strings"
 
 	scraper "github.com/promptapi/scraper-go"
 )
@@ -26,6 +27,7 @@ var (
 	optCookie             *string
 	optReferer            *string
 	optSelector           *string
+	optHeaders            headersFlag
 
 	usage = `
 usage: scraper-cli [flags...]
@@ -58,6 +60,7 @@ usage: scraper-cli [flags...]
   -referer      HTTP referer header
   -selector     CSS style selector path such as: a.btn div li
   -version      display version information
+  -header       request header(s)
   -help, -h     display help
 
 
@@ -70,11 +73,24 @@ usage: scraper-cli [flags...]
 
   $ PROMPTAPI_TOKEN="your-api-key" scraper-cli -url "https://promptapi.com"
   $ scraper-cli -url "https://promptapi.com" -token "your-api-key"
+  $ scraper-cli -url "https://promptapi.com" -token "your-api-key" -header "X-Referer: https://www.google.com"
+  $ scraper-cli -url "https://promptapi.com" -token "your-api-key" -header "X-Referer: https://www.google.com" -header "X-Custom-Header: Hello"
 
 
 `
 )
 
+type headersFlag []string
+
+func (h *headersFlag) String() string {
+	return "headers"
+}
+
+func (h *headersFlag) Set(value string) error {
+	*h = append(*h, strings.TrimSpace(value))
+	return nil
+}
+
 // CLIApplication represents app structure
 type CLIApplication struct {
 	Out io.Writer
@@ -94,6 +110,7 @@ func NewCLIApplication() *CLIApplication {
 	optCookie = flag.String("cookie", "n/a", "URL Encoded cookie header")
 	optReferer = flag.String("referer", "n/a", "HTTP referer header")
 	optSelector = flag.String("selector", "n/a", "CSS style selector path such as: a.btn div li")
+	flag.Var(&optHeaders, "header", "")
 
 	flag.Parse()
 
@@ -164,7 +181,19 @@ func (c *CLIApplication) Scrape() error {
 	}
 
 	result := new(scraper.Result)
-	err := s.Scrape(params, result)
+	extraHeaders := []*scraper.ExtraHeader{}
+
+	if len(optHeaders) > 0 {
+		for _, headerValue := range optHeaders {
+			valuePair := strings.Split(headerValue, ":")
+			reqHeader := new(scraper.ExtraHeader)
+			reqHeader.Name = valuePair[0]
+			reqHeader.Value = valuePair[1]
+			extraHeaders = append(extraHeaders, reqHeader)
+		}
+	}
+
+	err := s.Scrape(params, extraHeaders, result)
 	if err != nil {
 		return err
 	}