-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
60 lines (48 loc) · 1.16 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
package main
import (
"encoding/json"
"fmt"
"github.com/PuerkitoBio/goquery"
"github.com/rakeshkumargupt/scrape-go/utils"
"io"
"os"
)
func main() {
fmt.Println("Starting main..")
// Set header in request with key & value[OPTIONAL]
m := map[string]string{}
// Method to call http
method := "GET"
// OPTIONAL body if method is POST or ...
var body io.Reader
_ = body
// URL to call for scrapping
// Here choose here different category for getting data about product
url := "http://www.amazon.com/gp/bestsellers/beauty#1"
// Getting http response
resp, err := utils.CallHttp(url, method, nil, m)
if err != nil {
fmt.Println(err)
return
}
// Getting goQuery document for scrapping
doc, err := utils.GetDocument(resp)
if err != nil {
fmt.Println(err)
return
}
// Getting product information of scapped webpage
if doc != nil {
doc.Find("div[class='zg_itemImmersion']").Each(func(i int, s *goquery.Selection) {
prod := utils.CreateTrendingProductListItem(s)
if prod.ProductID != "" {
b, err := json.MarshalIndent(prod, "", " ")
if err != nil {
panic(err)
}
b2 := append(b, '\n')
os.Stdout.Write(b2)
}
})
}
}