mirror of
https://github.com/NohamR/papeer.git
synced 2026-05-24 20:00:45 +00:00
refacto, better scrape strategy
This commit is contained in:
lapwat
14
book/book.go
14
book/book.go
@@ -1,27 +1,27 @@
|
||||
package book
|
||||
|
||||
type Book struct {
|
||||
type book struct {
|
||||
name string
|
||||
author string
|
||||
chapters []chapter
|
||||
}
|
||||
|
||||
func New(name, author string) Book {
|
||||
return Book{name, author, []chapter{}}
|
||||
func New(name, author string) book {
|
||||
return book{name, author, []chapter{}}
|
||||
}
|
||||
|
||||
func (b *Book) AddChapter(c chapter) {
|
||||
func (b *book) AddChapter(c chapter) {
|
||||
b.chapters = append(b.chapters, c)
|
||||
}
|
||||
|
||||
func (b Book) Name() string {
|
||||
func (b book) Name() string {
|
||||
return b.name
|
||||
}
|
||||
|
||||
func (b Book) Author() string {
|
||||
func (b book) Author() string {
|
||||
return b.name
|
||||
}
|
||||
|
||||
func (b *Book) Chapters() []chapter {
|
||||
func (b *book) Chapters() []chapter {
|
||||
return b.chapters
|
||||
}
|
||||
|
||||
23
book/link.go
Normal file
23
book/link.go
Normal file
@@ -0,0 +1,23 @@
|
||||
package book
|
||||
|
||||
type link struct {
|
||||
href string
|
||||
text string
|
||||
class string
|
||||
}
|
||||
|
||||
func NewLink(href, text, class string) link {
|
||||
return link{href, text, class}
|
||||
}
|
||||
|
||||
func (c link) Href() string {
|
||||
return c.href
|
||||
}
|
||||
|
||||
func (c link) Text() string {
|
||||
return c.text
|
||||
}
|
||||
|
||||
func (c link) Class() string {
|
||||
return c.class
|
||||
}
|
||||
160
book/scraper.go
160
book/scraper.go
@@ -9,28 +9,32 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
readability "github.com/go-shiori/go-readability"
|
||||
colly "github.com/gocolly/colly/v2"
|
||||
"github.com/gosuri/uiprogress"
|
||||
)
|
||||
|
||||
type scraper struct {
|
||||
url string
|
||||
}
|
||||
func NewBookFromURL(url, selector string, recursive, include bool, limit, delay int) book {
|
||||
if recursive {
|
||||
home := NewChapterFromURL(url)
|
||||
b := New(home.Name(), home.Author())
|
||||
|
||||
func NewBookFromURL(url, selector string, include bool, limit, delay int) Book {
|
||||
home := NewChapterFromURL(url)
|
||||
b := New(home.Name(), home.Author())
|
||||
chapters := tableOfContent(url, selector, limit, delay)
|
||||
if include {
|
||||
b.AddChapter(home)
|
||||
}
|
||||
for _, c := range chapters {
|
||||
b.AddChapter(c)
|
||||
}
|
||||
|
||||
chapters := tableOfContent(url, selector, limit, delay)
|
||||
if include {
|
||||
b.AddChapter(home)
|
||||
}
|
||||
for _, c := range chapters {
|
||||
return b
|
||||
} else {
|
||||
c := NewChapterFromURL(url)
|
||||
b := New(c.Name(), c.Author())
|
||||
b.AddChapter(c)
|
||||
return b
|
||||
}
|
||||
|
||||
return b
|
||||
}
|
||||
|
||||
func NewChapterFromURL(url string) chapter {
|
||||
@@ -39,57 +43,22 @@ func NewChapterFromURL(url string) chapter {
|
||||
log.Fatalf("failed to parse %s, %v\n", url, err)
|
||||
}
|
||||
|
||||
// metadata := fmt.Sprintf("URL : %s\nTitle : %s\nAuthor : %s\nLength : %d\nExcerpt : %s\nSiteName: %s\nImage : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
|
||||
// fmt.Println(metadata)
|
||||
|
||||
return chapter{article.Title, article.Byline, article.Content}
|
||||
}
|
||||
|
||||
func tableOfContent(url, selector string, limit, delay int) []chapter {
|
||||
c := colly.NewCollector()
|
||||
|
||||
classesLinks := map[string][]map[string]string{}
|
||||
classesCount := map[string]int{}
|
||||
classMax := ""
|
||||
|
||||
selectorSet := true
|
||||
if selector == "" {
|
||||
selector = "a"
|
||||
selectorSet = false
|
||||
base, err := urllib.Parse(url)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
c.OnHTML(selector, func(e *colly.HTMLElement) {
|
||||
href := e.Attr("href")
|
||||
text := strings.TrimSpace(e.Text)
|
||||
class := e.Attr("class")
|
||||
|
||||
if selectorSet || class != "" && text != "" {
|
||||
classesLinks[class] = append(classesLinks[class], map[string]string{
|
||||
"href": href,
|
||||
"text": text,
|
||||
})
|
||||
|
||||
classesCount[class]++
|
||||
|
||||
if classesCount[class] > classesCount[classMax] {
|
||||
classMax = class
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
c.Visit(url)
|
||||
|
||||
links := classesLinks[classMax]
|
||||
links := GetLinks(base, selector)
|
||||
if limit != -1 {
|
||||
limit = int(math.Min(float64(limit), float64(len(links))))
|
||||
links = links[:limit]
|
||||
}
|
||||
|
||||
chapters := make([]chapter, len(links))
|
||||
base, err := urllib.Parse(url)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// init global progress bar
|
||||
uiprogress.Start()
|
||||
@@ -102,7 +71,7 @@ func tableOfContent(url, selector string, limit, delay int) []chapter {
|
||||
bars := []*uiprogress.Bar{}
|
||||
for index, link := range links {
|
||||
bar := uiprogress.AddBar(1).AppendCompleted().PrependElapsed()
|
||||
barText := fmt.Sprintf("%d. %s", index+1, link["text"])
|
||||
barText := fmt.Sprintf("%d. %s", index+1, link.text)
|
||||
bar.AppendFunc(func(b *uiprogress.Bar) string {
|
||||
return barText
|
||||
})
|
||||
@@ -112,7 +81,7 @@ func tableOfContent(url, selector string, limit, delay int) []chapter {
|
||||
if delay >= 0 {
|
||||
for index, link := range links {
|
||||
// and then use it to parse relative URLs
|
||||
u, err := base.Parse(link["href"])
|
||||
u, err := base.Parse(link.href)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -131,14 +100,14 @@ func tableOfContent(url, selector string, limit, delay int) []chapter {
|
||||
|
||||
} else {
|
||||
var wg sync.WaitGroup
|
||||
for index, link := range links {
|
||||
for index, l := range links {
|
||||
|
||||
wg.Add(1)
|
||||
go func(index int, link map[string]string) {
|
||||
go func(index int, l link) {
|
||||
defer wg.Done()
|
||||
|
||||
// and then use it to parse relative URLs
|
||||
u, err := base.Parse(link["href"])
|
||||
u, err := base.Parse(l.href)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -147,9 +116,84 @@ func tableOfContent(url, selector string, limit, delay int) []chapter {
|
||||
|
||||
bars[index].Incr()
|
||||
barGlobal.Incr()
|
||||
}(index, link)
|
||||
}(index, l)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return chapters
|
||||
}
|
||||
|
||||
func GetPath(elm *goquery.Selection) string {
|
||||
path := []string{}
|
||||
|
||||
for {
|
||||
selector := strings.ToLower(goquery.NodeName(elm))
|
||||
if selector == "" {
|
||||
break
|
||||
}
|
||||
|
||||
path = append(path, selector)
|
||||
elm = elm.Parent()
|
||||
}
|
||||
|
||||
join := strings.Join(path, "<")
|
||||
return join
|
||||
}
|
||||
|
||||
|
||||
func GetLinks(url *urllib.URL, selector string) []link {
|
||||
selectorSet := true
|
||||
if selector == "" {
|
||||
selector = "a"
|
||||
selectorSet = false
|
||||
}
|
||||
|
||||
// visit and count link classes
|
||||
pathLinks := map[string][]link{}
|
||||
pathCount := map[string]int{}
|
||||
pathMax := ""
|
||||
|
||||
c := colly.NewCollector()
|
||||
c.OnHTML(selector, func(e *colly.HTMLElement) {
|
||||
href := e.Attr("href")
|
||||
text := strings.TrimSpace(e.Text)
|
||||
path := GetPath(e.DOM)
|
||||
class := e.Attr("class")
|
||||
key := fmt.Sprintf("%s.%s", path, class)
|
||||
|
||||
if selectorSet || text != "" {
|
||||
pathLinks[key] = append(pathLinks[key], NewLink(href, text, class))
|
||||
pathCount[key] += len(text)
|
||||
// pathCount[key]++
|
||||
|
||||
if pathCount[key] > pathCount[pathMax] {
|
||||
pathMax = key
|
||||
}
|
||||
}
|
||||
})
|
||||
c.Visit(url.String())
|
||||
return pathLinks[pathMax]
|
||||
|
||||
// // visit and count link classes
|
||||
// classesLinks := map[string][]link{}
|
||||
// classesCount := map[string]int{}
|
||||
// classMax := ""
|
||||
|
||||
// c := colly.NewCollector()
|
||||
// c.OnHTML(selector, func(e *colly.HTMLElement) {
|
||||
// href := e.Attr("href")
|
||||
// text := strings.TrimSpace(e.Text)
|
||||
// class := e.Attr("class")
|
||||
|
||||
// if selectorSet || class != "" && text != "" {
|
||||
// classesLinks[class] = append(classesLinks[class], NewLink(href, text))
|
||||
// classesCount[class]++
|
||||
|
||||
// if classesCount[class] > classesCount[classMax] {
|
||||
// classMax = class
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
// c.Visit(url.String())
|
||||
// return classesLinks[classMax]
|
||||
}
|
||||
|
||||
15
cmd/get.go
15
cmd/get.go
@@ -68,20 +68,7 @@ var getCmd = &cobra.Command{
|
||||
},
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
url := args[0]
|
||||
var b book.Book
|
||||
|
||||
if recursive {
|
||||
b = book.NewBookFromURL(url, selector, include, limit, delay)
|
||||
} else {
|
||||
c := book.NewChapterFromURL(url)
|
||||
b = book.New(c.Name(), c.Author())
|
||||
b.AddChapter(c)
|
||||
}
|
||||
|
||||
// if quiet == false {
|
||||
// metadata := fmt.Sprintf("URL : %s\nTitle : %s\nAuthor : %s\nLength : %d\nExcerpt : %s\nSiteName: %s\nImage : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
|
||||
// fmt.Println(metadata)
|
||||
// }
|
||||
b := book.NewBookFromURL(url, selector, recursive, include, limit, delay)
|
||||
|
||||
if len(output) == 0 {
|
||||
// set default output
|
||||
|
||||
47
cmd/list.go
47
cmd/list.go
@@ -2,13 +2,14 @@ package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
urllib "net/url"
|
||||
"strings"
|
||||
"os"
|
||||
|
||||
colly "github.com/gocolly/colly/v2"
|
||||
"github.com/jedib0t/go-pretty/v6/table"
|
||||
cobra "github.com/spf13/cobra"
|
||||
|
||||
"github.com/lapwat/papeer/book"
|
||||
)
|
||||
|
||||
var listCmd = &cobra.Command{
|
||||
@@ -26,44 +27,22 @@ var listCmd = &cobra.Command{
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if selector == "" {
|
||||
selector = "a"
|
||||
}
|
||||
links := book.GetLinks(base, selector)
|
||||
|
||||
// visit and count link classes
|
||||
classesLinks := map[string][]map[string]string{}
|
||||
classesCount := map[string]int{}
|
||||
classMax := ""
|
||||
t := table.NewWriter()
|
||||
t.SetOutputMirror(os.Stdout)
|
||||
t.AppendHeader(table.Row{"#", "Name", "Url", "Class"})
|
||||
|
||||
c := colly.NewCollector()
|
||||
c.OnHTML(selector, func(e *colly.HTMLElement) {
|
||||
href := e.Attr("href")
|
||||
text := strings.TrimSpace(e.Text)
|
||||
class := e.Attr("class")
|
||||
|
||||
if cmd.Flags().Changed("selector") || class != "" && text != "" {
|
||||
classesLinks[class] = append(classesLinks[class], map[string]string{
|
||||
"href": href,
|
||||
"text": text,
|
||||
})
|
||||
|
||||
classesCount[class]++
|
||||
|
||||
if classesCount[class] > classesCount[classMax] {
|
||||
classMax = class
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
c.Visit(base.String())
|
||||
for index, link := range classesLinks[classMax] {
|
||||
u, err := base.Parse(link["href"])
|
||||
for index, link := range links {
|
||||
u, err := base.Parse(link.Href())
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("Chapter %d: %s %s\n", index+1, link["text"], u.String())
|
||||
t.AppendRow([]interface{}{index + 1, link.Text(), u.String(), link.Class()})
|
||||
}
|
||||
|
||||
t.Render()
|
||||
|
||||
},
|
||||
}
|
||||
|
||||
@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
|
||||
Use: "version",
|
||||
Short: "Print the version number of papeer",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
fmt.Println("papeer v0.0.2")
|
||||
fmt.Println("papeer v0.1.1")
|
||||
},
|
||||
}
|
||||
|
||||
4
go.mod
4
go.mod
@@ -10,13 +10,13 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.7.1
|
||||
github.com/gocolly/colly/v2 v2.1.0
|
||||
github.com/gosuri/uilive v0.0.4 // indirect
|
||||
github.com/gosuri/uiprogress v0.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.7.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.2.0 // indirect
|
||||
github.com/antchfx/htmlquery v1.2.3 // indirect
|
||||
github.com/antchfx/xmlquery v1.3.6 // indirect
|
||||
@@ -28,8 +28,10 @@ require (
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
||||
github.com/jedib0t/go-pretty/v6 v6.2.4 // indirect
|
||||
github.com/kennygrant/sanitize v1.2.4 // indirect
|
||||
github.com/mattn/go-isatty v0.0.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.9 // indirect
|
||||
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
|
||||
16
go.sum
16
go.sum
@@ -42,7 +42,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
|
||||
github.com/JohannesKaufmann/html-to-markdown v1.3.0 h1:K/p4cq8Ib13hcSVcKQNfKCSWw93CYW5pAjY0fl85has=
|
||||
github.com/JohannesKaufmann/html-to-markdown v1.3.0/go.mod h1:JNSClIRYICFDiFhw6RBhBeWGnMSSKVZ6sPQA+TK4tyM=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
|
||||
github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
|
||||
github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
|
||||
github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
|
||||
@@ -53,12 +52,10 @@ github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5
|
||||
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
|
||||
github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
|
||||
github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
|
||||
github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
|
||||
github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
|
||||
github.com/antchfx/xmlquery v1.3.6 h1:kaEVzH1mNo/2AJZrhZjAaAUTy2Nn2zxGfYYU8jWfXOo=
|
||||
github.com/antchfx/xmlquery v1.3.6/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
|
||||
github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk=
|
||||
github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
|
||||
github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
|
||||
@@ -106,6 +103,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@@ -125,7 +123,6 @@ github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuu
|
||||
github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
|
||||
github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gofrs/uuid v3.1.0+incompatible h1:q2rtkjaKT4YEr6E1kamy0Ha4RtepWlQBedyHx0uzKwA=
|
||||
github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
|
||||
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
@@ -138,7 +135,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
|
||||
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
@@ -236,6 +232,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
|
||||
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
|
||||
github.com/jedib0t/go-pretty/v6 v6.2.4 h1:wdaj2KHD2W+mz8JgJ/Q6L/T5dB7kyqEFI16eLq7GEmk=
|
||||
github.com/jedib0t/go-pretty/v6 v6.2.4/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
@@ -256,10 +254,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
@@ -281,6 +280,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
|
||||
github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
@@ -341,7 +341,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
|
||||
github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
|
||||
github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
|
||||
@@ -486,6 +485,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -532,7 +532,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
|
||||
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -709,7 +708,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
|
||||
Reference in New Issue
Block a user