diff --git a/book/book.go b/book/book.go index 6013e66..ffdc068 100644 --- a/book/book.go +++ b/book/book.go @@ -1,27 +1,27 @@ package book -type Book struct { +type book struct { name string author string chapters []chapter } -func New(name, author string) Book { - return Book{name, author, []chapter{}} +func New(name, author string) book { + return book{name, author, []chapter{}} } -func (b *Book) AddChapter(c chapter) { +func (b *book) AddChapter(c chapter) { b.chapters = append(b.chapters, c) } -func (b Book) Name() string { +func (b book) Name() string { return b.name } -func (b Book) Author() string { +func (b book) Author() string { return b.name } -func (b *Book) Chapters() []chapter { +func (b *book) Chapters() []chapter { return b.chapters } diff --git a/book/link.go b/book/link.go new file mode 100644 index 0000000..4f004a9 --- /dev/null +++ b/book/link.go @@ -0,0 +1,23 @@ +package book + +type link struct { + href string + text string + class string +} + +func NewLink(href, text, class string) link { + return link{href, text, class} +} + +func (c link) Href() string { + return c.href +} + +func (c link) Text() string { + return c.text +} + +func (c link) Class() string { + return c.class +} diff --git a/book/scraper.go b/book/scraper.go index b43a8e4..3a1027f 100644 --- a/book/scraper.go +++ b/book/scraper.go @@ -9,28 +9,32 @@ import ( "sync" "time" + "github.com/PuerkitoBio/goquery" readability "github.com/go-shiori/go-readability" colly "github.com/gocolly/colly/v2" "github.com/gosuri/uiprogress" ) -type scraper struct { - url string -} +func NewBookFromURL(url, selector string, recursive, include bool, limit, delay int) book { + if recursive { + home := NewChapterFromURL(url) + b := New(home.Name(), home.Author()) -func NewBookFromURL(url, selector string, include bool, limit, delay int) Book { - home := NewChapterFromURL(url) - b := New(home.Name(), home.Author()) + chapters := tableOfContent(url, selector, limit, delay) + if include { + b.AddChapter(home) + } + for _, c := range chapters { + b.AddChapter(c) + } - chapters := tableOfContent(url, selector, limit, delay) - if include { - b.AddChapter(home) - } - for _, c := range chapters { + return b + } else { + c := NewChapterFromURL(url) + b := New(c.Name(), c.Author()) b.AddChapter(c) + return b } - - return b } func NewChapterFromURL(url string) chapter { @@ -39,57 +43,22 @@ func NewChapterFromURL(url string) chapter { log.Fatalf("failed to parse %s, %v\n", url, err) } - // metadata := fmt.Sprintf("URL : %s\nTitle : %s\nAuthor : %s\nLength : %d\nExcerpt : %s\nSiteName: %s\nImage : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon) - // fmt.Println(metadata) - return chapter{article.Title, article.Byline, article.Content} } func tableOfContent(url, selector string, limit, delay int) []chapter { - c := colly.NewCollector() - - classesLinks := map[string][]map[string]string{} - classesCount := map[string]int{} - classMax := "" - - selectorSet := true - if selector == "" { - selector = "a" - selectorSet = false + base, err := urllib.Parse(url) + if err != nil { + log.Fatal(err) } - c.OnHTML(selector, func(e *colly.HTMLElement) { - href := e.Attr("href") - text := strings.TrimSpace(e.Text) - class := e.Attr("class") - - if selectorSet || class != "" && text != "" { - classesLinks[class] = append(classesLinks[class], map[string]string{ - "href": href, - "text": text, - }) - - classesCount[class]++ - - if classesCount[class] > classesCount[classMax] { - classMax = class - } - } - - }) - c.Visit(url) - - links := classesLinks[classMax] + links := GetLinks(base, selector) if limit != -1 { limit = int(math.Min(float64(limit), float64(len(links)))) links = links[:limit] } chapters := make([]chapter, len(links)) - base, err := urllib.Parse(url) - if err != nil { - log.Fatal(err) - } // init global progress bar uiprogress.Start() @@ -102,7 +71,7 @@ func tableOfContent(url, selector string, limit, delay int) []chapter { bars := []*uiprogress.Bar{} for index, link := range links { bar := uiprogress.AddBar(1).AppendCompleted().PrependElapsed() - barText := fmt.Sprintf("%d. %s", index+1, link["text"]) + barText := fmt.Sprintf("%d. %s", index+1, link.text) bar.AppendFunc(func(b *uiprogress.Bar) string { return barText }) @@ -112,7 +81,7 @@ func tableOfContent(url, selector string, limit, delay int) []chapter { if delay >= 0 { for index, link := range links { // and then use it to parse relative URLs - u, err := base.Parse(link["href"]) + u, err := base.Parse(link.href) if err != nil { log.Fatal(err) } @@ -131,14 +100,14 @@ func tableOfContent(url, selector string, limit, delay int) []chapter { } else { var wg sync.WaitGroup - for index, link := range links { + for index, l := range links { wg.Add(1) - go func(index int, link map[string]string) { + go func(index int, l link) { defer wg.Done() // and then use it to parse relative URLs - u, err := base.Parse(link["href"]) + u, err := base.Parse(l.href) if err != nil { log.Fatal(err) } @@ -147,9 +116,84 @@ func tableOfContent(url, selector string, limit, delay int) []chapter { bars[index].Incr() barGlobal.Incr() - }(index, link) + }(index, l) } wg.Wait() } return chapters } + +func GetPath(elm *goquery.Selection) string { + path := []string{} + + for { + selector := strings.ToLower(goquery.NodeName(elm)) + if selector == "" { + break + } + + path = append(path, selector) + elm = elm.Parent() + } + + join := strings.Join(path, "<") + return join +} + + +func GetLinks(url *urllib.URL, selector string) []link { + selectorSet := true + if selector == "" { + selector = "a" + selectorSet = false + } + + // visit and count link classes + pathLinks := map[string][]link{} + pathCount := map[string]int{} + pathMax := "" + + c := colly.NewCollector() + c.OnHTML(selector, func(e *colly.HTMLElement) { + href := e.Attr("href") + text := strings.TrimSpace(e.Text) + path := GetPath(e.DOM) + class := e.Attr("class") + key := fmt.Sprintf("%s.%s", path, class) + + if selectorSet || text != "" { + pathLinks[key] = append(pathLinks[key], NewLink(href, text, class)) + pathCount[key] += len(text) + // pathCount[key]++ + + if pathCount[key] > pathCount[pathMax] { + pathMax = key + } + } + }) + c.Visit(url.String()) + return pathLinks[pathMax] + + // // visit and count link classes + // classesLinks := map[string][]link{} + // classesCount := map[string]int{} + // classMax := "" + + // c := colly.NewCollector() + // c.OnHTML(selector, func(e *colly.HTMLElement) { + // href := e.Attr("href") + // text := strings.TrimSpace(e.Text) + // class := e.Attr("class") + + // if selectorSet || class != "" && text != "" { + // classesLinks[class] = append(classesLinks[class], NewLink(href, text)) + // classesCount[class]++ + + // if classesCount[class] > classesCount[classMax] { + // classMax = class + // } + // } + // }) + // c.Visit(url.String()) + // return classesLinks[classMax] +} diff --git a/cmd/get.go b/cmd/get.go index 1d2affb..f02b796 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -68,20 +68,7 @@ var getCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { url := args[0] - var b book.Book - - if recursive { - b = book.NewBookFromURL(url, selector, include, limit, delay) - } else { - c := book.NewChapterFromURL(url) - b = book.New(c.Name(), c.Author()) - b.AddChapter(c) - } - - // if quiet == false { - // metadata := fmt.Sprintf("URL : %s\nTitle : %s\nAuthor : %s\nLength : %d\nExcerpt : %s\nSiteName: %s\nImage : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon) - // fmt.Println(metadata) - // } + b := book.NewBookFromURL(url, selector, recursive, include, limit, delay) if len(output) == 0 { // set default output diff --git a/cmd/list.go b/cmd/list.go index bbe98a2..28f3806 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -2,13 +2,14 @@ package cmd import ( "errors" - "fmt" "log" urllib "net/url" - "strings" + "os" - colly "github.com/gocolly/colly/v2" + "github.com/jedib0t/go-pretty/v6/table" cobra "github.com/spf13/cobra" + + "github.com/lapwat/papeer/book" ) var listCmd = &cobra.Command{ @@ -26,44 +27,22 @@ var listCmd = &cobra.Command{ log.Fatal(err) } - if selector == "" { - selector = "a" - } + links := book.GetLinks(base, selector) - // visit and count link classes - classesLinks := map[string][]map[string]string{} - classesCount := map[string]int{} - classMax := "" + t := table.NewWriter() + t.SetOutputMirror(os.Stdout) + t.AppendHeader(table.Row{"#", "Name", "Url", "Class"}) - c := colly.NewCollector() - c.OnHTML(selector, func(e *colly.HTMLElement) { - href := e.Attr("href") - text := strings.TrimSpace(e.Text) - class := e.Attr("class") - - if cmd.Flags().Changed("selector") || class != "" && text != "" { - classesLinks[class] = append(classesLinks[class], map[string]string{ - "href": href, - "text": text, - }) - - classesCount[class]++ - - if classesCount[class] > classesCount[classMax] { - classMax = class - } - } - }) - - c.Visit(base.String()) - for index, link := range classesLinks[classMax] { - u, err := base.Parse(link["href"]) + for index, link := range links { + u, err := base.Parse(link.Href()) if err != nil { log.Fatal(err) } - fmt.Printf("Chapter %d: %s %s\n", index+1, link["text"], u.String()) + t.AppendRow([]interface{}{index + 1, link.Text(), u.String(), link.Class()}) } + t.Render() + }, } diff --git a/cmd/version.go b/cmd/version.go index 3d97d23..7336701 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of papeer", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("papeer v0.0.2") + fmt.Println("papeer v0.1.1") }, } diff --git a/go.mod b/go.mod index bc27619..2dca8c4 100644 --- a/go.mod +++ b/go.mod @@ -10,13 +10,13 @@ require ( ) require ( + github.com/PuerkitoBio/goquery v1.7.1 github.com/gocolly/colly/v2 v2.1.0 github.com/gosuri/uilive v0.0.4 // indirect github.com/gosuri/uiprogress v0.0.1 ) require ( - github.com/PuerkitoBio/goquery v1.7.1 // indirect github.com/andybalholm/cascadia v1.2.0 // indirect github.com/antchfx/htmlquery v1.2.3 // indirect github.com/antchfx/xmlquery v1.3.6 // indirect @@ -28,8 +28,10 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jedib0t/go-pretty/v6 v6.2.4 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 81eb8b8..aae6a73 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/JohannesKaufmann/html-to-markdown v1.3.0 h1:K/p4cq8Ib13hcSVcKQNfKCSWw93CYW5pAjY0fl85has= github.com/JohannesKaufmann/html-to-markdown v1.3.0/go.mod h1:JNSClIRYICFDiFhw6RBhBeWGnMSSKVZ6sPQA+TK4tyM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4= github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY= @@ -53,12 +52,10 @@ github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5 github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= -github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4= github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= github.com/antchfx/xmlquery v1.3.6 h1:kaEVzH1mNo/2AJZrhZjAaAUTy2Nn2zxGfYYU8jWfXOo= github.com/antchfx/xmlquery v1.3.6/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk= github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= @@ -106,6 +103,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -125,7 +123,6 @@ github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuu github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/uuid v3.1.0+incompatible h1:q2rtkjaKT4YEr6E1kamy0Ha4RtepWlQBedyHx0uzKwA= github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -138,7 +135,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -236,6 +232,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= +github.com/jedib0t/go-pretty/v6 v6.2.4 h1:wdaj2KHD2W+mz8JgJ/Q6L/T5dB7kyqEFI16eLq7GEmk= +github.com/jedib0t/go-pretty/v6 v6.2.4/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -256,10 +254,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -281,6 +280,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -341,7 +341,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= @@ -486,6 +485,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -532,7 +532,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -709,7 +708,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=