refacto progress bars, images option

refacto, better scrape strategy
better display, release script
2026-05-25 12:27:20 +00:00 · 2021-10-06 23:41:23 +02:00 · 2021-09-24 02:09:22 +02:00 · 2021-09-21 19:55:37 +02:00 · 2021-09-15 20:51:16 +02:00 · 2021-09-15 20:47:53 +02:00
12 changed files with 390 additions and 224 deletions
--- a/README.md
+++ b/README.md
@@ -1,42 +1,54 @@
 ```
-❯ papeer get --format epub --recursive --delay 500 --limit 10 https://news.ycombinator.com/
-   6s [===============================================>--------------------]  70% Status: 7 out of 10 chapters
-   0s [====================================================================] 100% 1. Three ex-US intelligence officers admit hacking for UAE
-   0s [====================================================================] 100% 2. Show HN: Time Travel Debugger
-   0s [====================================================================] 100% 3. How much faster is Java 17?
-   0s [====================================================================] 100% 4. The First Webcam Was Invented to Keep an Eye on a Coffee Pot
-   0s [====================================================================] 100% 5. Nikon's 2021 Photomicrography Competition Winners
-   0s [====================================================================] 100% 6. HTTP Status 418 – I'm a teapot
-   0s [====================================================================] 100% 7. H3: Hexagonal hierarchical geospatial indexing system
-  --- [--------------------------------------------------------------------]   0% 8. Automatic cipher suite ordering in Go’s crypto/tls
-  --- [--------------------------------------------------------------------]   0% 9. Find engineering roles at over 800 YC-funded startups
-  --- [--------------------------------------------------------------------]   0% 10. Futarchy: Robin Hanson on prediction markets
+❯ papeer get --format=epub --recursive --delay=500 --limit=10 https://news.ycombinator.com/
+[===============================================>--------------------] Chapters 7 / 10
+[====================================================================] 1. Three ex-US intelligence officers admit hacking for UAE
+[====================================================================] 2. Show HN: Time Travel Debugger
+[====================================================================] 3. How much faster is Java 17?
+[====================================================================] 4. The First Webcam Was Invented to Keep an Eye on a Coffee Pot
+[====================================================================] 5. Nikon's 2021 Photomicrography Competition Winners
+[====================================================================] 6. HTTP Status 418 – I'm a teapot
+[====================================================================] 7. H3: Hexagonal hierarchical geospatial indexing system
+[--------------------------------------------------------------------] 8. Automatic cipher suite ordering in Go’s crypto/tls
+[--------------------------------------------------------------------] 9. Find engineering roles at over 800 YC-funded startups
+[--------------------------------------------------------------------] 10. Futarchy: Robin Hanson on prediction markets
 Ebook saved to "Hacker_News.epub"
 ```

 # Installation

-## From binary
-
-```sh
-curl https://github.com/lapwat/papeer/releases/download/v0.0.1/papeer-v0.0.1 > papeer
-chmod +x papeer
-sudo mv papeer /usr/local/bin
-```
-
-```sh
-# (optional) install kindlegen to export ebooks to MOBI format
-curl https://github.com/lapwat/papeer/raw/master/bin/kindlegen_linux_2.6_i386_v2_9.tar.gz > kindlegen
-chmod +x kindlegen
-sudo mv kindlegen /usr/local/bin
-```
-
 ## From source

 ```sh
 go get -u github.com/lapwat/papeer
 ```

+## From binary
+
+### On Linux / MacOS
+
+```sh
+platform=linux
+# platform=darwin for MacOS
+curl -L https://github.com/lapwat/papeer/releases/download/v0.2.0/papeer-v0.2.0-$platform-amd64 > papeer
+chmod +x papeer
+sudo mv papeer /usr/local/bin
+```
+
+### On Windows
+
+Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.2.0/papeer-v0.2.0-windows-amd64.exe).
+
+## Install kindlegen to export websites to MOBI (optional)
+
+```sh
+TMPDIR=$(mktemp -d -t papeer-XXXXX)
+curl -L https://github.com/lapwat/papeer/releases/download/kindlegen/kindlegen_linux_2.6_i386_v2_9.tar.gz > $TMPDIR/kindlegen.tar.gz
+tar xzvf $TMPDIR/kindlegen.tar.gz -C $TMPDIR
+chmod +x $TMPDIR/kindlegen
+sudo mv $TMPDIR/kindlegen /usr/local/bin
+rm -rf $TMPDIR
+```
+
 # Usage

 ```
@@ -54,12 +66,14 @@ Available Commands:
  version     Print the version number of papeer

 Flags:
-  -d, --delay int         wait before downloading next chapter, in milliseconds (default -1)
+  -d, --delay int         time to wait before downloading next chapter, in milliseconds (default -1)
  -f, --format string     file format [md, epub, mobi] (default "md")
  -h, --help              help for papeer
+      --images            retrieve images only
  -i, --include           include URL as first chapter, in resursive mode
-  -o, --output string     output file
-  -q, --quiet             do not show progress bars
+  -l, --limit int         limit number of chapters, in recursive mode (default -1)
+  -o, --offset int        skip first chapters, in recursive mode
+      --output string     output file
  -r, --recursive         create one chapter per natigation item
  -s, --selector string   table of content CSS selector
      --stdout            print to standard output
--- a/book/book.go
+++ b/book/book.go
@@ -1,27 +1,27 @@
 package book

-type Book struct {
+type book struct {
 	name     string
 	author   string
 	chapters []chapter
 }

-func New(name, author string) Book {
-	return Book{name, author, []chapter{}}
+func New(name, author string) book {
+	return book{name, author, []chapter{}}
 }

-func (b *Book) AddChapter(c chapter) {
+func (b *book) AddChapter(c chapter) {
 	b.chapters = append(b.chapters, c)
 }

-func (b Book) Name() string {
+func (b book) Name() string {
 	return b.name
 }

-func (b Book) Author() string {
+func (b book) Author() string {
 	return b.name
 }

-func (b *Book) Chapters() []chapter {
+func (b *book) Chapters() []chapter {
 	return b.chapters
 }
--- a/book/link.go
+++ b/book/link.go
@@ -0,0 +1,18 @@
+package book
+
+type link struct {
+	href string
+	text string
+}
+
+func NewLink(href, text string) link {
+	return link{href, text}
+}
+
+func (c link) Href() string {
+	return c.href
+}
+
+func (c link) Text() string {
+	return c.text
+}
--- a/book/progress.go
+++ b/book/progress.go
@@ -0,0 +1,47 @@
+package book
+
+import (
+	"fmt"
+
+	"github.com/gosuri/uiprogress"
+)
+
+type progress struct {
+	global      *uiprogress.Bar
+	individuals []*uiprogress.Bar
+}
+
+func NewProgress(links []link) progress {
+	uiprogress.Start()
+
+	global := uiprogress.AddBar(len(links))
+	global.AppendFunc(func(b *uiprogress.Bar) string {
+		return fmt.Sprintf("Chapters %d / %d", b.Current(), len(links))
+	})
+
+	individuals := []*uiprogress.Bar{}
+	// hide individual bars if more than 50 chapters
+	if len(links) <= 50 {
+		for index, link := range links {
+			bar := uiprogress.AddBar(1)
+			barText := fmt.Sprintf("%d. %s", index+1, link.text)
+			bar.AppendFunc(func(b *uiprogress.Bar) string {
+				return barText
+			})
+			individuals = append(individuals, bar)
+		}
+	}
+
+	return progress{global, individuals}
+}
+
+func (p *progress) IncrGlobal() {
+	p.global.Incr()
+}
+
+func (p *progress) Incr(index int) {
+	p.global.Incr()
+	if len(p.individuals) > index {
+		p.individuals[index].Incr()
+	}
+}
--- a/book/scraper.go
+++ b/book/scraper.go
@@ -9,147 +9,181 @@ import (
 	"sync"
 	"time"

+	"github.com/PuerkitoBio/goquery"
 	readability "github.com/go-shiori/go-readability"
 	colly "github.com/gocolly/colly/v2"
-	"github.com/gosuri/uiprogress"
 )

-type scraper struct {
-	url string
-}
+func NewBookFromURL(url, selector string, recursive, include, images bool, limit, offset, delay int) book {
+	if recursive {
+		chapters := tableOfContent(url, selector, limit, offset, delay, include, images)

-func NewBookFromURL(url, selector string, include bool, limit, delay int) Book {
-	home := NewChapterFromURL(url)
-	b := New(home.Name(), home.Author())
+		b := New(chapters[0].Name(), chapters[0].Author())
+		for _, c := range chapters {
+			b.AddChapter(c)
+		}

-	chapters := tableOfContent(url, selector, limit, delay)
-	if include {
-		b.AddChapter(home)
-	}
-	for _, c := range chapters {
+		return b
+	} else {
+		c := NewChapterFromURL(url, images)
+		b := New(c.Name(), c.Author())
 		b.AddChapter(c)
+		return b
 	}
-
-	return b
 }

-func NewChapterFromURL(url string) chapter {
+func NewChapterFromURL(url string, images bool) chapter {
 	article, err := readability.FromURL(url, 30*time.Second)
 	if err != nil {
 		log.Fatalf("failed to parse %s, %v\n", url, err)
 	}

-	// metadata := fmt.Sprintf("URL     : %s\nTitle   : %s\nAuthor  : %s\nLength  : %d\nExcerpt : %s\nSiteName: %s\nImage   : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
-	// fmt.Println(metadata)
+	content := strings.ReplaceAll(article.Content, "\n", "")

-	return chapter{article.Title, article.Byline, article.Content}
+	if images {
+		// Load the HTML document
+		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Find the review items
+		doc.Find("img").Each(func(i int, s *goquery.Selection) {
+			content, _ = goquery.OuterHtml(s)
+		})
+	}
+
+	return chapter{article.Title, article.Byline, content}
 }

-func tableOfContent(url, selector string, limit, delay int) []chapter {
-	c := colly.NewCollector()
+func tableOfContent(url, selector string, limit, offset, delay int, include, images bool) []chapter {
+	base, err := urllib.Parse(url)
+	if err != nil {
+		log.Fatal(err)
+	}

-	classesLinks := map[string][]map[string]string{}
-	classesCount := map[string]int{}
-	classMax := ""
+	links, err := GetLinks(base, selector, limit, offset, include)
+	if err != nil {
+		log.Fatal(err)
+	}

+	chapters := make([]chapter, len(links))
+	progress := NewProgress(links)
+
+	if delay >= 0 {
+
+		for index, link := range links {
+			// and then use it to parse relative URLs
+			u, err := base.Parse(link.href)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			chapters[index] = NewChapterFromURL(u.String(), images)
+			progress.Incr(index)
+
+			// short sleep for last chapter to let the progress bar update
+			if index == len(links)-1 {
+				delay = 100
+			}
+
+			time.Sleep(time.Duration(delay) * time.Millisecond)
+		}
+
+	} else {
+		var wg sync.WaitGroup
+		for index, l := range links {
+
+			wg.Add(1)
+			go func(index int, l link) {
+				defer wg.Done()
+
+				// and then use it to parse relative URLs
+				u, err := base.Parse(l.href)
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				chapters[index] = NewChapterFromURL(u.String(), images)
+				progress.Incr(index)
+
+			}(index, l)
+		}
+		wg.Wait()
+	}
+	return chapters
+}
+
+func GetPath(elm *goquery.Selection) string {
+	path := []string{}
+
+	for {
+		selector := strings.ToLower(goquery.NodeName(elm))
+		if selector == "" {
+			break
+		}
+
+		path = append(path, selector)
+		elm = elm.Parent()
+	}
+
+	join := strings.Join(path, "<")
+	return join
+}
+
+func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, error) {
 	selectorSet := true
 	if selector == "" {
 		selector = "a"
 		selectorSet = false
 	}

+	// visit and count link classes
+	pathLinks := map[string][]link{}
+	pathCount := map[string]int{}
+	pathMax := ""
+
+	c := colly.NewCollector()
 	c.OnHTML(selector, func(e *colly.HTMLElement) {
 		href := e.Attr("href")
 		text := strings.TrimSpace(e.Text)
-		class := e.Attr("class")
+		path := GetPath(e.DOM)
+		key := path

-		if selectorSet || class != "" && text != "" {
-			classesLinks[class] = append(classesLinks[class], map[string]string{
-				"href": href,
-				"text": text,
-			})
-
-			classesCount[class]++
-
-			if classesCount[class] > classesCount[classMax] {
-				classMax = class
-			}
+		// include element class in key if selector is set
+		if !selectorSet {
+			class := e.Attr("class")
+			key = fmt.Sprintf("%s.%s", path, class)
 		}

-	})
-	c.Visit(url)
+		if selectorSet || text != "" {
+			pathLinks[key] = append(pathLinks[key], NewLink(href, text))
+			pathCount[key] += len(text)

-	links := classesLinks[classMax]
+			if pathCount[key] > pathCount[pathMax] {
+				pathMax = key
+			}
+		}
+	})
+	c.Visit(url.String())
+
+	links := pathLinks[pathMax]
+	if len(links) == 0 {
+		return []link{}, fmt.Errorf("no link found for selector: %s", selector)
+	}
+
+	end := len(links)
 	if limit != -1 {
-		limit = int(math.Min(float64(limit), float64(len(links))))
-		links = links[:limit]
+		end = int(math.Min(float64(limit+offset), float64(len(links))))
 	}

-	chapters := make([]chapter, len(links))
-	base, err := urllib.Parse(url)
-	if err != nil {
-		log.Fatal(err)
+	links = links[offset:end]
+
+	if include {
+		c := NewChapterFromURL(url.String(), false)
+		l := NewLink(url.String(), c.Name())
+		links = append([]link{l}, links...)
 	}

-	// init global progress bar
-	uiprogress.Start()
-	barGlobal := uiprogress.AddBar(len(links)).AppendCompleted().PrependElapsed()
-	barGlobal.AppendFunc(func(b *uiprogress.Bar) string {
-		return fmt.Sprintf("Status: %d out of %d chapters", b.Current(), len(links))
-	})
-
-	// init progress bars
-	bars := []*uiprogress.Bar{}
-	for index, link := range links {
-		bar := uiprogress.AddBar(1).AppendCompleted().PrependElapsed()
-		barText := fmt.Sprintf("%d. %s", index+1, link["text"])
-		bar.AppendFunc(func(b *uiprogress.Bar) string {
-			return barText
-		})
-		bars = append(bars, bar)
-	}
-
-	if delay >= 0 {
-		for index, link := range links {
-			// and then use it to parse relative URLs
-			u, err := base.Parse(link["href"])
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			chapters[index] = NewChapterFromURL(u.String())
-
-			bars[index].Incr()
-			barGlobal.Incr()
-
-			// do not wait after downloading last chapter
-			if index < len(links)-1 {
-				time.Sleep(time.Duration(delay) * time.Millisecond)
-			}
-
-		}
-
-	} else {
-		var wg sync.WaitGroup
-		for index, link := range links {
-
-			wg.Add(1)
-			go func(index int, link map[string]string) {
-				defer wg.Done()
-
-				// and then use it to parse relative URLs
-				u, err := base.Parse(link["href"])
-				if err != nil {
-					log.Fatal(err)
-				}
-
-				chapters[index] = NewChapterFromURL(u.String())
-
-				bars[index].Incr()
-				barGlobal.Incr()
-			}(index, link)
-		}
-		wg.Wait()
-	}
-	return chapters
+	return links, nil
 }
--- a/cmd/get.go
+++ b/cmd/get.go
@@ -15,9 +15,9 @@ import (
 	"github.com/lapwat/papeer/book"
 )

-var quiet, stdout, recursive, include bool
+var stdout, recursive, include, images bool
 var format, output, selector string
-var limit, delay int
+var limit, offset, delay int

 var getCmd = &cobra.Command{
 	Use:   "get",
@@ -60,6 +60,10 @@ var getCmd = &cobra.Command{
 			return errors.New("cannot use limit option if not in recursive mode")
 		}

+		if cmd.Flags().Changed("offset") && recursive == false {
+			return errors.New("cannot use offset option if not in recursive mode")
+		}
+
 		if cmd.Flags().Changed("delay") && recursive == false {
 			return errors.New("cannot use delay option if not in recursive mode")
 		}
@@ -68,20 +72,7 @@ var getCmd = &cobra.Command{
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		url := args[0]
-		var b book.Book
-
-		if recursive {
-			b = book.NewBookFromURL(url, selector, include, limit, delay)
-		} else {
-			c := book.NewChapterFromURL(url)
-			b = book.New(c.Name(), c.Author())
-			b.AddChapter(c)
-		}
-
-		// if quiet == false {
-		// 	metadata := fmt.Sprintf("URL     : %s\nTitle   : %s\nAuthor  : %s\nLength  : %d\nExcerpt : %s\nSiteName: %s\nImage   : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
-		// 	fmt.Println(metadata)
-		// }
+		b := book.NewBookFromURL(url, selector, recursive, include, images, limit, offset, delay)

 		if len(output) == 0 {
 			// set default output
@@ -91,12 +82,16 @@ var getCmd = &cobra.Command{
 		}

 		if format == "md" {
-			f, err := os.Create(output)
-			if err != nil {
-				log.Fatal(err)
-			}
+			var f *os.File
+			var err error

-			defer f.Close()
+			if !stdout {
+				f, err = os.Create(output)
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer f.Close()
+			}

 			for _, c := range b.Chapters() {
 				content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
@@ -104,11 +99,12 @@ var getCmd = &cobra.Command{
 					log.Fatal(err)
 				}

-				if stdout {
-					fmt.Println(content)
-				} else {
+				text := fmt.Sprintf("%s\n%s\n\n%s\n\n\n", c.Name(), strings.Repeat("=", len(c.Name())), content)

-					_, err := f.WriteString(content)
+				if stdout {
+					fmt.Println(text)
+				} else {
+					_, err := f.WriteString(text)
 					if err != nil {
 						log.Fatal(err)
 					}
@@ -126,7 +122,16 @@ var getCmd = &cobra.Command{
 			e.SetAuthor(b.Author())

 			for _, c := range b.Chapters() {
-				e.AddSection(c.Content(), c.Name(), "", "")
+				if images {
+					e.AddSection(c.Content(), "", "", "")
+				} else {
+					html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), c.Content())
+
+					_, err := e.AddSection(html, c.Name(), "", "")
+					if err != nil {
+						log.Fatal(err)
+					}
+				}
 			}

 			err := e.Write(output)
@@ -162,7 +167,7 @@ var getCmd = &cobra.Command{

 			err2 := os.Remove(outputEPUB)
 			if err2 != nil {
-				log.Fatal(err)
+				log.Fatal(err2)
 			}
 		}
 	},
--- a/cmd/list.go
+++ b/cmd/list.go
@@ -2,13 +2,14 @@ package cmd

 import (
 	"errors"
-	"fmt"
 	"log"
 	urllib "net/url"
-	"strings"
+	"os"

-	colly "github.com/gocolly/colly/v2"
+	"github.com/jedib0t/go-pretty/v6/table"
 	cobra "github.com/spf13/cobra"
+
+	"github.com/lapwat/papeer/book"
 )

 var listCmd = &cobra.Command{
@@ -26,44 +27,28 @@ var listCmd = &cobra.Command{
 			log.Fatal(err)
 		}

-		if selector == "" {
-			selector = "a"
+		links, err := book.GetLinks(base, selector, limit, offset, include)
+		if err != nil {
+			log.Fatal(err)
 		}

-		// visit and count link classes
-		classesLinks := map[string][]map[string]string{}
-		classesCount := map[string]int{}
-		classMax := ""
+		t := table.NewWriter()
+		t.SetOutputMirror(os.Stdout)
+		t.Style().Options.DrawBorder = false
+		t.Style().Options.SeparateColumns = false
+		t.Style().Options.SeparateHeader = false
+		t.AppendHeader(table.Row{"#", "Name", "Url"})

-		c := colly.NewCollector()
-		c.OnHTML(selector, func(e *colly.HTMLElement) {
-			href := e.Attr("href")
-			text := strings.TrimSpace(e.Text)
-			class := e.Attr("class")
-
-			if cmd.Flags().Changed("selector") || class != "" && text != "" {
-				classesLinks[class] = append(classesLinks[class], map[string]string{
-					"href": href,
-					"text": text,
-				})
-
-				classesCount[class]++
-
-				if classesCount[class] > classesCount[classMax] {
-					classMax = class
-				}
-			}
-		})
-
-		c.Visit(base.String())
-		for index, link := range classesLinks[classMax] {
-			u, err := base.Parse(link["href"])
+		for index, link := range links {
+			u, err := base.Parse(link.Href())
 			if err != nil {
 				log.Fatal(err)
 			}

-			fmt.Printf("Chapter %d: %s %s\n", index+1, link["text"], u.String())
+			t.AppendRow([]interface{}{index + 1, link.Text(), u.String()})
 		}

+		t.Render()
+
 	},
 }
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -24,13 +24,14 @@ func Execute() {

 func init() {
 	rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "md", "file format [md, epub, mobi]")
-	rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "", "output file")
-	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector")
+	rootCmd.PersistentFlags().StringVarP(&output, "output", "", "", "output file")
+	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector, in resursive mode")
 	rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "create one chapter per natigation item")
 	rootCmd.PersistentFlags().BoolVarP(&include, "include", "i", false, "include URL as first chapter, in resursive mode")
-	rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "do not show logs")
 	rootCmd.PersistentFlags().BoolVarP(&stdout, "stdout", "", false, "print to standard output")
+	rootCmd.PersistentFlags().BoolVarP(&images, "images", "", false, "retrieve images only")
 	rootCmd.PersistentFlags().IntVarP(&limit, "limit", "l", -1, "limit number of chapters, in recursive mode")
+	rootCmd.PersistentFlags().IntVarP(&offset, "offset", "o", 0, "skip first chapters, in recursive mode")
 	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "time to wait before downloading next chapter, in milliseconds")

 	rootCmd.AddCommand(getCmd)
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print the version number of papeer",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("papeer v0.0.2")
+		fmt.Println("papeer v0.1.1")
 	},
 }
--- a/go.mod
+++ b/go.mod
@@ -10,33 +10,40 @@ require (
 )

 require (
+	github.com/PuerkitoBio/goquery v1.7.1
 	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gosuri/uilive v0.0.4 // indirect
 	github.com/gosuri/uiprogress v0.0.1
 )

 require (
-	github.com/PuerkitoBio/goquery v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.2.0 // indirect
 	github.com/antchfx/htmlquery v1.2.3 // indirect
-	github.com/antchfx/xmlquery v1.2.4 // indirect
-	github.com/antchfx/xpath v1.1.8 // indirect
+	github.com/antchfx/xmlquery v1.3.6 // indirect
+	github.com/antchfx/xpath v1.2.0 // indirect
 	github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gofrs/uuid v3.1.0+incompatible // indirect
+	github.com/gofrs/uuid v4.0.0+incompatible // indirect
 	github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jedib0t/go-pretty/v6 v6.2.4 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
-	github.com/mattn/go-isatty v0.0.3 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/schollz/progressbar/v3 v3.8.3 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/temoto/robotstxt v1.1.1 // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
-	golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect
+	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -42,8 +42,9 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/JohannesKaufmann/html-to-markdown v1.3.0 h1:K/p4cq8Ib13hcSVcKQNfKCSWw93CYW5pAjY0fl85has=
 github.com/JohannesKaufmann/html-to-markdown v1.3.0/go.mod h1:JNSClIRYICFDiFhw6RBhBeWGnMSSKVZ6sPQA+TK4tyM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
+github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
@@ -51,11 +52,14 @@ github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5
 github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
 github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
 github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
-github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
 github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
+github.com/antchfx/xmlquery v1.3.6 h1:kaEVzH1mNo/2AJZrhZjAaAUTy2Nn2zxGfYYU8jWfXOo=
+github.com/antchfx/xmlquery v1.3.6/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
 github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
-github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk=
 github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
+github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -99,6 +103,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -118,8 +123,9 @@ github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuu
 github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
 github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/uuid v3.1.0+incompatible h1:q2rtkjaKT4YEr6E1kamy0Ha4RtepWlQBedyHx0uzKwA=
 github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
+github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -129,8 +135,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -225,12 +232,15 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
+github.com/jedib0t/go-pretty/v6 v6.2.4 h1:wdaj2KHD2W+mz8JgJ/Q6L/T5dB7kyqEFI16eLq7GEmk=
+github.com/jedib0t/go-pretty/v6 v6.2.4/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -245,11 +255,18 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -268,6 +285,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -282,6 +300,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -289,6 +309,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8=
+github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sebdah/goldie/v2 v2.5.1 h1:hh70HvG4n3T3MNRJN2z/baxPR8xutxo7JVxyi2svl+s=
 github.com/sebdah/goldie/v2 v2.5.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
@@ -328,8 +350,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
 github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
+github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -366,6 +389,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -435,6 +460,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -470,6 +496,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -516,9 +543,15 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -691,8 +724,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
--- a/release.sh
+++ b/release.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+version=$1
+platforms=("linux/amd64" "darwin/amd64" "windows/amd64")
+
+for platform in "${platforms[@]}"
+do
+    platform_split=(${platform//\// })
+    GOOS=${platform_split[0]}
+    GOARCH=${platform_split[1]}
+    output_name='papeer-'$version'-'$GOOS'-'$GOARCH
+    if [ $GOOS = "windows" ]; then
+        output_name+='.exe'
+    fi
+
+    env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name
+    if [ $? -ne 0 ]; then
+        echo 'An error has occurred! Aborting the script execution...'
+        exit 1
+    fi
+done
Author	SHA1	Message	Date
lapwat	2d1d5a964a	refacto progress bars, images option	2021-10-06 23:41:23 +02:00
lapwat	ee41e49dd1	refacto, better scrape strategy	2021-09-24 02:09:22 +02:00
lapwat	e50adf5e03	better display, release script	2021-09-21 19:55:37 +02:00
lapwat	079122f4a0	docs	2021-09-15 20:51:16 +02:00
lapwat	9ee5f43748	docs	2021-09-15 20:47:53 +02:00
lapwat	e3833fdddd	docs	2021-09-15 20:47:22 +02:00