refacto progress bars, images option

README.md

@@ -1,16 +1,16 @@
 ```
-❯ papeer get --format epub --recursive --delay 500 --limit 10 https://news.ycombinator.com/
+❯ papeer get --format=epub --recursive --delay=500 --limit=10 https://news.ycombinator.com/
-   6s [===============================================>--------------------]  70% Status: 7 out of 10 chapters
+[===============================================>--------------------] Chapters 7 / 10
-   0s [====================================================================] 100% 1. Three ex-US intelligence officers admit hacking for UAE
+[====================================================================] 1. Three ex-US intelligence officers admit hacking for UAE
-   0s [====================================================================] 100% 2. Show HN: Time Travel Debugger
+[====================================================================] 2. Show HN: Time Travel Debugger
-   0s [====================================================================] 100% 3. How much faster is Java 17?
+[====================================================================] 3. How much faster is Java 17?
-   0s [====================================================================] 100% 4. The First Webcam Was Invented to Keep an Eye on a Coffee Pot
+[====================================================================] 4. The First Webcam Was Invented to Keep an Eye on a Coffee Pot
-   0s [====================================================================] 100% 5. Nikon's 2021 Photomicrography Competition Winners
+[====================================================================] 5. Nikon's 2021 Photomicrography Competition Winners
-   0s [====================================================================] 100% 6. HTTP Status 418 – I'm a teapot
+[====================================================================] 6. HTTP Status 418 – I'm a teapot
-   0s [====================================================================] 100% 7. H3: Hexagonal hierarchical geospatial indexing system
+[====================================================================] 7. H3: Hexagonal hierarchical geospatial indexing system
-  --- [--------------------------------------------------------------------]   0% 8. Automatic cipher suite ordering in Go’s crypto/tls
+[--------------------------------------------------------------------] 8. Automatic cipher suite ordering in Go’s crypto/tls
-  --- [--------------------------------------------------------------------]   0% 9. Find engineering roles at over 800 YC-funded startups
+[--------------------------------------------------------------------] 9. Find engineering roles at over 800 YC-funded startups
-  --- [--------------------------------------------------------------------]   0% 10. Futarchy: Robin Hanson on prediction markets
+[--------------------------------------------------------------------] 10. Futarchy: Robin Hanson on prediction markets
 Ebook saved to "Hacker_News.epub"
 ```
 
@@ -29,14 +29,14 @@ go get -u github.com/lapwat/papeer
 ```sh
 platform=linux
 # platform=darwin for MacOS
-curl -L https://github.com/lapwat/papeer/releases/download/v0.1.0/papeer-v0.1.0-$platform-amd64 > papeer
+curl -L https://github.com/lapwat/papeer/releases/download/v0.2.0/papeer-v0.2.0-$platform-amd64 > papeer
 chmod +x papeer
 sudo mv papeer /usr/local/bin
 ```
 
 ### On Windows
 
-Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.1.0/papeer-v0.1.0-windows-amd64.exe).
+Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.2.0/papeer-v0.2.0-windows-amd64.exe).
 
 ## Install kindlegen to export websites to MOBI (optional)
 
@@ -69,10 +69,11 @@ Flags:
   -d, --delay int         time to wait before downloading next chapter, in milliseconds (default -1)
   -f, --format string     file format [md, epub, mobi] (default "md")
   -h, --help              help for papeer
+      --images            retrieve images only
   -i, --include           include URL as first chapter, in resursive mode
   -l, --limit int         limit number of chapters, in recursive mode (default -1)
-  -o, --output string     output file
+  -o, --offset int        skip first chapters, in recursive mode
-  -q, --quiet             do not show logs
+      --output string     output file
   -r, --recursive         create one chapter per natigation item
   -s, --selector string   table of content CSS selector
       --stdout            print to standard output

book/link.go

@@ -3,11 +3,10 @@ package book
 type link struct {
 	href string
 	text string
-	class string
 }
 
-func NewLink(href, text, class string) link {
+func NewLink(href, text string) link {
-	return link{href, text, class}
+	return link{href, text}
 }
 
 func (c link) Href() string {
@@ -17,7 +16,3 @@ func (c link) Href() string {
 func (c link) Text() string {
 	return c.text
 }
-
-func (c link) Class() string {
-	return c.class
-}

book/progress.go

@@ -0,0 +1,47 @@
+package book
+
+import (
+	"fmt"
+
+	"github.com/gosuri/uiprogress"
+)
+
+type progress struct {
+	global      *uiprogress.Bar
+	individuals []*uiprogress.Bar
+}
+
+func NewProgress(links []link) progress {
+	uiprogress.Start()
+
+	global := uiprogress.AddBar(len(links))
+	global.AppendFunc(func(b *uiprogress.Bar) string {
+		return fmt.Sprintf("Chapters %d / %d", b.Current(), len(links))
+	})
+
+	individuals := []*uiprogress.Bar{}
+	// hide individual bars if more than 50 chapters
+	if len(links) <= 50 {
+		for index, link := range links {
+			bar := uiprogress.AddBar(1)
+			barText := fmt.Sprintf("%d. %s", index+1, link.text)
+			bar.AppendFunc(func(b *uiprogress.Bar) string {
+				return barText
+			})
+			individuals = append(individuals, bar)
+		}
+	}
+
+	return progress{global, individuals}
+}
+
+func (p *progress) IncrGlobal() {
+	p.global.Incr()
+}
+
+func (p *progress) Incr(index int) {
+	p.global.Incr()
+	if len(p.individuals) > index {
+		p.individuals[index].Incr()
+	}
+}

book/scraper.go

@@ -12,73 +12,66 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	readability "github.com/go-shiori/go-readability"
 	colly "github.com/gocolly/colly/v2"
-	"github.com/gosuri/uiprogress"
 )
 
-func NewBookFromURL(url, selector string, recursive, include bool, limit, delay int) book {
+func NewBookFromURL(url, selector string, recursive, include, images bool, limit, offset, delay int) book {
 	if recursive {
-		home := NewChapterFromURL(url)
+		chapters := tableOfContent(url, selector, limit, offset, delay, include, images)
-		b := New(home.Name(), home.Author())
 
-		chapters := tableOfContent(url, selector, limit, delay)
+		b := New(chapters[0].Name(), chapters[0].Author())
-		if include {
-			b.AddChapter(home)
-		}
 		for _, c := range chapters {
 			b.AddChapter(c)
 		}
 
 		return b
 	} else {
-		c := NewChapterFromURL(url)
+		c := NewChapterFromURL(url, images)
 		b := New(c.Name(), c.Author())
 		b.AddChapter(c)
 		return b
 	}
 }
 
-func NewChapterFromURL(url string) chapter {
+func NewChapterFromURL(url string, images bool) chapter {
 	article, err := readability.FromURL(url, 30*time.Second)
 	if err != nil {
 		log.Fatalf("failed to parse %s, %v\n", url, err)
 	}
 
-	return chapter{article.Title, article.Byline, article.Content}
+	content := strings.ReplaceAll(article.Content, "\n", "")
+
+	if images {
+		// Load the HTML document
+		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// Find the review items
+		doc.Find("img").Each(func(i int, s *goquery.Selection) {
+			content, _ = goquery.OuterHtml(s)
+		})
+	}
+
+	return chapter{article.Title, article.Byline, content}
 }
 
-func tableOfContent(url, selector string, limit, delay int) []chapter {
+func tableOfContent(url, selector string, limit, offset, delay int, include, images bool) []chapter {
 	base, err := urllib.Parse(url)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	links := GetLinks(base, selector)
+	links, err := GetLinks(base, selector, limit, offset, include)
-	if limit != -1 {
+	if err != nil {
-		limit = int(math.Min(float64(limit), float64(len(links))))
+		log.Fatal(err)
-		links = links[:limit]
 	}
 
 	chapters := make([]chapter, len(links))
-
+	progress := NewProgress(links)
-	// init global progress bar
-	uiprogress.Start()
-	barGlobal := uiprogress.AddBar(len(links)).AppendCompleted().PrependElapsed()
-	barGlobal.AppendFunc(func(b *uiprogress.Bar) string {
-		return fmt.Sprintf("Status: %d out of %d chapters", b.Current(), len(links))
-	})
-
-	// init progress bars
-	bars := []*uiprogress.Bar{}
-	for index, link := range links {
-		bar := uiprogress.AddBar(1).AppendCompleted().PrependElapsed()
-		barText := fmt.Sprintf("%d. %s", index+1, link.text)
-		bar.AppendFunc(func(b *uiprogress.Bar) string {
-			return barText
-		})
-		bars = append(bars, bar)
-	}
 
 	if delay >= 0 {
+
 		for index, link := range links {
 			// and then use it to parse relative URLs
 			u, err := base.Parse(link.href)
@@ -86,16 +79,15 @@ func tableOfContent(url, selector string, limit, delay int) []chapter {
 				log.Fatal(err)
 			}
 
-			chapters[index] = NewChapterFromURL(u.String())
+			chapters[index] = NewChapterFromURL(u.String(), images)
+			progress.Incr(index)
 
-			bars[index].Incr()
+			// short sleep for last chapter to let the progress bar update
-			barGlobal.Incr()
+			if index == len(links)-1 {
-
+				delay = 100
-			// do not wait after downloading last chapter
-			if index < len(links)-1 {
-				time.Sleep(time.Duration(delay) * time.Millisecond)
 			}
 
+			time.Sleep(time.Duration(delay) * time.Millisecond)
 		}
 
 	} else {
@@ -112,10 +104,9 @@ func tableOfContent(url, selector string, limit, delay int) []chapter {
 					log.Fatal(err)
 				}
 
-				chapters[index] = NewChapterFromURL(u.String())
+				chapters[index] = NewChapterFromURL(u.String(), images)
+				progress.Incr(index)
 
-				bars[index].Incr()
-				barGlobal.Incr()
 			}(index, l)
 		}
 		wg.Wait()
@@ -140,8 +131,7 @@ func GetPath(elm *goquery.Selection) string {
 	return join
 }
 
-
+func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, error) {
-func GetLinks(url *urllib.URL, selector string) []link {
 	selectorSet := true
 	if selector == "" {
 		selector = "a"
@@ -158,13 +148,17 @@ func GetLinks(url *urllib.URL, selector string) []link {
 		href := e.Attr("href")
 		text := strings.TrimSpace(e.Text)
 		path := GetPath(e.DOM)
-		class := e.Attr("class")
+		key := path
-		key := fmt.Sprintf("%s.%s", path, class)
+
+		// include element class in key if selector is set
+		if !selectorSet {
+			class := e.Attr("class")
+			key = fmt.Sprintf("%s.%s", path, class)
+		}
 
 		if selectorSet || text != "" {
-			pathLinks[key] = append(pathLinks[key], NewLink(href, text, class))
+			pathLinks[key] = append(pathLinks[key], NewLink(href, text))
 			pathCount[key] += len(text)
-			// pathCount[key]++
 
 			if pathCount[key] > pathCount[pathMax] {
 				pathMax = key
@@ -172,28 +166,24 @@ func GetLinks(url *urllib.URL, selector string) []link {
 		}
 	})
 	c.Visit(url.String())
-	return pathLinks[pathMax]
 
-	// // visit and count link classes
+	links := pathLinks[pathMax]
-	// classesLinks := map[string][]link{}
+	if len(links) == 0 {
-	// classesCount := map[string]int{}
+		return []link{}, fmt.Errorf("no link found for selector: %s", selector)
-	// classMax := ""
+	}
 
-	// c := colly.NewCollector()
+	end := len(links)
-	// c.OnHTML(selector, func(e *colly.HTMLElement) {
+	if limit != -1 {
-	// 	href := e.Attr("href")
+		end = int(math.Min(float64(limit+offset), float64(len(links))))
-	// 	text := strings.TrimSpace(e.Text)
+	}
-	// 	class := e.Attr("class")
 
-	// 	if selectorSet || class != "" && text != "" {
+	links = links[offset:end]
-	// 		classesLinks[class] = append(classesLinks[class], NewLink(href, text))
-	// 		classesCount[class]++
 
-	// 		if classesCount[class] > classesCount[classMax] {
+	if include {
-	// 			classMax = class
+		c := NewChapterFromURL(url.String(), false)
-	// 		}
+		l := NewLink(url.String(), c.Name())
-	// 	}
+		links = append([]link{l}, links...)
-	// })
+	}
-	// c.Visit(url.String())
+
-	// return classesLinks[classMax]
+	return links, nil
 }

cmd/get.go

@@ -15,9 +15,9 @@ import (
 	"github.com/lapwat/papeer/book"
 )
 
-var quiet, stdout, recursive, include bool
+var stdout, recursive, include, images bool
 var format, output, selector string
-var limit, delay int
+var limit, offset, delay int
 
 var getCmd = &cobra.Command{
 	Use:   "get",
@@ -60,6 +60,10 @@ var getCmd = &cobra.Command{
 			return errors.New("cannot use limit option if not in recursive mode")
 		}
 
+		if cmd.Flags().Changed("offset") && recursive == false {
+			return errors.New("cannot use offset option if not in recursive mode")
+		}
+
 		if cmd.Flags().Changed("delay") && recursive == false {
 			return errors.New("cannot use delay option if not in recursive mode")
 		}
@@ -68,7 +72,7 @@ var getCmd = &cobra.Command{
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		url := args[0]
-		b := book.NewBookFromURL(url, selector, recursive, include, limit, delay)
+		b := book.NewBookFromURL(url, selector, recursive, include, images, limit, offset, delay)
 
 		if len(output) == 0 {
 			// set default output
@@ -78,12 +82,16 @@ var getCmd = &cobra.Command{
 		}
 
 		if format == "md" {
-			f, err := os.Create(output)
+			var f *os.File
-			if err != nil {
+			var err error
-				log.Fatal(err)
-			}
 
-			defer f.Close()
+			if !stdout {
+				f, err = os.Create(output)
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer f.Close()
+			}
 
 			for _, c := range b.Chapters() {
 				content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
@@ -96,7 +104,6 @@ var getCmd = &cobra.Command{
 				if stdout {
 					fmt.Println(text)
 				} else {
-
 					_, err := f.WriteString(text)
 					if err != nil {
 						log.Fatal(err)
@@ -115,8 +122,16 @@ var getCmd = &cobra.Command{
 			e.SetAuthor(b.Author())
 
 			for _, c := range b.Chapters() {
-				html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), c.Content())
+				if images {
-				e.AddSection(html, c.Name(), "", "")
+					e.AddSection(c.Content(), "", "", "")
+				} else {
+					html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), c.Content())
+
+					_, err := e.AddSection(html, c.Name(), "", "")
+					if err != nil {
+						log.Fatal(err)
+					}
+				}
 			}
 
 			err := e.Write(output)
@@ -152,7 +167,7 @@ var getCmd = &cobra.Command{
 
 			err2 := os.Remove(outputEPUB)
 			if err2 != nil {
-				log.Fatal(err)
+				log.Fatal(err2)
 			}
 		}
 	},

cmd/list.go

@@ -27,11 +27,17 @@ var listCmd = &cobra.Command{
 			log.Fatal(err)
 		}
 
-		links := book.GetLinks(base, selector)
+		links, err := book.GetLinks(base, selector, limit, offset, include)
+		if err != nil {
+			log.Fatal(err)
+		}
 
 		t := table.NewWriter()
 		t.SetOutputMirror(os.Stdout)
-		t.AppendHeader(table.Row{"#", "Name", "Url", "Class"})
+		t.Style().Options.DrawBorder = false
+		t.Style().Options.SeparateColumns = false
+		t.Style().Options.SeparateHeader = false
+		t.AppendHeader(table.Row{"#", "Name", "Url"})
 
 		for index, link := range links {
 			u, err := base.Parse(link.Href())
@@ -39,7 +45,7 @@ var listCmd = &cobra.Command{
 				log.Fatal(err)
 			}
 
-			t.AppendRow([]interface{}{index + 1, link.Text(), u.String(), link.Class()})
+			t.AppendRow([]interface{}{index + 1, link.Text(), u.String()})
 		}
 
 		t.Render()

cmd/root.go

@@ -24,13 +24,14 @@ func Execute() {
 
 func init() {
 	rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "md", "file format [md, epub, mobi]")
-	rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "", "output file")
+	rootCmd.PersistentFlags().StringVarP(&output, "output", "", "", "output file")
-	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector")
+	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector, in resursive mode")
 	rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "create one chapter per natigation item")
 	rootCmd.PersistentFlags().BoolVarP(&include, "include", "i", false, "include URL as first chapter, in resursive mode")
-	rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "do not show logs")
 	rootCmd.PersistentFlags().BoolVarP(&stdout, "stdout", "", false, "print to standard output")
+	rootCmd.PersistentFlags().BoolVarP(&images, "images", "", false, "retrieve images only")
 	rootCmd.PersistentFlags().IntVarP(&limit, "limit", "l", -1, "limit number of chapters, in recursive mode")
+	rootCmd.PersistentFlags().IntVarP(&offset, "offset", "o", 0, "skip first chapters, in recursive mode")
 	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "time to wait before downloading next chapter, in milliseconds")
 
 	rootCmd.AddCommand(getCmd)

go.mod

@@ -31,13 +31,18 @@ require (
 	github.com/jedib0t/go-pretty/v6 v6.2.4 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/schollz/progressbar/v3 v3.8.3 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect

go.sum

@@ -240,6 +240,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -259,9 +260,13 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -295,6 +300,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -302,6 +309,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8=
+github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sebdah/goldie/v2 v2.5.1 h1:hh70HvG4n3T3MNRJN2z/baxPR8xutxo7JVxyi2svl+s=
 github.com/sebdah/goldie/v2 v2.5.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
@@ -380,6 +389,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -533,9 +544,14 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=