bump version in readme

Makefile

@@ -0,0 +1,10 @@
+install:
+	go install
+
+format:
+	gofmt -s -w .
+
+clean:
+	find . -maxdepth 1 -not -name 'README.md' -name '*.md' -delete
+	find . -maxdepth 1 -name '*.epub' -delete
+	find . -maxdepth 1 -name '*.mobi' -delete

README.md

@@ -1,12 +1,10 @@
-# Installation
+# Papeer
 
-```sh
+Papeer is a tool that lets you scrape content from the internet. It can scrape any web page, keeping only relevant content (formatted text and images) and removing ads and menus. You can export the content to Markdown, EPUB or MOBI files.
-go install github.com/lapwat/papeer
-```
 
 # Usage
 
-```sh
+```
 Browse the web in the eink era
 
 Usage:
@@ -21,23 +19,121 @@ Available Commands:
   version     Print the version number of papeer
 
 Flags:
-  -d, --delay int         wait before downloading next chapter, in milliseconds (default -1)
+  -a, --author string     book author
-  -f, --format string     file format [md, epub, mobi] (default "md")
+  -d, --delay int         time to wait before downloading next chapter, in milliseconds (default -1)
+  -f, --format string     file format [stdout, md, epub, mobi] (default "stdout")
   -h, --help              help for papeer
+      --images            retrieve images only
   -i, --include           include URL as first chapter, in resursive mode
-  -o, --output string     output file
+  -l, --limit int         limit number of chapters, in recursive mode (default -1)
-  -q, --quiet             do not show progress bars
+  -n, --name string       book name (default: page title)
+  -o, --offset int        skip first chapters, in recursive mode
+      --output string     file name (default: book name)
   -r, --recursive         create one chapter per natigation item
-  -s, --selector string   table of content CSS selector
+  -s, --selector string   table of content CSS selector, in resursive mode
-      --stdout            print to standard output
+  -t, --threads int       download concurrency, in recursive mode (default -1)
 
 Use "papeer [command] --help" for more information about a command.
 ```
 
+# Examples
+
+## Grab a single page
+
+The `get` command lets you retrieve the content of a web page.
+
+```sh
+papeer get https://www.eff.org/cyberspace-independence
+# A Declaration of the Independence of Cyberspace
+# ===============================================
+
+# Governments of the Industrial World, you weary giants of flesh and steel, I come from Cyberspace, the new home of Mind. On behalf of the future, I ask you of the past to leave us alone. You are not welcome among us. You have no sovereignty where we gather...
+```
+
+## Grab several pages (recursive mode)
+
+The `recursive` option lets you extract the table of content of a website, then scrape the content of each one of its pages.
+
+### Display table of content
+
+Before trying the `recursive` option, it is a good idea to use the `ls` option, which lets you vizualize the content that will be retrieved. You can use several options to customize the table of content extraction, such as `selector`, `limit`, `offset` and `include`. Type `papeer help` for more information about those options.
+
+```sh
+papeer ls https://news.ycombinator.com/ --limit=5
+#  #  NAME                                                      URL                                                                                             
+#  1  Tailwind CSS v3.0                                         https://tailwindcss.com/blog/tailwindcss-v3                                                     
+#  2  A molten salt storage solution using sodium hydroxide     https://sifted.eu/articles/salt-energy-storage-seaborg-hyme/                                    
+#  3  HashiCorp IPO today                                       https://www.hashicorp.com/blog/a-new-chapter-for-hashicorp                                      
+#  4  Stack Graphs                                              https://github.blog/2021-12-09-introducing-stack-graphs/
+#  5  ‘Tipping point’ makes partisan polarization irreversible  https://news.cornell.edu/stories/2021/12/tipping-point-makes-partisan-polarization-irreversible 
+```
+
+### Scrape time
+
+Once you are satisfied with the table of content listed by the `ls` command, you can actually scrape the content of those pages. You can use the same options that you specified for the `ls` command. In recursive mode, you also have the possibility to use `delay` and `threads` options.
+
+```sh
+papeer get https://news.ycombinator.com/ --recursive --delay=500 --limit=5 --format=md
+# [========================================>---------------------------] Chapters 3 / 5
+# [====================================================================] 1. Tailwind CSS v3.0
+# [====================================================================] 2. A molten salt storage solution using sodium hydroxide
+# [====================================================================] 3. HashiCorp IPO today
+# [--------------------------------------------------------------------] 4. Stack Graphs
+# [--------------------------------------------------------------------] 5. ‘Tipping point’ makes partisan polarization irreversible
+# Markdown saved to "Hacker News.md"
+```
+
+# Installation
+
+## From source
+
+```sh
+go get -u github.com/lapwat/papeer
+```
+
+## From binary
+
+### On Linux / MacOS
+
+```sh
+platform=linux # use platform=darwin for MacOS
+curl -L https://github.com/lapwat/papeer/releases/download/v0.3.1/papeer-v0.3.1-$platform-amd64 > papeer
+chmod +x papeer
+sudo mv papeer /usr/local/bin
+```
+
+### On Windows
+
+Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.3.1/papeer-v0.3.1-windows-amd64.exe).
+
+## Install kindlegen to export websites to MOBI (optional)
+
+```sh
+TMPDIR=$(mktemp -d -t papeer-XXXXX)
+curl -L https://github.com/lapwat/papeer/releases/download/kindlegen/kindlegen_linux_2.6_i386_v2_9.tar.gz > $TMPDIR/kindlegen.tar.gz
+tar xzvf $TMPDIR/kindlegen.tar.gz -C $TMPDIR
+chmod +x $TMPDIR/kindlegen
+sudo mv $TMPDIR/kindlegen /usr/local/bin
+rm -rf $TMPDIR
+```
+
 # Autocompletion
 
 Execute this command in your current shell, or add it to your `.bashrc`.
 
 ```sh
-. <(papeer completion [bash|fish|powershell|zsh])
+. <(papeer completion bash)
 ```
+
+Type `papeer completion bash -h` for more information.
+
+You can replace `bash` by your own shell (zsh, fish or powershell).
+
+# Dependencies
+
+- `cobra` command line interface
+- `go-readability` extract content from HTML
+- `html-to-markdown` convert HTML to Markdown
+- `go-epub` convert HTML to EPUB
+- `colly` query HTML trees
+- `uiprogress` display progress bars

book/book.go

@@ -1,27 +1,27 @@
 package book
 
-type Book struct {
+type book struct {
 	name     string
 	author   string
 	chapters []chapter
 }
 
-func New(name, author string) Book {
+func New(name, author string) book {
-	return Book{name, author, []chapter{}}
+	return book{name, author, []chapter{}}
 }
 
-func (b *Book) AddChapter(c chapter) {
+func (b *book) AddChapter(c chapter) {
 	b.chapters = append(b.chapters, c)
 }
 
-func (b Book) Name() string {
+func (b book) Name() string {
 	return b.name
 }
 
-func (b Book) Author() string {
+func (b book) Author() string {
 	return b.name
 }
 
-func (b *Book) Chapters() []chapter {
+func (b *book) Chapters() []chapter {
 	return b.chapters
 }

book/link.go

@@ -0,0 +1,18 @@
+package book
+
+type link struct {
+	href string
+	text string
+}
+
+func NewLink(href, text string) link {
+	return link{href, text}
+}
+
+func (c link) Href() string {
+	return c.href
+}
+
+func (c link) Text() string {
+	return c.text
+}

book/progress.go

@@ -0,0 +1,47 @@
+package book
+
+import (
+	"fmt"
+
+	"github.com/gosuri/uiprogress"
+)
+
+type progress struct {
+	global      *uiprogress.Bar
+	individuals []*uiprogress.Bar
+}
+
+func NewProgress(links []link) progress {
+	uiprogress.Start()
+
+	global := uiprogress.AddBar(len(links))
+	global.AppendFunc(func(b *uiprogress.Bar) string {
+		return fmt.Sprintf("Chapters %d / %d", b.Current(), len(links))
+	})
+
+	// hide individual bars if more than 50 chapters
+	individuals := []*uiprogress.Bar{}
+	if len(links) <= 50 {
+		for index, link := range links {
+			bar := uiprogress.AddBar(1)
+			barText := fmt.Sprintf("%d. %s", index+1, link.text)
+			bar.AppendFunc(func(b *uiprogress.Bar) string {
+				return barText
+			})
+			individuals = append(individuals, bar)
+		}
+	}
+
+	return progress{global, individuals}
+}
+
+func (p *progress) IncrGlobal() {
+	p.global.Incr()
+}
+
+func (p *progress) Incr(index int) {
+	p.global.Incr()
+	if len(p.individuals) > index {
+		p.individuals[index].Incr()
+	}
+}

book/scraper.go

@@ -3,28 +3,37 @@ package book
 import (
 	"fmt"
 	"log"
+	"math"
 	urllib "net/url"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/PuerkitoBio/goquery"
 	readability "github.com/go-shiori/go-readability"
 	colly "github.com/gocolly/colly/v2"
-	"github.com/gosuri/uiprogress"
 )
 
-type scraper struct {
+func NewBookFromURL(url, selector, name, author string, recursive, include bool, limit, offset, delay, threads int) book {
-	url string
+	var chapters []chapter
-}
+	var home chapter
 
-func NewBookFromURL(url, selector string, include bool, delay int) Book {
+	if recursive {
-	home := NewChapterFromURL(url)
+		chapters, home = tableOfContent(url, selector, limit, offset, delay, threads, include)
-	b := New(home.Name(), home.Author())
+	} else {
-
+		chapters = []chapter{NewChapterFromURL(url)}
-	chapters := tableOfContent(url, selector, delay)
+		home = chapters[0]
-	if include {
-		b.AddChapter(home)
 	}
+
+	if len(name) == 0 {
+		name = home.Name()
+	}
+
+	if len(author) == 0 {
+		author = home.Author()
+	}
+
+	b := New(name, author)
 	for _, c := range chapters {
 		b.AddChapter(c)
 	}
@@ -38,128 +47,167 @@ func NewChapterFromURL(url string) chapter {
 		log.Fatalf("failed to parse %s, %v\n", url, err)
 	}
 
-	// metadata := fmt.Sprintf("URL     : %s\nTitle   : %s\nAuthor  : %s\nLength  : %d\nExcerpt : %s\nSiteName: %s\nImage   : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
+	content := strings.ReplaceAll(article.Content, "\n", "")
-	// fmt.Println(metadata)
 
-	return chapter{article.Title, article.Byline, article.Content}
+	// if images {
+	// 	// parse html content
+	// 	doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
+	// 	if err != nil {
+	// 		log.Fatal(err)
+	// 	}
+
+	// 	// extract images only
+	// 	content = ""
+	// 	doc.Find("img").Each(func(i int, s *goquery.Selection) {
+	// 		newContent, _ := goquery.OuterHtml(s)
+	// 		content += newContent
+	// 	})
+	// }
+
+	return chapter{article.Title, article.Byline, content}
 }
 
-func tableOfContent(url, selector string, delay int) []chapter {
+func tableOfContent(url, selector string, limit, offset, delay, threads int, include bool) ([]chapter, chapter) {
-	c := colly.NewCollector()
-
-	classesLinks := map[string][]map[string]string{}
-	classesCount := map[string]int{}
-	classMax := ""
-
-	if selector == "" {
-		c.OnHTML("a", func(e *colly.HTMLElement) {
-			href := e.Attr("href")
-			text := strings.TrimSpace(e.Text)
-			class := e.Attr("class")
-
-			//if class != "" && text != "" {
-				classesLinks[class] = append(classesLinks[class], map[string]string{
-					"href": href,
-					"text": text,
-				})
-
-				classesCount[class]++
-
-				if classesCount[class] > classesCount[classMax] {
-					classMax = class
-				}
-			//}
-
-		})
-	} else {
-		c.OnHTML(selector, func(e *colly.HTMLElement) {
-			href := e.Attr("href")
-			text := strings.TrimSpace(e.Text)
-			class := e.Attr("class")
-
-			//if class != "" && text != "" {
-				classesLinks[class] = append(classesLinks[class], map[string]string{
-					"href": href,
-					"text": text,
-				})
-
-				classesCount[class]++
-
-				if classesCount[class] > classesCount[classMax] {
-					classMax = class
-				}
-			//}
-		})
-	}
-	c.Visit(url)
-	fmt.Println(classesCount)
-	links := classesLinks[classMax]
-
-	chapters := make([]chapter, len(links))
 	base, err := urllib.Parse(url)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	// init global progress bar
+	links, home, err := GetLinks(base, selector, limit, offset, include)
-	uiprogress.Start()
+	if err != nil {
-	barGlobal := uiprogress.AddBar(len(links)).AppendCompleted().PrependElapsed()
+		log.Fatal(err)
-	barGlobal.AppendFunc(func(b *uiprogress.Bar) string {
-		return fmt.Sprintf("Status: %d out of %d chapters", b.Current(), len(links))
-	})
-
-	// init progress bars
-	bars := []*uiprogress.Bar{}
-	for index, link := range links {
-		bar := uiprogress.AddBar(1).AppendCompleted().PrependElapsed()
-		barText := fmt.Sprintf("%d. %s", index+1, link["text"])
-		bar.AppendFunc(func(b *uiprogress.Bar) string {
-			return barText
-		})
-		bars = append(bars, bar)
 	}
 
+	chapters := make([]chapter, len(links))
+	progress := NewProgress(links)
+
 	if delay >= 0 {
+		// synchronous mode
+
 		for index, link := range links {
 			// and then use it to parse relative URLs
-			u, err := base.Parse(link["href"])
+			u, err := base.Parse(link.href)
 			if err != nil {
 				log.Fatal(err)
 			}
 
 			chapters[index] = NewChapterFromURL(u.String())
+			progress.Incr(index)
 
-			bars[index].Incr()
+			// short sleep for last chapter to let the progress bar update
-			barGlobal.Incr()
+			if index == len(links)-1 {
-
+				delay = 100
-			// do not wait after downloading last chapter
-			if index < len(links)-1 {
-				time.Sleep(time.Duration(delay) * time.Millisecond)
 			}
 
+			time.Sleep(time.Duration(delay) * time.Millisecond)
 		}
 
 	} else {
+		// asynchronous mode
 		var wg sync.WaitGroup
-		for index, link := range links {
+
+		if threads == -1 {
+			threads = len(links)
+		}
+		semaphore := make(chan bool, threads)
+
+		for index, l := range links {
 
 			wg.Add(1)
-			go func(index int, link map[string]string) {
+			semaphore <- true
+
+			go func(index int, l link) {
 				defer wg.Done()
 
 				// and then use it to parse relative URLs
-				u, err := base.Parse(link["href"])
+				u, err := base.Parse(l.href)
 				if err != nil {
 					log.Fatal(err)
 				}
 
 				chapters[index] = NewChapterFromURL(u.String())
+				progress.Incr(index)
 
-				bars[index].Incr()
+				<-semaphore
-				barGlobal.Incr()
+			}(index, l)
-			}(index, link)
 		}
 		wg.Wait()
 	}
-	return chapters
+
+	return chapters, home
+}
+
+func GetPath(elm *goquery.Selection) string {
+	path := []string{}
+
+	for {
+		selector := strings.ToLower(goquery.NodeName(elm))
+		if selector == "" {
+			break
+		}
+
+		path = append(path, selector)
+		elm = elm.Parent()
+	}
+
+	join := strings.Join(path, "<")
+	return join
+}
+
+func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, chapter, error) {
+	selectorSet := true
+	if selector == "" {
+		selector = "a"
+		selectorSet = false
+	}
+
+	// visit and count link classes
+	pathLinks := map[string][]link{}
+	pathCount := map[string]int{}
+	pathMax := ""
+
+	c := colly.NewCollector()
+	c.OnHTML(selector, func(e *colly.HTMLElement) {
+		href := e.Attr("href")
+		text := strings.TrimSpace(e.Text)
+		path := GetPath(e.DOM)
+		key := path
+
+		// include element class in key if selector is set
+		if !selectorSet {
+			class := e.Attr("class")
+			key = fmt.Sprintf("%s.%s", path, class)
+		}
+
+		if selectorSet || text != "" {
+			pathLinks[key] = append(pathLinks[key], NewLink(href, text))
+			pathCount[key] += len(text)
+
+			if pathCount[key] > pathCount[pathMax] {
+				pathMax = key
+			}
+		}
+	})
+	c.Visit(url.String())
+
+	links := pathLinks[pathMax]
+	if len(links) == 0 {
+		return []link{}, chapter{}, fmt.Errorf("no link found for selector: %s", selector)
+	}
+
+	end := len(links)
+	if limit != -1 {
+		end = int(math.Min(float64(limit+offset), float64(len(links))))
+	}
+
+	links = links[offset:end]
+
+	home := NewChapterFromURL(url.String())
+
+	if include {
+		l := NewLink(url.String(), home.Name())
+		links = append([]link{l}, links...)
+	}
+
+	return links, home, nil
 }

cmd/get.go

@@ -9,15 +9,16 @@ import (
 	"strings"
 
 	md "github.com/JohannesKaufmann/html-to-markdown"
+	"github.com/PuerkitoBio/goquery"
 	epub "github.com/bmaupin/go-epub"
-	cobra "github.com/spf13/cobra"
+	"github.com/spf13/cobra"
 
 	"github.com/lapwat/papeer/book"
 )
 
-var quiet, stdout, recursive, include bool
+var recursive, include, images bool
-var format, output, selector string
+var format, output, selector, name, author string
-var delay int
+var limit, offset, delay, threads int
 
 var getCmd = &cobra.Command{
 	Use:   "get",
@@ -28,48 +29,55 @@ var getCmd = &cobra.Command{
 		}
 
 		formatEnum := map[string]bool{
-			"md":   true,
+			"stdout": true,
-			"epub": true,
+			"md":     true,
-			"mobi": true,
+			"epub":   true,
+			"mobi":   true,
 		}
 		if formatEnum[format] != true {
 			return fmt.Errorf("invalid format specified: %s", format)
 		}
 
-		if format == "epub" || format == "mobi" {
+		// add .mobi to filename if not specified
-			if stdout {
-				return errors.New("cannot print EPUB/MOBI file to standard output")
-			}
-		}
-
 		if format == "mobi" {
 			if len(output) > 0 && strings.HasSuffix(output, ".mobi") == false {
 				output = fmt.Sprintf("%s.mobi", output)
 			}
 		}
 
-		if include && recursive == false {
+		if cmd.Flags().Changed("selector") && recursive == false {
+			return errors.New("cannot use selector option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("include") && recursive == false {
 			return errors.New("cannot use include option if not in recursive mode")
 		}
 
+		if cmd.Flags().Changed("limit") && recursive == false {
+			return errors.New("cannot use limit option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("offset") && recursive == false {
+			return errors.New("cannot use offset option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("delay") && recursive == false {
+			return errors.New("cannot use delay option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("threads") && recursive == false {
+			return errors.New("cannot use threads option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("delay") && cmd.Flags().Changed("threads") {
+			return errors.New("cannot use delay and threads options at the same time")
+		}
+
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		url := args[0]
-		var b book.Book
+		b := book.NewBookFromURL(url, selector, name, author, recursive, include, limit, offset, delay, threads)
-
-		if recursive {
-			b = book.NewBookFromURL(url, selector, include, delay)
-		} else {
-			c := book.NewChapterFromURL(url)
-			b = book.New(c.Name(), c.Author())
-			b.AddChapter(c)
-		}
-
-		// if quiet == false {
-		// 	metadata := fmt.Sprintf("URL     : %s\nTitle   : %s\nAuthor  : %s\nLength  : %d\nExcerpt : %s\nSiteName: %s\nImage   : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
-		// 	fmt.Println(metadata)
-		// }
 
 		if len(output) == 0 {
 			// set default output
@@ -78,29 +86,45 @@ var getCmd = &cobra.Command{
 			output = fmt.Sprintf("%s.%s", output, format)
 		}
 
-		if format == "md" {
+		if format == "stdout" {
-			f, err := os.Create(output)
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			defer f.Close()
 
 			for _, c := range b.Chapters() {
+				// convert to markdown
 				content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
 				if err != nil {
 					log.Fatal(err)
 				}
 
-				if stdout {
+				text := fmt.Sprintf("%s\n%s\n\n%s\n\n\n", c.Name(), strings.Repeat("=", len(c.Name())), content)
-					fmt.Println(content)
-				} else {
 
-					_, err := f.WriteString(content)
+				// write to stdout
-					if err != nil {
+				fmt.Println(text)
-						log.Fatal(err)
+			}
-					}
 
+		}
+
+		if format == "md" {
+
+			// create markdown file
+			f, err := os.Create(output)
+			if err != nil {
+				log.Fatal(err)
+			}
+			defer f.Close()
+
+			for _, c := range b.Chapters() {
+				// convert to markdown
+				content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				text := fmt.Sprintf("%s\n%s\n\n%s\n\n\n", c.Name(), strings.Repeat("=", len(c.Name())), content)
+
+				// write to markdown file
+				_, err = f.WriteString(text)
+				if err != nil {
+					log.Fatal(err)
 				}
 			}
 
@@ -112,7 +136,36 @@ var getCmd = &cobra.Command{
 			e.SetAuthor(b.Author())
 
 			for _, c := range b.Chapters() {
-				e.AddSection(c.Content(), c.Name(), "", "")
+				var content string
+
+				if images == false {
+					content = c.Content()
+				}
+
+				// parse content
+				doc, err := goquery.NewDocumentFromReader(strings.NewReader(c.Content()))
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				// retrieve images and download it
+				doc.Find("img").Each(func(i int, s *goquery.Selection) {
+					src, _ := s.Attr("src")
+					imagePath, _ := e.AddImage(src, "")
+
+					if images {
+						imageTag, _ := goquery.OuterHtml(s)
+						content += imageTag
+					}
+
+					content = strings.ReplaceAll(content, src, imagePath)
+				})
+
+				html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), content)
+				_, err = e.AddSection(html, c.Name(), "", "")
+				if err != nil {
+					log.Fatal(err)
+				}
 			}
 
 			err := e.Write(output)
@@ -127,8 +180,37 @@ var getCmd = &cobra.Command{
 			e := epub.NewEpub(b.Name())
 			e.SetAuthor(b.Author())
 
-			for _, chapter := range b.Chapters() {
+			for _, c := range b.Chapters() {
-				e.AddSection(chapter.Content(), chapter.Name(), "", "")
+				var content string
+
+				if images == false {
+					content = c.Content()
+				}
+
+				// parse content
+				doc, err := goquery.NewDocumentFromReader(strings.NewReader(c.Content()))
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				// retrieve images and download it
+				doc.Find("img").Each(func(i int, s *goquery.Selection) {
+					src, _ := s.Attr("src")
+					imagePath, _ := e.AddImage(src, "")
+
+					if images {
+						imageTag, _ := goquery.OuterHtml(s)
+						content += imageTag
+					}
+
+					content = strings.ReplaceAll(content, src, imagePath)
+				})
+
+				html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), content)
+				_, err = e.AddSection(html, c.Name(), "", "")
+				if err != nil {
+					log.Fatal(err)
+				}
 			}
 
 			outputEPUB := strings.ReplaceAll(output, ".mobi", ".epub")
@@ -139,15 +221,15 @@ var getCmd = &cobra.Command{
 			}
 
 			exec.Command("kindlegen", outputEPUB).Run()
-			// exec command always return status 1 even if it fails
+			// exec command always return status 1 even if it succeed
 			// if err != nil {
 			// 	log.Fatal(err)
 			// }
 
 			fmt.Printf("Ebook saved to \"%s\"\n", output)
 
-			err2 := os.Remove(outputEPUB)
+			err = os.Remove(outputEPUB)
-			if err2 != nil {
+			if err != nil {
 				log.Fatal(err)
 			}
 		}

cmd/list.go

@@ -2,11 +2,14 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
+	"log"
-	"strings"
+	urllib "net/url"
+	"os"
 
-	colly "github.com/gocolly/colly/v2"
+	"github.com/jedib0t/go-pretty/v6/table"
 	cobra "github.com/spf13/cobra"
+
+	"github.com/lapwat/papeer/book"
 )
 
 var listCmd = &cobra.Command{
@@ -19,37 +22,33 @@ var listCmd = &cobra.Command{
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		url := args[0]
+		base, err := urllib.Parse(args[0])
-		c := colly.NewCollector()
+		if err != nil {
-
+			log.Fatal(err)
-		// visit and count link classes
-		classesLinks := map[string][]map[string]string{}
-		classesCount := map[string]int{}
-		classMax := ""
-
-		c.OnHTML(selector, func(e *colly.HTMLElement) {
-			href := e.Attr("href")
-			text := strings.TrimSpace(e.Text)
-			class := e.Attr("class")
-
-			// if class != "" && text != "" {
-				classesLinks[class] = append(classesLinks[class], map[string]string{
-					"href": href,
-					"text": text,
-				})
-
-				classesCount[class]++
-
-				if classesCount[class] > classesCount[classMax] {
-					classMax = class
-				}
-			// }
-		})
-
-		c.Visit(url)
-		for index, link := range classesLinks[classMax] {
-			fmt.Printf("Chapter %d: %s %s\n", index+1, link["text"], link["href"])
 		}
 
+		links, _, err := book.GetLinks(base, selector, limit, offset, include)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		t := table.NewWriter()
+		t.SetOutputMirror(os.Stdout)
+		t.Style().Options.DrawBorder = false
+		t.Style().Options.SeparateColumns = false
+		t.Style().Options.SeparateHeader = false
+		t.AppendHeader(table.Row{"#", "Name", "Url"})
+
+		for index, link := range links {
+			u, err := base.Parse(link.Href())
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			t.AppendRow([]interface{}{index + 1, link.Text(), u.String()})
+		}
+
+		t.Render()
+
 	},
 }

cmd/root.go

@@ -23,14 +23,18 @@ func Execute() {
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "md", "file format [md, epub, mobi]")
+	rootCmd.PersistentFlags().StringVarP(&name, "name", "n", "", "book name (default: page title)")
-	rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "", "output file")
+	rootCmd.PersistentFlags().StringVarP(&author, "author", "a", "", "book author")
-	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector")
+	rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "stdout", "file format [stdout, md, epub, mobi]")
+	rootCmd.PersistentFlags().StringVarP(&output, "output", "", "", "file name (default: book name)")
+	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector, in resursive mode")
 	rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "create one chapter per natigation item")
 	rootCmd.PersistentFlags().BoolVarP(&include, "include", "i", false, "include URL as first chapter, in resursive mode")
-	rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "do not show progress bars")
+	rootCmd.PersistentFlags().BoolVarP(&images, "images", "", false, "retrieve images only")
-	rootCmd.PersistentFlags().BoolVarP(&stdout, "stdout", "", false, "print to standard output")
+	rootCmd.PersistentFlags().IntVarP(&limit, "limit", "l", -1, "limit number of chapters, in recursive mode")
-	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "wait before downloading next chapter, in milliseconds")
+	rootCmd.PersistentFlags().IntVarP(&offset, "offset", "o", 0, "skip first chapters, in recursive mode")
+	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "time to wait before downloading next chapter, in milliseconds")
+	rootCmd.PersistentFlags().IntVarP(&threads, "threads", "t", -1, "download concurrency, in recursive mode")
 
 	rootCmd.AddCommand(getCmd)
 	rootCmd.AddCommand(listCmd)

cmd/utils.go

@@ -1,5 +0,0 @@
-package cmd
-
-func getTableOfContent() {
-
-}

cmd/version.go

@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print the version number of papeer",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("papeer v0.0.1")
+		fmt.Println("papeer v0.3.1")
 	},
 }

go.mod

@@ -10,33 +10,40 @@ require (
 )
 
 require (
+	github.com/PuerkitoBio/goquery v1.7.1
 	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gosuri/uilive v0.0.4 // indirect
 	github.com/gosuri/uiprogress v0.0.1
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.2.0 // indirect
 	github.com/antchfx/htmlquery v1.2.3 // indirect
-	github.com/antchfx/xmlquery v1.2.4 // indirect
+	github.com/antchfx/xmlquery v1.3.6 // indirect
-	github.com/antchfx/xpath v1.1.8 // indirect
+	github.com/antchfx/xpath v1.2.0 // indirect
 	github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gofrs/uuid v3.1.0+incompatible // indirect
+	github.com/gofrs/uuid v4.0.0+incompatible // indirect
 	github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jedib0t/go-pretty/v6 v6.2.4 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
-	github.com/mattn/go-isatty v0.0.3 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.13 // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/schollz/progressbar/v3 v3.8.3 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/temoto/robotstxt v1.1.1 // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
+	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
-	golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect
+	golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 )

go.sum

@@ -42,8 +42,9 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/JohannesKaufmann/html-to-markdown v1.3.0 h1:K/p4cq8Ib13hcSVcKQNfKCSWw93CYW5pAjY0fl85has=
 github.com/JohannesKaufmann/html-to-markdown v1.3.0/go.mod h1:JNSClIRYICFDiFhw6RBhBeWGnMSSKVZ6sPQA+TK4tyM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
+github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
@@ -51,11 +52,14 @@ github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5
 github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
 github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
 github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
-github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
 github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
+github.com/antchfx/xmlquery v1.3.6 h1:kaEVzH1mNo/2AJZrhZjAaAUTy2Nn2zxGfYYU8jWfXOo=
+github.com/antchfx/xmlquery v1.3.6/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
 github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
-github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk=
 github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
+github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -99,6 +103,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -118,8 +123,9 @@ github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuu
 github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
 github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/uuid v3.1.0+incompatible h1:q2rtkjaKT4YEr6E1kamy0Ha4RtepWlQBedyHx0uzKwA=
 github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
+github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -129,8 +135,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -225,12 +232,15 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
+github.com/jedib0t/go-pretty/v6 v6.2.4 h1:wdaj2KHD2W+mz8JgJ/Q6L/T5dB7kyqEFI16eLq7GEmk=
+github.com/jedib0t/go-pretty/v6 v6.2.4/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -245,11 +255,18 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
+github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -268,6 +285,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -282,6 +300,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -289,6 +309,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI=
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
+github.com/schollz/progressbar/v3 v3.8.3 h1:FnLGl3ewlDUP+YdSwveXBaXs053Mem/du+wr7XSYKl8=
+github.com/schollz/progressbar/v3 v3.8.3/go.mod h1:pWnVCjSBZsT2X3nx9HfRdnCDrpbevliMeoEVhStwHko=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sebdah/goldie/v2 v2.5.1 h1:hh70HvG4n3T3MNRJN2z/baxPR8xutxo7JVxyi2svl+s=
 github.com/sebdah/goldie/v2 v2.5.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
@@ -328,8 +350,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
 github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
+github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -366,6 +389,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
+golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -435,6 +460,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -470,6 +496,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -516,9 +543,15 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0 h1:xrCZDmdtoloIiooiA9q0OQb9r8HejIHYoHGhGCe1pGg=
+golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -691,8 +724,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

release.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+version=$1
+platforms=("linux/amd64" "darwin/amd64" "windows/amd64")
+
+for platform in "${platforms[@]}"
+do
+    platform_split=(${platform//\// })
+    GOOS=${platform_split[0]}
+    GOARCH=${platform_split[1]}
+    output_name='papeer-'$version'-'$GOOS'-'$GOARCH
+    if [ $GOOS = "windows" ]; then
+        output_name+='.exe'
+    fi
+
+    env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name
+    if [ $? -ne 0 ]; then
+        echo 'An error has occurred! Aborting the script execution...'
+        exit 1
+    fi
+done