refacto, better scrape strategy

better display, release script
docs
2026-05-25 12:27:20 +00:00 · 2021-09-24 02:09:22 +02:00 · 2021-09-21 19:55:37 +02:00 · 2021-09-15 20:51:16 +02:00 · 2021-09-15 20:47:53 +02:00 · 2021-09-15 20:47:22 +02:00
11 changed files with 315 additions and 160 deletions
--- a/README.md
+++ b/README.md
@@ -1,12 +1,57 @@
+```
+❯ papeer get --format epub --recursive --delay 500 --limit 10 https://news.ycombinator.com/
+   6s [===============================================>--------------------]  70% Status: 7 out of 10 chapters
+   0s [====================================================================] 100% 1. Three ex-US intelligence officers admit hacking for UAE
+   0s [====================================================================] 100% 2. Show HN: Time Travel Debugger
+   0s [====================================================================] 100% 3. How much faster is Java 17?
+   0s [====================================================================] 100% 4. The First Webcam Was Invented to Keep an Eye on a Coffee Pot
+   0s [====================================================================] 100% 5. Nikon's 2021 Photomicrography Competition Winners
+   0s [====================================================================] 100% 6. HTTP Status 418 – I'm a teapot
+   0s [====================================================================] 100% 7. H3: Hexagonal hierarchical geospatial indexing system
+  --- [--------------------------------------------------------------------]   0% 8. Automatic cipher suite ordering in Go’s crypto/tls
+  --- [--------------------------------------------------------------------]   0% 9. Find engineering roles at over 800 YC-funded startups
+  --- [--------------------------------------------------------------------]   0% 10. Futarchy: Robin Hanson on prediction markets
+Ebook saved to "Hacker_News.epub"
+```
+
 # Installation

+## From source
+
 ```sh
-go install github.com/lapwat/papeer
+go get -u github.com/lapwat/papeer
+```
+
+## From binary
+
+### On Linux / MacOS
+
+```sh
+platform=linux
+# platform=darwin for MacOS
+curl -L https://github.com/lapwat/papeer/releases/download/v0.1.0/papeer-v0.1.0-$platform-amd64 > papeer
+chmod +x papeer
+sudo mv papeer /usr/local/bin
+```
+
+### On Windows
+
+Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.1.0/papeer-v0.1.0-windows-amd64.exe).
+
+## Install kindlegen to export websites to MOBI (optional)
+
+```sh
+TMPDIR=$(mktemp -d -t papeer-XXXXX)
+curl -L https://github.com/lapwat/papeer/releases/download/kindlegen/kindlegen_linux_2.6_i386_v2_9.tar.gz > $TMPDIR/kindlegen.tar.gz
+tar xzvf $TMPDIR/kindlegen.tar.gz -C $TMPDIR
+chmod +x $TMPDIR/kindlegen
+sudo mv $TMPDIR/kindlegen /usr/local/bin
+rm -rf $TMPDIR
 ```

 # Usage

-```sh
+```
 Browse the web in the eink era

 Usage:
@@ -21,12 +66,13 @@ Available Commands:
  version     Print the version number of papeer

 Flags:
-  -d, --delay int         wait before downloading next chapter, in milliseconds (default -1)
+  -d, --delay int         time to wait before downloading next chapter, in milliseconds (default -1)
  -f, --format string     file format [md, epub, mobi] (default "md")
  -h, --help              help for papeer
  -i, --include           include URL as first chapter, in resursive mode
+  -l, --limit int         limit number of chapters, in recursive mode (default -1)
  -o, --output string     output file
-  -q, --quiet             do not show progress bars
+  -q, --quiet             do not show logs
  -r, --recursive         create one chapter per natigation item
  -s, --selector string   table of content CSS selector
      --stdout            print to standard output
@@ -39,5 +85,18 @@ Use "papeer [command] --help" for more information about a command.
 Execute this command in your current shell, or add it to your `.bashrc`.

 ```sh
-. <(papeer completion [bash|fish|powershell|zsh])
+. <(papeer completion bash)
 ```
+
+Type `papeer completion bash -h` for more information.
+
+You can replace `bash` by your own shell (zsh, fish or powershell).
+
+# Dependencies
+
+- `cobra` command line interface
+- `go-readability` extract content from HTML
+- `html-to-markdown` convert HTML to Markdown
+- `go-epub` convert HTML to EPUB
+- `colly` query HTML trees
+- `uiprogress` display progress bars
--- a/book/book.go
+++ b/book/book.go
@@ -1,27 +1,27 @@
 package book

-type Book struct {
+type book struct {
 	name     string
 	author   string
 	chapters []chapter
 }

-func New(name, author string) Book {
-	return Book{name, author, []chapter{}}
+func New(name, author string) book {
+	return book{name, author, []chapter{}}
 }

-func (b *Book) AddChapter(c chapter) {
+func (b *book) AddChapter(c chapter) {
 	b.chapters = append(b.chapters, c)
 }

-func (b Book) Name() string {
+func (b book) Name() string {
 	return b.name
 }

-func (b Book) Author() string {
+func (b book) Author() string {
 	return b.name
 }

-func (b *Book) Chapters() []chapter {
+func (b *book) Chapters() []chapter {
 	return b.chapters
 }
--- a/book/link.go
+++ b/book/link.go
@@ -0,0 +1,23 @@
+package book
+
+type link struct {
+	href string
+	text string
+	class string
+}
+
+func NewLink(href, text, class string) link {
+	return link{href, text, class}
+}
+
+func (c link) Href() string {
+	return c.href
+}
+
+func (c link) Text() string {
+	return c.text
+}
+
+func (c link) Class() string {
+	return c.class
+}
--- a/book/scraper.go
+++ b/book/scraper.go
@@ -3,33 +3,38 @@ package book
 import (
 	"fmt"
 	"log"
+	"math"
 	urllib "net/url"
 	"strings"
 	"sync"
 	"time"

+	"github.com/PuerkitoBio/goquery"
 	readability "github.com/go-shiori/go-readability"
 	colly "github.com/gocolly/colly/v2"
 	"github.com/gosuri/uiprogress"
 )

-type scraper struct {
-	url string
-}
+func NewBookFromURL(url, selector string, recursive, include bool, limit, delay int) book {
+	if recursive {
+		home := NewChapterFromURL(url)
+		b := New(home.Name(), home.Author())

-func NewBookFromURL(url, selector string, include bool, delay int) Book {
-	home := NewChapterFromURL(url)
-	b := New(home.Name(), home.Author())
+		chapters := tableOfContent(url, selector, limit, delay)
+		if include {
+			b.AddChapter(home)
+		}
+		for _, c := range chapters {
+			b.AddChapter(c)
+		}

-	chapters := tableOfContent(url, selector, delay)
-	if include {
-		b.AddChapter(home)
-	}
-	for _, c := range chapters {
+		return b
+	} else {
+		c := NewChapterFromURL(url)
+		b := New(c.Name(), c.Author())
 		b.AddChapter(c)
+		return b
 	}
-
-	return b
 }

 func NewChapterFromURL(url string) chapter {
@@ -38,69 +43,23 @@ func NewChapterFromURL(url string) chapter {
 		log.Fatalf("failed to parse %s, %v\n", url, err)
 	}

-	// metadata := fmt.Sprintf("URL     : %s\nTitle   : %s\nAuthor  : %s\nLength  : %d\nExcerpt : %s\nSiteName: %s\nImage   : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
-	// fmt.Println(metadata)
-
 	return chapter{article.Title, article.Byline, article.Content}
 }

-func tableOfContent(url, selector string, delay int) []chapter {
-	c := colly.NewCollector()
-
-	classesLinks := map[string][]map[string]string{}
-	classesCount := map[string]int{}
-	classMax := ""
-
-	if selector == "" {
-		c.OnHTML("a", func(e *colly.HTMLElement) {
-			href := e.Attr("href")
-			text := strings.TrimSpace(e.Text)
-			class := e.Attr("class")
-
-			//if class != "" && text != "" {
-				classesLinks[class] = append(classesLinks[class], map[string]string{
-					"href": href,
-					"text": text,
-				})
-
-				classesCount[class]++
-
-				if classesCount[class] > classesCount[classMax] {
-					classMax = class
-				}
-			//}
-
-		})
-	} else {
-		c.OnHTML(selector, func(e *colly.HTMLElement) {
-			href := e.Attr("href")
-			text := strings.TrimSpace(e.Text)
-			class := e.Attr("class")
-
-			//if class != "" && text != "" {
-				classesLinks[class] = append(classesLinks[class], map[string]string{
-					"href": href,
-					"text": text,
-				})
-
-				classesCount[class]++
-
-				if classesCount[class] > classesCount[classMax] {
-					classMax = class
-				}
-			//}
-		})
-	}
-	c.Visit(url)
-	fmt.Println(classesCount)
-	links := classesLinks[classMax]
-
-	chapters := make([]chapter, len(links))
+func tableOfContent(url, selector string, limit, delay int) []chapter {
 	base, err := urllib.Parse(url)
 	if err != nil {
 		log.Fatal(err)
 	}

+	links := GetLinks(base, selector)
+	if limit != -1 {
+		limit = int(math.Min(float64(limit), float64(len(links))))
+		links = links[:limit]
+	}
+
+	chapters := make([]chapter, len(links))
+
 	// init global progress bar
 	uiprogress.Start()
 	barGlobal := uiprogress.AddBar(len(links)).AppendCompleted().PrependElapsed()
@@ -112,7 +71,7 @@ func tableOfContent(url, selector string, delay int) []chapter {
 	bars := []*uiprogress.Bar{}
 	for index, link := range links {
 		bar := uiprogress.AddBar(1).AppendCompleted().PrependElapsed()
-		barText := fmt.Sprintf("%d. %s", index+1, link["text"])
+		barText := fmt.Sprintf("%d. %s", index+1, link.text)
 		bar.AppendFunc(func(b *uiprogress.Bar) string {
 			return barText
 		})
@@ -122,7 +81,7 @@ func tableOfContent(url, selector string, delay int) []chapter {
 	if delay >= 0 {
 		for index, link := range links {
 			// and then use it to parse relative URLs
-			u, err := base.Parse(link["href"])
+			u, err := base.Parse(link.href)
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -141,14 +100,14 @@ func tableOfContent(url, selector string, delay int) []chapter {

 	} else {
 		var wg sync.WaitGroup
-		for index, link := range links {
+		for index, l := range links {

 			wg.Add(1)
-			go func(index int, link map[string]string) {
+			go func(index int, l link) {
 				defer wg.Done()

 				// and then use it to parse relative URLs
-				u, err := base.Parse(link["href"])
+				u, err := base.Parse(l.href)
 				if err != nil {
 					log.Fatal(err)
 				}
@@ -157,9 +116,84 @@ func tableOfContent(url, selector string, delay int) []chapter {

 				bars[index].Incr()
 				barGlobal.Incr()
-			}(index, link)
+			}(index, l)
 		}
 		wg.Wait()
 	}
 	return chapters
 }
+
+func GetPath(elm *goquery.Selection) string {
+	path := []string{}
+
+	for {
+		selector := strings.ToLower(goquery.NodeName(elm))
+		if selector == "" {
+			break
+		}
+
+		path = append(path, selector)
+		elm = elm.Parent()
+	}
+
+	join := strings.Join(path, "<")
+	return join
+}
+
+
+func GetLinks(url *urllib.URL, selector string) []link {
+	selectorSet := true
+	if selector == "" {
+		selector = "a"
+		selectorSet = false
+	}
+
+	// visit and count link classes
+	pathLinks := map[string][]link{}
+	pathCount := map[string]int{}
+	pathMax := ""
+
+	c := colly.NewCollector()
+	c.OnHTML(selector, func(e *colly.HTMLElement) {
+		href := e.Attr("href")
+		text := strings.TrimSpace(e.Text)
+		path := GetPath(e.DOM)
+		class := e.Attr("class")
+		key := fmt.Sprintf("%s.%s", path, class)
+
+		if selectorSet || text != "" {
+			pathLinks[key] = append(pathLinks[key], NewLink(href, text, class))
+			pathCount[key] += len(text)
+			// pathCount[key]++
+
+			if pathCount[key] > pathCount[pathMax] {
+				pathMax = key
+			}
+		}
+	})
+	c.Visit(url.String())
+	return pathLinks[pathMax]
+
+	// // visit and count link classes
+	// classesLinks := map[string][]link{}
+	// classesCount := map[string]int{}
+	// classMax := ""
+
+	// c := colly.NewCollector()
+	// c.OnHTML(selector, func(e *colly.HTMLElement) {
+	// 	href := e.Attr("href")
+	// 	text := strings.TrimSpace(e.Text)
+	// 	class := e.Attr("class")
+
+	// 	if selectorSet || class != "" && text != "" {
+	// 		classesLinks[class] = append(classesLinks[class], NewLink(href, text))
+	// 		classesCount[class]++
+
+	// 		if classesCount[class] > classesCount[classMax] {
+	// 			classMax = class
+	// 		}
+	// 	}
+	// })
+	// c.Visit(url.String())
+	// return classesLinks[classMax]
+}
--- a/cmd/get.go
+++ b/cmd/get.go
@@ -17,7 +17,7 @@ import (

 var quiet, stdout, recursive, include bool
 var format, output, selector string
-var delay int
+var limit, delay int

 var getCmd = &cobra.Command{
 	Use:   "get",
@@ -48,28 +48,27 @@ var getCmd = &cobra.Command{
 			}
 		}

-		if include && recursive == false {
+		if cmd.Flags().Changed("selector") && recursive == false {
+			return errors.New("cannot use selector option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("include") && recursive == false {
 			return errors.New("cannot use include option if not in recursive mode")
 		}

+		if cmd.Flags().Changed("limit") && recursive == false {
+			return errors.New("cannot use limit option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("delay") && recursive == false {
+			return errors.New("cannot use delay option if not in recursive mode")
+		}
+
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		url := args[0]
-		var b book.Book
-
-		if recursive {
-			b = book.NewBookFromURL(url, selector, include, delay)
-		} else {
-			c := book.NewChapterFromURL(url)
-			b = book.New(c.Name(), c.Author())
-			b.AddChapter(c)
-		}
-
-		// if quiet == false {
-		// 	metadata := fmt.Sprintf("URL     : %s\nTitle   : %s\nAuthor  : %s\nLength  : %d\nExcerpt : %s\nSiteName: %s\nImage   : %s\nFavicon : %s", url, article.Title, article.Byline, article.Length, article.Excerpt, article.SiteName, article.Image, article.Favicon)
-		// 	fmt.Println(metadata)
-		// }
+		b := book.NewBookFromURL(url, selector, recursive, include, limit, delay)

 		if len(output) == 0 {
 			// set default output
@@ -92,11 +91,13 @@ var getCmd = &cobra.Command{
 					log.Fatal(err)
 				}

+				text := fmt.Sprintf("%s\n%s\n\n%s\n\n\n", c.Name(), strings.Repeat("=", len(c.Name())), content)
+
 				if stdout {
-					fmt.Println(content)
+					fmt.Println(text)
 				} else {

-					_, err := f.WriteString(content)
+					_, err := f.WriteString(text)
 					if err != nil {
 						log.Fatal(err)
 					}
@@ -104,7 +105,9 @@ var getCmd = &cobra.Command{
 				}
 			}

-			fmt.Printf("Markdown saved to \"%s\"\n", output)
+			if stdout == false {
+				fmt.Printf("Markdown saved to \"%s\"\n", output)
+			}
 		}

 		if format == "epub" {
@@ -112,7 +115,8 @@ var getCmd = &cobra.Command{
 			e.SetAuthor(b.Author())

 			for _, c := range b.Chapters() {
-				e.AddSection(c.Content(), c.Name(), "", "")
+				html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), c.Content())
+				e.AddSection(html, c.Name(), "", "")
 			}

 			err := e.Write(output)
--- a/cmd/list.go
+++ b/cmd/list.go
@@ -2,11 +2,14 @@ package cmd

 import (
 	"errors"
-	"fmt"
-	"strings"
+	"log"
+	urllib "net/url"
+	"os"

-	colly "github.com/gocolly/colly/v2"
+	"github.com/jedib0t/go-pretty/v6/table"
 	cobra "github.com/spf13/cobra"
+
+	"github.com/lapwat/papeer/book"
 )

 var listCmd = &cobra.Command{
@@ -19,37 +22,27 @@ var listCmd = &cobra.Command{
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		url := args[0]
-		c := colly.NewCollector()
-
-		// visit and count link classes
-		classesLinks := map[string][]map[string]string{}
-		classesCount := map[string]int{}
-		classMax := ""
-
-		c.OnHTML(selector, func(e *colly.HTMLElement) {
-			href := e.Attr("href")
-			text := strings.TrimSpace(e.Text)
-			class := e.Attr("class")
-
-			// if class != "" && text != "" {
-				classesLinks[class] = append(classesLinks[class], map[string]string{
-					"href": href,
-					"text": text,
-				})
-
-				classesCount[class]++
-
-				if classesCount[class] > classesCount[classMax] {
-					classMax = class
-				}
-			// }
-		})
-
-		c.Visit(url)
-		for index, link := range classesLinks[classMax] {
-			fmt.Printf("Chapter %d: %s %s\n", index+1, link["text"], link["href"])
+		base, err := urllib.Parse(args[0])
+		if err != nil {
+			log.Fatal(err)
 		}

+		links := book.GetLinks(base, selector)
+
+		t := table.NewWriter()
+		t.SetOutputMirror(os.Stdout)
+		t.AppendHeader(table.Row{"#", "Name", "Url", "Class"})
+
+		for index, link := range links {
+			u, err := base.Parse(link.Href())
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			t.AppendRow([]interface{}{index + 1, link.Text(), u.String(), link.Class()})
+		}
+
+		t.Render()
+
 	},
 }
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -28,9 +28,10 @@ func init() {
 	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector")
 	rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "create one chapter per natigation item")
 	rootCmd.PersistentFlags().BoolVarP(&include, "include", "i", false, "include URL as first chapter, in resursive mode")
-	rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "do not show progress bars")
+	rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "do not show logs")
 	rootCmd.PersistentFlags().BoolVarP(&stdout, "stdout", "", false, "print to standard output")
-	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "wait before downloading next chapter, in milliseconds")
+	rootCmd.PersistentFlags().IntVarP(&limit, "limit", "l", -1, "limit number of chapters, in recursive mode")
+	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "time to wait before downloading next chapter, in milliseconds")

 	rootCmd.AddCommand(getCmd)
 	rootCmd.AddCommand(listCmd)
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print the version number of papeer",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("papeer v0.0.1")
+		fmt.Println("papeer v0.1.1")
 	},
 }
--- a/go.mod
+++ b/go.mod
@@ -10,33 +10,35 @@ require (
 )

 require (
+	github.com/PuerkitoBio/goquery v1.7.1
 	github.com/gocolly/colly/v2 v2.1.0
 	github.com/gosuri/uilive v0.0.4 // indirect
 	github.com/gosuri/uiprogress v0.0.1
 )

 require (
-	github.com/PuerkitoBio/goquery v1.5.1 // indirect
 	github.com/andybalholm/cascadia v1.2.0 // indirect
 	github.com/antchfx/htmlquery v1.2.3 // indirect
-	github.com/antchfx/xmlquery v1.2.4 // indirect
-	github.com/antchfx/xpath v1.1.8 // indirect
+	github.com/antchfx/xmlquery v1.3.6 // indirect
+	github.com/antchfx/xpath v1.2.0 // indirect
 	github.com/go-shiori/dom v0.0.0-20210627111528-4e4722cd0d65 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gofrs/uuid v3.1.0+incompatible // indirect
+	github.com/gofrs/uuid v4.0.0+incompatible // indirect
 	github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/jedib0t/go-pretty/v6 v6.2.4 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
-	github.com/mattn/go-isatty v0.0.3 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/temoto/robotstxt v1.1.1 // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
-	golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 // indirect
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/text v0.3.6 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -42,8 +42,9 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/JohannesKaufmann/html-to-markdown v1.3.0 h1:K/p4cq8Ib13hcSVcKQNfKCSWw93CYW5pAjY0fl85has=
 github.com/JohannesKaufmann/html-to-markdown v1.3.0/go.mod h1:JNSClIRYICFDiFhw6RBhBeWGnMSSKVZ6sPQA+TK4tyM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
+github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
@@ -51,11 +52,14 @@ github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5
 github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
 github.com/antchfx/htmlquery v1.2.3 h1:sP3NFDneHx2stfNXCKbhHFo8XgNjCACnU/4AO5gWz6M=
 github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0=
-github.com/antchfx/xmlquery v1.2.4 h1:T/SH1bYdzdjTMoz2RgsfVKbM5uWh3gjDYYepFqQmFv4=
 github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM=
+github.com/antchfx/xmlquery v1.3.6 h1:kaEVzH1mNo/2AJZrhZjAaAUTy2Nn2zxGfYYU8jWfXOo=
+github.com/antchfx/xmlquery v1.3.6/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc=
 github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
-github.com/antchfx/xpath v1.1.8 h1:PcL6bIX42Px5usSx6xRYw/wjB3wYGkj0MJ9MBzEKVgk=
 github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
+github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8=
+github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -99,6 +103,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -118,8 +123,9 @@ github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuu
 github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs=
 github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/uuid v3.1.0+incompatible h1:q2rtkjaKT4YEr6E1kamy0Ha4RtepWlQBedyHx0uzKwA=
 github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
+github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
@@ -129,8 +135,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -225,6 +232,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
+github.com/jedib0t/go-pretty/v6 v6.2.4 h1:wdaj2KHD2W+mz8JgJ/Q6L/T5dB7kyqEFI16eLq7GEmk=
+github.com/jedib0t/go-pretty/v6 v6.2.4/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -245,8 +254,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -268,6 +280,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -328,8 +341,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/temoto/robotstxt v1.1.1 h1:Gh8RCs8ouX3hRSxxK7B1mO5RFByQ4CmJZDwgom++JaA=
 github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
+github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
+github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -435,6 +449,7 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -470,6 +485,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -516,8 +532,9 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -691,8 +708,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
--- a/release.sh
+++ b/release.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+version=$1
+platforms=("linux/amd64" "darwin/amd64" "windows/amd64")
+
+for platform in "${platforms[@]}"
+do
+    platform_split=(${platform//\// })
+    GOOS=${platform_split[0]}
+    GOARCH=${platform_split[1]}
+    output_name='papeer-'$version'-'$GOOS'-'$GOARCH
+    if [ $GOOS = "windows" ]; then
+        output_name+='.exe'
+    fi
+
+    env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name
+    if [ $? -ne 0 ]; then
+        echo 'An error has occurred! Aborting the script execution...'
+        exit 1
+    fi
+done
Author	SHA1	Message	Date
lapwat	ee41e49dd1	refacto, better scrape strategy	2021-09-24 02:09:22 +02:00
lapwat	e50adf5e03	better display, release script	2021-09-21 19:55:37 +02:00
lapwat	079122f4a0	docs	2021-09-15 20:51:16 +02:00
lapwat	9ee5f43748	docs	2021-09-15 20:47:53 +02:00
lapwat	e3833fdddd	docs	2021-09-15 20:47:22 +02:00
lapwat	a7d14e7372	bump version	2021-09-15 20:26:50 +02:00
lapwat	ac1fb3dd51	limit option, docs	2021-09-15 20:25:23 +02:00
lapwat	bb1df3b6a3	update readme	2021-09-14 23:28:22 +02:00
lapwat	83adc22e2c	update readme	2021-09-14 23:27:26 +02:00
lapwat	73441aa7fb	update readme	2021-09-14 23:18:32 +02:00