refacto get command, fix: images option

Makefile

@@ -0,0 +1,14 @@
+format:
+	gofmt -s -w .
+
+test:
+	go test github.com/lapwat/papeer/book
+
+install:
+	go install
+
+clean:
+	find . -maxdepth 1 -not -name 'README.md' -name '*.md' -delete
+	find . -maxdepth 1 -name '*.epub' -delete
+	find . -maxdepth 1 -name '*.mobi' -delete
+	find . -maxdepth 1 -name 'papeer-v*' -delete

README.md

@@ -1,53 +1,6 @@
-```
+# Papeer
-❯ papeer get --format=epub --recursive --delay=500 --limit=10 https://news.ycombinator.com/
-[===============================================>--------------------] Chapters 7 / 10
-[====================================================================] 1. Three ex-US intelligence officers admit hacking for UAE
-[====================================================================] 2. Show HN: Time Travel Debugger
-[====================================================================] 3. How much faster is Java 17?
-[====================================================================] 4. The First Webcam Was Invented to Keep an Eye on a Coffee Pot
-[====================================================================] 5. Nikon's 2021 Photomicrography Competition Winners
-[====================================================================] 6. HTTP Status 418 – I'm a teapot
-[====================================================================] 7. H3: Hexagonal hierarchical geospatial indexing system
-[--------------------------------------------------------------------] 8. Automatic cipher suite ordering in Go’s crypto/tls
-[--------------------------------------------------------------------] 9. Find engineering roles at over 800 YC-funded startups
-[--------------------------------------------------------------------] 10. Futarchy: Robin Hanson on prediction markets
-Ebook saved to "Hacker_News.epub"
-```
 
-# Installation
+Papeer is a tool that lets you scrape content from the internet. It can scrape any web page, keeping only relevant content (formatted text and images) and removing ads and menus. You can export the content to Markdown, EPUB or MOBI files.
-
-## From source
-
-```sh
-go get -u github.com/lapwat/papeer
-```
-
-## From binary
-
-### On Linux / MacOS
-
-```sh
-platform=linux
-# platform=darwin for MacOS
-curl -L https://github.com/lapwat/papeer/releases/download/v0.2.0/papeer-v0.2.0-$platform-amd64 > papeer
-chmod +x papeer
-sudo mv papeer /usr/local/bin
-```
-
-### On Windows
-
-Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.2.0/papeer-v0.2.0-windows-amd64.exe).
-
-## Install kindlegen to export websites to MOBI (optional)
-
-```sh
-TMPDIR=$(mktemp -d -t papeer-XXXXX)
-curl -L https://github.com/lapwat/papeer/releases/download/kindlegen/kindlegen_linux_2.6_i386_v2_9.tar.gz > $TMPDIR/kindlegen.tar.gz
-tar xzvf $TMPDIR/kindlegen.tar.gz -C $TMPDIR
-chmod +x $TMPDIR/kindlegen
-sudo mv $TMPDIR/kindlegen /usr/local/bin
-rm -rf $TMPDIR
-```
 
 # Usage
 
@@ -66,21 +19,119 @@ Available Commands:
   version     Print the version number of papeer
 
 Flags:
+  -a, --author string     book author
   -d, --delay int         time to wait before downloading next chapter, in milliseconds (default -1)
-  -f, --format string     file format [md, epub, mobi] (default "md")
+  -f, --format string     file format [stdout, md, epub, mobi] (default "stdout")
   -h, --help              help for papeer
       --images            retrieve images only
   -i, --include           include URL as first chapter, in resursive mode
   -l, --limit int         limit number of chapters, in recursive mode (default -1)
+  -n, --name string       book name (default: page title)
   -o, --offset int        skip first chapters, in recursive mode
-      --output string     output file
+      --output string     file name (default: book name)
   -r, --recursive         create one chapter per natigation item
-  -s, --selector string   table of content CSS selector
+  -s, --selector string   table of content CSS selector, in resursive mode
-      --stdout            print to standard output
+  -t, --threads int       download concurrency, in recursive mode (default -1)
 
 Use "papeer [command] --help" for more information about a command.
 ```
 
+# Examples
+
+## Grab a single page
+
+The `get` command lets you retrieve the content of a web page.
+
+```sh
+papeer get https://www.eff.org/cyberspace-independence
+# A Declaration of the Independence of Cyberspace
+# ===============================================
+
+# Governments of the Industrial World, you weary giants of flesh and steel, I come from Cyberspace, the new home of Mind. On behalf of the future, I ask you of the past to leave us alone. You are not welcome among us. You have no sovereignty where we gather...
+```
+
+## Grab several pages (recursive mode)
+
+The `recursive` option lets you extract the table of content of a website, then scrape the content of each one of its pages.
+
+### Display table of content
+
+Before trying the `recursive` option, it is a good idea to use the `ls` option, which lets you vizualize the content that will be retrieved. You can use several options to customize the table of content extraction, such as `selector`, `limit`, `offset` and `include`. Type `papeer help` for more information about those options.
+
+```sh
+papeer ls https://12factor.net/ -s 'section.concrete > article > h2 > a'
+#  #  NAME                    URL                                    
+#  1  I. Codebase             https://12factor.net/codebase          
+#  2  II. Dependencies        https://12factor.net/dependencies      
+#  3  III. Config             https://12factor.net/config            
+#  4  IV. Backing services    https://12factor.net/backing-services  
+#  5  V. Build, release, run  https://12factor.net/build-release-run 
+#  6  VI. Processes           https://12factor.net/processes         
+#  7  VII. Port binding       https://12factor.net/port-binding      
+#  8  VIII. Concurrency       https://12factor.net/concurrency       
+#  9  IX. Disposability       https://12factor.net/disposability     
+# 10  X. Dev/prod parity      https://12factor.net/dev-prod-parity   
+# 11  XI. Logs                https://12factor.net/logs              
+# 12  XII. Admin processes    https://12factor.net/admin-processes
+```
+
+### Scrape time
+
+Once you are satisfied with the table of content listed by the `ls` command, you can actually scrape the content of those pages. You can use the same options that you specified for the `ls` command. In recursive mode, you also have the possibility to use `delay` and `threads` options.
+
+```sh
+papeer get https://12factor.net/ --recursive -s 'section.concrete > article > h2 > a' --format=md
+# [======================================>-----------------------------] Chapters 7 / 12
+# [====================================================================] 1. I. Codebase
+# [====================================================================] 2. II. Dependencies
+# [====================================================================] 3. III. Config
+# [====================================================================] 4. IV. Backing services
+# [====================================================================] 5. V. Build, release, run
+# [====================================================================] 6. VI. Processes
+# [====================================================================] 7. VII. Port binding
+# [--------------------------------------------------------------------] 8. VIII. Concurrency
+# [--------------------------------------------------------------------] 9. IX. Disposability
+# [--------------------------------------------------------------------] 10. X. Dev/prod parity
+# [--------------------------------------------------------------------] 11. XI. Logs
+# [--------------------------------------------------------------------] 12. XII. Admin processes
+# Markdown saved to "The_Twelve-Factor_App.md"
+```
+
+# Installation
+
+## From source
+
+```sh
+go get -u github.com/lapwat/papeer
+```
+
+## From binary
+
+### On Linux / MacOS
+
+```sh
+platform=linux # use platform=darwin for MacOS
+release=0.3.2
+curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64 > papeer
+chmod +x papeer
+sudo mv papeer /usr/local/bin
+```
+
+### On Windows
+
+Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.3.2/papeer-v0.3.2-windows-amd64.exe).
+
+## Install kindlegen to export websites to MOBI (optional)
+
+```sh
+TMPDIR=$(mktemp -d -t papeer-XXXXX)
+curl -L https://github.com/lapwat/papeer/releases/download/kindlegen/kindlegen_linux_2.6_i386_v2_9.tar.gz > $TMPDIR/kindlegen.tar.gz
+tar xzvf $TMPDIR/kindlegen.tar.gz -C $TMPDIR
+chmod +x $TMPDIR/kindlegen
+sudo mv $TMPDIR/kindlegen /usr/local/bin
+rm -rf $TMPDIR
+```
+
 # Autocompletion
 
 Execute this command in your current shell, or add it to your `.bashrc`.

book/chapter.go

@@ -1,13 +1,20 @@
 package book
 
 type chapter struct {
-	name    string
+	body        string
-	author  string
+	name        string
-	content string
+	author      string
+	content     string
+	subChapters []chapter
+	config      *ScrapeConfig
 }
 
-func NewChapter(name, author, content string) chapter {
+func NewChapter(body, name, author, content string, subChapters []chapter, config *ScrapeConfig) chapter {
-	return chapter{name, author, content}
+	return chapter{body, name, author, content, subChapters, config}
+}
+
+func (c chapter) Body() string {
+	return c.body
 }
 
 func (c chapter) Name() string {
@@ -21,3 +28,7 @@ func (c chapter) Author() string {
 func (c chapter) Content() string {
 	return c.content
 }
+
+func (c chapter) SubChapters() []chapter {
+	return c.subChapters
+}

book/format.go

@@ -0,0 +1,143 @@
+package book
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+
+	md "github.com/JohannesKaufmann/html-to-markdown"
+	"github.com/PuerkitoBio/goquery"
+	epub "github.com/bmaupin/go-epub"
+)
+
+func Filename(name string) string {
+	filename := name
+
+	filename = strings.ReplaceAll(filename, " ", "_")
+	filename = strings.ReplaceAll(filename, "/", "")
+
+	return filename
+}
+
+func ToMarkdown(c chapter) string {
+	markdown := ""
+
+	if c.config.include {
+		// title
+		markdown += fmt.Sprintf("%s\n", c.Name())
+		markdown += fmt.Sprintf("%s\n\n", strings.Repeat("=", len(c.Name())))
+
+		// convert content to markdown
+		content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
+		if err != nil {
+			log.Fatal(err)
+		}
+		markdown += fmt.Sprintf("%s\n\n\n", content)
+	}
+
+	for _, sc := range c.SubChapters() {
+		// subchapters content
+		markdown += fmt.Sprintf("%s\n\n\n", ToMarkdown(sc))
+	}
+
+	return markdown
+}
+
+func ToEpub(c chapter, filename string) string {
+	if len(filename) == 0 {
+		filename = fmt.Sprintf("%s.epub", Filename(c.Name()))
+	}
+
+	// init ebook
+	e := epub.NewEpub(c.Name())
+	e.SetAuthor(c.Author())
+
+	AppendToEpub(e, c)
+
+	err := e.Write(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return filename
+}
+
+func AppendToEpub(e *epub.Epub, c chapter) {
+	content := ""
+
+	if c.config.include {
+
+		if c.config.imagesOnly == false {
+			content = c.Content()
+		}
+
+		// parse content
+		doc, err := goquery.NewDocumentFromReader(strings.NewReader(c.Content()))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		// download images and replace src in img tags of content
+		doc.Find("img").Each(func(i int, s *goquery.Selection) {
+			src, _ := s.Attr("src")
+			src = strings.Split(src, "?")[0] // remove query part
+			imagePath, _ := e.AddImage(src, "")
+
+			if c.config.imagesOnly {
+				imageTag, _ := goquery.OuterHtml(s)
+				content += strings.Replace(imageTag, src, imagePath, 1)
+			} else {
+				content = strings.Replace(content, src, imagePath, 1)
+			}
+		})
+
+		html := ""
+		// add title only if imagesOnly = false
+		if c.config.imagesOnly == false {
+			html += fmt.Sprintf("<h1>%s</h1>", c.Name())
+		}
+		html += content
+
+		//  write to epub file
+		_, err = e.AddSection(html, c.Name(), "", "")
+		if err != nil {
+			log.Fatal(err)
+		}
+
+	}
+
+	for _, sc := range c.SubChapters() {
+		AppendToEpub(e, sc)
+	}
+}
+
+func ToMobi(c chapter, filename string) string {
+	if len(filename) == 0 {
+		filename = fmt.Sprintf("%s.mobi", Filename(c.Name()))
+	} else {
+
+		// add .mobi extension if not specified
+		if strings.HasSuffix(filename, ".mobi") == false {
+			filename = fmt.Sprintf("%s.mobi", filename)
+		}
+
+	}
+
+	filenameEPUB := strings.ReplaceAll(filename, ".mobi", ".epub")
+	ToEpub(c, filenameEPUB)
+
+	exec.Command("kindlegen", filenameEPUB).Run()
+	// exec command always return status 1 even if it succeed
+	// if err != nil {
+	// 	log.Fatal(err)
+	// }
+
+	err := os.Remove(filenameEPUB)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return filename
+}

book/format_test.go

@@ -0,0 +1,79 @@
+package book
+
+import (
+	"errors"
+	"os"
+	"testing"
+)
+
+func TestFilename(t *testing.T) {
+
+	got := Filename("This is a chapter / book")
+	want := "This_is_a_chapter__book"
+
+	if got != want {
+		t.Errorf("got %q, wanted %q", got, want)
+	}
+
+}
+
+func TestToMarkdown(t *testing.T) {
+
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
+
+	got := ToMarkdown(c)
+	want := "Books\n=====\n\n- [Discours de la Méthode](https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/)clock 98 min read -\n1637\n\n- [The Twelve-Factor App](https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/)clock 22 min read -\n2011\n\n\n"
+
+	if got != want {
+		t.Errorf("got %q, wanted %q", got, want)
+	}
+
+}
+
+func TestToEpub(t *testing.T) {
+
+	filename := "Books.epub"
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
+	ToEpub(c, "")
+
+	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
+		t.Errorf("%s does not exist: %v", filename, err)
+	} else {
+		if err := os.Remove(filename); err != nil {
+			t.Errorf("cannot remove %v: %v", filename, err)
+		}
+	}
+
+}
+
+func TestToEpubFilename(t *testing.T) {
+
+	filename := "ebook.epub"
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
+	ToEpub(c, filename)
+
+	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
+		t.Errorf("%s does not exist: %v", filename, err)
+	} else {
+		if err := os.Remove(filename); err != nil {
+			t.Errorf("cannot remove %v: %v", filename, err)
+		}
+	}
+
+}
+
+func TestToMobi(t *testing.T) {
+
+	filename := "ebook.mobi"
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
+	ToMobi(c, filename)
+
+	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
+		t.Errorf("%s does not exist: %v", filename, err)
+	} else {
+		if err := os.Remove(filename); err != nil {
+			t.Errorf("cannot remove %v: %v", filename, err)
+		}
+	}
+
+}

book/progress.go

@@ -2,6 +2,7 @@ package book
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/gosuri/uiprogress"
 )
@@ -11,20 +12,22 @@ type progress struct {
 	individuals []*uiprogress.Bar
 }
 
-func NewProgress(links []link) progress {
+func NewProgress(links []link, parent string, depth int) progress {
 	uiprogress.Start()
 
 	global := uiprogress.AddBar(len(links))
+	indentGlobal := strings.Repeat("> ", depth)
 	global.AppendFunc(func(b *uiprogress.Bar) string {
-		return fmt.Sprintf("Chapters %d / %d", b.Current(), len(links))
+		return fmt.Sprintf("%v%v (%v / %v)", indentGlobal, parent, b.Current(), len(links))
 	})
 
-	individuals := []*uiprogress.Bar{}
 	// hide individual bars if more than 50 chapters
+	individuals := []*uiprogress.Bar{}
+	indent := strings.Repeat("- ", depth)
 	if len(links) <= 50 {
 		for index, link := range links {
 			bar := uiprogress.AddBar(1)
-			barText := fmt.Sprintf("%d. %s", index+1, link.text)
+			barText := fmt.Sprintf("%v#%v %v", indent, index+1, link.Text())
 			bar.AppendFunc(func(b *uiprogress.Bar) string {
 				return barText
 			})
@@ -35,13 +38,22 @@ func NewProgress(links []link) progress {
 	return progress{global, individuals}
 }
 
-func (p *progress) IncrGlobal() {
+func (p *progress) IncrementGlobal() {
 	p.global.Incr()
 }
 
-func (p *progress) Incr(index int) {
+func (p *progress) Increment(index int) {
-	p.global.Incr()
+	p.IncrementGlobal()
 	if len(p.individuals) > index {
 		p.individuals[index].Incr()
 	}
 }
+
+func (p *progress) UpdateName(index int, name string) {
+	if len(p.individuals) > index {
+		barText := fmt.Sprintf("%s", name)
+		p.individuals[index].AppendFunc(func(b *uiprogress.Bar) string {
+			return barText
+		})
+	}
+}

book/scraper.go

@@ -1,9 +1,12 @@
 package book
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"log"
 	"math"
+	"net/http"
 	urllib "net/url"
 	"strings"
 	"sync"
@@ -14,63 +17,246 @@ import (
 	colly "github.com/gocolly/colly/v2"
 )
 
-func NewBookFromURL(url, selector string, recursive, include, images bool, limit, offset, delay int) book {
+type ScrapeConfig struct {
+	depth      int
+	selector   string
+	limit      int
+	offset     int
+	delay      int
+	threads    int
+	include    bool
+	imagesOnly bool
+}
+
+func NewScrapeConfig() *ScrapeConfig {
+	return &ScrapeConfig{0, "", -1, 0, -1, -1, true, false}
+}
+
+func NewScrapeConfigsAjin() []*ScrapeConfig {
+	config0 := NewScrapeConfig()
+	config0.depth = 0
+	config0.selector = ".dt>a"
+	config0.limit = 3
+	config0.offset = 0
+	config0.delay = 5000
+	config0.include = false
+
+	config1 := NewScrapeConfig()
+	config1.depth = 1
+	config1.selector = ".nav_apb>a"
+	config1.limit = 3
+	config1.offset = 1
+	config1.delay = 5000
+	config1.include = false
+
+	config2 := NewScrapeConfig()
+	config2.depth = 2
+	config2.imagesOnly = true
+
+	return []*ScrapeConfig{config0, config1, config2}
+}
+
+func NewScrapeConfigsWikipedia() []*ScrapeConfig {
+	config0 := NewScrapeConfig()
+	config0.depth = 0
+	config0.threads = -1
+	config0.include = true
+
+	config1 := NewScrapeConfig()
+	config1.depth = 1
+	config1.include = true
+
+	return []*ScrapeConfig{config0, config1}
+}
+
+func NewScrapeConfigFake() *ScrapeConfig {
+	config := NewScrapeConfig()
+	config.include = false
+
+	return config
+}
+
+func NewBookFromURL(url, selector, name, author string, recursive, include, imagesOnly bool, limit, offset, delay, threads int) book {
+	config1 := NewScrapeConfig()
+	config1.imagesOnly = imagesOnly
+
+	var chapters []chapter
+	var home chapter
+
 	if recursive {
-		chapters := tableOfContent(url, selector, limit, offset, delay, include, images)
+		config2 := NewScrapeConfig()
-
+		config2.selector = selector
-		b := New(chapters[0].Name(), chapters[0].Author())
+		config2.limit = limit
-		for _, c := range chapters {
+		config2.offset = offset
-			b.AddChapter(c)
+		config2.delay = delay
-		}
+		config2.threads = threads
-
+		config2.include = include
-		return b
+		config2.imagesOnly = imagesOnly
+		chapters, home = tableOfContent(url, config2, config1)
 	} else {
-		c := NewChapterFromURL(url, images)
+		chapters = []chapter{NewChapterFromURL(url, []*ScrapeConfig{config1}, 0, func(index int, name string) {})}
-		b := New(c.Name(), c.Author())
+		home = chapters[0]
+	}
+
+	if len(name) == 0 {
+		name = home.Name()
+	}
+
+	if len(author) == 0 {
+		author = home.Author()
+	}
+
+	b := New(name, author)
+	for _, c := range chapters {
 		b.AddChapter(c)
-		return b
 	}
+
+	return b
 }
 
-func NewChapterFromURL(url string, images bool) chapter {
+func NewChapterFromURL(url string, configs []*ScrapeConfig, index int, updateProgressBarName func(index int, name string)) chapter {
-	article, err := readability.FromURL(url, 30*time.Second)
+	config := configs[0]
-	if err != nil {
-		log.Fatalf("failed to parse %s, %v\n", url, err)
-	}
 
-	content := strings.ReplaceAll(article.Content, "\n", "")
-
-	if images {
-		// Load the HTML document
-		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		// Find the review items
-		doc.Find("img").Each(func(i int, s *goquery.Selection) {
-			content, _ = goquery.OuterHtml(s)
-		})
-	}
-
-	return chapter{article.Title, article.Byline, content}
-}
-
-func tableOfContent(url, selector string, limit, offset, delay int, include, images bool) []chapter {
 	base, err := urllib.Parse(url)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	links, err := GetLinks(base, selector, limit, offset, include)
+	// get page body
+	response, err := http.Get(url)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer response.Body.Close()
+
+	// duplicate response stream
+	readabilityReader := &bytes.Buffer{}
+	bodyReader := io.TeeReader(response.Body, readabilityReader)
+
+	// extract HTML body
+	body, err := io.ReadAll(bodyReader)
+
+	// extract article content and metadata
+	article, err := readability.FromReader(readabilityReader, base)
+	if err != nil {
+		log.Fatalf("failed to parse %s, %v\n", url, err)
+	}
+	name := article.Title
+
+	// notify progress bar with new name
+	updateProgressBarName(index, name)
+
+	subchapters := []chapter{}
+	if len(configs) > 1 {
+		// add subchapters
+
+		links, _, err := GetLinks(base, config.selector, config.limit, config.offset, false)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		subchapters = make([]chapter, len(links))
+		progress := NewProgress(links, name, config.depth)
+
+		if config.delay >= 0 {
+
+			// synchronous mode
+			for index, link := range links {
+				// and then use it to parse relative URLs
+				u, err := base.Parse(link.href)
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				sc := NewChapterFromURL(u.String(), configs[1:], index, progress.UpdateName)
+				subchapters[index] = sc
+				progress.Increment(index)
+
+				time.Sleep(time.Duration(config.delay) * time.Millisecond)
+			}
+
+		} else {
+			// asynchronous mode
+			var wg sync.WaitGroup
+
+			threads := config.threads
+			if threads == -1 {
+				threads = len(links)
+			}
+			semaphore := make(chan bool, threads)
+
+			for index, l := range links {
+
+				wg.Add(1)
+				semaphore <- true
+
+				go func(index int, l link) {
+					defer wg.Done()
+
+					// and then use it to parse relative URLs
+					u, err := base.Parse(l.href)
+					if err != nil {
+						log.Fatal(err)
+					}
+
+					sc := NewChapterFromURL(u.String(), configs[1:], index, progress.UpdateName)
+					subchapters[index] = sc
+					progress.Increment(index)
+
+					<-semaphore
+				}(index, l)
+			}
+			wg.Wait()
+		}
+	}
+
+	content := ""
+	if config.include {
+
+		// we care about the content only if we include this level
+		content = article.Content
+
+		// extract images
+		if config.imagesOnly {
+
+			// parse HTML
+			doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			// append every image to content
+			content = ""
+			doc.Find("img").Each(func(i int, s *goquery.Selection) {
+				imageTag, _ := goquery.OuterHtml(s)
+				imageTag = strings.ReplaceAll(imageTag, "\n", "")
+
+				content += imageTag
+			})
+
+		}
+	}
+
+	return chapter{string(body), name, article.Byline, content, subchapters, config}
+}
+
+func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig) ([]chapter, chapter) {
+	base, err := urllib.Parse(url)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	links, home, err := GetLinks(base, config.selector, config.limit, config.offset, config.include)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	chapters := make([]chapter, len(links))
-	progress := NewProgress(links)
+	// progress := NewProgress(links, "", 0)
+	delay := config.delay
 
 	if delay >= 0 {
+		// synchronous mode
 
 		for index, link := range links {
 			// and then use it to parse relative URLs
@@ -79,8 +265,9 @@ func tableOfContent(url, selector string, limit, offset, delay int, include, ima
 				log.Fatal(err)
 			}
 
-			chapters[index] = NewChapterFromURL(u.String(), images)
+			sc := NewChapterFromURL(u.String(), []*ScrapeConfig{subConfig}, 0, func(index int, name string) {})
-			progress.Incr(index)
+			chapters[index] = sc
+			// progress.Increment(index)
 
 			// short sleep for last chapter to let the progress bar update
 			if index == len(links)-1 {
@@ -91,10 +278,20 @@ func tableOfContent(url, selector string, limit, offset, delay int, include, ima
 		}
 
 	} else {
+		// asynchronous mode
 		var wg sync.WaitGroup
+
+		threads := config.threads
+		if threads == -1 {
+			threads = len(links)
+		}
+		semaphore := make(chan bool, threads)
+
 		for index, l := range links {
 
 			wg.Add(1)
+			semaphore <- true
+
 			go func(index int, l link) {
 				defer wg.Done()
 
@@ -104,14 +301,17 @@ func tableOfContent(url, selector string, limit, offset, delay int, include, ima
 					log.Fatal(err)
 				}
 
-				chapters[index] = NewChapterFromURL(u.String(), images)
+				sc := NewChapterFromURL(u.String(), []*ScrapeConfig{subConfig}, 0, func(index int, name string) {})
-				progress.Incr(index)
+				chapters[index] = sc
+				// progress.Increment(index)
 
+				<-semaphore
 			}(index, l)
 		}
 		wg.Wait()
 	}
-	return chapters
+
+	return chapters, home
 }
 
 func GetPath(elm *goquery.Selection) string {
@@ -131,7 +331,7 @@ func GetPath(elm *goquery.Selection) string {
 	return join
 }
 
-func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, error) {
+func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, chapter, error) {
 	selectorSet := true
 	if selector == "" {
 		selector = "a"
@@ -169,7 +369,7 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool)
 
 	links := pathLinks[pathMax]
 	if len(links) == 0 {
-		return []link{}, fmt.Errorf("no link found for selector: %s", selector)
+		return []link{}, chapter{}, fmt.Errorf("no link found for selector: %s", selector)
 	}
 
 	end := len(links)
@@ -179,11 +379,12 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool)
 
 	links = links[offset:end]
 
+	home := NewChapterFromURL(url.String(), []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
+
 	if include {
-		c := NewChapterFromURL(url.String(), false)
+		l := NewLink(url.String(), home.Name())
-		l := NewLink(url.String(), c.Name())
 		links = append([]link{l}, links...)
 	}
 
-	return links, nil
+	return links, home, nil
 }

book/scraper_test.go

@@ -0,0 +1,184 @@
+package book
+
+import (
+	"testing"
+	"time"
+)
+
+func TestBody(t *testing.T) {
+
+	config := NewScrapeConfig()
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {})
+
+	got := c.Body()
+	want := "<!doctype html>\n<html lang=\"en-us\">\n  <head>\n    <title>Books</title>\n    <link rel=\"shortcut icon\" href=\"/favicon.ico\" />\n    <meta charset=\"utf-8\" />\n    <meta name=\"generator\" content=\"Hugo 0.59.1\" />\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n    <meta name=\"author\" content=\"John Doe\" />\n    <meta name=\"description\" content=\" \" />\n    <link rel=\"stylesheet\" href=\"https://books.lapw.at/css/main.min.88e7083eff65effb7485b6e6f38d10afbec25093a6fac42d734ce9024d3defbd.css\" />\n\n    \n    <meta name=\"twitter:card\" content=\"summary\"/>\n<meta name=\"twitter:title\" content=\"Books\"/>\n<meta name=\"twitter:description\" content=\" \"/>\n\n    <meta property=\"og:title\" content=\"Books\" />\n<meta property=\"og:description\" content=\" \" />\n<meta property=\"og:type\" content=\"website\" />\n<meta property=\"og:url\" content=\"https://books.lapw.at/\" />\n\n\n\n  </head>\n  <body>\n    <header class=\"app-header\">\n      <a href=\"https://books.lapw.at/\"><img class=\"app-header-avatar\" src=\"/book.svg\" alt=\"John Doe\" /></a>\n      <h1>Books</h1>\n      <p> </p>\n      <div class=\"app-header-social\">\n        \n      </div>\n    </header>\n    <main class=\"app-container\">\n      \n  <article>\n    <h1>Books</h1>\n    <ul class=\"posts-list\">\n      \n        <li class=\"posts-list-item\">\n          <a class=\"posts-list-item-title\" href=\"https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/\">Discours de la Méthode</a>\n          <span class=\"posts-list-item-description\">\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\" class=\"icon icon-clock\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 98 min read -\n            1637\n          </span>\n        </li>\n      \n        <li class=\"posts-list-item\">\n          <a class=\"posts-list-item-title\" href=\"https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/\">The Twelve-Factor App</a>\n          <span class=\"posts-list-item-description\">\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\" class=\"icon icon-clock\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 22 min read -\n            2011\n          </span>\n        </li>\n      \n    </ul>\n    \n\n\n\n  </article>\n\n    </main>\n  </body>\n</html>\n"
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestName(t *testing.T) {
+
+	config := NewScrapeConfig()
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {})
+
+	got := c.Name()
+	want := "Books"
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestAuthor(t *testing.T) {
+
+	config := NewScrapeConfig()
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {})
+
+	got := c.Author()
+	want := "John Doe"
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestContent(t *testing.T) {
+
+	config := NewScrapeConfig()
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {})
+
+	got := c.Content()
+	want := "<div id=\"readability-page-1\" class=\"page\">\n    \n    <main>\n      \n  <article>\n    \n    <ul>\n      \n        <li>\n          <a href=\"https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/\">Discours de la Méthode</a>\n          <span>\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 98 min read -\n            1637\n          </span>\n        </li>\n      \n        <li>\n          <a href=\"https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/\">The Twelve-Factor App</a>\n          <span>\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 22 min read -\n            2011\n          </span>\n        </li>\n      \n    </ul>\n    \n\n\n\n  </article>\n\n    </main>\n  \n\n</div>"
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestDelay(t *testing.T) {
+
+	config0 := NewScrapeConfig()
+	config0.delay = 500
+
+	config1 := NewScrapeConfig()
+
+	start := time.Now()
+	NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
+	elapsed := time.Since(start)
+
+	got := elapsed
+	want := time.Duration(500) * time.Millisecond
+
+	if got < want {
+		t.Errorf("got %v, wanted min %v", got, want)
+	}
+
+}
+
+func TestContentImagesOnly(t *testing.T) {
+
+	config := NewScrapeConfig()
+	config.imagesOnly = true
+
+	c := NewChapterFromURL("https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/", []*ScrapeConfig{config}, 0, func(index int, name string) {})
+
+	got := c.Content()
+	want := "<img src=\"https://books.lapw.at/images/codebase-deploys.png\" alt=\"One codebase maps to many deploys\"/><img src=\"https://books.lapw.at/images/attached-resources.png\" alt=\"A production deploy attached to four backing services.\"/><img src=\"https://books.lapw.at/images/release.png\" alt=\"Code becomes a build, which is combined with config to create a release.\"/><img src=\"https://books.lapw.at/images/process-types.png\" alt=\"Scale is expressed as running processes, workload diversity is expressed as process types.\"/>"
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestSubChapters(t *testing.T) {
+
+	config0 := NewScrapeConfig()
+	config1 := NewScrapeConfig()
+
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
+
+	got := len(c.SubChapters())
+	want := 2
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestSubChaptersSelector(t *testing.T) {
+
+	config0 := NewScrapeConfig()
+	config0.selector = "section.concrete > article > h2 > a"
+
+	config1 := NewScrapeConfig()
+
+	c := NewChapterFromURL("https://12factor.net/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
+
+	got := len(c.SubChapters())
+	want := 12
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestSubChaptersLimit(t *testing.T) {
+
+	config0 := NewScrapeConfig()
+	config0.limit = 1
+
+	config1 := NewScrapeConfig()
+
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
+
+	got := len(c.SubChapters())
+	want := 1
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestSubChaptersLimitOver(t *testing.T) {
+
+	config0 := NewScrapeConfig()
+	config0.limit = 3
+
+	config1 := NewScrapeConfig()
+
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
+
+	got := len(c.SubChapters())
+	want := 2
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}
+
+func TestNotInclude(t *testing.T) {
+
+	config := NewScrapeConfig()
+	config.include = false
+
+	c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {})
+
+	got := c.Content()
+	want := ""
+
+	if got != want {
+		t.Errorf("got %v, wanted %v", got, want)
+	}
+
+}

cmd/get.go

@@ -5,19 +5,16 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"os/exec"
 	"strings"
 
-	md "github.com/JohannesKaufmann/html-to-markdown"
+	"github.com/spf13/cobra"
-	epub "github.com/bmaupin/go-epub"
-	cobra "github.com/spf13/cobra"
 
 	"github.com/lapwat/papeer/book"
 )
 
-var stdout, recursive, include, images bool
+var recursive, include, images, quiet bool
-var format, output, selector string
+var format, output, selector, name, author string
-var limit, offset, delay int
+var limit, offset, delay, threads int
 
 var getCmd = &cobra.Command{
 	Use:   "get",
@@ -28,20 +25,16 @@ var getCmd = &cobra.Command{
 		}
 
 		formatEnum := map[string]bool{
-			"md":   true,
+			"stdout": true,
-			"epub": true,
+			"md":     true,
-			"mobi": true,
+			"epub":   true,
+			"mobi":   true,
 		}
 		if formatEnum[format] != true {
 			return fmt.Errorf("invalid format specified: %s", format)
 		}
 
-		if format == "epub" || format == "mobi" {
+		// add .mobi to filename if not specified
-			if stdout {
-				return errors.New("cannot print EPUB/MOBI file to standard output")
-			}
-		}
-
 		if format == "mobi" {
 			if len(output) > 0 && strings.HasSuffix(output, ".mobi") == false {
 				output = fmt.Sprintf("%s.mobi", output)
@@ -68,107 +61,60 @@ var getCmd = &cobra.Command{
 			return errors.New("cannot use delay option if not in recursive mode")
 		}
 
+		if cmd.Flags().Changed("threads") && recursive == false {
+			return errors.New("cannot use threads option if not in recursive mode")
+		}
+
+		if cmd.Flags().Changed("delay") && cmd.Flags().Changed("threads") {
+			return errors.New("cannot use delay and threads options at the same time")
+		}
+
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		url := args[0]
-		b := book.NewBookFromURL(url, selector, recursive, include, images, limit, offset, delay)
+		b := book.NewBookFromURL(url, selector, name, author, recursive, include, images, limit, offset, delay, threads)
 
-		if len(output) == 0 {
+		fakeConfig := book.NewScrapeConfigFake()
-			// set default output
+		fakeChapter := book.NewChapter("", b.Name(), b.Author(), "", b.Chapters(), fakeConfig)
-			output = strings.ReplaceAll(b.Name(), " ", "_")
+
-			output = strings.ReplaceAll(output, "/", "")
+		if format == "stdout" {
-			output = fmt.Sprintf("%s.%s", output, format)
+			// TODO: ToMarkdownString
+			markdown := book.ToMarkdown(fakeChapter)
+			fmt.Println(markdown)
 		}
 
 		if format == "md" {
-			var f *os.File
+			// TODO: ToMarkdownFile
-			var err error
+			markdown := book.ToMarkdown(fakeChapter)
 
-			if !stdout {
+			if len(output) == 0 {
-				f, err = os.Create(output)
+				filename := book.Filename(fakeChapter.Name())
-				if err != nil {
+				output = fmt.Sprintf("%s.md", filename)
-					log.Fatal(err)
-				}
-				defer f.Close()
 			}
 
-			for _, c := range b.Chapters() {
+			// write to file
-				content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
+			f, err := os.Create(output)
-				if err != nil {
-					log.Fatal(err)
-				}
-
-				text := fmt.Sprintf("%s\n%s\n\n%s\n\n\n", c.Name(), strings.Repeat("=", len(c.Name())), content)
-
-				if stdout {
-					fmt.Println(text)
-				} else {
-					_, err := f.WriteString(text)
-					if err != nil {
-						log.Fatal(err)
-					}
-
-				}
-			}
-
-			if stdout == false {
-				fmt.Printf("Markdown saved to \"%s\"\n", output)
-			}
-		}
-
-		if format == "epub" {
-			e := epub.NewEpub(b.Name())
-			e.SetAuthor(b.Author())
-
-			for _, c := range b.Chapters() {
-				if images {
-					e.AddSection(c.Content(), "", "", "")
-				} else {
-					html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), c.Content())
-
-					_, err := e.AddSection(html, c.Name(), "", "")
-					if err != nil {
-						log.Fatal(err)
-					}
-				}
-			}
-
-			err := e.Write(output)
 			if err != nil {
 				log.Fatal(err)
 			}
+			_, err2 := f.WriteString(markdown)
+			if err2 != nil {
+				log.Fatal(err2)
+			}
+			f.Close()
 
+			fmt.Printf("Markdown saved to \"%s\"\n", output)
+		}
+
+		if format == "epub" {
+			output = book.ToEpub(fakeChapter, output)
 			fmt.Printf("Ebook saved to \"%s\"\n", output)
 		}
 
 		if format == "mobi" {
-			e := epub.NewEpub(b.Name())
+			output = book.ToMobi(fakeChapter, output)
-			e.SetAuthor(b.Author())
-
-			for _, chapter := range b.Chapters() {
-				e.AddSection(chapter.Content(), chapter.Name(), "", "")
-			}
-
-			outputEPUB := strings.ReplaceAll(output, ".mobi", ".epub")
-
-			err := e.Write(outputEPUB)
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			exec.Command("kindlegen", outputEPUB).Run()
-			// exec command always return status 1 even if it fails
-			// if err != nil {
-			// 	log.Fatal(err)
-			// }
-
 			fmt.Printf("Ebook saved to \"%s\"\n", output)
-
-			err2 := os.Remove(outputEPUB)
-			if err2 != nil {
-				log.Fatal(err2)
-			}
 		}
 	},
 }

cmd/list.go

@@ -27,7 +27,7 @@ var listCmd = &cobra.Command{
 			log.Fatal(err)
 		}
 
-		links, err := book.GetLinks(base, selector, limit, offset, include)
+		links, _, err := book.GetLinks(base, selector, limit, offset, include)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/root.go

@@ -23,16 +23,18 @@ func Execute() {
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "md", "file format [md, epub, mobi]")
+	rootCmd.PersistentFlags().StringVarP(&name, "name", "n", "", "book name (default: page title)")
-	rootCmd.PersistentFlags().StringVarP(&output, "output", "", "", "output file")
+	rootCmd.PersistentFlags().StringVarP(&author, "author", "a", "", "book author")
+	rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "stdout", "file format [stdout, md, epub, mobi]")
+	rootCmd.PersistentFlags().StringVarP(&output, "output", "", "", "file name (default: book name)")
 	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector, in resursive mode")
 	rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "create one chapter per natigation item")
 	rootCmd.PersistentFlags().BoolVarP(&include, "include", "i", false, "include URL as first chapter, in resursive mode")
-	rootCmd.PersistentFlags().BoolVarP(&stdout, "stdout", "", false, "print to standard output")
 	rootCmd.PersistentFlags().BoolVarP(&images, "images", "", false, "retrieve images only")
 	rootCmd.PersistentFlags().IntVarP(&limit, "limit", "l", -1, "limit number of chapters, in recursive mode")
 	rootCmd.PersistentFlags().IntVarP(&offset, "offset", "o", 0, "skip first chapters, in recursive mode")
 	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "time to wait before downloading next chapter, in milliseconds")
+	rootCmd.PersistentFlags().IntVarP(&threads, "threads", "t", -1, "download concurrency, in recursive mode")
 
 	rootCmd.AddCommand(getCmd)
 	rootCmd.AddCommand(listCmd)

cmd/utils.go

@@ -1,5 +0,0 @@
-package cmd
-
-func getTableOfContent() {
-
-}

cmd/version.go

@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print the version number of papeer",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("papeer v0.1.1")
+		fmt.Println("papeer v0.3.2")
 	},
 }

release.sh

@@ -3,12 +3,18 @@
 version=$1
 platforms=("linux/amd64" "darwin/amd64" "windows/amd64")
 
+if [ "$#" -ne 1 ]; then
+    echo "Illegal number of parameters"
+    echo "Usage: ./release.sh X.X.X"
+    exit 1
+fi
+
 for platform in "${platforms[@]}"
 do
     platform_split=(${platform//\// })
     GOOS=${platform_split[0]}
     GOARCH=${platform_split[1]}
-    output_name='papeer-'$version'-'$GOOS'-'$GOARCH
+    output_name='papeer-v'$version'-'$GOOS'-'$GOARCH
     if [ $GOOS = "windows" ]; then
         output_name+='.exe'
     fi