fix: display images in epub

Makefile

@@ -1,14 +0,0 @@
-format:
-	gofmt -s -w .
-
-test:
-	go test github.com/lapwat/papeer/book
-
-install:
-	go install
-
-clean:
-	find . -maxdepth 1 -not -name 'README.md' -name '*.md' -delete
-	find . -maxdepth 1 -name '*.epub' -delete
-	find . -maxdepth 1 -name '*.mobi' -delete
-	find . -maxdepth 1 -name 'papeer-v*' -delete

README.md

@@ -1,125 +1,84 @@
 # Papeer
 
-Papeer is a powerful **ereader internet vacuum**. It can scrape any website, removing ads and keeping only the relevant content (formatted text and images). You can export the content to Markdown, EPUB or MOBI files.
+Papeer is a tool that lets you scrape content from the internet. It can scrape any web page, keeping only relevant content (formatted text and images) and removing ads and menus. You can export the content to Markdown, EPUB or MOBI files.
-
-# Table of contents
-
-- [Usage](#usage)
-  * [Scrape a web page](#scrape-a-web-page)
-  * [Scrape a whole website](#scrape-a-whole-website)
-    + [`depth` option](#-depth--option)
-    + [`selector` option](#-selector--option)
-    + [Display the table of contents](#display-the-table-of-contents)
-    + [Scrape time](#scrape-time)
-- [Installation](#installation)
-  * [From source](#from-source)
-  * [From binary](#from-binary)
-    + [Linux / MacOS](#linux---macos)
-    + [Windows](#windows)
-  * [MOBI support](#mobi-support)
-- [Autocompletion](#autocompletion)
-- [Dependencies](#dependencies)
 
 # Usage
 
-## Scrape a web page
-
-The `get` command lets you retrieve the content of any web page.
-
 ```
-Scrape URL content
+Browse the web in the eink era
 
 Usage:
-  papeer get URL [flags]
+  papeer [flags]
+  papeer [command]
 
-Examples:
+Available Commands:
-papeer get https://www.eff.org/cyberspace-independence
+  completion  generate the autocompletion script for the specified shell
+  get         Scrape URL content
+  help        Help about any command
+  ls          Print table of content
+  version     Print the version number of papeer
 
 Flags:
-  -a, --author string      book author
+  -d, --delay int         time to wait before downloading next chapter, in milliseconds (default -1)
-      --delay int          time in milliseconds to wait before downloading next chapter, use with depth/selector (default -1)
+  -f, --format string     file format [stdout, md, epub, mobi] (default "stdout")
-  -d, --depth int          scraping depth
+  -h, --help              help for papeer
-  -f, --format string      file format [stdout, md, epub, mobi] (default "md")
+      --images            retrieve images only
-  -h, --help               help for get
+  -i, --include           include URL as first chapter, in resursive mode
-      --images             retrieve images only
+  -l, --limit int         limit number of chapters, in recursive mode (default -1)
-  -i, --include            include URL as first chapter, use with depth/selector
+  -o, --offset int        skip first chapters, in recursive mode
-  -l, --limit int          limit number of chapters, use with depth/selector (default -1)
+      --output string     output file
-  -n, --name string        book name (default: page title)
+  -r, --recursive         create one chapter per natigation item
-  -o, --offset int         skip first chapters, use with depth/selector
+  -s, --selector string   table of content CSS selector, in resursive mode
-      --output string      file name (default: book name)
+  -t, --threads int       download concurrency, in recursive mode (default -1)
-  -q, --quiet              hide progress bar
+
-  -s, --selector strings   table of contents CSS selector
+Use "papeer [command] --help" for more information about a command.
-  -t, --threads int        download concurrency, use with depth/selector (default -1)
-      --use-link-name      use link name for chapter title
 ```
 
-## Scrape a whole website
+# Examples
 
-If a navigation menu is present on a website, you can scrape the content of each page.
+## Grab a single page
 
-You can activate this mode by using the `depth` or `selector` options.
+The `get` command lets you retrieve the content of a web page.
-
-### `depth` option
-
-This option defaults to 0, `papeer` will grab only the main page.
-
-If you specify a value greater than 0, `papeer` will grab pages as deep as the value you specify.
-
-> Using `include` option will include all intermediary levels into the book.
-
-### `selector` option
-
-If this option is not specified, `papeer` will grab only the one page.
-
-If this option is specified, `papeer` will select the links (a HTML tag) present on the main page, then grab each one of them.
-
-You can chain this option to grab several level of pages with diferent selectors for each level.
-
-### Display the table of contents
-
-Before actually scraping a whole website, it is a good idea to use the `list` command. This command is like a **dry run**, which lets you vizualize the content before actually retrieving it. You can use several options to customize the table of contents extraction, such as `selector`, `limit`, `offset` and `include`. Type `papeer list --help` for more information about those options.
 
 ```sh
-papeer list https://12factor.net/ -s 'section.concrete>article>h2>a'
+papeer get https://www.eff.org/cyberspace-independence
+# A Declaration of the Independence of Cyberspace
+# ===============================================
+
+# Governments of the Industrial World, you weary giants of flesh and steel, I come from Cyberspace, the new home of Mind. On behalf of the future, I ask you of the past to leave us alone. You are not welcome among us. You have no sovereignty where we gather...
 ```
-```
+
- #  NAME                    URL                                    
+## Grab several pages (recursive mode)
- 1  I. Codebase             https://12factor.net/codebase          
+
- 2  II. Dependencies        https://12factor.net/dependencies      
+The `recursive` option lets you extract the table of content of a website, then scrape the content of each one of its pages.
- 3  III. Config             https://12factor.net/config            
+
- 4  IV. Backing services    https://12factor.net/backing-services  
+### Display table of content
- 5  V. Build, release, run  https://12factor.net/build-release-run 
+
- 6  VI. Processes           https://12factor.net/processes         
+Before trying the `recursive` option, it is a good idea to use the `ls` option, which lets you vizualize the content that will be retrieved. You can use several options to customize the table of content extraction, such as `selector`, `limit`, `offset` and `include`. Type `papeer help` for more information about those options.
- 7  VII. Port binding       https://12factor.net/port-binding      
+
- 8  VIII. Concurrency       https://12factor.net/concurrency       
+```sh
- 9  IX. Disposability       https://12factor.net/disposability     
+papeer ls https://news.ycombinator.com/ --limit=5
-10  X. Dev/prod parity      https://12factor.net/dev-prod-parity   
+#  #  NAME                                                      URL                                                                                             
-11  XI. Logs                https://12factor.net/logs              
+#  1  Tailwind CSS v3.0                                         https://tailwindcss.com/blog/tailwindcss-v3                                                     
-12  XII. Admin processes    https://12factor.net/admin-processes
+#  2  A molten salt storage solution using sodium hydroxide     https://sifted.eu/articles/salt-energy-storage-seaborg-hyme/                                    
+#  3  HashiCorp IPO today                                       https://www.hashicorp.com/blog/a-new-chapter-for-hashicorp                                      
+#  4  Stack Graphs                                              https://github.blog/2021-12-09-introducing-stack-graphs/
+#  5  ‘Tipping point’ makes partisan polarization irreversible  https://news.cornell.edu/stories/2021/12/tipping-point-makes-partisan-polarization-irreversible 
 ```
 
 ### Scrape time
 
-Once you are satisfied with the table of contents listed by the `ls` command, you can actually scrape the content of those pages. You can use the same options that you specified for the `ls` command. You can specify `delay` and `threads` options when using `selector` or `depth` options.
+Once you are satisfied with the table of content listed by the `ls` command, you can actually scrape the content of those pages. You can use the same options that you specified for the `ls` command. In recursive mode, you also have the possibility to use `delay` and `threads` options.
 
 ```sh
-papeer get https://12factor.net/ --selector='section.concrete>article>h2>a'
+papeer get https://news.ycombinator.com/ --recursive --delay=500 --limit=5 --format=md
-```
+# [========================================>---------------------------] Chapters 3 / 5
-```
+# [====================================================================] 1. Tailwind CSS v3.0
-[======================================>-----------------------------] Chapters 7 / 12
+# [====================================================================] 2. A molten salt storage solution using sodium hydroxide
-[====================================================================] 1. I. Codebase
+# [====================================================================] 3. HashiCorp IPO today
-[====================================================================] 2. II. Dependencies
+# [--------------------------------------------------------------------] 4. Stack Graphs
-[====================================================================] 3. III. Config
+# [--------------------------------------------------------------------] 5. ‘Tipping point’ makes partisan polarization irreversible
-[====================================================================] 4. IV. Backing services
+# Markdown saved to "Hacker News.md"
-[====================================================================] 5. V. Build, release, run
-[====================================================================] 6. VI. Processes
-[====================================================================] 7. VII. Port binding
-[--------------------------------------------------------------------] 8. VIII. Concurrency
-[--------------------------------------------------------------------] 9. IX. Disposability
-[--------------------------------------------------------------------] 10. X. Dev/prod parity
-[--------------------------------------------------------------------] 11. XI. Logs
-[--------------------------------------------------------------------] 12. XII. Admin processes
-Markdown saved to "The_Twelve-Factor_App.md"
 ```
 
 # Installation
@@ -132,29 +91,20 @@ go get -u github.com/lapwat/papeer
 
 ## From binary
 
-### Linux / MacOS
+### On Linux / MacOS
 
 ```sh
-# use platform=darwin for MacOS
+platform=linux # use platform=darwin for MacOS
-platform=linux
+curl -L https://github.com/lapwat/papeer/releases/download/v0.3.0/papeer-v0.3.0-$platform-amd64 > papeer
-release=0.4.1
+chmod +x papeer
-
-# download and extract
-curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64.tar.gz > papeer.tar.gz
-tar xzvf papeer.tar.gz
-rm papeer.tar.gz
-
-# move to user binaries
 sudo mv papeer /usr/local/bin
 ```
 
-### Windows
+### On Windows
 
-Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.4.1/papeer-v0.4.1-windows-amd64.exe.zip).
+Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.3.0/papeer-v0.3.0-windows-amd64.exe).
 
-## MOBI support
+## Install kindlegen to export websites to MOBI (optional)
-
-Install kindlegen to convert websites, Linux only
 
 ```sh
 TMPDIR=$(mktemp -d -t papeer-XXXXX)
@@ -173,10 +123,10 @@ Execute this command in your current shell, or add it to your `.bashrc`.
 . <(papeer completion bash)
 ```
 
-You can replace `bash` by your own shell (zsh, fish or powershell).
-
 Type `papeer completion bash -h` for more information.
 
+You can replace `bash` by your own shell (zsh, fish or powershell).
+
 # Dependencies
 
 - `cobra` command line interface

book/chapter.go

@@ -1,20 +1,13 @@
 package book
 
 type chapter struct {
-	body        string
+	name    string
-	name        string
+	author  string
-	author      string
+	content string
-	content     string
-	subChapters []chapter
-	config      *ScrapeConfig
 }
 
-func NewChapter(body, name, author, content string, subChapters []chapter, config *ScrapeConfig) chapter {
+func NewChapter(name, author, content string) chapter {
-	return chapter{body, name, author, content, subChapters, config}
+	return chapter{name, author, content}
-}
-
-func (c chapter) Body() string {
-	return c.body
 }
 
 func (c chapter) Name() string {
@@ -28,7 +21,3 @@ func (c chapter) Author() string {
 func (c chapter) Content() string {
 	return c.content
 }
-
-func (c chapter) SubChapters() []chapter {
-	return c.subChapters
-}

book/format.go

@@ -1,164 +0,0 @@
-package book
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"strings"
-
-	md "github.com/JohannesKaufmann/html-to-markdown"
-	"github.com/PuerkitoBio/goquery"
-	epub "github.com/bmaupin/go-epub"
-)
-
-func Filename(name string) string {
-	filename := name
-
-	filename = strings.ReplaceAll(filename, " ", "_")
-	filename = strings.ReplaceAll(filename, "/", "")
-
-	return filename
-}
-
-func ToMarkdownString(c chapter) string {
-	markdown := ""
-
-	if c.config.Include {
-		// title
-		markdown += fmt.Sprintf("%s\n", c.Name())
-		markdown += fmt.Sprintf("%s\n\n", strings.Repeat("=", len(c.Name())))
-
-		// convert content to markdown
-		content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
-		if err != nil {
-			log.Fatal(err)
-		}
-		markdown += fmt.Sprintf("%s\n\n\n", content)
-	}
-
-	for _, sc := range c.SubChapters() {
-		// subchapters content
-		markdown += fmt.Sprintf("%s\n\n\n", ToMarkdownString(sc))
-	}
-
-	return markdown
-}
-
-func ToMarkdown(c chapter, filename string) string {
-	if len(filename) == 0 {
-		filename = fmt.Sprintf("%s.md", Filename(c.Name()))
-	}
-
-	markdown := ToMarkdownString(c)
-
-	// write to file
-	f, err := os.Create(filename)
-	if err != nil {
-		log.Fatal(err)
-	}
-	_, err2 := f.WriteString(markdown)
-	if err2 != nil {
-		log.Fatal(err2)
-	}
-	f.Close()
-
-	return filename
-}
-
-func ToEpub(c chapter, filename string) string {
-	if len(filename) == 0 {
-		filename = fmt.Sprintf("%s.epub", Filename(c.Name()))
-	}
-
-	// init ebook
-	e := epub.NewEpub(c.Name())
-	e.SetAuthor(c.Author())
-
-	AppendToEpub(e, c)
-
-	err := e.Write(filename)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return filename
-}
-
-func AppendToEpub(e *epub.Epub, c chapter) {
-	content := ""
-
-	if c.config.Include {
-
-		if c.config.ImagesOnly == false {
-			content = c.Content()
-		}
-
-		// parse content
-		doc, err := goquery.NewDocumentFromReader(strings.NewReader(c.Content()))
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		// download images and replace src in img tags of content
-		doc.Find("img").Each(func(i int, s *goquery.Selection) {
-			src, _ := s.Attr("src")
-			src = strings.Split(src, "?")[0] // remove query part
-			imagePath, _ := e.AddImage(src, "")
-
-			if c.config.ImagesOnly {
-				imageTag, _ := goquery.OuterHtml(s)
-				content += strings.Replace(imageTag, src, imagePath, 1)
-			} else {
-				content = strings.Replace(content, src, imagePath, 1)
-			}
-		})
-
-		html := ""
-		// add title only if ImagesOnly = false
-		if c.config.ImagesOnly == false {
-			html += fmt.Sprintf("<h1>%s</h1>", c.Name())
-		}
-		html += content
-
-		//  write to epub file
-		_, err = e.AddSection(html, c.Name(), "", "")
-		if err != nil {
-			log.Fatal(err)
-		}
-
-	}
-
-	for _, sc := range c.SubChapters() {
-		AppendToEpub(e, sc)
-	}
-}
-
-func ToMobi(c chapter, filename string) string {
-	if len(filename) == 0 {
-		filename = fmt.Sprintf("%s.mobi", Filename(c.Name()))
-	} else {
-
-		// add .mobi extension if not specified
-		if strings.HasSuffix(filename, ".mobi") == false {
-			filename = fmt.Sprintf("%s.mobi", filename)
-		}
-
-	}
-
-	filenameEPUB := strings.ReplaceAll(filename, ".mobi", ".epub")
-	ToEpub(c, filenameEPUB)
-
-	exec.Command("kindlegen", filenameEPUB).Run()
-	// exec command always return status 1 even if it succeed
-	// if err != nil {
-	// 	log.Fatal(err)
-	// }
-
-	err := os.Remove(filenameEPUB)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return filename
-}

book/format_test.go

@@ -1,127 +0,0 @@
-package book
-
-import (
-	"errors"
-	"os"
-	"testing"
-)
-
-func TestFilename(t *testing.T) {
-
-	got := Filename("This is a chapter / book")
-	want := "This_is_a_chapter__book"
-
-	if got != want {
-		t.Errorf("got %q, wanted %q", got, want)
-	}
-
-}
-
-func TestToMarkdownString(t *testing.T) {
-
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-
-	got := ToMarkdownString(c)
-	want := "Books\n=====\n\n- [Discours de la Méthode](https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/)clock 98 min read -\n1637\n\n- [The Twelve-Factor App](https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/)clock 22 min read -\n2011\n\n\n"
-
-	if got != want {
-		t.Errorf("got %q, wanted %q", got, want)
-	}
-
-}
-
-func TestToMarkdown(t *testing.T) {
-
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-	ToMarkdown(c, "")
-
-	filename := "Books.md"
-	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
-		t.Errorf("%s does not exist: %v", filename, err)
-	} else {
-		if err := os.Remove(filename); err != nil {
-			t.Errorf("cannot remove %v: %v", filename, err)
-		}
-	}
-
-}
-
-func TestToMarkdownFilename(t *testing.T) {
-
-	filename := "ebook.md"
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-	ToMarkdown(c, filename)
-
-	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
-		t.Errorf("%s does not exist: %v", filename, err)
-	} else {
-		if err := os.Remove(filename); err != nil {
-			t.Errorf("cannot remove %v: %v", filename, err)
-		}
-	}
-
-}
-
-func TestToEpub(t *testing.T) {
-
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-	ToEpub(c, "")
-
-	filename := "Books.epub"
-	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
-		t.Errorf("%s does not exist: %v", filename, err)
-	} else {
-		if err := os.Remove(filename); err != nil {
-			t.Errorf("cannot remove %v: %v", filename, err)
-		}
-	}
-
-}
-
-func TestToEpubFilename(t *testing.T) {
-
-	filename := "ebook.epub"
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-	ToEpub(c, filename)
-
-	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
-		t.Errorf("%s does not exist: %v", filename, err)
-	} else {
-		if err := os.Remove(filename); err != nil {
-			t.Errorf("cannot remove %v: %v", filename, err)
-		}
-	}
-
-}
-
-func TestToMobi(t *testing.T) {
-
-	filename := "ebook.mobi"
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-	ToMobi(c, filename)
-
-	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
-		t.Errorf("%s does not exist: %v", filename, err)
-	} else {
-		if err := os.Remove(filename); err != nil {
-			t.Errorf("cannot remove %v: %v", filename, err)
-		}
-	}
-
-}
-
-func TestToMobiFilename(t *testing.T) {
-
-	filename := "ebook.mobi"
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-	ToMobi(c, filename)
-
-	if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) {
-		t.Errorf("%s does not exist: %v", filename, err)
-	} else {
-		if err := os.Remove(filename); err != nil {
-			t.Errorf("cannot remove %v: %v", filename, err)
-		}
-	}
-
-}

book/progress.go

@@ -2,7 +2,6 @@ package book
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/gosuri/uiprogress"
 )
@@ -12,22 +11,20 @@ type progress struct {
 	individuals []*uiprogress.Bar
 }
 
-func NewProgress(links []link, parent string, depth int) progress {
+func NewProgress(links []link) progress {
 	uiprogress.Start()
 
 	global := uiprogress.AddBar(len(links))
-	indentGlobal := strings.Repeat("> ", depth)
 	global.AppendFunc(func(b *uiprogress.Bar) string {
-		return fmt.Sprintf("%v%v (%v / %v)", indentGlobal, parent, b.Current(), len(links))
+		return fmt.Sprintf("Chapters %d / %d", b.Current(), len(links))
 	})
 
 	// hide individual bars if more than 50 chapters
 	individuals := []*uiprogress.Bar{}
-	indent := strings.Repeat("- ", depth)
 	if len(links) <= 50 {
 		for index, link := range links {
 			bar := uiprogress.AddBar(1)
-			barText := fmt.Sprintf("%v#%v %v", indent, index+1, link.Text())
+			barText := fmt.Sprintf("%d. %s", index+1, link.text)
 			bar.AppendFunc(func(b *uiprogress.Bar) string {
 				return barText
 			})
@@ -38,22 +35,13 @@ func NewProgress(links []link, parent string, depth int) progress {
 	return progress{global, individuals}
 }
 
-func (p *progress) IncrementGlobal() {
+func (p *progress) IncrGlobal() {
 	p.global.Incr()
 }
 
-func (p *progress) Increment(index int) {
+func (p *progress) Incr(index int) {
-	p.IncrementGlobal()
+	p.global.Incr()
 	if len(p.individuals) > index {
 		p.individuals[index].Incr()
 	}
 }
-
-func (p *progress) UpdateName(index int, name string) {
-	if len(p.individuals) > index {
-		barText := fmt.Sprintf("%s", name)
-		p.individuals[index].AppendFunc(func(b *uiprogress.Bar) string {
-			return barText
-		})
-	}
-}

book/scraper.go

@@ -1,12 +1,9 @@
 package book
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"log"
 	"math"
-	"net/http"
 	urllib "net/url"
 	"strings"
 	"sync"
@@ -17,294 +14,76 @@ import (
 	colly "github.com/gocolly/colly/v2"
 )
 
-type ScrapeConfig struct {
+func NewBookFromURL(url, selector string, recursive, include, images bool, limit, offset, delay, threads int) book {
-	Depth       int
+	if recursive {
-	Selector    string
+		chapters := tableOfContent(url, selector, limit, offset, delay, threads, include, images)
-	Quiet       bool
-	Limit       int
-	Offset      int
-	Delay       int
-	Threads     int
-	Include     bool
-	ImagesOnly  bool
-	UseLinkName bool
-}
 
-func NewScrapeConfig() *ScrapeConfig {
+		b := New(chapters[0].Name(), chapters[0].Author())
-	return &ScrapeConfig{0, "", false, -1, 0, -1, -1, true, false, false}
+		for _, c := range chapters {
-}
+			b.AddChapter(c)
+		}
 
-func NewScrapeConfigs(selectors []string) []*ScrapeConfig {
+		return b
-	configs := []*ScrapeConfig{}
-
-	for _, s := range selectors {
-		config := NewScrapeConfig()
-		config.Selector = s
-
-		configs = append(configs, config)
-	}
-
-	return configs
-}
-
-func NewScrapeConfigsAjin() []*ScrapeConfig {
-	config0 := NewScrapeConfig()
-	config0.Depth = 0
-	config0.Selector = ".dt>a"
-	config0.Limit = 3
-	config0.Offset = 0
-	config0.Delay = 5000
-	config0.Include = false
-
-	config1 := NewScrapeConfig()
-	config1.Depth = 1
-	config1.Selector = ".nav_apb>a"
-	config1.Limit = 3
-	config1.Offset = 1
-	config1.Delay = 5000
-	config1.Include = false
-
-	config2 := NewScrapeConfig()
-	config2.Depth = 2
-	config2.ImagesOnly = true
-
-	return []*ScrapeConfig{config0, config1, config2}
-}
-
-func NewScrapeConfigsWikipedia() []*ScrapeConfig {
-	config0 := NewScrapeConfig()
-	config0.Depth = 0
-	config0.Threads = -1
-	config0.Include = true
-
-	config1 := NewScrapeConfig()
-	config1.Depth = 1
-	config1.Include = true
-
-	return []*ScrapeConfig{config0, config1}
-}
-
-func NewScrapeConfigFake() *ScrapeConfig {
-	config := NewScrapeConfig()
-	config.Include = false
-
-	return config
-}
-
-func NewBookFromURL(url string, selector []string, name, author string, include, ImagesOnly, useLinkName, quiet bool, limit, offset, delay, threads int) book {
-	config1 := NewScrapeConfig()
-	config1.ImagesOnly = ImagesOnly
-	config1.UseLinkName = useLinkName
-
-	var chapters []chapter
-	var home chapter
-
-	if len(selector) > 0 {
-		config2 := NewScrapeConfig()
-		config2.Selector = selector[0]
-		config2.Limit = limit
-		config2.Offset = offset
-		config2.Delay = delay
-		config2.Threads = threads
-		config2.Include = include
-		config2.ImagesOnly = ImagesOnly
-		config2.UseLinkName = useLinkName
-		chapters, home = tableOfContent(url, config2, config1, quiet)
 	} else {
-		chapters = []chapter{NewChapterFromURL(url, "", []*ScrapeConfig{config1}, 0, func(index int, name string) {})}
+		c := NewChapterFromURL(url, images)
-		home = chapters[0]
+		b := New(c.Name(), c.Author())
-	}
-
-	if len(name) == 0 {
-		name = home.Name()
-	}
-
-	if len(author) == 0 {
-		author = home.Author()
-	}
-
-	b := New(name, author)
-	for _, c := range chapters {
 		b.AddChapter(c)
+		return b
 	}
-
-	return b
 }
 
-func NewChapterFromURL(url, linkName string, configs []*ScrapeConfig, index int, updateProgressBarName func(index int, name string)) chapter {
+func NewChapterFromURL(url string, images bool) chapter {
-	config := configs[0]
+	article, err := readability.FromURL(url, 30*time.Second)
-
-	base, err := urllib.Parse(url)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	// get page body
-	response, err := http.Get(url)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer response.Body.Close()
-
-	// duplicate response stream
-	readabilityReader := &bytes.Buffer{}
-	bodyReader := io.TeeReader(response.Body, readabilityReader)
-
-	// extract HTML body
-	body, err := io.ReadAll(bodyReader)
-
-	// extract article content and metadata
-	article, err := readability.FromReader(readabilityReader, base)
 	if err != nil {
 		log.Fatalf("failed to parse %s, %v\n", url, err)
 	}
 
-	name := linkName
+	content := strings.ReplaceAll(article.Content, "\n", "")
-	if config.UseLinkName == false {
-		name = article.Title
 
-		// notify progressbar with new name
+	if images {
-		updateProgressBarName(index, name)
+		// parse html content
-	}
+		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
-
-	subchapters := []chapter{}
-	if len(configs) > 1 {
-		// add subchapters
-
-		links, _, _, err := GetLinks(base, config.Selector, config.Limit, config.Offset, false)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		subchapters = make([]chapter, len(links))
+		// extract images only
-		var p progress
+		content = ""
-		if config.Quiet == false {
+		doc.Find("img").Each(func(i int, s *goquery.Selection) {
-			p = NewProgress(links, name, config.Depth)
+			newContent, _ := goquery.OuterHtml(s)
-		}
+			content += newContent
-
+		})
-		if config.Delay >= 0 {
-
-			// synchronous mode
-			for index, link := range links {
-				// and then use it to parse relative URLs
-				u, err := base.Parse(link.href)
-				if err != nil {
-					log.Fatal(err)
-				}
-
-				sc := NewChapterFromURL(u.String(), link.text, configs[1:], index, p.UpdateName)
-				subchapters[index] = sc
-				if config.Quiet == false {
-					p.Increment(index)
-				}
-
-				time.Sleep(time.Duration(config.Delay) * time.Millisecond)
-			}
-
-		} else {
-			// asynchronous mode
-			var wg sync.WaitGroup
-
-			threads := config.Threads
-			if threads == -1 {
-				threads = len(links)
-			}
-			semaphore := make(chan bool, threads)
-
-			for index, l := range links {
-
-				wg.Add(1)
-				semaphore <- true
-
-				go func(index int, l link) {
-					defer wg.Done()
-
-					// and then use it to parse relative URLs
-					u, err := base.Parse(l.href)
-					if err != nil {
-						log.Fatal(err)
-					}
-
-					sc := NewChapterFromURL(u.String(), l.text, configs[1:], index, p.UpdateName)
-					subchapters[index] = sc
-
-					if config.Quiet == false {
-						p.Increment(index)
-					}
-
-					<-semaphore
-				}(index, l)
-			}
-			wg.Wait()
-		}
 	}
 
-	content := ""
+	return chapter{article.Title, article.Byline, content}
-	if config.Include {
-
-		// we care about the content only if:
-		// - we include this level
-		// - we use the page name
-		content = article.Content
-
-		// extract images
-		if config.ImagesOnly {
-
-			// parse HTML
-			doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			// append every image to content
-			content = ""
-			doc.Find("img").Each(func(i int, s *goquery.Selection) {
-				imageTag, _ := goquery.OuterHtml(s)
-				imageTag = strings.ReplaceAll(imageTag, "\n", "")
-
-				content += imageTag
-			})
-
-		}
-	}
-
-	return chapter{string(body), name, article.Byline, content, subchapters, config}
 }
 
-func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, quiet bool) ([]chapter, chapter) {
+func tableOfContent(url, selector string, limit, offset, delay, threads int, include, images bool) []chapter {
 	base, err := urllib.Parse(url)
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	links, _, home, err := GetLinks(base, config.Selector, config.Limit, config.Offset, config.Include)
+	links, err := GetLinks(base, selector, limit, offset, include)
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	chapters := make([]chapter, len(links))
-	delay := config.Delay
+	progress := NewProgress(links)
-
-	var p progress
-	if quiet == false {
-		p = NewProgress(links, "", 0)
-	}
 
 	if delay >= 0 {
 		// synchronous mode
 
-		for index, l := range links {
+		for index, link := range links {
 			// and then use it to parse relative URLs
-			u, err := base.Parse(l.href)
+			u, err := base.Parse(link.href)
 			if err != nil {
 				log.Fatal(err)
 			}
 
-			chapters[index] = NewChapterFromURL(u.String(), l.text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {})
+			chapters[index] = NewChapterFromURL(u.String(), images)
-
+			progress.Incr(index)
-			if quiet == false {
-				p.Increment(index)
-			}
 
 			// short sleep for last chapter to let the progress bar update
 			if index == len(links)-1 {
@@ -318,7 +97,6 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q
 		// asynchronous mode
 		var wg sync.WaitGroup
 
-		threads := config.Threads
 		if threads == -1 {
 			threads = len(links)
 		}
@@ -338,19 +116,15 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q
 					log.Fatal(err)
 				}
 
-				chapters[index] = NewChapterFromURL(u.String(), l.text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {})
+				chapters[index] = NewChapterFromURL(u.String(), images)
-
+				progress.Incr(index)
-				if quiet == false {
-					p.Increment(index)
-				}
 
 				<-semaphore
 			}(index, l)
 		}
 		wg.Wait()
 	}
-
+	return chapters
-	return chapters, home
 }
 
 func GetPath(elm *goquery.Selection) string {
@@ -358,7 +132,7 @@ func GetPath(elm *goquery.Selection) string {
 
 	for {
 		selector := strings.ToLower(goquery.NodeName(elm))
-		if len(selector) == 0 {
+		if selector == "" {
 			break
 		}
 
@@ -370,18 +144,18 @@ func GetPath(elm *goquery.Selection) string {
 	return join
 }
 
-func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, string, chapter, error) {
+func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, error) {
 	selectorSet := true
-	if len(selector) == 0 {
+	if selector == "" {
 		selector = "a"
 		selectorSet = false
 	}
 
+	// visit and count link classes
 	pathLinks := map[string][]link{}
 	pathCount := map[string]int{}
 	pathMax := ""
 
-	// visit and count link classes
 	c := colly.NewCollector()
 	c.OnHTML(selector, func(e *colly.HTMLElement) {
 		href := e.Attr("href")
@@ -389,40 +163,26 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool)
 		path := GetPath(e.DOM)
 		key := path
 
-		if selectorSet {
+		// include element class in key if selector is set
-
+		if !selectorSet {
-			// if selector is set, we use the selector specified by the user
-
-			key = selector
-			pathLinks[key] = append(pathLinks[key], NewLink(href, text))
-			pathCount[key] += 1
-			pathMax = key
-
-		} else {
-
-			// if selector is not set, we compute the selector ourselves
-
 			class := e.Attr("class")
-			// include the element class to make sure we have the same exact path for every link in the table of content
 			key = fmt.Sprintf("%s.%s", path, class)
+		}
 
-			// we count this key if the link text is not empty
+		if selectorSet || text != "" {
-			if text != "" {
+			pathLinks[key] = append(pathLinks[key], NewLink(href, text))
-				pathLinks[key] = append(pathLinks[key], NewLink(href, text))
+			pathCount[key] += len(text)
-				pathCount[key] += len(text)
 
-				if pathCount[key] > pathCount[pathMax] {
+			if pathCount[key] > pathCount[pathMax] {
-					pathMax = key
+				pathMax = key
-				}
 			}
-
 		}
 	})
 	c.Visit(url.String())
 
 	links := pathLinks[pathMax]
 	if len(links) == 0 {
-		return []link{}, pathMax, chapter{}, fmt.Errorf("no link found for selector: %s", selector)
+		return []link{}, fmt.Errorf("no link found for selector: %s", selector)
 	}
 
 	end := len(links)
@@ -432,12 +192,11 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool)
 
 	links = links[offset:end]
 
-	home := NewChapterFromURL(url.String(), "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {})
-
 	if include {
-		l := NewLink(url.String(), home.Name())
+		c := NewChapterFromURL(url.String(), false)
+		l := NewLink(url.String(), c.Name())
 		links = append([]link{l}, links...)
 	}
 
-	return links, pathMax, home, nil
+	return links, nil
 }

book/scraper_test.go

@@ -1,199 +0,0 @@
-package book
-
-import (
-	"testing"
-	"time"
-)
-
-func TestBody(t *testing.T) {
-
-	config := NewScrapeConfig()
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {})
-
-	got := c.Body()
-	want := "<!doctype html>\n<html lang=\"en-us\">\n  <head>\n    <title>Books</title>\n    <link rel=\"shortcut icon\" href=\"/favicon.ico\" />\n    <meta charset=\"utf-8\" />\n    <meta name=\"generator\" content=\"Hugo 0.59.1\" />\n    <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" />\n    <meta name=\"author\" content=\"John Doe\" />\n    <meta name=\"description\" content=\" \" />\n    <link rel=\"stylesheet\" href=\"https://books.lapw.at/css/main.min.88e7083eff65effb7485b6e6f38d10afbec25093a6fac42d734ce9024d3defbd.css\" />\n\n    \n    <meta name=\"twitter:card\" content=\"summary\"/>\n<meta name=\"twitter:title\" content=\"Books\"/>\n<meta name=\"twitter:description\" content=\" \"/>\n\n    <meta property=\"og:title\" content=\"Books\" />\n<meta property=\"og:description\" content=\" \" />\n<meta property=\"og:type\" content=\"website\" />\n<meta property=\"og:url\" content=\"https://books.lapw.at/\" />\n\n\n\n  </head>\n  <body>\n    <header class=\"app-header\">\n      <a href=\"https://books.lapw.at/\"><img class=\"app-header-avatar\" src=\"/book.svg\" alt=\"John Doe\" /></a>\n      <h1>Books</h1>\n      <p> </p>\n      <div class=\"app-header-social\">\n        \n      </div>\n    </header>\n    <main class=\"app-container\">\n      \n  <article>\n    <h1>Books</h1>\n    <ul class=\"posts-list\">\n      \n        <li class=\"posts-list-item\">\n          <a class=\"posts-list-item-title\" href=\"https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/\">Discours de la Méthode</a>\n          <span class=\"posts-list-item-description\">\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\" class=\"icon icon-clock\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 98 min read -\n            1637\n          </span>\n        </li>\n      \n        <li class=\"posts-list-item\">\n          <a class=\"posts-list-item-title\" href=\"https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/\">The Twelve-Factor App</a>\n          <span class=\"posts-list-item-description\">\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\" class=\"icon icon-clock\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 22 min read -\n            2011\n          </span>\n        </li>\n      \n    </ul>\n    \n\n\n\n  </article>\n\n    </main>\n  </body>\n</html>\n"
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestName(t *testing.T) {
-
-	config := NewScrapeConfig()
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {})
-
-	got := c.Name()
-	want := "Books"
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestCustomName(t *testing.T) {
-
-	config := NewScrapeConfig()
-	config.UseLinkName = true
-	c := NewChapterFromURL("https://books.lapw.at/", "Custom Name", []*ScrapeConfig{config}, 0, func(index int, name string) {})
-
-	got := c.Name()
-	want := "Custom Name"
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestAuthor(t *testing.T) {
-
-	config := NewScrapeConfig()
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {})
-
-	got := c.Author()
-	want := "John Doe"
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestContent(t *testing.T) {
-
-	config := NewScrapeConfig()
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {})
-
-	got := c.Content()
-	want := "<div id=\"readability-page-1\" class=\"page\">\n    \n    <main>\n      \n  <article>\n    \n    <ul>\n      \n        <li>\n          <a href=\"https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/\">Discours de la Méthode</a>\n          <span>\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 98 min read -\n            1637\n          </span>\n        </li>\n      \n        <li>\n          <a href=\"https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/\">The Twelve-Factor App</a>\n          <span>\n            <svg xmlns=\"http://www.w3.org/2000/svg\" width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" fill=\"none\" stroke=\"currentColor\" stroke-width=\"2\" stroke-linecap=\"round\" stroke-linejoin=\"round\">\n  <title>clock</title>\n  <circle cx=\"12\" cy=\"12\" r=\"10\"></circle><polyline points=\"12 6 12 12 16 14\"></polyline>\n</svg> 22 min read -\n            2011\n          </span>\n        </li>\n      \n    </ul>\n    \n\n\n\n  </article>\n\n    </main>\n  \n\n</div>"
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestDelay(t *testing.T) {
-
-	config0 := NewScrapeConfig()
-	config0.Delay = 500
-
-	config1 := NewScrapeConfig()
-
-	start := time.Now()
-	NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
-	elapsed := time.Since(start)
-
-	got := elapsed
-	want := time.Duration(500) * time.Millisecond
-
-	if got < want {
-		t.Errorf("got %v, wanted min %v", got, want)
-	}
-
-}
-
-func TestContentImagesOnly(t *testing.T) {
-
-	config := NewScrapeConfig()
-	config.ImagesOnly = true
-
-	c := NewChapterFromURL("https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {})
-
-	got := c.Content()
-	want := "<img src=\"https://books.lapw.at/images/codebase-deploys.png\" alt=\"One codebase maps to many deploys\"/><img src=\"https://books.lapw.at/images/attached-resources.png\" alt=\"A production deploy attached to four backing services.\"/><img src=\"https://books.lapw.at/images/release.png\" alt=\"Code becomes a build, which is combined with config to create a release.\"/><img src=\"https://books.lapw.at/images/process-types.png\" alt=\"Scale is expressed as running processes, workload diversity is expressed as process types.\"/>"
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestSubChapters(t *testing.T) {
-
-	config0 := NewScrapeConfig()
-	config1 := NewScrapeConfig()
-
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
-
-	got := len(c.SubChapters())
-	want := 2
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestSubChaptersSelector(t *testing.T) {
-
-	config0 := NewScrapeConfig()
-	config0.Selector = "section.concrete > article > h2 > a"
-
-	config1 := NewScrapeConfig()
-
-	c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
-
-	got := len(c.SubChapters())
-	want := 12
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestSubChaptersLimit(t *testing.T) {
-
-	config0 := NewScrapeConfig()
-	config0.Limit = 1
-
-	config1 := NewScrapeConfig()
-
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
-
-	got := len(c.SubChapters())
-	want := 1
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestSubChaptersLimitOver(t *testing.T) {
-
-	config0 := NewScrapeConfig()
-	config0.Limit = 3
-
-	config1 := NewScrapeConfig()
-
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {})
-
-	got := len(c.SubChapters())
-	want := 2
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}
-
-func TestNotInclude(t *testing.T) {
-
-	config := NewScrapeConfig()
-	config.Include = false
-
-	c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {})
-
-	got := c.Content()
-	want := ""
-
-	if got != want {
-		t.Errorf("got %v, wanted %v", got, want)
-	}
-
-}

cmd/get.go

@@ -3,64 +3,26 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"log"
+	"os"
+	"os/exec"
 	"strings"
 
+	md "github.com/JohannesKaufmann/html-to-markdown"
+	"github.com/PuerkitoBio/goquery"
+	epub "github.com/bmaupin/go-epub"
 	"github.com/spf13/cobra"
 
 	"github.com/lapwat/papeer/book"
 )
 
-type GetOptions struct {
+var recursive, include, images bool
-	// url string
+var format, output, selector string
-
+var limit, offset, delay, threads int
-	name   string
-	author string
-	Format string
-	output string
-	images bool
-	// ImagesOnly bool
-	quiet bool
-
-	Selector []string
-	depth    int
-	limit    int
-	offset   int
-	delay    int
-	threads  int
-	// includeUrl bool
-	include     bool
-	useLinkName bool
-}
-
-var getOpts *GetOptions
-
-func init() {
-	getOpts = &GetOptions{}
-
-	getCmd.PersistentFlags().StringVarP(&getOpts.name, "name", "n", "", "book name (default: page title)")
-	getCmd.PersistentFlags().StringVarP(&getOpts.author, "author", "a", "", "book author")
-	getCmd.PersistentFlags().StringVarP(&getOpts.Format, "format", "f", "md", "file format [stdout, md, epub, mobi]")
-	getCmd.PersistentFlags().StringVarP(&getOpts.output, "output", "", "", "file name (default: book name)")
-	getCmd.PersistentFlags().BoolVarP(&getOpts.images, "images", "", false, "retrieve images only")
-	getCmd.PersistentFlags().BoolVarP(&getOpts.quiet, "quiet", "q", false, "hide progress bar")
-
-	// common with list command
-	getCmd.Flags().StringSliceVarP(&getOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector")
-	getCmd.Flags().IntVarP(&getOpts.depth, "depth", "d", 0, "scraping depth")
-	getCmd.Flags().IntVarP(&getOpts.limit, "limit", "l", -1, "limit number of chapters, use with depth/selector")
-	getCmd.Flags().IntVarP(&getOpts.offset, "offset", "o", 0, "skip first chapters, use with depth/selector")
-	getCmd.Flags().IntVarP(&getOpts.delay, "delay", "", -1, "time in milliseconds to wait before downloading next chapter, use with depth/selector")
-	getCmd.Flags().IntVarP(&getOpts.threads, "threads", "t", -1, "download concurrency, use with depth/selector")
-	getCmd.Flags().BoolVarP(&getOpts.include, "include", "i", false, "include URL as first chapter, use with depth/selector")
-	getCmd.Flags().BoolVarP(&getOpts.useLinkName, "use-link-name", "", false, "use link name for chapter title")
-
-	rootCmd.AddCommand(getCmd)
-}
 
 var getCmd = &cobra.Command{
-	Use:     "get URL",
+	Use:   "get",
-	Short:   "Scrape URL content",
+	Short: "Scrape URL content",
-	Example: "papeer get https://www.eff.org/cyberspace-independence",
 	Args: func(cmd *cobra.Command, args []string) error {
 		if len(args) < 1 {
 			return errors.New("requires an URL argument")
@@ -72,40 +34,39 @@ var getCmd = &cobra.Command{
 			"epub":   true,
 			"mobi":   true,
 		}
-
+		if formatEnum[format] != true {
-		if formatEnum[getOpts.Format] != true {
+			return fmt.Errorf("invalid format specified: %s", format)
-			return fmt.Errorf("invalid format specified: %s", getOpts.Format)
 		}
 
 		// add .mobi to filename if not specified
-		if getOpts.Format == "mobi" {
+		if format == "mobi" {
-			if len(getOpts.output) > 0 && strings.HasSuffix(getOpts.output, ".mobi") == false {
+			if len(output) > 0 && strings.HasSuffix(output, ".mobi") == false {
-				getOpts.output = fmt.Sprintf("%s.mobi", getOpts.output)
+				output = fmt.Sprintf("%s.mobi", output)
 			}
 		}
 
-		if cmd.Flags().Changed("include") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
+		if cmd.Flags().Changed("selector") && recursive == false {
-			return errors.New("cannot use include option if depth/selector is not specified")
+			return errors.New("cannot use selector option if not in recursive mode")
 		}
 
-		if cmd.Flags().Changed("limit") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
+		if cmd.Flags().Changed("include") && recursive == false {
-			return errors.New("cannot use limit option if depth/selector is not specified")
+			return errors.New("cannot use include option if not in recursive mode")
 		}
 
-		if cmd.Flags().Changed("offset") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
+		if cmd.Flags().Changed("limit") && recursive == false {
-			return errors.New("cannot use offset option if depth/selector is not specified")
+			return errors.New("cannot use limit option if not in recursive mode")
 		}
 
-		if cmd.Flags().Changed("delay") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
+		if cmd.Flags().Changed("offset") && recursive == false {
-			return errors.New("cannot use delay option if depth/selector is not specified")
+			return errors.New("cannot use offset option if not in recursive mode")
 		}
 
-		if cmd.Flags().Changed("threads") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
+		if cmd.Flags().Changed("delay") && recursive == false {
-			return errors.New("cannot use threads option if depth/selector is not specified")
+			return errors.New("cannot use delay option if not in recursive mode")
 		}
 
-		if cmd.Flags().Changed("use-link-name") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
+		if cmd.Flags().Changed("threads") && recursive == false {
-			return errors.New("cannot use use-link-name option if depth/selector is not specified")
+			return errors.New("cannot use threads option if not in recursive mode")
 		}
 
 		if cmd.Flags().Changed("delay") && cmd.Flags().Changed("threads") {
@@ -116,61 +77,123 @@ var getCmd = &cobra.Command{
 	},
 	Run: func(cmd *cobra.Command, args []string) {
 		url := args[0]
+		b := book.NewBookFromURL(url, selector, recursive, include, images, limit, offset, delay, threads)
 
-		// fill selector array with empty selectors to match depth
+		if len(output) == 0 {
-		getOpts.Selector = append(getOpts.Selector, "")
+			// set default output
-		for len(getOpts.Selector) < getOpts.depth+1 {
+			output = strings.ReplaceAll(b.Name(), " ", "_")
-			getOpts.Selector = append(getOpts.Selector, "")
+			output = strings.ReplaceAll(output, "/", "")
+			output = fmt.Sprintf("%s.%s", output, format)
 		}
-		fmt.Println(len(getOpts.Selector))
 
-		// generate config for each level
+		if format == "stdout" {
-		configs := make([]*book.ScrapeConfig, len(getOpts.Selector))
-		for index, s := range getOpts.Selector {
-			config := book.NewScrapeConfig()
-			config.Selector = s
-			config.Quiet = getOpts.quiet
-			config.Limit = getOpts.limit
-			config.Offset = getOpts.offset
-			config.Delay = getOpts.delay
-			config.Threads = getOpts.threads
-			config.ImagesOnly = getOpts.images
-			config.Include = getOpts.include
-			config.UseLinkName = getOpts.useLinkName
 
-			// do not use link name for root level as there is not parent link
+			for _, c := range b.Chapters() {
-			if index == 0 {
+				// convert to markdown
-				config.UseLinkName = false
+				content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				text := fmt.Sprintf("%s\n%s\n\n%s\n\n\n", c.Name(), strings.Repeat("=", len(c.Name())), content)
+
+				// write to stdout
+				fmt.Println(text)
 			}
 
-			// always include last level by default
+		}
-			if index == len(getOpts.Selector)-1 {
+
-				config.Include = true
+		if format == "md" {
+
+			// create markdown file
+			f, err := os.Create(output)
+			if err != nil {
+				log.Fatal(err)
+			}
+			defer f.Close()
+
+			for _, c := range b.Chapters() {
+				// convert to markdown
+				content, err := md.NewConverter("", true, nil).ConvertString(c.Content())
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				text := fmt.Sprintf("%s\n%s\n\n%s\n\n\n", c.Name(), strings.Repeat("=", len(c.Name())), content)
+
+				// write to markdown file
+				_, err = f.WriteString(text)
+				if err != nil {
+					log.Fatal(err)
+				}
 			}
 
-			configs[index] = config
+			fmt.Printf("Markdown saved to \"%s\"\n", output)
 		}
 
-		c := book.NewChapterFromURL(url, "", configs, 0, func(index int, name string) {})
+		if format == "epub" {
+			e := epub.NewEpub(b.Name())
+			e.SetAuthor(b.Author())
 
-		if getOpts.Format == "stdout" {
+			for _, c := range b.Chapters() {
-			markdown := book.ToMarkdownString(c)
+				// parse content 
-			fmt.Println(markdown)
+				doc, err := goquery.NewDocumentFromReader(strings.NewReader(c.Content()))
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				// retrieve images and download it
+				contentWithLocalImages := c.Content()
+				doc.Find("img").Each(func(i int, s *goquery.Selection) {
+					src, _ := s.Attr("src")
+					imagePath, _ := e.AddImage(src, "")
+
+					contentWithLocalImages = strings.ReplaceAll(contentWithLocalImages, src, imagePath)
+				})
+
+				html := fmt.Sprintf("<h1>%s</h1>%s", c.Name(), contentWithLocalImages)
+				_, err = e.AddSection(html, c.Name(), "", "")
+				if err != nil {
+					log.Fatal(err)
+				}
+
+			}
+
+			err := e.Write(output)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			fmt.Printf("Ebook saved to \"%s\"\n", output)
 		}
 
-		if getOpts.Format == "md" {
+		if format == "mobi" {
-			filename := book.ToMarkdown(c, getOpts.output)
+			e := epub.NewEpub(b.Name())
-			fmt.Printf("Markdown saved to \"%s\"\n", filename)
+			e.SetAuthor(b.Author())
-		}
 
-		if getOpts.Format == "epub" {
+			for _, chapter := range b.Chapters() {
-			filename := book.ToEpub(c, getOpts.output)
+				e.AddSection(chapter.Content(), chapter.Name(), "", "")
-			fmt.Printf("Ebook saved to \"%s\"\n", filename)
+			}
-		}
 
-		if getOpts.Format == "mobi" {
+			outputEPUB := strings.ReplaceAll(output, ".mobi", ".epub")
-			filename := book.ToMobi(c, getOpts.output)
+
-			fmt.Printf("Ebook saved to \"%s\"\n", filename)
+			err := e.Write(outputEPUB)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			exec.Command("kindlegen", outputEPUB).Run()
+			// exec command always return status 1 even if it fails
+			// if err != nil {
+			// 	log.Fatal(err)
+			// }
+
+			fmt.Printf("Ebook saved to \"%s\"\n", output)
+
+			err2 := os.Remove(outputEPUB)
+			if err2 != nil {
+				log.Fatal(err2)
+			}
 		}
 	},
 }

cmd/list.go

@@ -2,11 +2,9 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
 	"log"
 	urllib "net/url"
 	"os"
-	"strings"
 
 	"github.com/jedib0t/go-pretty/v6/table"
 	cobra "github.com/spf13/cobra"
@@ -14,42 +12,9 @@ import (
 	"github.com/lapwat/papeer/book"
 )
 
-type ListOptions struct {
-	// url string
-
-	Selector []string
-	depth    int
-	limit    int
-	offset   int
-	delay    int
-	threads  int
-	// includeUrl bool
-	include     bool
-	useLinkName bool
-}
-
-var listOpts *ListOptions
-
-func init() {
-	listOpts = &ListOptions{}
-
-	listCmd.Flags().StringSliceVarP(&listOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector")
-	listCmd.Flags().IntVarP(&listOpts.depth, "depth", "d", 0, "scraping depth")
-	listCmd.Flags().IntVarP(&listOpts.limit, "limit", "l", -1, "limit number of chapters, use with depth/selector")
-	listCmd.Flags().IntVarP(&listOpts.offset, "offset", "o", 0, "skip first chapters, use with depth/selector")
-	listCmd.Flags().IntVarP(&listOpts.delay, "delay", "", -1, "time in milliseconds to wait before downloading next chapter, use with depth/selector")
-	listCmd.Flags().IntVarP(&listOpts.threads, "threads", "t", -1, "download concurrency, use with depth/selector")
-	listCmd.Flags().BoolVarP(&listOpts.include, "include", "i", false, "include URL as first chapter, use with depth/selector")
-	listCmd.Flags().BoolVarP(&listOpts.useLinkName, "use-link-name", "", false, "use link name for chapter title")
-
-	rootCmd.AddCommand(listCmd)
-}
-
 var listCmd = &cobra.Command{
-	Use:     "list URL",
+	Use:   "ls",
-	Aliases: []string{"ls"},
+	Short: "Print table of content",
-	Short:   "Print URL table of contents",
-	Example: "papeer list https://12factor.net/ -s 'section.concrete>article>h2>a'",
 	Args: func(cmd *cobra.Command, args []string) error {
 		if len(args) < 1 {
 			return errors.New("requires an URL argument")
@@ -57,16 +22,12 @@ var listCmd = &cobra.Command{
 		return nil
 	},
 	Run: func(cmd *cobra.Command, args []string) {
-		if len(listOpts.Selector) == 0 {
-			listOpts.Selector = []string{""}
-		}
-
 		base, err := urllib.Parse(args[0])
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		links, path, _, err := book.GetLinks(base, listOpts.Selector[0], listOpts.limit, listOpts.offset, listOpts.include)
+		links, err := book.GetLinks(base, selector, limit, offset, include)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -76,16 +37,7 @@ var listCmd = &cobra.Command{
 		t.Style().Options.DrawBorder = false
 		t.Style().Options.SeparateColumns = false
 		t.Style().Options.SeparateHeader = false
-
+		t.AppendHeader(table.Row{"#", "Name", "Url"})
-		// format selector path
-		pathArray := strings.Split(path, "<")
-		// reverse path
-		for i, j := 0, len(pathArray)-1; i < j; i, j = i+1, j-1 {
-			pathArray[i], pathArray[j] = pathArray[j], pathArray[i]
-		}
-		pathFormatted := strings.Join(pathArray, ">")
-
-		t.AppendHeader(table.Row{"#", "Name", fmt.Sprintf("Url [%s]", pathFormatted)})
 
 		for index, link := range links {
 			u, err := base.Parse(link.Href())

cmd/root.go

@@ -21,3 +21,19 @@ func Execute() {
 		os.Exit(1)
 	}
 }
+
+func init() {
+	rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "stdout", "file format [stdout, md, epub, mobi]")
+	rootCmd.PersistentFlags().StringVarP(&output, "output", "", "", "output file")
+	rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector, in resursive mode")
+	rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "create one chapter per natigation item")
+	rootCmd.PersistentFlags().BoolVarP(&include, "include", "i", false, "include URL as first chapter, in resursive mode")
+	rootCmd.PersistentFlags().BoolVarP(&images, "images", "", false, "retrieve images only")
+	rootCmd.PersistentFlags().IntVarP(&limit, "limit", "l", -1, "limit number of chapters, in recursive mode")
+	rootCmd.PersistentFlags().IntVarP(&offset, "offset", "o", 0, "skip first chapters, in recursive mode")
+	rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "time to wait before downloading next chapter, in milliseconds")
+	rootCmd.PersistentFlags().IntVarP(&threads, "threads", "t", -1, "download concurrency, in recursive mode")
+
+	rootCmd.AddCommand(getCmd)
+	rootCmd.AddCommand(listCmd)
+}

cmd/version.go

@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print the version number of papeer",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("papeer v0.4.1")
+		fmt.Println("papeer v0.3.0")
 	},
 }

release.sh

@@ -3,31 +3,19 @@
 version=$1
 platforms=("linux/amd64" "darwin/amd64" "windows/amd64")
 
-if [ "$#" -ne 1 ]; then
-    echo "Illegal number of parameters"
-    echo "Usage: ./release.sh X.X.X"
-    exit 1
-fi
-
 for platform in "${platforms[@]}"
 do
     platform_split=(${platform//\// })
     GOOS=${platform_split[0]}
     GOARCH=${platform_split[1]}
-    output_name=papeer
+    output_name='papeer-'$version'-'$GOOS'-'$GOARCH
-
     if [ $GOOS = "windows" ]; then
-        env GOOS=$GOOS GOARCH=$GOARCH go build -o "$output_name.exe"
+        output_name+='.exe'
-        zip "$output_name-v$version-$GOOS-$GOARCH.exe.zip" "$output_name.exe"
-        rm "$output_name.exe"
-    else
-        env GOOS=$GOOS GOARCH=$GOARCH go build -o "$output_name"
-        tar czvf "$output_name-v$version-$GOOS-$GOARCH.tar.gz" "$output_name"
-        rm "$output_name"
     fi
 
-    # if [ $? -ne 0 ]; then
+    env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name
-    #     echo 'An error has occurred! Aborting the script execution...'
+    if [ $? -ne 0 ]; then
-    #     exit 1
+        echo 'An error has occurred! Aborting the script execution...'
-    # fi
+        exit 1
+    fi
 done