From 4b760c956291ba9d033932e7e917746cc12e7e9d Mon Sep 17 00:00:00 2001 From: lapwat Date: Fri, 4 Feb 2022 19:42:27 +0100 Subject: [PATCH] chain selctors, depth & quiet options, split main commands --- README.md | 200 +++++++++++++++++++++++------------------ book/format.go | 37 ++++++-- book/format_test.go | 62 +++++++++++-- book/scraper.go | 205 ++++++++++++++++++++++++++----------------- book/scraper_test.go | 49 +++++++---- cmd/get.go | 168 ++++++++++++++++++++++------------- cmd/list.go | 56 +++++++++++- cmd/root.go | 19 ---- cmd/version.go | 2 +- release.sh | 9 +- 10 files changed, 528 insertions(+), 279 deletions(-) diff --git a/README.md b/README.md index 79135a4..97bbf9e 100644 --- a/README.md +++ b/README.md @@ -1,101 +1,126 @@ # Papeer -Papeer is a tool that lets you scrape content from the internet. It can scrape any web page, keeping only relevant content (formatted text and images) and removing ads and menus. You can export the content to Markdown, EPUB or MOBI files. +Papeer is a powerful an **ereader internet vacuum**. It can scrape any website, removing ads and keeping only the relevant content (formatted text and images). You can export the content to Markdown, EPUB or MOBI files. + +# Table of contents + +- [Usage](#usage) +- [Examples](#examples) + * [Grab a single page](#grab-a-single-page) + * [Grab several pages](#grab-several-pages) + + [`selector` option](#-selector--option) + + [`depth` option](#-depth--option) + + [Display table of contents](#display-table-of-contents) + + [Scrape time](#scrape-time) +- [Installation](#installation) + * [From source](#from-source) + * [From binary](#from-binary) + + [Linux / MacOS](#linux---macos) + + [Windows](#windows) + * [MOBI support (optional)](#mobi-support--optional-) +- [Autocompletion](#autocompletion) +- [Dependencies](#dependencies) # Usage +## Scrape a web page + +The `get` command lets you retrieve the content of any web page. + ``` -Browse the web in the eink era +Scrape URL content Usage: - papeer [flags] - papeer [command] + papeer get URL [flags] -Available Commands: - completion generate the autocompletion script for the specified shell - get Scrape URL content - help Help about any command - ls Print table of content - version Print the version number of papeer +Examples: +papeer get https://www.eff.org/cyberspace-independence Flags: - -a, --author string book author - -d, --delay int time to wait before downloading next chapter, in milliseconds (default -1) - -f, --format string file format [stdout, md, epub, mobi] (default "stdout") - -h, --help help for papeer - --images retrieve images only - -i, --include include URL as first chapter, in resursive mode - -l, --limit int limit number of chapters, in recursive mode (default -1) - -n, --name string book name (default: page title) - -o, --offset int skip first chapters, in recursive mode - --output string file name (default: book name) - -q, --quiet hide progress bar - -r, --recursive create one chapter per natigation item - -s, --selector string table of content CSS selector, in resursive mode - -t, --threads int download concurrency, in recursive mode (default -1) - -Use "papeer [command] --help" for more information about a command. + -a, --author string book author + --delay int time in milliseconds to wait before downloading next chapter, use with depth/selector (default -1) + -d, --depth int scraping depth + -f, --format string file format [stdout, md, epub, mobi] (default "md") + -h, --help help for get + --images retrieve images only + -i, --include include URL as first chapter, use with depth/selector + -l, --limit int limit number of chapters, use with depth/selector (default -1) + -n, --name string book name (default: page title) + -o, --offset int skip first chapters, use with depth/selector + --output string file name (default: book name) + -q, --quiet hide progress bar + -s, --selector strings table of contents CSS selector + -t, --threads int download concurrency, use with depth/selector (default -1) + --use-link-name use link name for chapter title ``` -# Examples +## Scrape a whole website -## Grab a single page +If a navigation menu is present on a website, you can scrape the content of each page. -The `get` command lets you retrieve the content of a web page. +You can activate this mode by using the `depth` or `selector` options. + +### `depth` option + +This option defaults to 0, `papeer` will grab only the main page. + +If you specify a value greater than 0, `papeer` will grab only the pages as deep as the value you specify. + +> Using `include` option will include all intermediary levels. + +### `selector` option + +If this option is not specified, `papeer` will grab only the one page. + +If this option is specified, `papeer` will select the links (a HTML tag) present on the main page, then grab each one of them. + +You can chain this option to grab several level of pages with diferent selectors for each level. + +## Display the table of contents + +Before actually scraping a whole website, it is a good idea to use the `list` command. This command is like a **dry run**, which lets you vizualize the content before actually retrieving it. You can use several options to customize the table of contents extraction, such as `selector`, `limit`, `offset` and `include`. Type `papeer list --help` for more information about those options. ```sh -papeer get https://www.eff.org/cyberspace-independence -# A Declaration of the Independence of Cyberspace -# =============================================== - -# Governments of the Industrial World, you weary giants of flesh and steel, I come from Cyberspace, the new home of Mind. On behalf of the future, I ask you of the past to leave us alone. You are not welcome among us. You have no sovereignty where we gather... +papeer list https://12factor.net/ -s 'section.concrete>article>h2>a' ``` - -## Grab several pages (recursive mode) - -The `recursive` option lets you extract the table of content of a website, then scrape the content of each one of its pages. - -### Display table of content - -Before trying the `recursive` option, it is a good idea to use the `ls` option, which lets you vizualize the content that will be retrieved. You can use several options to customize the table of content extraction, such as `selector`, `limit`, `offset` and `include`. Type `papeer help` for more information about those options. - -```sh -papeer ls https://12factor.net/ -s 'section.concrete > article > h2 > a' -# # NAME URL -# 1 I. Codebase https://12factor.net/codebase -# 2 II. Dependencies https://12factor.net/dependencies -# 3 III. Config https://12factor.net/config -# 4 IV. Backing services https://12factor.net/backing-services -# 5 V. Build, release, run https://12factor.net/build-release-run -# 6 VI. Processes https://12factor.net/processes -# 7 VII. Port binding https://12factor.net/port-binding -# 8 VIII. Concurrency https://12factor.net/concurrency -# 9 IX. Disposability https://12factor.net/disposability -# 10 X. Dev/prod parity https://12factor.net/dev-prod-parity -# 11 XI. Logs https://12factor.net/logs -# 12 XII. Admin processes https://12factor.net/admin-processes +``` + # NAME URL + 1 I. Codebase https://12factor.net/codebase + 2 II. Dependencies https://12factor.net/dependencies + 3 III. Config https://12factor.net/config + 4 IV. Backing services https://12factor.net/backing-services + 5 V. Build, release, run https://12factor.net/build-release-run + 6 VI. Processes https://12factor.net/processes + 7 VII. Port binding https://12factor.net/port-binding + 8 VIII. Concurrency https://12factor.net/concurrency + 9 IX. Disposability https://12factor.net/disposability +10 X. Dev/prod parity https://12factor.net/dev-prod-parity +11 XI. Logs https://12factor.net/logs +12 XII. Admin processes https://12factor.net/admin-processes ``` ### Scrape time -Once you are satisfied with the table of content listed by the `ls` command, you can actually scrape the content of those pages. You can use the same options that you specified for the `ls` command. In recursive mode, you also have the possibility to use `delay` and `threads` options. +Once you are satisfied with the table of contents listed by the `ls` command, you can actually scrape the content of those pages. You can use the same options that you specified for the `ls` command. You can specify `delay` and `threads` options when using `selector` or `depth` options. ```sh -papeer get https://12factor.net/ --recursive -s 'section.concrete > article > h2 > a' --format=md -# [======================================>-----------------------------] Chapters 7 / 12 -# [====================================================================] 1. I. Codebase -# [====================================================================] 2. II. Dependencies -# [====================================================================] 3. III. Config -# [====================================================================] 4. IV. Backing services -# [====================================================================] 5. V. Build, release, run -# [====================================================================] 6. VI. Processes -# [====================================================================] 7. VII. Port binding -# [--------------------------------------------------------------------] 8. VIII. Concurrency -# [--------------------------------------------------------------------] 9. IX. Disposability -# [--------------------------------------------------------------------] 10. X. Dev/prod parity -# [--------------------------------------------------------------------] 11. XI. Logs -# [--------------------------------------------------------------------] 12. XII. Admin processes -# Markdown saved to "The_Twelve-Factor_App.md" +papeer get https://12factor.net/ --selector='section.concrete>article>h2>a' +``` +``` +[======================================>-----------------------------] Chapters 7 / 12 +[====================================================================] 1. I. Codebase +[====================================================================] 2. II. Dependencies +[====================================================================] 3. III. Config +[====================================================================] 4. IV. Backing services +[====================================================================] 5. V. Build, release, run +[====================================================================] 6. VI. Processes +[====================================================================] 7. VII. Port binding +[--------------------------------------------------------------------] 8. VIII. Concurrency +[--------------------------------------------------------------------] 9. IX. Disposability +[--------------------------------------------------------------------] 10. X. Dev/prod parity +[--------------------------------------------------------------------] 11. XI. Logs +[--------------------------------------------------------------------] 12. XII. Admin processes +Markdown saved to "The_Twelve-Factor_App.md" ``` # Installation @@ -108,21 +133,30 @@ go get -u github.com/lapwat/papeer ## From binary -### On Linux / MacOS +### Linux / MacOS ```sh -platform=linux # use platform=darwin for MacOS -release=0.3.3 -curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64 > papeer +# use platform=darwin for MacOS +platform=linux +release=0.4.0 + +# download and extract +curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64.tar.gz > papeer.tar.gz +tar xzvf papeer.tar.gz +rm papeer.tar.gz + +# move to user binaries chmod +x papeer sudo mv papeer /usr/local/bin ``` -### On Windows +### Windows -Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.3.3/papeer-v0.3.3-windows-amd64.exe). +Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.4.0/papeer-v0.4.0-windows-amd64.exe). -## Install kindlegen to export websites to MOBI (optional) +## (optional) MOBI support + +Install kindlegen to convert websites, Linux only ```sh TMPDIR=$(mktemp -d -t papeer-XXXXX) @@ -141,10 +175,10 @@ Execute this command in your current shell, or add it to your `.bashrc`. . <(papeer completion bash) ``` -Type `papeer completion bash -h` for more information. - You can replace `bash` by your own shell (zsh, fish or powershell). +Type `papeer completion bash -h` for more information. + # Dependencies - `cobra` command line interface diff --git a/book/format.go b/book/format.go index 6a7d576..ee60278 100644 --- a/book/format.go +++ b/book/format.go @@ -21,10 +21,10 @@ func Filename(name string) string { return filename } -func ToMarkdown(c chapter) string { +func ToMarkdownString(c chapter) string { markdown := "" - if c.config.include { + if c.config.Include { // title markdown += fmt.Sprintf("%s\n", c.Name()) markdown += fmt.Sprintf("%s\n\n", strings.Repeat("=", len(c.Name()))) @@ -39,12 +39,33 @@ func ToMarkdown(c chapter) string { for _, sc := range c.SubChapters() { // subchapters content - markdown += fmt.Sprintf("%s\n\n\n", ToMarkdown(sc)) + markdown += fmt.Sprintf("%s\n\n\n", ToMarkdownString(sc)) } return markdown } +func ToMarkdown(c chapter, filename string) string { + if len(filename) == 0 { + filename = fmt.Sprintf("%s.md", Filename(c.Name())) + } + + markdown := ToMarkdownString(c) + + // write to file + f, err := os.Create(filename) + if err != nil { + log.Fatal(err) + } + _, err2 := f.WriteString(markdown) + if err2 != nil { + log.Fatal(err2) + } + f.Close() + + return filename +} + func ToEpub(c chapter, filename string) string { if len(filename) == 0 { filename = fmt.Sprintf("%s.epub", Filename(c.Name())) @@ -67,9 +88,9 @@ func ToEpub(c chapter, filename string) string { func AppendToEpub(e *epub.Epub, c chapter) { content := "" - if c.config.include { + if c.config.Include { - if c.config.imagesOnly == false { + if c.config.ImagesOnly == false { content = c.Content() } @@ -85,7 +106,7 @@ func AppendToEpub(e *epub.Epub, c chapter) { src = strings.Split(src, "?")[0] // remove query part imagePath, _ := e.AddImage(src, "") - if c.config.imagesOnly { + if c.config.ImagesOnly { imageTag, _ := goquery.OuterHtml(s) content += strings.Replace(imageTag, src, imagePath, 1) } else { @@ -94,8 +115,8 @@ func AppendToEpub(e *epub.Epub, c chapter) { }) html := "" - // add title only if imagesOnly = false - if c.config.imagesOnly == false { + // add title only if ImagesOnly = false + if c.config.ImagesOnly == false { html += fmt.Sprintf("

%s

", c.Name()) } html += content diff --git a/book/format_test.go b/book/format_test.go index 1bb4b85..34cccd6 100644 --- a/book/format_test.go +++ b/book/format_test.go @@ -17,11 +17,11 @@ func TestFilename(t *testing.T) { } -func TestToMarkdown(t *testing.T) { +func TestToMarkdownString(t *testing.T) { - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) - got := ToMarkdown(c) + got := ToMarkdownString(c) want := "Books\n=====\n\n- [Discours de la Méthode](https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/)clock 98 min read -\n1637\n\n- [The Twelve-Factor App](https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/)clock 22 min read -\n2011\n\n\n" if got != want { @@ -30,12 +30,44 @@ func TestToMarkdown(t *testing.T) { } +func TestToMarkdown(t *testing.T) { + + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + ToMarkdown(c, "") + + filename := "Books.md" + if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { + t.Errorf("%s does not exist: %v", filename, err) + } else { + if err := os.Remove(filename); err != nil { + t.Errorf("cannot remove %v: %v", filename, err) + } + } + +} + +func TestToMarkdownFilename(t *testing.T) { + + filename := "ebook.md" + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + ToMarkdown(c, filename) + + if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { + t.Errorf("%s does not exist: %v", filename, err) + } else { + if err := os.Remove(filename); err != nil { + t.Errorf("cannot remove %v: %v", filename, err) + } + } + +} + func TestToEpub(t *testing.T) { - filename := "Books.epub" - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToEpub(c, "") + filename := "Books.epub" if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { t.Errorf("%s does not exist: %v", filename, err) } else { @@ -49,7 +81,7 @@ func TestToEpub(t *testing.T) { func TestToEpubFilename(t *testing.T) { filename := "ebook.epub" - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToEpub(c, filename) if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { @@ -65,7 +97,23 @@ func TestToEpubFilename(t *testing.T) { func TestToMobi(t *testing.T) { filename := "ebook.mobi" - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + ToMobi(c, filename) + + if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { + t.Errorf("%s does not exist: %v", filename, err) + } else { + if err := os.Remove(filename); err != nil { + t.Errorf("cannot remove %v: %v", filename, err) + } + } + +} + +func TestToMobiFilename(t *testing.T) { + + filename := "ebook.mobi" + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToMobi(c, filename) if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { diff --git a/book/scraper.go b/book/scraper.go index 0d34a95..041640f 100644 --- a/book/scraper.go +++ b/book/scraper.go @@ -18,83 +18,100 @@ import ( ) type ScrapeConfig struct { - depth int - selector string - limit int - offset int - delay int - threads int - include bool - imagesOnly bool + Depth int + Selector string + Quiet bool + Limit int + Offset int + Delay int + Threads int + Include bool + ImagesOnly bool + UseLinkName bool } func NewScrapeConfig() *ScrapeConfig { - return &ScrapeConfig{0, "", -1, 0, -1, -1, true, false} + return &ScrapeConfig{0, "", false, -1, 0, -1, -1, true, false, false} +} + +func NewScrapeConfigs(selectors []string) []*ScrapeConfig { + configs := []*ScrapeConfig{} + + for _, s := range selectors { + config := NewScrapeConfig() + config.Selector = s + + configs = append(configs, config) + } + + return configs } func NewScrapeConfigsAjin() []*ScrapeConfig { config0 := NewScrapeConfig() - config0.depth = 0 - config0.selector = ".dt>a" - config0.limit = 3 - config0.offset = 0 - config0.delay = 5000 - config0.include = false + config0.Depth = 0 + config0.Selector = ".dt>a" + config0.Limit = 3 + config0.Offset = 0 + config0.Delay = 5000 + config0.Include = false config1 := NewScrapeConfig() - config1.depth = 1 - config1.selector = ".nav_apb>a" - config1.limit = 3 - config1.offset = 1 - config1.delay = 5000 - config1.include = false + config1.Depth = 1 + config1.Selector = ".nav_apb>a" + config1.Limit = 3 + config1.Offset = 1 + config1.Delay = 5000 + config1.Include = false config2 := NewScrapeConfig() - config2.depth = 2 - config2.imagesOnly = true + config2.Depth = 2 + config2.ImagesOnly = true return []*ScrapeConfig{config0, config1, config2} } func NewScrapeConfigsWikipedia() []*ScrapeConfig { config0 := NewScrapeConfig() - config0.depth = 0 - config0.threads = -1 - config0.include = true + config0.Depth = 0 + config0.Threads = -1 + config0.Include = true config1 := NewScrapeConfig() - config1.depth = 1 - config1.include = true + config1.Depth = 1 + config1.Include = true return []*ScrapeConfig{config0, config1} } func NewScrapeConfigFake() *ScrapeConfig { config := NewScrapeConfig() - config.include = false + config.Include = false return config } -func NewBookFromURL(url, selector, name, author string, recursive, include, imagesOnly, quiet bool, limit, offset, delay, threads int) book { +func NewBookFromURL(url string, selector []string, name, author string, include, ImagesOnly, useLinkName, quiet bool, limit, offset, delay, threads int) book { config1 := NewScrapeConfig() - config1.imagesOnly = imagesOnly + config1.ImagesOnly = ImagesOnly + config1.UseLinkName = useLinkName var chapters []chapter var home chapter - if recursive { + if len(selector) > 0 { config2 := NewScrapeConfig() - config2.selector = selector - config2.limit = limit - config2.offset = offset - config2.delay = delay - config2.threads = threads - config2.include = include - config2.imagesOnly = imagesOnly + config2.Selector = selector[0] + config2.Limit = limit + config2.Offset = offset + config2.Delay = delay + config2.Threads = threads + config2.Include = include + config2.ImagesOnly = ImagesOnly + config2.UseLinkName = useLinkName chapters, home = tableOfContent(url, config2, config1, quiet) } else { - chapters = []chapter{NewChapterFromURL(url, []*ScrapeConfig{config1}, 0, func(index int, name string) {})} + chapters = []chapter{NewChapterFromURL(url, "", []*ScrapeConfig{config1}, 0, func(index int, name string) {})} home = chapters[0] } @@ -114,7 +131,7 @@ func NewBookFromURL(url, selector, name, author string, recursive, include, imag return b } -func NewChapterFromURL(url string, configs []*ScrapeConfig, index int, updateProgressBarName func(index int, name string)) chapter { +func NewChapterFromURL(url, linkName string, configs []*ScrapeConfig, index int, updateProgressBarName func(index int, name string)) chapter { config := configs[0] base, err := urllib.Parse(url) @@ -141,24 +158,31 @@ func NewChapterFromURL(url string, configs []*ScrapeConfig, index int, updatePro if err != nil { log.Fatalf("failed to parse %s, %v\n", url, err) } - name := article.Title - // notify progress bar with new name - updateProgressBarName(index, name) + name := linkName + if config.UseLinkName == false { + name = article.Title + + // notify progressbar with new name + updateProgressBarName(index, name) + } subchapters := []chapter{} if len(configs) > 1 { // add subchapters - links, _, err := GetLinks(base, config.selector, config.limit, config.offset, false) + links, _, _, err := GetLinks(base, config.Selector, config.Limit, config.Offset, false) if err != nil { log.Fatal(err) } subchapters = make([]chapter, len(links)) - progress := NewProgress(links, name, config.depth) + var p progress + if config.Quiet == false { + p = NewProgress(links, name, config.Depth) + } - if config.delay >= 0 { + if config.Delay >= 0 { // synchronous mode for index, link := range links { @@ -168,18 +192,20 @@ func NewChapterFromURL(url string, configs []*ScrapeConfig, index int, updatePro log.Fatal(err) } - sc := NewChapterFromURL(u.String(), configs[1:], index, progress.UpdateName) + sc := NewChapterFromURL(u.String(), link.text, configs[1:], index, p.UpdateName) subchapters[index] = sc - progress.Increment(index) + if config.Quiet == false { + p.Increment(index) + } - time.Sleep(time.Duration(config.delay) * time.Millisecond) + time.Sleep(time.Duration(config.Delay) * time.Millisecond) } } else { // asynchronous mode var wg sync.WaitGroup - threads := config.threads + threads := config.Threads if threads == -1 { threads = len(links) } @@ -199,9 +225,12 @@ func NewChapterFromURL(url string, configs []*ScrapeConfig, index int, updatePro log.Fatal(err) } - sc := NewChapterFromURL(u.String(), configs[1:], index, progress.UpdateName) + sc := NewChapterFromURL(u.String(), l.text, configs[1:], index, p.UpdateName) subchapters[index] = sc - progress.Increment(index) + + if config.Quiet == false { + p.Increment(index) + } <-semaphore }(index, l) @@ -211,13 +240,15 @@ func NewChapterFromURL(url string, configs []*ScrapeConfig, index int, updatePro } content := "" - if config.include { + if config.Include { - // we care about the content only if we include this level + // we care about the content only if: + // - we include this level + // - we use the page name content = article.Content // extract images - if config.imagesOnly { + if config.ImagesOnly { // parse HTML doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) @@ -246,13 +277,13 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q log.Fatal(err) } - links, home, err := GetLinks(base, config.selector, config.limit, config.offset, config.include) + links, _, home, err := GetLinks(base, config.Selector, config.Limit, config.Offset, config.Include) if err != nil { log.Fatal(err) } chapters := make([]chapter, len(links)) - delay := config.delay + delay := config.Delay var p progress if quiet == false { @@ -262,15 +293,15 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q if delay >= 0 { // synchronous mode - for index, link := range links { + for index, l := range links { // and then use it to parse relative URLs - u, err := base.Parse(link.href) + u, err := base.Parse(l.href) if err != nil { log.Fatal(err) } - chapters[index] = NewChapterFromURL(u.String(), []*ScrapeConfig{subConfig}, 0, func(index int, name string) {}) - + chapters[index] = NewChapterFromURL(u.String(), l.text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {}) + if quiet == false { p.Increment(index) } @@ -287,7 +318,7 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q // asynchronous mode var wg sync.WaitGroup - threads := config.threads + threads := config.Threads if threads == -1 { threads = len(links) } @@ -307,7 +338,7 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q log.Fatal(err) } - chapters[index] = NewChapterFromURL(u.String(), []*ScrapeConfig{subConfig}, 0, func(index int, name string) {}) + chapters[index] = NewChapterFromURL(u.String(), l.text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {}) if quiet == false { p.Increment(index) @@ -327,7 +358,7 @@ func GetPath(elm *goquery.Selection) string { for { selector := strings.ToLower(goquery.NodeName(elm)) - if selector == "" { + if len(selector) == 0 { break } @@ -339,18 +370,18 @@ func GetPath(elm *goquery.Selection) string { return join } -func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, chapter, error) { +func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) ([]link, string, chapter, error) { selectorSet := true - if selector == "" { + if len(selector) == 0 { selector = "a" selectorSet = false } - // visit and count link classes pathLinks := map[string][]link{} pathCount := map[string]int{} pathMax := "" + // visit and count link classes c := colly.NewCollector() c.OnHTML(selector, func(e *colly.HTMLElement) { href := e.Attr("href") @@ -358,26 +389,40 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) path := GetPath(e.DOM) key := path - // include element class in key if selector is set - if !selectorSet { - class := e.Attr("class") - key = fmt.Sprintf("%s.%s", path, class) - } + if selectorSet { - if selectorSet || text != "" { + // if selector is set, we use the selector specified by the user + + key = selector pathLinks[key] = append(pathLinks[key], NewLink(href, text)) - pathCount[key] += len(text) + pathCount[key] += 1 + pathMax = key - if pathCount[key] > pathCount[pathMax] { - pathMax = key + } else { + + // if selector is not set, we compute the selector ourselves + + class := e.Attr("class") + // include the element class to make sure we have the same exact path for every link in the table of content + key = fmt.Sprintf("%s.%s", path, class) + + // we count this key if the link text is not empty + if text != "" { + pathLinks[key] = append(pathLinks[key], NewLink(href, text)) + pathCount[key] += len(text) + + if pathCount[key] > pathCount[pathMax] { + pathMax = key + } } + } }) c.Visit(url.String()) links := pathLinks[pathMax] if len(links) == 0 { - return []link{}, chapter{}, fmt.Errorf("no link found for selector: %s", selector) + return []link{}, pathMax, chapter{}, fmt.Errorf("no link found for selector: %s", selector) } end := len(links) @@ -387,12 +432,12 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, include bool) links = links[offset:end] - home := NewChapterFromURL(url.String(), []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + home := NewChapterFromURL(url.String(), "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) if include { l := NewLink(url.String(), home.Name()) links = append([]link{l}, links...) } - return links, home, nil + return links, pathMax, home, nil } diff --git a/book/scraper_test.go b/book/scraper_test.go index 46e97bd..f013161 100644 --- a/book/scraper_test.go +++ b/book/scraper_test.go @@ -8,7 +8,7 @@ import ( func TestBody(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Body() want := "\n\n \n Books\n \n \n \n \n \n \n \n\n \n \n\n\n\n \n\n\n\n\n\n\n \n \n
\n \"John\n

Books

\n

\n
\n \n
\n
\n
\n \n \n\n
\n \n\n" @@ -22,7 +22,7 @@ func TestBody(t *testing.T) { func TestName(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Name() want := "Books" @@ -33,10 +33,25 @@ func TestName(t *testing.T) { } +func TestCustomName(t *testing.T) { + + config := NewScrapeConfig() + config.UseLinkName = true + c := NewChapterFromURL("https://books.lapw.at/", "Custom Name", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + + got := c.Name() + want := "Custom Name" + + if got != want { + t.Errorf("got %v, wanted %v", got, want) + } + +} + func TestAuthor(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Author() want := "John Doe" @@ -50,7 +65,7 @@ func TestAuthor(t *testing.T) { func TestContent(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Content() want := "
\n \n
\n \n \n\n
\n \n\n
" @@ -64,12 +79,12 @@ func TestContent(t *testing.T) { func TestDelay(t *testing.T) { config0 := NewScrapeConfig() - config0.delay = 500 + config0.Delay = 500 config1 := NewScrapeConfig() start := time.Now() - NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) elapsed := time.Since(start) got := elapsed @@ -84,9 +99,9 @@ func TestDelay(t *testing.T) { func TestContentImagesOnly(t *testing.T) { config := NewScrapeConfig() - config.imagesOnly = true + config.ImagesOnly = true - c := NewChapterFromURL("https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Content() want := "\"One\"A\"Code\"Scale" @@ -102,7 +117,7 @@ func TestSubChapters(t *testing.T) { config0 := NewScrapeConfig() config1 := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) want := 2 @@ -116,11 +131,11 @@ func TestSubChapters(t *testing.T) { func TestSubChaptersSelector(t *testing.T) { config0 := NewScrapeConfig() - config0.selector = "section.concrete > article > h2 > a" + config0.Selector = "section.concrete > article > h2 > a" config1 := NewScrapeConfig() - c := NewChapterFromURL("https://12factor.net/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) want := 12 @@ -134,11 +149,11 @@ func TestSubChaptersSelector(t *testing.T) { func TestSubChaptersLimit(t *testing.T) { config0 := NewScrapeConfig() - config0.limit = 1 + config0.Limit = 1 config1 := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) want := 1 @@ -152,11 +167,11 @@ func TestSubChaptersLimit(t *testing.T) { func TestSubChaptersLimitOver(t *testing.T) { config0 := NewScrapeConfig() - config0.limit = 3 + config0.Limit = 3 config1 := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) want := 2 @@ -170,9 +185,9 @@ func TestSubChaptersLimitOver(t *testing.T) { func TestNotInclude(t *testing.T) { config := NewScrapeConfig() - config.include = false + config.Include = false - c := NewChapterFromURL("https://books.lapw.at/", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Content() want := "" diff --git a/cmd/get.go b/cmd/get.go index cdbfef8..cf61a7c 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -3,8 +3,6 @@ package cmd import ( "errors" "fmt" - "log" - "os" "strings" "github.com/spf13/cobra" @@ -12,13 +10,57 @@ import ( "github.com/lapwat/papeer/book" ) -var recursive, include, images, quiet bool -var format, output, selector, name, author string -var limit, offset, delay, threads int +type GetOptions struct { + // url string + + name string + author string + Format string + output string + images bool + // ImagesOnly bool + quiet bool + + Selector []string + depth int + limit int + offset int + delay int + threads int + // includeUrl bool + include bool + useLinkName bool +} + +var getOpts *GetOptions + +func init() { + getOpts = &GetOptions{} + + getCmd.PersistentFlags().StringVarP(&getOpts.name, "name", "n", "", "book name (default: page title)") + getCmd.PersistentFlags().StringVarP(&getOpts.author, "author", "a", "", "book author") + getCmd.PersistentFlags().StringVarP(&getOpts.Format, "format", "f", "md", "file format [stdout, md, epub, mobi]") + getCmd.PersistentFlags().StringVarP(&getOpts.output, "output", "", "", "file name (default: book name)") + getCmd.PersistentFlags().BoolVarP(&getOpts.images, "images", "", false, "retrieve images only") + getCmd.PersistentFlags().BoolVarP(&getOpts.quiet, "quiet", "q", false, "hide progress bar") + + // common with list command + getCmd.Flags().StringSliceVarP(&getOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector") + getCmd.Flags().IntVarP(&getOpts.depth, "depth", "d", 0, "scraping depth") + getCmd.Flags().IntVarP(&getOpts.limit, "limit", "l", -1, "limit number of chapters, use with depth/selector") + getCmd.Flags().IntVarP(&getOpts.offset, "offset", "o", 0, "skip first chapters, use with depth/selector") + getCmd.Flags().IntVarP(&getOpts.delay, "delay", "", -1, "time in milliseconds to wait before downloading next chapter, use with depth/selector") + getCmd.Flags().IntVarP(&getOpts.threads, "threads", "t", -1, "download concurrency, use with depth/selector") + getCmd.Flags().BoolVarP(&getOpts.include, "include", "i", false, "include URL as first chapter, use with depth/selector") + getCmd.Flags().BoolVarP(&getOpts.useLinkName, "use-link-name", "", false, "use link name for chapter title") + + rootCmd.AddCommand(getCmd) +} var getCmd = &cobra.Command{ - Use: "get", - Short: "Scrape URL content", + Use: "get URL", + Short: "Scrape URL content", + Example: "papeer get https://www.eff.org/cyberspace-independence", Args: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { return errors.New("requires an URL argument") @@ -30,39 +72,36 @@ var getCmd = &cobra.Command{ "epub": true, "mobi": true, } - if formatEnum[format] != true { - return fmt.Errorf("invalid format specified: %s", format) + + if formatEnum[getOpts.Format] != true { + return fmt.Errorf("invalid format specified: %s", getOpts.Format) } // add .mobi to filename if not specified - if format == "mobi" { - if len(output) > 0 && strings.HasSuffix(output, ".mobi") == false { - output = fmt.Sprintf("%s.mobi", output) + if getOpts.Format == "mobi" { + if len(getOpts.output) > 0 && strings.HasSuffix(getOpts.output, ".mobi") == false { + getOpts.output = fmt.Sprintf("%s.mobi", getOpts.output) } } - if cmd.Flags().Changed("selector") && recursive == false { - return errors.New("cannot use selector option if not in recursive mode") + if cmd.Flags().Changed("include") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { + return errors.New("cannot use include option if depth/selector is not specified") } - if cmd.Flags().Changed("include") && recursive == false { - return errors.New("cannot use include option if not in recursive mode") + if cmd.Flags().Changed("limit") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { + return errors.New("cannot use limit option if depth/selector is not specified") } - if cmd.Flags().Changed("limit") && recursive == false { - return errors.New("cannot use limit option if not in recursive mode") + if cmd.Flags().Changed("offset") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { + return errors.New("cannot use offset option if depth/selector is not specified") } - if cmd.Flags().Changed("offset") && recursive == false { - return errors.New("cannot use offset option if not in recursive mode") + if cmd.Flags().Changed("delay") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { + return errors.New("cannot use delay option if depth/selector is not specified") } - if cmd.Flags().Changed("delay") && recursive == false { - return errors.New("cannot use delay option if not in recursive mode") - } - - if cmd.Flags().Changed("threads") && recursive == false { - return errors.New("cannot use threads option if not in recursive mode") + if cmd.Flags().Changed("threads") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { + return errors.New("cannot use threads option if depth/selector is not specified") } if cmd.Flags().Changed("delay") && cmd.Flags().Changed("threads") { @@ -73,48 +112,59 @@ var getCmd = &cobra.Command{ }, Run: func(cmd *cobra.Command, args []string) { url := args[0] - b := book.NewBookFromURL(url, selector, name, author, recursive, include, images, quiet, limit, offset, delay, threads) - fakeConfig := book.NewScrapeConfigFake() - fakeChapter := book.NewChapter("", b.Name(), b.Author(), "", b.Chapters(), fakeConfig) + // fill selector array with empty selectors to match depth + for len(getOpts.Selector) < getOpts.depth+2 { + getOpts.Selector = append(getOpts.Selector, "") + } - if format == "stdout" { - // TODO: ToMarkdownString - markdown := book.ToMarkdown(fakeChapter) + // generate config for each level + configs := make([]*book.ScrapeConfig, len(getOpts.Selector)) + for index, s := range getOpts.Selector { + config := book.NewScrapeConfig() + config.Selector = s + config.Quiet = getOpts.quiet + config.Limit = getOpts.limit + config.Offset = getOpts.offset + config.Delay = getOpts.delay + config.Threads = getOpts.threads + config.ImagesOnly = getOpts.images + config.Include = getOpts.include + config.UseLinkName = getOpts.useLinkName + + // do not use link name for root level as there is not parent link + if index == 0 { + config.UseLinkName = false + } + + // always include last level by default + if index == len(getOpts.Selector)-1 { + config.Include = true + } + + configs[index] = config + } + + c := book.NewChapterFromURL(url, "", configs, 0, func(index int, name string) {}) + + if getOpts.Format == "stdout" { + markdown := book.ToMarkdownString(c) fmt.Println(markdown) } - if format == "md" { - // TODO: ToMarkdownFile - markdown := book.ToMarkdown(fakeChapter) - - if len(output) == 0 { - filename := book.Filename(fakeChapter.Name()) - output = fmt.Sprintf("%s.md", filename) - } - - // write to file - f, err := os.Create(output) - if err != nil { - log.Fatal(err) - } - _, err2 := f.WriteString(markdown) - if err2 != nil { - log.Fatal(err2) - } - f.Close() - - fmt.Printf("Markdown saved to \"%s\"\n", output) + if getOpts.Format == "md" { + filename := book.ToMarkdown(c, getOpts.output) + fmt.Printf("Markdown saved to \"%s\"\n", filename) } - if format == "epub" { - output = book.ToEpub(fakeChapter, output) - fmt.Printf("Ebook saved to \"%s\"\n", output) + if getOpts.Format == "epub" { + filename := book.ToEpub(c, getOpts.output) + fmt.Printf("Ebook saved to \"%s\"\n", filename) } - if format == "mobi" { - output = book.ToMobi(fakeChapter, output) - fmt.Printf("Ebook saved to \"%s\"\n", output) + if getOpts.Format == "mobi" { + filename := book.ToMobi(c, getOpts.output) + fmt.Printf("Ebook saved to \"%s\"\n", filename) } }, } diff --git a/cmd/list.go b/cmd/list.go index 4ab8c06..867e75a 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -2,9 +2,11 @@ package cmd import ( "errors" + "fmt" "log" urllib "net/url" "os" + "strings" "github.com/jedib0t/go-pretty/v6/table" cobra "github.com/spf13/cobra" @@ -12,9 +14,42 @@ import ( "github.com/lapwat/papeer/book" ) +type ListOptions struct { + // url string + + Selector []string + depth int + limit int + offset int + delay int + threads int + // includeUrl bool + include bool + useLinkName bool +} + +var listOpts *ListOptions + +func init() { + listOpts = &ListOptions{} + + listCmd.Flags().StringSliceVarP(&listOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector") + listCmd.Flags().IntVarP(&listOpts.depth, "depth", "d", 0, "scraping depth") + listCmd.Flags().IntVarP(&listOpts.limit, "limit", "l", -1, "limit number of chapters, use with depth/selector") + listCmd.Flags().IntVarP(&listOpts.offset, "offset", "o", 0, "skip first chapters, use with depth/selector") + listCmd.Flags().IntVarP(&listOpts.delay, "delay", "", -1, "time in milliseconds to wait before downloading next chapter, use with depth/selector") + listCmd.Flags().IntVarP(&listOpts.threads, "threads", "t", -1, "download concurrency, use with depth/selector") + listCmd.Flags().BoolVarP(&listOpts.include, "include", "i", false, "include URL as first chapter, use with depth/selector") + listCmd.Flags().BoolVarP(&listOpts.useLinkName, "use-link-name", "", false, "use link name for chapter title") + + rootCmd.AddCommand(listCmd) +} + var listCmd = &cobra.Command{ - Use: "ls", - Short: "Print table of content", + Use: "list URL", + Aliases: []string{"ls"}, + Short: "Print URL table of contents", + Example: "papeer list https://12factor.net/ -s 'section.concrete>article>h2>a'", Args: func(cmd *cobra.Command, args []string) error { if len(args) < 1 { return errors.New("requires an URL argument") @@ -22,12 +57,16 @@ var listCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { + if len(listOpts.Selector) == 0 { + listOpts.Selector = []string{""} + } + base, err := urllib.Parse(args[0]) if err != nil { log.Fatal(err) } - links, _, err := book.GetLinks(base, selector, limit, offset, include) + links, path, _, err := book.GetLinks(base, listOpts.Selector[0], listOpts.limit, listOpts.offset, listOpts.include) if err != nil { log.Fatal(err) } @@ -37,7 +76,16 @@ var listCmd = &cobra.Command{ t.Style().Options.DrawBorder = false t.Style().Options.SeparateColumns = false t.Style().Options.SeparateHeader = false - t.AppendHeader(table.Row{"#", "Name", "Url"}) + + // format selector path + pathArray := strings.Split(path, "<") + // reverse path + for i, j := 0, len(pathArray)-1; i < j; i, j = i+1, j-1 { + pathArray[i], pathArray[j] = pathArray[j], pathArray[i] + } + pathFormatted := strings.Join(pathArray, ">") + + t.AppendHeader(table.Row{"#", "Name", fmt.Sprintf("Url [%s]", pathFormatted)}) for index, link := range links { u, err := base.Parse(link.Href()) diff --git a/cmd/root.go b/cmd/root.go index 517429e..bf61248 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,22 +21,3 @@ func Execute() { os.Exit(1) } } - -func init() { - rootCmd.PersistentFlags().StringVarP(&name, "name", "n", "", "book name (default: page title)") - rootCmd.PersistentFlags().StringVarP(&author, "author", "a", "", "book author") - rootCmd.PersistentFlags().StringVarP(&format, "format", "f", "stdout", "file format [stdout, md, epub, mobi]") - rootCmd.PersistentFlags().StringVarP(&output, "output", "", "", "file name (default: book name)") - rootCmd.PersistentFlags().StringVarP(&selector, "selector", "s", "", "table of content CSS selector, in resursive mode") - rootCmd.PersistentFlags().BoolVarP(&recursive, "recursive", "r", false, "create one chapter per natigation item") - rootCmd.PersistentFlags().BoolVarP(&include, "include", "i", false, "include URL as first chapter, in resursive mode") - rootCmd.PersistentFlags().BoolVarP(&images, "images", "", false, "retrieve images only") - rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "hide progress bar") - rootCmd.PersistentFlags().IntVarP(&limit, "limit", "l", -1, "limit number of chapters, in recursive mode") - rootCmd.PersistentFlags().IntVarP(&offset, "offset", "o", 0, "skip first chapters, in recursive mode") - rootCmd.PersistentFlags().IntVarP(&delay, "delay", "d", -1, "time to wait before downloading next chapter, in milliseconds") - rootCmd.PersistentFlags().IntVarP(&threads, "threads", "t", -1, "download concurrency, in recursive mode") - - rootCmd.AddCommand(getCmd) - rootCmd.AddCommand(listCmd) -} diff --git a/cmd/version.go b/cmd/version.go index 612dc86..f54165a 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of papeer", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("papeer v0.3.3") + fmt.Println("papeer v0.4.0") }, } diff --git a/release.sh b/release.sh index cdd6d2a..c17a1ae 100755 --- a/release.sh +++ b/release.sh @@ -18,10 +18,17 @@ do if [ $GOOS = "windows" ]; then output_name+='.exe' fi - + env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name if [ $? -ne 0 ]; then echo 'An error has occurred! Aborting the script execution...' exit 1 fi + + if [ $GOOS = "windows" ]; then + zip "$output_name.exe.zip" "$output_name" + else + tar czvf "$output_name.tar.gz" "$output_name" + fi + rm "$output_name" done