noham/papeer
1
0
Fork 0
mirror of https://github.com/NohamR/papeer.git synced 2026-05-25 20:00:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: default depth when limit is specified

Browse Source
This commit is contained in:
lapwat
2022-05-10 15:09:07 +02:00
parent 740b66669f
commit 6d795f3033
5 changed files with 30 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -64,6 +64,8 @@ You can activate this mode by using the `depth` or `selector` options.
This option defaults to 0, `papeer` will grab only the main page. This option defaults to 0, `papeer` will grab only the main page.
This option defaults to 1 if the `limit` option is specified.
If you specify a value greater than 0, `papeer` will grab pages as deep as the value you specify. If you specify a value greater than 0, `papeer` will grab pages as deep as the value you specify.
> Using `include` option will include all intermediary levels into the book. > Using `include` option will include all intermediary levels into the book.
@@ -128,7 +130,7 @@ Markdown saved to "The_Twelve-Factor_App.md"
## From source ## From source
```sh ```sh
go get -u github.com/lapwat/papeer go install github.com/lapwat/papeer@latest
``` ```
## From binary ## From binary
@@ -138,7 +140,7 @@ go get -u github.com/lapwat/papeer
```sh ```sh
# use platform=darwin for MacOS # use platform=darwin for MacOS
platform=linux platform=linux
release=0.5.2 release=0.5.4
# download and extract # download and extract
curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64.tar.gz > papeer.tar.gz curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64.tar.gz > papeer.tar.gz
@@ -151,7 +153,7 @@ sudo mv papeer /usr/local/bin
### Windows ### Windows
Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.5.2/papeer-v0.5.2-windows-amd64.exe.zip). Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.5.4/papeer-v0.5.4-windows-amd64.exe.zip).
## MOBI support ## MOBI support

29
cmd/get.go
View File

@@ -3,9 +3,9 @@ package cmd
import ( import (
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"log" "log"
"strings" "strings"
"io/ioutil"
"github.com/spf13/cobra" "github.com/spf13/cobra"
@@ -73,9 +73,9 @@ var getCmd = &cobra.Command{
} }
formatEnum := map[string]bool{ formatEnum := map[string]bool{
"md": true, "md": true,
"epub": true, "epub": true,
"mobi": true, "mobi": true,
} }
if formatEnum[getOpts.Format] != true { if formatEnum[getOpts.Format] != true {
@@ -89,12 +89,19 @@ var getCmd = &cobra.Command{
} }
} }
if cmd.Flags().Changed("include") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { // increase depth to match limit
return errors.New("cannot use include option if depth/selector is not specified") if cmd.Flags().Changed("limit") && getOpts.depth == 0 {
getOpts.depth = 1
} }
if cmd.Flags().Changed("limit") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { // fill selector array with empty selectors to match depth
return errors.New("cannot use limit option if depth/selector is not specified") getOpts.Selector = append(getOpts.Selector, "")
for len(getOpts.Selector) < getOpts.depth+1 {
getOpts.Selector = append(getOpts.Selector, "")
}
if cmd.Flags().Changed("include") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
return errors.New("cannot use include option if depth/selector is not specified")
} }
if cmd.Flags().Changed("offset") && getOpts.depth == 0 && len(getOpts.Selector) == 0 { if cmd.Flags().Changed("offset") && getOpts.depth == 0 && len(getOpts.Selector) == 0 {
@@ -126,12 +133,6 @@ var getCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
url := args[0] url := args[0]
// fill selector array with empty selectors to match depth
getOpts.Selector = append(getOpts.Selector, "")
for len(getOpts.Selector) < getOpts.depth+1 {
getOpts.Selector = append(getOpts.Selector, "")
}
// generate config for each level // generate config for each level
configs := make([]*book.ScrapeConfig, len(getOpts.Selector)) configs := make([]*book.ScrapeConfig, len(getOpts.Selector))
for index, s := range getOpts.Selector { for index, s := range getOpts.Selector {

2
cmd/version.go
View File

@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
Use: "version", Use: "version",
Short: "Print the version number of papeer", Short: "Print the version number of papeer",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
fmt.Println("papeer v0.5.3") fmt.Println("papeer v0.5.4")
}, },
} }

6
go.mod
View File

@@ -6,7 +6,7 @@ require (
github.com/JohannesKaufmann/html-to-markdown v1.3.3 github.com/JohannesKaufmann/html-to-markdown v1.3.3
github.com/bmaupin/go-epub v0.11.0 github.com/bmaupin/go-epub v0.11.0
github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b github.com/go-shiori/go-readability v0.0.0-20220215145315-dd6828d2f09b
github.com/jedib0t/go-pretty/v6 v6.3.0 github.com/jedib0t/go-pretty/v6 v6.3.1
github.com/mmcdole/gofeed v1.1.3 github.com/mmcdole/gofeed v1.1.3
github.com/spf13/cobra v1.4.0 github.com/spf13/cobra v1.4.0
) )
@@ -44,8 +44,8 @@ require (
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/temoto/robotstxt v1.1.2 // indirect github.com/temoto/robotstxt v1.1.2 // indirect
github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
golang.org/x/text v0.3.7 // indirect golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.0 // indirect google.golang.org/protobuf v1.28.0 // indirect

12
go.sum
View File

@@ -111,8 +111,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg=
github.com/jedib0t/go-pretty/v6 v6.3.0 h1:QQ5yZPDUMEjbZRXDJtZlvwfDQqCYFaxV3yEzTkogUgk= github.com/jedib0t/go-pretty/v6 v6.3.1 h1:aOXiD9oqiuLH8btPQW6SfgtQN5zwhyfzZls8a6sPJ/I=
github.com/jedib0t/go-pretty/v6 v6.3.0/go.mod h1:FMkOpgGD3EZ91cW8g/96RfxoV7bdeJyzXPYgz1L1ln0= github.com/jedib0t/go-pretty/v6 v6.3.1/go.mod h1:FMkOpgGD3EZ91cW8g/96RfxoV7bdeJyzXPYgz1L1ln0=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -245,8 +245,8 @@ golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210505214959-0714010a04ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -267,8 +267,8 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=