noham/papeer
1
0
Fork 0
mirror of https://github.com/NohamR/papeer.git synced 2026-05-26 04:17:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

json output for list command

Browse Source
This commit is contained in:
lapwat
2023-02-17 19:47:23 +01:00
parent 23cc576926
commit efcc45316b
7 changed files with 89 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -141,7 +141,7 @@ go install github.com/lapwat/papeer@latest
```sh ```sh
# use platform=darwin for MacOS # use platform=darwin for MacOS
platform=linux platform=linux
release=0.6.0 release=0.6.1
# download and extract # download and extract
curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64.tar.gz > papeer.tar.gz curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64.tar.gz > papeer.tar.gz

12
book/link.go
View File

@@ -1,18 +1,10 @@
package book package book
type link struct { type link struct {
href string Href string `json:"href"`
text string Text string `json:"name"`
} }
func NewLink(href, text string) link { func NewLink(href, text string) link {
return link{href, text} return link{href, text}
} }
func (c link) Href() string {
return c.href
}
func (c link) Text() string {
return c.text
}

2
book/progress.go
View File

@@ -27,7 +27,7 @@ func NewProgress(links []link, parent string, depth int) progress {
if len(links) <= 50 { if len(links) <= 50 {
for index, link := range links { for index, link := range links {
bar := uiprogress.AddBar(1) bar := uiprogress.AddBar(1)
barText := fmt.Sprintf("%v#%v %v", indent, index+1, link.Text()) barText := fmt.Sprintf("%v#%v %v", indent, index+1, link.Text)
bar.AppendFunc(func(b *uiprogress.Bar) string { bar.AppendFunc(func(b *uiprogress.Bar) string {
return barText return barText
}) })

18
book/scraper.go
View File

@@ -156,12 +156,12 @@ func NewChapterFromURL(url, linkName string, configs []*ScrapeConfig, index int,
// synchronous mode // synchronous mode
for index, link := range links { for index, link := range links {
// and then use it to parse relative URLs // and then use it to parse relative URLs
u, err := base.Parse(link.href) u, err := base.Parse(link.Href)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
sc := NewChapterFromURL(u.String(), link.text, configs[1:], index, p.UpdateName) sc := NewChapterFromURL(u.String(), link.Text, configs[1:], index, p.UpdateName)
subchapters[index] = sc subchapters[index] = sc
if config.Quiet == false { if config.Quiet == false {
p.Increment(index) p.Increment(index)
@@ -189,12 +189,12 @@ func NewChapterFromURL(url, linkName string, configs []*ScrapeConfig, index int,
defer wg.Done() defer wg.Done()
// and then use it to parse relative URLs // and then use it to parse relative URLs
u, err := base.Parse(l.href) u, err := base.Parse(l.Href)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
sc := NewChapterFromURL(u.String(), l.text, configs[1:], index, p.UpdateName) sc := NewChapterFromURL(u.String(), l.Text, configs[1:], index, p.UpdateName)
subchapters[index] = sc subchapters[index] = sc
if config.Quiet == false { if config.Quiet == false {
@@ -279,12 +279,12 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q
for index, l := range links { for index, l := range links {
// and then use it to parse relative URLs // and then use it to parse relative URLs
u, err := base.Parse(l.href) u, err := base.Parse(l.Href)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
chapters[index] = NewChapterFromURL(u.String(), l.text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {}) chapters[index] = NewChapterFromURL(u.String(), l.Text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {})
if quiet == false { if quiet == false {
p.Increment(index) p.Increment(index)
@@ -317,12 +317,12 @@ func tableOfContent(url string, config *ScrapeConfig, subConfig *ScrapeConfig, q
defer wg.Done() defer wg.Done()
// and then use it to parse relative URLs // and then use it to parse relative URLs
u, err := base.Parse(l.href) u, err := base.Parse(l.Href)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
chapters[index] = NewChapterFromURL(u.String(), l.text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {}) chapters[index] = NewChapterFromURL(u.String(), l.Text, []*ScrapeConfig{subConfig}, 0, func(index int, name string) {})
if quiet == false { if quiet == false {
p.Increment(index) p.Increment(index)
@@ -361,8 +361,6 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, reverse, incl
parser := gofeed.NewParser() parser := gofeed.NewParser()
feed, err := parser.ParseURL(url.String()) feed, err := parser.ParseURL(url.String())
fmt.Println(feed, url.String(), err)
if err == nil { if err == nil {
// RSS feed // RSS feed

34
cmd/get.go
View File

@@ -21,17 +21,15 @@ type GetOptions struct {
output string output string
stdout bool stdout bool
images bool images bool
// ImagesOnly bool quiet bool
quiet bool
Selector []string Selector []string
depth int depth int
limit int limit int
offset int offset int
reverse bool reverse bool
delay int delay int
threads int threads int
// includeUrl bool
include bool include bool
useLinkName bool useLinkName bool
} }
@@ -41,13 +39,13 @@ var getOpts *GetOptions
func init() { func init() {
getOpts = &GetOptions{} getOpts = &GetOptions{}
getCmd.PersistentFlags().StringVarP(&getOpts.name, "name", "n", "", "book name (default: page title)") getCmd.Flags().StringVarP(&getOpts.name, "name", "n", "", "book name (default: page title)")
getCmd.PersistentFlags().StringVarP(&getOpts.author, "author", "a", "", "book author") getCmd.Flags().StringVarP(&getOpts.author, "author", "a", "", "book author")
getCmd.PersistentFlags().StringVarP(&getOpts.Format, "format", "f", "md", "file format [md, html, epub, mobi]") getCmd.Flags().StringVarP(&getOpts.Format, "format", "f", "md", "file format [md, html, epub, mobi]")
getCmd.PersistentFlags().StringVarP(&getOpts.output, "output", "", "", "file name (default: book name)") getCmd.Flags().StringVarP(&getOpts.output, "output", "", "", "file name (default: book name)")
getCmd.PersistentFlags().BoolVarP(&getOpts.stdout, "stdout", "", false, "print to standard output") getCmd.Flags().BoolVarP(&getOpts.stdout, "stdout", "", false, "print to standard output")
getCmd.PersistentFlags().BoolVarP(&getOpts.images, "images", "", false, "retrieve images only") getCmd.Flags().BoolVarP(&getOpts.images, "images", "", false, "retrieve images only")
getCmd.PersistentFlags().BoolVarP(&getOpts.quiet, "quiet", "q", false, "hide progress bar") getCmd.Flags().BoolVarP(&getOpts.quiet, "quiet", "q", false, "hide progress bar")
// common with list command // common with list command
getCmd.Flags().StringSliceVarP(&getOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector") getCmd.Flags().StringSliceVarP(&getOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector")
@@ -72,13 +70,13 @@ var getCmd = &cobra.Command{
return errors.New("requires an URL argument") return errors.New("requires an URL argument")
} }
// check provided format is in list
formatEnum := map[string]bool{ formatEnum := map[string]bool{
"md": true, "md": true,
"html": true, "html": true,
"epub": true, "epub": true,
"mobi": true, "mobi": true,
} }
if formatEnum[getOpts.Format] != true { if formatEnum[getOpts.Format] != true {
return fmt.Errorf("invalid format specified: %s", getOpts.Format) return fmt.Errorf("invalid format specified: %s", getOpts.Format)
} }

88
cmd/list.go
View File

@@ -1,6 +1,7 @@
package cmd package cmd
import ( import (
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"log" "log"
@@ -17,14 +18,15 @@ import (
type ListOptions struct { type ListOptions struct {
// url string // url string
Selector []string output string
depth int
limit int Selector []string
offset int depth int
reverse bool limit int
delay int offset int
threads int reverse bool
// includeUrl bool delay int
threads int
include bool include bool
useLinkName bool useLinkName bool
} }
@@ -34,11 +36,13 @@ var listOpts *ListOptions
func init() { func init() {
listOpts = &ListOptions{} listOpts = &ListOptions{}
listCmd.Flags().StringVarP(&listOpts.output, "output", "o", "table", "file format [table, json]")
// common with get command // common with get command
listCmd.Flags().StringSliceVarP(&listOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector") listCmd.Flags().StringSliceVarP(&listOpts.Selector, "selector", "s", []string{}, "table of contents CSS selector")
listCmd.Flags().IntVarP(&listOpts.depth, "depth", "d", 0, "scraping depth") listCmd.Flags().IntVarP(&listOpts.depth, "depth", "d", 0, "scraping depth")
listCmd.Flags().IntVarP(&listOpts.limit, "limit", "l", -1, "limit number of chapters, use with depth/selector") listCmd.Flags().IntVarP(&listOpts.limit, "limit", "l", -1, "limit number of chapters, use with depth/selector")
listCmd.Flags().IntVarP(&listOpts.offset, "offset", "o", 0, "skip first chapters, use with depth/selector") listCmd.Flags().IntVarP(&listOpts.offset, "offset", "", 0, "skip first chapters, use with depth/selector")
listCmd.Flags().BoolVarP(&listOpts.reverse, "reverse", "r", false, "reverse chapter order") listCmd.Flags().BoolVarP(&listOpts.reverse, "reverse", "r", false, "reverse chapter order")
listCmd.Flags().IntVarP(&listOpts.delay, "delay", "", -1, "time in milliseconds to wait before downloading next chapter, use with depth/selector") listCmd.Flags().IntVarP(&listOpts.delay, "delay", "", -1, "time in milliseconds to wait before downloading next chapter, use with depth/selector")
listCmd.Flags().IntVarP(&listOpts.threads, "threads", "t", -1, "download concurrency, use with depth/selector") listCmd.Flags().IntVarP(&listOpts.threads, "threads", "t", -1, "download concurrency, use with depth/selector")
@@ -57,6 +61,16 @@ var listCmd = &cobra.Command{
if len(args) < 1 { if len(args) < 1 {
return errors.New("requires an URL argument") return errors.New("requires an URL argument")
} }
// check provided output is in list
outputEnum := map[string]bool{
"table": true,
"json": true,
}
if outputEnum[listOpts.output] != true {
return fmt.Errorf("invalid output specified: %s", listOpts.output)
}
return nil return nil
}, },
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
@@ -74,34 +88,52 @@ var listCmd = &cobra.Command{
log.Fatal(err) log.Fatal(err)
} }
t := table.NewWriter() switch listOpts.output {
t.SetOutputMirror(os.Stdout)
t.Style().Options.DrawBorder = false
t.Style().Options.SeparateColumns = false
t.Style().Options.SeparateHeader = false
t.SetTitle(home.Name()) // render as table
case "table":
t := table.NewWriter()
t.SetOutputMirror(os.Stdout)
t.Style().Options.DrawBorder = false
t.Style().Options.SeparateColumns = false
t.Style().Options.SeparateHeader = false
// format selector path t.SetTitle(home.Name())
pathArray := strings.Split(path, "<")
// reverse path
for i, j := 0, len(pathArray)-1; i < j; i, j = i+1, j-1 {
pathArray[i], pathArray[j] = pathArray[j], pathArray[i]
}
pathFormatted := strings.Join(pathArray, ">")
t.AppendHeader(table.Row{"#", "Name", fmt.Sprintf("Url [%s]", pathFormatted)}) // format selector path
pathArray := strings.Split(path, "<")
// reverse path
for i, j := 0, len(pathArray)-1; i < j; i, j = i+1, j-1 {
pathArray[i], pathArray[j] = pathArray[j], pathArray[i]
}
pathFormatted := strings.Join(pathArray, ">")
for index, link := range links { t.AppendHeader(table.Row{"#", "Name", fmt.Sprintf("Url [%s]", pathFormatted)})
u, err := base.Parse(link.Href())
for index, link := range links {
u, err := base.Parse(link.Href)
if err != nil {
log.Fatal(err)
}
t.AppendRow([]interface{}{index + 1, link.Text, u.String()})
}
t.Render()
// render as json
case "json":
book := make(map[string]interface{})
book["name"] = home.Name()
book["chapters"] = links
bookJson, err := json.Marshal(book)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
t.AppendRow([]interface{}{index + 1, link.Text(), u.String()}) fmt.Println(string(bookJson))
} }
t.Render()
}, },
} }

2
cmd/version.go
View File

@@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{
Use: "version", Use: "version",
Short: "Print the version number of papeer", Short: "Print the version number of papeer",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
fmt.Println("papeer v0.6.0") fmt.Println("papeer v0.6.1")
}, },
} }