From ab5e2ede077a827447a79ae883a3e51e1751a4aa Mon Sep 17 00:00:00 2001 From: lapwat Date: Wed, 18 Jan 2023 00:52:48 +0100 Subject: [PATCH] accept several urls, fix tests, remove unused book struct --- README.md | 4 ++-- book/book.go | 27 ---------------------- book/chapter.go | 12 ++++++++++ book/format_test.go | 34 ++++++++++++++-------------- book/scraper.go | 46 +++++--------------------------------- book/scraper_test.go | 53 +++++++++++++++++++++++--------------------- cmd/get.go | 9 ++++++-- cmd/version.go | 2 +- 8 files changed, 73 insertions(+), 114 deletions(-) delete mode 100644 book/book.go diff --git a/README.md b/README.md index 27dc8ca..79c5af0 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ go install github.com/lapwat/papeer@latest ```sh # use platform=darwin for MacOS platform=linux -release=0.5.6 +release=0.6.0 # download and extract curl -L https://github.com/lapwat/papeer/releases/download/v$release/papeer-v$release-$platform-amd64.tar.gz > papeer.tar.gz @@ -154,7 +154,7 @@ sudo mv papeer /usr/local/bin ### Windows -Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.5.6/papeer-v0.5.6-windows-amd64.zip). +Download [latest release](https://github.com/lapwat/papeer/releases/download/v0.6.0/papeer-v0.6.0-windows-amd64.zip). ## MOBI support diff --git a/book/book.go b/book/book.go deleted file mode 100644 index ffdc068..0000000 --- a/book/book.go +++ /dev/null @@ -1,27 +0,0 @@ -package book - -type book struct { - name string - author string - chapters []chapter -} - -func New(name, author string) book { - return book{name, author, []chapter{}} -} - -func (b *book) AddChapter(c chapter) { - b.chapters = append(b.chapters, c) -} - -func (b book) Name() string { - return b.name -} - -func (b book) Author() string { - return b.name -} - -func (b *book) Chapters() []chapter { - return b.chapters -} diff --git a/book/chapter.go b/book/chapter.go index b9fb2a7..9c82b1b 100644 --- a/book/chapter.go +++ b/book/chapter.go @@ -9,6 +9,10 @@ type chapter struct { config *ScrapeConfig } +func NewEmptyChapter() chapter { + return chapter{"", "", "", "", []chapter{}, NewScrapeConfigNoInclude()} +} + func NewChapter(body, name, author, content string, subChapters []chapter, config *ScrapeConfig) chapter { return chapter{body, name, author, content, subChapters, config} } @@ -21,6 +25,10 @@ func (c chapter) Name() string { return c.name } +func (c *chapter) SetName(name string) { + c.name = name +} + func (c chapter) Author() string { return c.author } @@ -32,3 +40,7 @@ func (c chapter) Content() string { func (c chapter) SubChapters() []chapter { return c.subChapters } + +func (c *chapter) AddSubChapter(newChapter chapter) { + c.subChapters = append(c.subChapters, newChapter) +} diff --git a/book/format_test.go b/book/format_test.go index 94fda74..351b823 100644 --- a/book/format_test.go +++ b/book/format_test.go @@ -19,10 +19,10 @@ func TestFilename(t *testing.T) { func TestToMarkdownString(t *testing.T) { - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) got := ToMarkdownString(c) - want := "Books\n=====\n\n- [Discours de la Méthode](https://books.lapw.at/posts/ren%C3%A9-descartes-discours-de-la-m%C3%A9thode/)clock 98 min read -\n 1637\n\n- [The Twelve-Factor App](https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/)clock 22 min read -\n 2011\n\n\n" + want := "The Twelve-Factor App\n=====================\n\nIn the modern era, software is commonly delivered as a service: called _web apps_, or _software-as-a-service_. The twelve-factor app is a methodology for building software-as-a-service apps that:\n\n- Use **declarative** formats for setup automation, to minimize time and cost for new developers joining the project;\n- Have a **clean contract** with the underlying operating system, offering **maximum portability** between execution environments;\n- Are suitable for **deployment** on modern **cloud platforms**, obviating the need for servers and systems administration;\n- **Minimize divergence** between development and production, enabling **continuous deployment** for maximum agility;\n- And can **scale up** without significant changes to tooling, architecture, or development practices.\n\nThe twelve-factor methodology can be applied to apps written in any programming language, and which use any combination of backing services (database, queue, memory cache, etc).\n\nThe contributors to this document have been directly involved in the development and deployment of hundreds of apps, and indirectly witnessed the development, operation, and scaling of hundreds of thousands of apps via our work on the [Heroku](http://www.heroku.com/) platform.\n\nThis document synthesizes all of our experience and observations on a wide variety of software-as-a-service apps in the wild. It is a triangulation on ideal practices for app development, paying particular attention to the dynamics of the organic growth of an app over time, the dynamics of collaboration between developers working on the app’s codebase, and [avoiding the cost of software erosion](http://blog.heroku.com/archives/2011/6/28/the_new_heroku_4_erosion_resistance_explicit_contracts/).\n\nOur motivation is to raise awareness of some systemic problems we’ve seen in modern application development, to provide a shared vocabulary for discussing those problems, and to offer a set of broad conceptual solutions to those problems with accompanying terminology. The format is inspired by Martin Fowler’s books _[Patterns of Enterprise Application Architecture](https://books.google.com/books/about/Patterns_of_enterprise_application_archi.html?id=FyWZt5DdvFkC)_ and _[Refactoring](https://books.google.com/books/about/Refactoring.html?id=1MsETFPD3I0C)_.\n\nAny developer building applications which run as a service. Ops engineers who deploy or manage such applications.\n\n## [I. Codebase](https://12factor.net/codebase)\n\n### One codebase tracked in revision control, many deploys\n\n## [II. Dependencies](https://12factor.net/dependencies)\n\n### Explicitly declare and isolate dependencies\n\n## [III. Config](https://12factor.net/config)\n\n### Store config in the environment\n\n## [IV. Backing services](https://12factor.net/backing-services)\n\n### Treat backing services as attached resources\n\n## [V. Build, release, run](https://12factor.net/build-release-run)\n\n### Strictly separate build and run stages\n\n## [VI. Processes](https://12factor.net/processes)\n\n### Execute the app as one or more stateless processes\n\n## [VII. Port binding](https://12factor.net/port-binding)\n\n### Export services via port binding\n\n## [VIII. Concurrency](https://12factor.net/concurrency)\n\n### Scale out via the process model\n\n## [IX. Disposability](https://12factor.net/disposability)\n\n### Maximize robustness with fast startup and graceful shutdown\n\n## [X. Dev/prod parity](https://12factor.net/dev-prod-parity)\n\n### Keep development, staging, and production as similar as possible\n\n## [XI. Logs](https://12factor.net/logs)\n\n### Treat logs as event streams\n\n## [XII. Admin processes](https://12factor.net/admin-processes)\n\n### Run admin/management tasks as one-off processes\n\n\n" if got != want { t.Errorf("got %q, wanted %q", got, want) @@ -32,10 +32,10 @@ func TestToMarkdownString(t *testing.T) { func TestToMarkdown(t *testing.T) { - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToMarkdown(c, "") - filename := "Books.md" + filename := "The_Twelve-Factor_App.md" if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { t.Errorf("%s does not exist: %v", filename, err) } else { @@ -49,7 +49,7 @@ func TestToMarkdown(t *testing.T) { func TestToMarkdownFilename(t *testing.T) { filename := "ebook.md" - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToMarkdown(c, filename) if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { @@ -64,10 +64,10 @@ func TestToMarkdownFilename(t *testing.T) { func TestToHtmlString(t *testing.T) { - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) got := ToHtmlString(c) - want := "

Books

\n \n
\n \n \n\n
\n \n\n" + want := "

The Twelve-Factor App

\n \n\n
\n \n
\n\n
\n
\n\n\n

In the modern era, software is commonly delivered as a service: called web apps, or software-as-a-service. The twelve-factor app is a methodology for building software-as-a-service apps that:

\n\n
    \n
  • Use declarative formats for setup automation, to minimize time and cost for new developers joining the project;
    • \n\n
  • Have a clean contract with the underlying operating system, offering maximum portability between execution environments;
    • \n\n
  • Are suitable for deployment on modern cloud platforms, obviating the need for servers and systems administration;
    • \n\n
  • Minimize divergence between development and production, enabling continuous deployment for maximum agility;
    • \n\n
  • And can scale up without significant changes to tooling, architecture, or development practices.
    • \n
\n\n

The twelve-factor methodology can be applied to apps written in any programming language, and which use any combination of backing services (database, queue, memory cache, etc).

\n
\n
\n\n\n

The contributors to this document have been directly involved in the development and deployment of hundreds of apps, and indirectly witnessed the development, operation, and scaling of hundreds of thousands of apps via our work on the Heroku platform.

\n\n

This document synthesizes all of our experience and observations on a wide variety of software-as-a-service apps in the wild. It is a triangulation on ideal practices for app development, paying particular attention to the dynamics of the organic growth of an app over time, the dynamics of collaboration between developers working on the app’s codebase, and avoiding the cost of software erosion.

\n\n

Our motivation is to raise awareness of some systemic problems we’ve seen in modern application development, to provide a shared vocabulary for discussing those problems, and to offer a set of broad conceptual solutions to those problems with accompanying terminology. The format is inspired by Martin Fowler’s books Patterns of Enterprise Application Architecture and Refactoring.

\n
\n
\n\n\n

Any developer building applications which run as a service. Ops engineers who deploy or manage such applications.

\n
\n
\n\n
\n
\n\n\n

I. Codebase

\n\n

One codebase tracked in revision control, many deploys

\n\n

II. Dependencies

\n\n

Explicitly declare and isolate dependencies

\n\n

III. Config

\n\n

Store config in the environment

\n\n

IV. Backing services

\n\n

Treat backing services as attached resources

\n\n

V. Build, release, run

\n\n

Strictly separate build and run stages

\n\n

VI. Processes

\n\n

Execute the app as one or more stateless processes

\n\n

VII. Port binding

\n\n

Export services via port binding

\n\n

VIII. Concurrency

\n\n

Scale out via the process model

\n\n

IX. Disposability

\n\n

Maximize robustness with fast startup and graceful shutdown

\n\n

X. Dev/prod parity

\n\n

Keep development, staging, and production as similar as possible

\n\n

XI. Logs

\n\n

Treat logs as event streams

\n\n

XII. Admin processes

\n\n

Run admin/management tasks as one-off processes

\n
\n
\n\n\n \n\n\n" if got != want { t.Errorf("got %q, wanted %q", got, want) @@ -77,10 +77,10 @@ func TestToHtmlString(t *testing.T) { func TestToHtml(t *testing.T) { - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToHtml(c, "") - filename := "Books.html" + filename := "The_Twelve-Factor_App.html" if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { t.Errorf("%s does not exist: %v", filename, err) } else { @@ -94,7 +94,7 @@ func TestToHtml(t *testing.T) { func TestToHtmlFilename(t *testing.T) { filename := "ebook.html" - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToHtml(c, filename) if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { @@ -109,10 +109,10 @@ func TestToHtmlFilename(t *testing.T) { func TestToEpub(t *testing.T) { - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToEpub(c, "") - filename := "Books.epub" + filename := "The_Twelve-Factor_App.epub" if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { t.Errorf("%s does not exist: %v", filename, err) } else { @@ -126,7 +126,7 @@ func TestToEpub(t *testing.T) { func TestToEpubFilename(t *testing.T) { filename := "ebook.epub" - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToEpub(c, filename) if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { @@ -141,10 +141,10 @@ func TestToEpubFilename(t *testing.T) { func TestToMobi(t *testing.T) { - filename := "ebook.mobi" - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) - ToMobi(c, filename) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + ToMobi(c, "") + filename := "The_Twelve-Factor_App.mobi" if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { t.Errorf("%s does not exist: %v", filename, err) } else { @@ -158,7 +158,7 @@ func TestToMobi(t *testing.T) { func TestToMobiFilename(t *testing.T) { filename := "ebook.mobi" - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{NewScrapeConfig()}, 0, func(index int, name string) {}) ToMobi(c, filename) if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { diff --git a/book/scraper.go b/book/scraper.go index e2492a3..af076d0 100644 --- a/book/scraper.go +++ b/book/scraper.go @@ -36,6 +36,10 @@ func NewScrapeConfig() *ScrapeConfig { return &ScrapeConfig{0, "", false, -1, 0, false, -1, -1, true, false, false} } +func NewScrapeConfigNoInclude() *ScrapeConfig { + return &ScrapeConfig{0, "", false, -1, 0, false, -1, -1, false, false, false} +} + func NewScrapeConfigs(selectors []string) []*ScrapeConfig { configs := []*ScrapeConfig{} @@ -93,46 +97,6 @@ func NewScrapeConfigFake() *ScrapeConfig { return config } -func NewBookFromURL(url string, selector []string, name, author string, include, ImagesOnly, useLinkName, quiet bool, limit, offset, delay, threads int) book { - config1 := NewScrapeConfig() - config1.ImagesOnly = ImagesOnly - config1.UseLinkName = useLinkName - - var chapters []chapter - var home chapter - - if len(selector) > 0 { - config2 := NewScrapeConfig() - config2.Selector = selector[0] - config2.Limit = limit - config2.Offset = offset - config2.Delay = delay - config2.Threads = threads - config2.Include = include - config2.ImagesOnly = ImagesOnly - config2.UseLinkName = useLinkName - chapters, home = tableOfContent(url, config2, config1, quiet) - } else { - chapters = []chapter{NewChapterFromURL(url, "", []*ScrapeConfig{config1}, 0, func(index int, name string) {})} - home = chapters[0] - } - - if len(name) == 0 { - name = home.Name() - } - - if len(author) == 0 { - author = home.Author() - } - - b := New(name, author) - for _, c := range chapters { - b.AddChapter(c) - } - - return b -} - func NewChapterFromURL(url, linkName string, configs []*ScrapeConfig, index int, updateProgressBarName func(index int, name string)) chapter { config := configs[0] @@ -397,6 +361,8 @@ func GetLinks(url *urllib.URL, selector string, limit, offset int, reverse, incl parser := gofeed.NewParser() feed, err := parser.ParseURL(url.String()) + fmt.Println(feed, url.String(), err) + if err == nil { // RSS feed diff --git a/book/scraper_test.go b/book/scraper_test.go index 27a0494..9d0127e 100644 --- a/book/scraper_test.go +++ b/book/scraper_test.go @@ -8,10 +8,10 @@ import ( func TestBody(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Body() - want := "\n\n \n Books\n \n \n \n \n \n \n \n\n \n \n\n\n\n \n\n\n\n\n\n\n \n \n
\n \"John\n

Books

\n

\n
\n \n
\n
\n
\n \n \n\n
\n \n\n" + want := "\n\n\n \n\n The Twelve-Factor App \n \n \n\n \n \n\n \n \n\n \n \n\n\n \n\n
\n

The Twelve-Factor App

\n
\n\n
\n
\n

Introduction

\n\n

In the modern era, software is commonly delivered as a service: called web apps, or software-as-a-service. The twelve-factor app is a methodology for building software-as-a-service apps that:

\n\n
    \n
  • Use declarative formats for setup automation, to minimize time and cost for new developers joining the project;
    • \n\n
  • Have a clean contract with the underlying operating system, offering maximum portability between execution environments;
    • \n\n
  • Are suitable for deployment on modern cloud platforms, obviating the need for servers and systems administration;
    • \n\n
  • Minimize divergence between development and production, enabling continuous deployment for maximum agility;
    • \n\n
  • And can scale up without significant changes to tooling, architecture, or development practices.
    • \n
\n\n

The twelve-factor methodology can be applied to apps written in any programming language, and which use any combination of backing services (database, queue, memory cache, etc).

\n
\n
\n

Background

\n\n

The contributors to this document have been directly involved in the development and deployment of hundreds of apps, and indirectly witnessed the development, operation, and scaling of hundreds of thousands of apps via our work on the Heroku platform.

\n\n

This document synthesizes all of our experience and observations on a wide variety of software-as-a-service apps in the wild. It is a triangulation on ideal practices for app development, paying particular attention to the dynamics of the organic growth of an app over time, the dynamics of collaboration between developers working on the app’s codebase, and avoiding the cost of software erosion.

\n\n

Our motivation is to raise awareness of some systemic problems we’ve seen in modern application development, to provide a shared vocabulary for discussing those problems, and to offer a set of broad conceptual solutions to those problems with accompanying terminology. The format is inspired by Martin Fowler’s books Patterns of Enterprise Application Architecture and Refactoring.

\n
\n
\n

Who should read this document?

\n\n

Any developer building applications which run as a service. Ops engineers who deploy or manage such applications.

\n
\n
\n\n
\n
\n

The Twelve Factors

\n\n

I. Codebase

\n\n

One codebase tracked in revision control, many deploys

\n\n

II. Dependencies

\n\n

Explicitly declare and isolate dependencies

\n\n

III. Config

\n\n

Store config in the environment

\n\n

IV. Backing services

\n\n

Treat backing services as attached resources

\n\n

V. Build, release, run

\n\n

Strictly separate build and run stages

\n\n

VI. Processes

\n\n

Execute the app as one or more stateless processes

\n\n

VII. Port binding

\n\n

Export services via port binding

\n\n

VIII. Concurrency

\n\n

Scale out via the process model

\n\n

IX. Disposability

\n\n

Maximize robustness with fast startup and graceful shutdown

\n\n

X. Dev/prod parity

\n\n

Keep development, staging, and production as similar as possible

\n\n

XI. Logs

\n\n

Treat logs as event streams

\n\n

XII. Admin processes

\n\n

Run admin/management tasks as one-off processes

\n
\n
\n\n\n \n\n\n" if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -22,10 +22,10 @@ func TestBody(t *testing.T) { func TestName(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Name() - want := "Books" + want := "The Twelve-Factor App" if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -37,7 +37,7 @@ func TestCustomName(t *testing.T) { config := NewScrapeConfig() config.UseLinkName = true - c := NewChapterFromURL("https://books.lapw.at/", "Custom Name", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "Custom Name", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Name() want := "Custom Name" @@ -51,10 +51,10 @@ func TestCustomName(t *testing.T) { func TestAuthor(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Author() - want := "John Doe" + want := "Adam Wiggins" if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -65,10 +65,10 @@ func TestAuthor(t *testing.T) { func TestContent(t *testing.T) { config := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Content() - want := "\n \n
\n \n \n\n
\n \n\n" + want := "\n \n\n
\n \n
\n\n
\n
\n\n\n

In the modern era, software is commonly delivered as a service: called web apps, or software-as-a-service. The twelve-factor app is a methodology for building software-as-a-service apps that:

\n\n
    \n
  • Use declarative formats for setup automation, to minimize time and cost for new developers joining the project;
    • \n\n
  • Have a clean contract with the underlying operating system, offering maximum portability between execution environments;
    • \n\n
  • Are suitable for deployment on modern cloud platforms, obviating the need for servers and systems administration;
    • \n\n
  • Minimize divergence between development and production, enabling continuous deployment for maximum agility;
    • \n\n
  • And can scale up without significant changes to tooling, architecture, or development practices.
    • \n
\n\n

The twelve-factor methodology can be applied to apps written in any programming language, and which use any combination of backing services (database, queue, memory cache, etc).

\n
\n
\n\n\n

The contributors to this document have been directly involved in the development and deployment of hundreds of apps, and indirectly witnessed the development, operation, and scaling of hundreds of thousands of apps via our work on the Heroku platform.

\n\n

This document synthesizes all of our experience and observations on a wide variety of software-as-a-service apps in the wild. It is a triangulation on ideal practices for app development, paying particular attention to the dynamics of the organic growth of an app over time, the dynamics of collaboration between developers working on the app’s codebase, and avoiding the cost of software erosion.

\n\n

Our motivation is to raise awareness of some systemic problems we’ve seen in modern application development, to provide a shared vocabulary for discussing those problems, and to offer a set of broad conceptual solutions to those problems with accompanying terminology. The format is inspired by Martin Fowler’s books Patterns of Enterprise Application Architecture and Refactoring.

\n
\n
\n\n\n

Any developer building applications which run as a service. Ops engineers who deploy or manage such applications.

\n
\n
\n\n
\n
\n\n\n

I. Codebase

\n\n

One codebase tracked in revision control, many deploys

\n\n

II. Dependencies

\n\n

Explicitly declare and isolate dependencies

\n\n

III. Config

\n\n

Store config in the environment

\n\n

IV. Backing services

\n\n

Treat backing services as attached resources

\n\n

V. Build, release, run

\n\n

Strictly separate build and run stages

\n\n

VI. Processes

\n\n

Execute the app as one or more stateless processes

\n\n

VII. Port binding

\n\n

Export services via port binding

\n\n

VIII. Concurrency

\n\n

Scale out via the process model

\n\n

IX. Disposability

\n\n

Maximize robustness with fast startup and graceful shutdown

\n\n

X. Dev/prod parity

\n\n

Keep development, staging, and production as similar as possible

\n\n

XI. Logs

\n\n

Treat logs as event streams

\n\n

XII. Admin processes

\n\n

Run admin/management tasks as one-off processes

\n
\n
\n\n\n \n\n\n" if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -84,7 +84,7 @@ func TestDelay(t *testing.T) { config1 := NewScrapeConfig() start := time.Now() - NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) elapsed := time.Since(start) got := elapsed @@ -101,10 +101,10 @@ func TestContentImagesOnly(t *testing.T) { config := NewScrapeConfig() config.ImagesOnly = true - c := NewChapterFromURL("https://books.lapw.at/posts/adam-wiggins-the-twelve-factor-app/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/codebase", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Content() - want := "\"One\"A\"Code\"Scale" + want := "\"One" if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -117,10 +117,10 @@ func TestSubChapters(t *testing.T) { config0 := NewScrapeConfig() config1 := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://atomicdesign.bradfrost.com/table-of-contents/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) - want := 2 + want := 9 if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -133,10 +133,10 @@ func TestSubChaptersRSS(t *testing.T) { config0 := NewScrapeConfig() config1 := NewScrapeConfig() - c := NewChapterFromURL("https://blog.lapw.at/rss", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://www.nginx.com/feed/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) - want := 8 + want := 10 if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -165,14 +165,15 @@ func TestSubChaptersSelector(t *testing.T) { func TestSubChaptersLimit(t *testing.T) { config0 := NewScrapeConfig() - config0.Limit = 1 + config0.Selector = "section.concrete > article > h2 > a" + config0.Limit = 2 config1 := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) - want := 1 + want := 2 if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -183,14 +184,15 @@ func TestSubChaptersLimit(t *testing.T) { func TestSubChaptersLimitOver(t *testing.T) { config0 := NewScrapeConfig() - config0.Limit = 3 + config0.Selector = "section.concrete > article > h2 > a" + config0.Limit = 13 config1 := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := len(c.SubChapters()) - want := 2 + want := 12 if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -205,10 +207,10 @@ func TestReverse(t *testing.T) { config1 := NewScrapeConfig() - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://atomicdesign.bradfrost.com/table-of-contents/", "", []*ScrapeConfig{config0, config1}, 0, func(index int, name string) {}) got := c.SubChapters()[0].Name() - want := "The Twelve-Factor App" + want := "About the Author | Atomic Design by Brad Frost" if got != want { t.Errorf("got %v, wanted %v", got, want) @@ -219,9 +221,10 @@ func TestReverse(t *testing.T) { func TestNotInclude(t *testing.T) { config := NewScrapeConfig() + config.Selector = "section.concrete > article > h2 > a" config.Include = false - c := NewChapterFromURL("https://books.lapw.at/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) + c := NewChapterFromURL("https://12factor.net/", "", []*ScrapeConfig{config}, 0, func(index int, name string) {}) got := c.Content() want := "" diff --git a/cmd/get.go b/cmd/get.go index 96fec66..c46d2e8 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -132,7 +132,6 @@ var getCmd = &cobra.Command{ return nil }, Run: func(cmd *cobra.Command, args []string) { - url := args[0] // generate config for each level configs := make([]*book.ScrapeConfig, len(getOpts.Selector)) @@ -162,7 +161,13 @@ var getCmd = &cobra.Command{ configs[index] = config } - c := book.NewChapterFromURL(url, "", configs, 0, func(index int, name string) {}) + // dummy root chapter to contain all subchapters + c := book.NewEmptyChapter() + for _, u := range args { + newChapter := book.NewChapterFromURL(u, "", configs, 0, func(index int, name string) {}) + c.AddSubChapter(newChapter) + } + c.SetName(c.SubChapters()[0].Name()) if getOpts.Format == "md" { filename := book.ToMarkdown(c, getOpts.output) diff --git a/cmd/version.go b/cmd/version.go index bbede9f..34b15b3 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -14,6 +14,6 @@ var versionCmd = &cobra.Command{ Use: "version", Short: "Print the version number of papeer", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("papeer v0.5.6") + fmt.Println("papeer v0.6.0") }, }