mirror of
https://github.com/NohamR/pdfy.git
synced 2026-10-11 02:39:58 +00:00
Export as a lib
This commit is contained in:
56
README.md
56
README.md
@@ -20,7 +20,7 @@ npm install
|
|||||||
|
|
||||||
The Reader View extension is downloaded and extracted automatically on first run.
|
The Reader View extension is downloaded and extracted automatically on first run.
|
||||||
|
|
||||||
## Usage
|
## CLI Usage
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
node src/index.js --help
|
node src/index.js --help
|
||||||
@@ -90,6 +90,60 @@ npm test
|
|||||||
|
|
||||||
The test script reads URLs from `src/test/test.txt` and runs multiple scenarios (default, dark, sepia, custom CSS, etc.).
|
The test script reads URLs from `src/test/test.txt` and runs multiple scenarios (default, dark, sepia, custom CSS, etc.).
|
||||||
|
|
||||||
|
## Library Usage
|
||||||
|
|
||||||
|
Use pdfy programmatically in your own Node.js applications:
|
||||||
|
|
||||||
|
```js
|
||||||
|
import { convert, launchBrowser, BotProtectionError, VALID_THEMES } from 'pdfy'
|
||||||
|
|
||||||
|
// Simple usage
|
||||||
|
const { title, pdfBuffer } = await convert('https://example.com/article')
|
||||||
|
fs.writeFileSync('article.pdf', pdfBuffer)
|
||||||
|
|
||||||
|
// With options
|
||||||
|
const result = await convert('https://example.com/article', {
|
||||||
|
theme: 'dark',
|
||||||
|
css: 'body { color: #333 }',
|
||||||
|
prefs: { fontSize: 20 },
|
||||||
|
output: './article.pdf' // saves to disk and returns buffer
|
||||||
|
})
|
||||||
|
console.log(`Generated: ${result.title}`)
|
||||||
|
|
||||||
|
// Reuse browser across conversions (for servers)
|
||||||
|
const browser = await launchBrowser()
|
||||||
|
const a = await convert('https://example.com/1', { browser, theme: 'dark' })
|
||||||
|
const b = await convert('https://example.com/2', { browser, theme: 'sepia' })
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
// Handle bot protection
|
||||||
|
try {
|
||||||
|
await convert('https://example.com/article')
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof BotProtectionError) {
|
||||||
|
// Prompt user to provide the article HTML manually
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### API
|
||||||
|
|
||||||
|
**`convert(url, options?)`** — returns `{ title, pdfBuffer, outputPath }`
|
||||||
|
|
||||||
|
| Option | Type | Default | Description |
|
||||||
|
|--------|------|---------|-------------|
|
||||||
|
| `theme` | string | `'light'` | Reader View theme |
|
||||||
|
| `css` | string | `null` | Custom CSS string |
|
||||||
|
| `prefs` | object | `{}` | Extension preferences |
|
||||||
|
| `output` | string | `null` | File path to write PDF (returns buffer when omitted) |
|
||||||
|
| `browser` | Browser | `null` | Reuse a Puppeteer browser instance |
|
||||||
|
| `html` | string | `null` | Pre-fetched HTML content (skips URL fetch) |
|
||||||
|
|
||||||
|
**`launchBrowser()`** — launches a Puppeteer browser with the Reader View extension loaded.
|
||||||
|
|
||||||
|
**`BotProtectionError`** — thrown when the target page appears to be behind bot protection.
|
||||||
|
|
||||||
|
|
||||||
## Credits
|
## Credits
|
||||||
|
|
||||||
- [Reader View](https://chromewebstore.google.com/detail/reader-view/ecabifbgmdmgdllomnfinbmaellmclnh) Chrome extension
|
- [Reader View](https://chromewebstore.google.com/detail/reader-view/ecabifbgmdmgdllomnfinbmaellmclnh) Chrome extension
|
||||||
|
|||||||
@@ -4,7 +4,10 @@
|
|||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"description": "Convert web articles to PDF using Reader View and Puppeteer",
|
"description": "Convert web articles to PDF using Reader View and Puppeteer",
|
||||||
"main": "src/index.js",
|
"main": "src/pdfy.js",
|
||||||
|
"exports": {
|
||||||
|
".": "./src/pdfy.js"
|
||||||
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node src/index.js",
|
"start": "node src/index.js",
|
||||||
"test": "node src/test/test.js"
|
"test": "node src/test/test.js"
|
||||||
|
|||||||
@@ -14,20 +14,17 @@ export async function checkExtension () {
|
|||||||
try {
|
try {
|
||||||
await downloadExtension({ logger })
|
await downloadExtension({ logger })
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.error(`Failed to download extension: ${err.message}`)
|
throw new Error(`Failed to download extension: ${err.message}`)
|
||||||
process.exit(1)
|
|
||||||
}
|
}
|
||||||
if (!fs.existsSync(MANIFEST_PATH)) {
|
if (!fs.existsSync(MANIFEST_PATH)) {
|
||||||
logger.error('Download completed but extension not found. Extraction may have failed.')
|
throw new Error('Download completed but extension not found. Extraction may have failed.')
|
||||||
process.exit(1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!fs.existsSync(READABILITY_PATH)) {
|
if (!fs.existsSync(READABILITY_PATH)) {
|
||||||
logger.error(
|
throw new Error(
|
||||||
`Readability.js not found at: "${READABILITY_PATH}"\n` +
|
`Readability.js not found at: "${READABILITY_PATH}"\n` +
|
||||||
'The extension may be incomplete.'
|
'The extension may be incomplete.'
|
||||||
)
|
)
|
||||||
process.exit(1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
169
src/index.js
169
src/index.js
@@ -2,31 +2,10 @@ import fs from 'node:fs'
|
|||||||
import process from 'node:process'
|
import process from 'node:process'
|
||||||
import path from 'node:path'
|
import path from 'node:path'
|
||||||
import readline from 'node:readline/promises'
|
import readline from 'node:readline/promises'
|
||||||
import { parseArgs, VALID_THEMES } from './cli.js'
|
import { parseArgs } from './cli.js'
|
||||||
import { loadConfigFile, loadPreferences, readCustomCss } from './config.js'
|
import { loadConfigFile, loadPreferences, readCustomCss } from './config.js'
|
||||||
|
import { convert, BotProtectionError } from './pdfy.js'
|
||||||
import logger from './logger.js'
|
import logger from './logger.js'
|
||||||
import {
|
|
||||||
checkExtension,
|
|
||||||
launchBrowser,
|
|
||||||
discoverExtensionId,
|
|
||||||
getReadabilityPath
|
|
||||||
} from './browser.js'
|
|
||||||
import {
|
|
||||||
extractArticle,
|
|
||||||
storeArticle,
|
|
||||||
setExtensionPreferences,
|
|
||||||
waitForReaderView,
|
|
||||||
extractRenderedContent,
|
|
||||||
getArticleTitle
|
|
||||||
} from './reader.js'
|
|
||||||
import {
|
|
||||||
getOutputFilename,
|
|
||||||
resolveOutputPath,
|
|
||||||
enhanceHtml,
|
|
||||||
generatePdf
|
|
||||||
} from './pdf.js'
|
|
||||||
|
|
||||||
const EXTENSION_BASE = 'chrome-extension://'
|
|
||||||
|
|
||||||
async function promptForInput () {
|
async function promptForInput () {
|
||||||
const rl = readline.createInterface({
|
const rl = readline.createInterface({
|
||||||
@@ -41,120 +20,58 @@ async function promptForInput () {
|
|||||||
async function main () {
|
async function main () {
|
||||||
const { url, opts } = parseArgs()
|
const { url, opts } = parseArgs()
|
||||||
|
|
||||||
await checkExtension()
|
await loadConfigFile(opts.config)
|
||||||
|
|
||||||
const config = await loadConfigFile(opts.config)
|
const prefsPath = opts.prefs || null
|
||||||
|
const cssPath = opts.css || null
|
||||||
|
const outputPath = opts.output || null
|
||||||
|
|
||||||
const prefsPath = opts.prefs || config.prefs || null
|
const preferences = prefsPath ? loadPreferences(path.resolve(prefsPath)) : {}
|
||||||
const cssPath = opts.css || config.css || null
|
const customCss = cssPath ? readCustomCss(path.resolve(cssPath)) : null
|
||||||
const outputPath = opts.output || config.output || null
|
|
||||||
|
|
||||||
let preferences = {}
|
let html
|
||||||
if (prefsPath) {
|
const currentUrl = url
|
||||||
preferences = loadPreferences(path.resolve(prefsPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
const theme = opts.theme || config.theme || preferences.mode || 'light'
|
for (let attempt = 0; attempt < 2; attempt++) {
|
||||||
if (!VALID_THEMES.includes(theme)) {
|
try {
|
||||||
logger.error(
|
const result = await convert(currentUrl, {
|
||||||
`Invalid theme "${theme}". Valid themes: ${VALID_THEMES.join(', ')}`
|
html,
|
||||||
)
|
theme: opts.theme || preferences.mode,
|
||||||
process.exit(1)
|
css: customCss,
|
||||||
}
|
prefs: preferences,
|
||||||
preferences.mode = theme
|
output: outputPath
|
||||||
logger.info(`Theme: ${theme}`)
|
})
|
||||||
|
|
||||||
let customCss = null
|
const sizeKb = (result.pdfBuffer.length / 1024).toFixed(0)
|
||||||
if (cssPath) {
|
logger.info(`PDF generated: "${result.title}" (${sizeKb} KB)`)
|
||||||
customCss = readCustomCss(path.resolve(cssPath))
|
if (result.outputPath) {
|
||||||
if (customCss !== null) {
|
logger.info(`Saved to: "${result.outputPath}"`)
|
||||||
preferences['user-css'] = customCss
|
|
||||||
logger.info(`Custom CSS loaded: "${path.resolve(cssPath)}"`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const browser = await launchBrowser()
|
|
||||||
|
|
||||||
try {
|
|
||||||
const extId = await discoverExtensionId(browser)
|
|
||||||
|
|
||||||
logger.info(`Fetching article: "${url}"`)
|
|
||||||
const articlePage = await browser.newPage()
|
|
||||||
await articlePage.goto(url, { waitUntil: 'networkidle0', timeout: 30000 })
|
|
||||||
let article = await extractArticle(articlePage, getReadabilityPath())
|
|
||||||
|
|
||||||
if (article.title === 'Just a moment...' || article.length < 500) {
|
|
||||||
logger.warn('Cloudflare or bot protection detected..')
|
|
||||||
logger.warn('Open the URL in your browser, save the page as complete HTML (File > Save Page As),')
|
|
||||||
logger.warn('then paste the saved file path below and press Enter:')
|
|
||||||
|
|
||||||
const htmlPath = await promptForInput()
|
|
||||||
const resolvedPath = path.resolve(htmlPath)
|
|
||||||
|
|
||||||
if (!fs.existsSync(resolvedPath)) {
|
|
||||||
logger.error(`File not found: "${resolvedPath}"`)
|
|
||||||
process.exit(1)
|
|
||||||
}
|
}
|
||||||
|
return
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof BotProtectionError && attempt === 0) {
|
||||||
|
logger.warn('Bot protection or low-content page detected.')
|
||||||
|
logger.warn('Open the URL in your browser, save the page as complete HTML (File > Save Page As),')
|
||||||
|
logger.warn('then paste the saved file path below and press Enter:')
|
||||||
|
|
||||||
logger.info(`Loading article from local file: "${resolvedPath}"`)
|
const htmlPath = await promptForInput()
|
||||||
await articlePage.goto(`file://${resolvedPath}`, { waitUntil: 'networkidle0', timeout: 30000 })
|
const resolvedPath = path.resolve(htmlPath)
|
||||||
article = await extractArticle(articlePage, getReadabilityPath())
|
|
||||||
|
if (!fs.existsSync(resolvedPath)) {
|
||||||
|
logger.error(`File not found: "${resolvedPath}"`)
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Loading article from local file: "${resolvedPath}"`)
|
||||||
|
html = fs.readFileSync(resolvedPath, 'utf-8')
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
throw err
|
||||||
}
|
}
|
||||||
|
|
||||||
await articlePage.close()
|
|
||||||
|
|
||||||
const extPage = await browser.newPage()
|
|
||||||
await storeArticle(extPage, extId, 1, article)
|
|
||||||
|
|
||||||
const prefsToSet = Object.fromEntries(
|
|
||||||
Object.entries(preferences).filter(
|
|
||||||
([_, v]) => v !== undefined && v !== null
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if (Object.keys(prefsToSet).length > 0) {
|
|
||||||
await setExtensionPreferences(extPage, extId, prefsToSet)
|
|
||||||
}
|
|
||||||
await extPage.close()
|
|
||||||
|
|
||||||
const readerUrl = [
|
|
||||||
`${EXTENSION_BASE}${extId}/data/reader/index.html`,
|
|
||||||
'?id=1',
|
|
||||||
`&url=${encodeURIComponent(url)}`
|
|
||||||
].join('')
|
|
||||||
logger.debug(`Opening Reader View: ${readerUrl}`)
|
|
||||||
|
|
||||||
const readerPage = await browser.newPage()
|
|
||||||
await readerPage.goto(readerUrl, { waitUntil: 'load', timeout: 30000 })
|
|
||||||
|
|
||||||
await waitForReaderView(readerPage)
|
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1000))
|
|
||||||
|
|
||||||
const title = await getArticleTitle(readerPage)
|
|
||||||
const outputFile = outputPath
|
|
||||||
? resolveOutputPath(outputPath)
|
|
||||||
: getOutputFilename(title)
|
|
||||||
|
|
||||||
let contentHtml = await extractRenderedContent(readerPage)
|
|
||||||
contentHtml = contentHtml.replace(
|
|
||||||
'<head>',
|
|
||||||
`<head><base href="${url}"><meta name="referrer" content="unsafe-url">`
|
|
||||||
)
|
|
||||||
contentHtml = contentHtml.replace(/ loading="lazy"/g, '')
|
|
||||||
await readerPage.close()
|
|
||||||
|
|
||||||
const enhancedHtml = enhanceHtml(contentHtml, customCss)
|
|
||||||
|
|
||||||
const pdfPage = await browser.newPage()
|
|
||||||
await pdfPage.setViewport({ width: 1280, height: 7200 })
|
|
||||||
await generatePdf(pdfPage, enhancedHtml, outputFile)
|
|
||||||
await pdfPage.close()
|
|
||||||
} finally {
|
|
||||||
await browser.close()
|
|
||||||
logger.info('Browser closed.')
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
main().catch((err) => {
|
main().catch((err) => {
|
||||||
logger.error(err)
|
logger.error(err.message || err)
|
||||||
process.exit(1)
|
process.exit(1)
|
||||||
})
|
})
|
||||||
|
|||||||
16
src/pdf.js
16
src/pdf.js
@@ -47,9 +47,7 @@ export async function generatePdf (page, html, outputPath) {
|
|||||||
() => Array.from(document.images).every(img => img.complete),
|
() => Array.from(document.images).every(img => img.complete),
|
||||||
{ timeout: 15000, polling: 200 }
|
{ timeout: 15000, polling: 200 }
|
||||||
).catch(() => logger.debug('Image wait timed out, continuing...'))
|
).catch(() => logger.debug('Image wait timed out, continuing...'))
|
||||||
logger.info(`Generating PDF: "${outputPath}"`)
|
const pdfOptions = {
|
||||||
await page.pdf({
|
|
||||||
path: outputPath,
|
|
||||||
format: 'A4',
|
format: 'A4',
|
||||||
printBackground: true,
|
printBackground: true,
|
||||||
margin: {
|
margin: {
|
||||||
@@ -58,6 +56,14 @@ export async function generatePdf (page, html, outputPath) {
|
|||||||
left: '0mm',
|
left: '0mm',
|
||||||
right: '0mm'
|
right: '0mm'
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
logger.info(`PDF saved: "${outputPath}"`)
|
if (outputPath) {
|
||||||
|
pdfOptions.path = outputPath
|
||||||
|
logger.info(`Generating PDF: "${outputPath}"`)
|
||||||
|
}
|
||||||
|
const buffer = await page.pdf(pdfOptions)
|
||||||
|
if (outputPath) {
|
||||||
|
logger.info(`PDF saved: "${outputPath}"`)
|
||||||
|
}
|
||||||
|
return buffer
|
||||||
}
|
}
|
||||||
|
|||||||
133
src/pdfy.js
Normal file
133
src/pdfy.js
Normal file
@@ -0,0 +1,133 @@
|
|||||||
|
import fs from 'node:fs'
|
||||||
|
import path from 'node:path'
|
||||||
|
import {
|
||||||
|
checkExtension,
|
||||||
|
launchBrowser,
|
||||||
|
discoverExtensionId,
|
||||||
|
getReadabilityPath
|
||||||
|
} from './browser.js'
|
||||||
|
import {
|
||||||
|
extractArticle,
|
||||||
|
storeArticle,
|
||||||
|
setExtensionPreferences,
|
||||||
|
waitForReaderView,
|
||||||
|
extractRenderedContent,
|
||||||
|
getArticleTitle
|
||||||
|
} from './reader.js'
|
||||||
|
import { enhanceHtml, generatePdf } from './pdf.js'
|
||||||
|
import { VALID_THEMES } from './cli.js'
|
||||||
|
import logger from './logger.js'
|
||||||
|
|
||||||
|
const EXTENSION_BASE = 'chrome-extension://'
|
||||||
|
|
||||||
|
export { launchBrowser, VALID_THEMES }
|
||||||
|
|
||||||
|
export class BotProtectionError extends Error {
|
||||||
|
constructor () {
|
||||||
|
super('Bot protection or low-content page detected. Provide the article as HTML.')
|
||||||
|
this.name = 'BotProtectionError'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function convert (url, options = {}) {
|
||||||
|
const {
|
||||||
|
html,
|
||||||
|
theme = 'light',
|
||||||
|
css = null,
|
||||||
|
prefs = {},
|
||||||
|
output = null,
|
||||||
|
browser: externalBrowser = null
|
||||||
|
} = options
|
||||||
|
|
||||||
|
if (!VALID_THEMES.includes(theme)) {
|
||||||
|
throw new Error(`Invalid theme "${theme}". Valid themes: ${VALID_THEMES.join(', ')}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Theme: ${theme}`)
|
||||||
|
|
||||||
|
await checkExtension()
|
||||||
|
|
||||||
|
let browser = externalBrowser
|
||||||
|
let ownBrowser = false
|
||||||
|
if (!browser) {
|
||||||
|
browser = await launchBrowser()
|
||||||
|
ownBrowser = true
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const extId = await discoverExtensionId(browser)
|
||||||
|
|
||||||
|
logger.info(`Fetching article: "${url}"`)
|
||||||
|
const articlePage = await browser.newPage()
|
||||||
|
if (html) {
|
||||||
|
await articlePage.setContent(html, { waitUntil: 'networkidle0' })
|
||||||
|
} else {
|
||||||
|
await articlePage.goto(url, { waitUntil: 'networkidle0', timeout: 30000 })
|
||||||
|
}
|
||||||
|
const article = await extractArticle(articlePage, getReadabilityPath())
|
||||||
|
await articlePage.close()
|
||||||
|
|
||||||
|
if (article.title === 'Just a moment...' || article.length < 500) {
|
||||||
|
throw new BotProtectionError()
|
||||||
|
}
|
||||||
|
|
||||||
|
const mergedPrefs = { ...prefs, mode: theme }
|
||||||
|
if (css) {
|
||||||
|
mergedPrefs['user-css'] = css
|
||||||
|
}
|
||||||
|
|
||||||
|
const extPage = await browser.newPage()
|
||||||
|
await storeArticle(extPage, extId, 1, article)
|
||||||
|
|
||||||
|
const prefsToSet = Object.fromEntries(
|
||||||
|
Object.entries(mergedPrefs).filter(([_, v]) => v !== undefined && v !== null)
|
||||||
|
)
|
||||||
|
if (Object.keys(prefsToSet).length > 0) {
|
||||||
|
await setExtensionPreferences(extPage, extId, prefsToSet)
|
||||||
|
}
|
||||||
|
await extPage.close()
|
||||||
|
|
||||||
|
const readerUrl = [
|
||||||
|
`${EXTENSION_BASE}${extId}/data/reader/index.html`,
|
||||||
|
'?id=1',
|
||||||
|
`&url=${encodeURIComponent(url)}`
|
||||||
|
].join('')
|
||||||
|
logger.debug(`Opening Reader View: ${readerUrl}`)
|
||||||
|
|
||||||
|
const readerPage = await browser.newPage()
|
||||||
|
await readerPage.goto(readerUrl, { waitUntil: 'load', timeout: 30000 })
|
||||||
|
await waitForReaderView(readerPage)
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1000))
|
||||||
|
|
||||||
|
const title = await getArticleTitle(readerPage)
|
||||||
|
|
||||||
|
let contentHtml = await extractRenderedContent(readerPage)
|
||||||
|
contentHtml = contentHtml.replace(
|
||||||
|
'<head>',
|
||||||
|
`<head><base href="${url}"><meta name="referrer" content="unsafe-url">`
|
||||||
|
)
|
||||||
|
contentHtml = contentHtml.replace(/ loading="lazy"/g, '')
|
||||||
|
await readerPage.close()
|
||||||
|
|
||||||
|
const enhancedHtml = enhanceHtml(contentHtml, css)
|
||||||
|
|
||||||
|
const pdfPage = await browser.newPage()
|
||||||
|
await pdfPage.setViewport({ width: 1280, height: 7200 })
|
||||||
|
|
||||||
|
let outputPath = null
|
||||||
|
if (output) {
|
||||||
|
outputPath = path.resolve(output)
|
||||||
|
fs.mkdirSync(path.dirname(outputPath), { recursive: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
const pdfBuffer = await generatePdf(pdfPage, enhancedHtml, outputPath)
|
||||||
|
await pdfPage.close()
|
||||||
|
|
||||||
|
return { title, pdfBuffer, outputPath }
|
||||||
|
} finally {
|
||||||
|
if (ownBrowser && browser) {
|
||||||
|
await browser.close()
|
||||||
|
logger.info('Browser closed.')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user