From 22d332a20f6ac19fed2750227a3b031d7b8e93d4 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 18 Jan 2025 14:22:53 -0500 Subject: [PATCH 01/79] Convert to ES modules and related changes --- .eslintrc.json | 13 +- index.js | 10 +- package-lock.json | 649 ++++++++++++++++++++++----------------------- package.json | 7 +- src/frontmatter.js | 81 ++++++ src/parser.js | 25 +- src/settings.js | 16 +- src/shared.js | 4 +- src/translator.js | 13 +- src/wizard.js | 19 +- src/writer.js | 22 +- 11 files changed, 449 insertions(+), 410 deletions(-) create mode 100644 src/frontmatter.js diff --git a/.eslintrc.json b/.eslintrc.json index 8c541d5..d29a87b 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,17 +1,10 @@ { "env": { - "commonjs": true, - "es6": true, + "es2022": true, "node": true }, "extends": "eslint:recommended", - "globals": { - "Atomics": "readonly", - "SharedArrayBuffer": "readonly" - }, "parserOptions": { - "ecmaVersion": 2018 - }, - "rules": { + "sourceType": "module" } -} \ No newline at end of file +} diff --git a/index.js b/index.js index ff993cc..1e3bdf5 100644 --- a/index.js +++ b/index.js @@ -1,11 +1,11 @@ #!/usr/bin/env node -const path = require('path'); -const process = require('process'); +import path from 'path'; +import process from 'process'; -const wizard = require('./src/wizard'); -const parser = require('./src/parser'); -const writer = require('./src/writer'); +import * as wizard from './src/wizard.js'; +import * as parser from './src/parser.js'; +import * as writer from './src/writer.js'; (async () => { // parse any command line arguments and run wizard diff --git a/package-lock.json b/package-lock.json index cb2be38..2a9e210 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,10 +10,10 @@ "license": "MIT", "dependencies": { "axios": "^1.7.9", - "camelcase": "^6.3.0", - "chalk": "^4.1.2", + "camelcase": "^8.0.0", + "chalk": "^5.4.1", "commander": "^13.0.0", - "inquirer": "^8.2.6", + "inquirer": "^12.3.2", "luxon": "^3.5.0", "require-directory": "^2.1.1", "turndown": "^7.2.0", @@ -124,6 +124,229 @@ "deprecated": "Use @eslint/object-schema instead", "dev": true }, + "node_modules/@inquirer/checkbox": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.0.6.tgz", + "integrity": "sha512-PgP35JfmGjHU0LSXOyRew0zHuA9N6OJwOlos1fZ20b7j8ISeAdib3L+n0jIxBtX958UeEpte6xhG/gxJ5iUqMw==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", + "ansi-escapes": "^4.3.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/confirm": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.3.tgz", + "integrity": "sha512-fuF9laMmHoOgWapF9h9hv6opA5WvmGFHsTYGCmuFxcghIhEhb3dN0CdQR4BUMqa2H506NCj8cGX4jwMsE4t6dA==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/core": { + "version": "10.1.4", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.4.tgz", + "integrity": "sha512-5y4/PUJVnRb4bwWY67KLdebWOhOc7xj5IP2J80oWXa64mVag24rwQ1VAdnj7/eDY/odhguW0zQ1Mp1pj6fO/2w==", + "dependencies": { + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", + "ansi-escapes": "^4.3.2", + "cli-width": "^4.1.0", + "mute-stream": "^2.0.0", + "signal-exit": "^4.1.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/editor": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.3.tgz", + "integrity": "sha512-S9KnIOJuTZpb9upeRSBBhoDZv7aSV3pG9TECrBj0f+ZsFwccz886hzKBrChGrXMJwd4NKY+pOA9Vy72uqnd6Eg==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", + "external-editor": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/expand": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.6.tgz", + "integrity": "sha512-TRTfi1mv1GeIZGyi9PQmvAaH65ZlG4/FACq6wSzs7Vvf1z5dnNWsAAXBjWMHt76l+1hUY8teIqJFrWBk5N6gsg==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.9.tgz", + "integrity": "sha512-BXvGj0ehzrngHTPTDqUoDT3NXL8U0RxUk2zJm2A66RhCEIWdtU1v6GuUqNAgArW4PQ9CinqIWyHdQgdwOj06zQ==", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/input": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.3.tgz", + "integrity": "sha512-zeo++6f7hxaEe7OjtMzdGZPHiawsfmCZxWB9X1NpmYgbeoyerIbWemvlBxxl+sQIlHC0WuSAG19ibMq3gbhaqQ==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/number": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.6.tgz", + "integrity": "sha512-xO07lftUHk1rs1gR0KbqB+LJPhkUNkyzV/KhH+937hdkMazmAYHLm1OIrNKpPelppeV1FgWrgFDjdUD8mM+XUg==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/password": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.6.tgz", + "integrity": "sha512-QLF0HmMpHZPPMp10WGXh6F+ZPvzWE7LX6rNoccdktv/Rov0B+0f+eyXkAcgqy5cH9V+WSpbLxu2lo3ysEVK91w==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", + "ansi-escapes": "^4.3.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/prompts": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.2.3.tgz", + "integrity": "sha512-hzfnm3uOoDySDXfDNOm9usOuYIaQvTgKp/13l1uJoe6UNY+Zpcn2RYt0jXz3yA+yemGHvDOxVzqWl3S5sQq53Q==", + "dependencies": { + "@inquirer/checkbox": "^4.0.6", + "@inquirer/confirm": "^5.1.3", + "@inquirer/editor": "^4.2.3", + "@inquirer/expand": "^4.0.6", + "@inquirer/input": "^4.1.3", + "@inquirer/number": "^3.0.6", + "@inquirer/password": "^4.0.6", + "@inquirer/rawlist": "^4.0.6", + "@inquirer/search": "^3.0.6", + "@inquirer/select": "^4.0.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/rawlist": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.6.tgz", + "integrity": "sha512-QoE4s1SsIPx27FO4L1b1mUjVcoHm1pWE/oCmm4z/Hl+V1Aw5IXl8FYYzGmfXaBT0l/sWr49XmNSiq7kg3Kd/Lg==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/search": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.6.tgz", + "integrity": "sha512-eFZ2hiAq0bZcFPuFFBmZEtXU1EarHLigE+ENCtpO+37NHCl4+Yokq1P/d09kUblObaikwfo97w+0FtG/EXl5Ng==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/select": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.0.6.tgz", + "integrity": "sha512-yANzIiNZ8fhMm4NORm+a74+KFYHmf7BZphSOBovIzYPVLquseTGEkU5l2UTnBOf5k0VLmTgPighNDLE9QtbViQ==", + "dependencies": { + "@inquirer/core": "^10.1.4", + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", + "ansi-escapes": "^4.3.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, + "node_modules/@inquirer/type": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.2.tgz", + "integrity": "sha512-ZhQ4TvhwHZF+lGhQ2O/rsjo80XoZR5/5qhOY3t6FJuX5XBg5Be8YzYTvaUGJnc12AUGI2nr4QSUE4PhKSigx7g==", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + } + }, "node_modules/@mixmark-io/domino": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", @@ -164,6 +387,15 @@ "node": ">= 8" } }, + "node_modules/@types/node": { + "version": "22.10.7", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.7.tgz", + "integrity": "sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==", + "peer": true, + "dependencies": { + "undici-types": "~6.20.0" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.1.tgz", @@ -281,35 +513,6 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -320,29 +523,6 @@ "concat-map": "0.0.1" } }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -353,26 +533,22 @@ } }, "node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-8.0.0.tgz", + "integrity": "sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==", "engines": { - "node": ">=10" + "node": ">=16" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", + "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", "engines": { - "node": ">=10" + "node": "^12.17.0 || ^14.13 || >=16.0.0" }, "funding": { "url": "https://github.com/chalk/chalk?sponsor=1" @@ -383,42 +559,12 @@ "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, - "node_modules/cli-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", - "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", - "dependencies": { - "restore-cursor": "^3.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/cli-spinners": { - "version": "2.9.2", - "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", - "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/cli-width": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", - "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", "engines": { - "node": ">= 10" - } - }, - "node_modules/clone": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", - "engines": { - "node": ">=0.8" + "node": ">= 12" } }, "node_modules/color-convert": { @@ -499,17 +645,6 @@ "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, - "node_modules/defaults": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", - "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", - "dependencies": { - "clone": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -631,6 +766,22 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/espree": { "version": "9.6.1", "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", @@ -730,28 +881,6 @@ "reusify": "^1.0.4" } }, - "node_modules/figures": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", - "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", - "dependencies": { - "escape-string-regexp": "^1.0.5" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/figures/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/file-entry-cache": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", @@ -896,6 +1025,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, "engines": { "node": ">=8" } @@ -911,25 +1041,6 @@ "node": ">=0.10.0" } }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -978,31 +1089,27 @@ "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true }, "node_modules/inquirer": { - "version": "8.2.6", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.6.tgz", - "integrity": "sha512-M1WuAmb7pn9zdFRtQYk26ZBoY043Sse0wVDdk4Bppr+JOXyQYybdtvK+l9wUibhtjdjvtoiNy8tk+EgsYIUqKg==", + "version": "12.3.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-12.3.2.tgz", + "integrity": "sha512-YjQCIcDd3yyDuQrbII0FBtm/ZqNoWtvaC71yeCnd5Vbg4EgzsAGaemzfpzmqfvIZEp2roSwuZZKdM0C65hA43g==", "dependencies": { - "ansi-escapes": "^4.2.1", - "chalk": "^4.1.1", - "cli-cursor": "^3.1.0", - "cli-width": "^3.0.0", - "external-editor": "^3.0.3", - "figures": "^3.0.0", - "lodash": "^4.17.21", - "mute-stream": "0.0.8", - "ora": "^5.4.1", - "run-async": "^2.4.0", - "rxjs": "^7.5.5", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0", - "through": "^2.3.6", - "wrap-ansi": "^6.0.1" + "@inquirer/core": "^10.1.4", + "@inquirer/prompts": "^7.2.3", + "@inquirer/type": "^3.0.2", + "ansi-escapes": "^4.3.2", + "mute-stream": "^2.0.0", + "run-async": "^3.0.0", + "rxjs": "^7.8.1" }, "engines": { - "node": ">=12.0.0" + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/is-extglob": { @@ -1034,14 +1141,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-interactive": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", - "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-path-inside": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", @@ -1051,17 +1150,6 @@ "node": ">=8" } }, - "node_modules/is-unicode-supported": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", - "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -1135,32 +1223,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash": { - "version": "4.17.21", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" - }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "node_modules/log-symbols": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", - "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dependencies": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/luxon": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", @@ -1188,14 +1256,6 @@ "node": ">= 0.6" } }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "engines": { - "node": ">=6" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -1215,9 +1275,12 @@ "dev": true }, "node_modules/mute-stream": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", + "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } }, "node_modules/natural-compare": { "version": "1.4.0", @@ -1234,20 +1297,6 @@ "wrappy": "1" } }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -1265,28 +1314,6 @@ "node": ">= 0.8.0" } }, - "node_modules/ora": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", - "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", - "dependencies": { - "bl": "^4.1.0", - "chalk": "^4.1.0", - "cli-cursor": "^3.1.0", - "cli-spinners": "^2.5.0", - "is-interactive": "^1.0.0", - "is-unicode-supported": "^0.1.0", - "log-symbols": "^4.1.0", - "strip-ansi": "^6.0.0", - "wcwidth": "^1.0.1" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -1407,19 +1434,6 @@ } ] }, - "node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -1437,18 +1451,6 @@ "node": ">=4" } }, - "node_modules/restore-cursor": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", - "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", - "dependencies": { - "onetime": "^5.1.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -1476,9 +1478,9 @@ } }, "node_modules/run-async": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", + "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", "engines": { "node": ">=0.12.0" } @@ -1514,25 +1516,6 @@ "tslib": "^2.1.0" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1565,16 +1548,14 @@ } }, "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/string-width": { @@ -1617,6 +1598,7 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -1630,11 +1612,6 @@ "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" - }, "node_modules/tmp": { "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", @@ -1688,6 +1665,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "peer": true + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -1697,19 +1680,6 @@ "punycode": "^2.1.0" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" - }, - "node_modules/wcwidth": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", - "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", - "dependencies": { - "defaults": "^1.0.3" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -1784,6 +1754,17 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/yoctocolors-cjs": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz", + "integrity": "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/package.json b/package.json index 10512ed..4abd834 100644 --- a/package.json +++ b/package.json @@ -19,12 +19,13 @@ "engines": { "node": ">= 18.0.0" }, + "type": "module", "dependencies": { "axios": "^1.7.9", - "camelcase": "^6.3.0", - "chalk": "^4.1.2", + "camelcase": "^8.0.0", + "chalk": "^5.4.1", "commander": "^13.0.0", - "inquirer": "^8.2.6", + "inquirer": "^12.3.2", "luxon": "^3.5.0", "require-directory": "^2.1.1", "turndown": "^7.2.0", diff --git a/src/frontmatter.js b/src/frontmatter.js new file mode 100644 index 0000000..43eb6dd --- /dev/null +++ b/src/frontmatter.js @@ -0,0 +1,81 @@ +import * as luxon from 'luxon'; + +import * as settings from './settings.js'; + +// get author, without decoding +// WordPress doesn't allow funky characters in usernames anyway +export function getAuthor(post) { + return post.data.creator[0]; +} + +// get array of decoded category names, filtered as specified in settings +export function getCategories(post) { + if (!post.data.category) { + return []; + } + + const categories = post.data.category + .filter(category => category.$.domain === 'category') + .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); + + return categories.filter(category => !settings.filter_categories.includes(category)); +} + +// get cover image filename, previously decoded and set on post.meta +// this one is unique as it relies on special logic executed by the parser +export function getCoverImage(post) { + return post.meta.coverImage; +} + +// get post date, optionally formatted as specified in settings +// this value is also used for year/month folders, date prefixes, etc. as needed +export function getDate(post) { + const dateTime = luxon.DateTime.fromRFC2822(post.data.pubDate[0], { zone: settings.custom_date_timezone }); + + if (settings.custom_date_formatting) { + return dateTime.toFormat(settings.custom_date_formatting); + } else if (settings.include_time_with_date) { + return dateTime.toISO(); + } else { + return dateTime.toISODate(); + } +} + +// get excerpt, not decoded, newlines collapsed +export function getExcerpt(post) { + return post.data.encoded[1].replace(/[\r\n]+/gm, ' '); +} + +// get ID +export function getId(post) { + return post.data.post_id[0]; +} + +// get slug, previously decoded and set on post.meta +export function getSlug(post) { + return post.meta.slug; +} + +// get array of decoded tag names +export function getTags(post) { + if (!post.data.category) { + return []; + } + + const categories = post.data.category + .filter(category => category.$.domain === 'post_tag') + .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); + + return categories; +} + +// get simple post title, but not decoded like other frontmatter string fields +export function getTitle(post) { + return post.data.title[0]; +} + +// get type, often this will always be "post" +// but can also be "page" or other custom types +export function getType(post) { + return post.data.post_type[0]; +} diff --git a/src/parser.js b/src/parser.js index 5fe5415..0e751db 100644 --- a/src/parser.js +++ b/src/parser.js @@ -1,15 +1,12 @@ -const fs = require('fs'); -const requireDirectory = require('require-directory'); -const xml2js = require('xml2js'); +import fs from 'fs'; +import xml2js from 'xml2js'; -const shared = require('./shared'); -const settings = require('./settings'); -const translator = require('./translator'); +import * as shared from './shared.js'; +import * as settings from './settings.js'; +import * as translator from './translator.js'; +import * as frontmatter from './frontmatter.js'; -// dynamically requires all frontmatter getters -const frontmatterGetters = requireDirectory(module, './frontmatter', { recurse: false }); - -async function parseFilePromise(config) { +export async function parseFilePromise(config) { console.log('\nParsing...'); const content = await fs.promises.readFile(config.input, 'utf8'); const allData = await xml2js.parseStringPromise(content, { @@ -174,19 +171,17 @@ function mergeImagesIntoPosts(images, posts) { function populateFrontmatter(posts) { posts.forEach(post => { - const frontmatter = {}; + post.frontmatter = {}; settings.frontmatter_fields.forEach(field => { const [key, alias] = field.split(':'); - let frontmatterGetter = frontmatterGetters[key]; + let frontmatterGetter = frontmatter['get' + key.replace(/^./, (match) => match.toUpperCase())]; if (!frontmatterGetter) { throw `Could not find a frontmatter getter named "${key}".`; } - frontmatter[alias || key] = frontmatterGetter(post); + post.frontmatter[alias || key] = frontmatterGetter(post); }); - post.frontmatter = frontmatter; }); } -exports.parseFilePromise = parseFilePromise; diff --git a/src/settings.js b/src/settings.js index 39e6334..195b8f8 100644 --- a/src/settings.js +++ b/src/settings.js @@ -2,7 +2,7 @@ // Order is preserved. If a field has an empty value, it will not be included. You can rename a // field by providing an alias after a ':'. For example, 'date:created' will include 'date' in // frontmatter, but renamed to 'created'. -exports.frontmatter_fields = [ +export const frontmatter_fields = [ 'title', 'date', 'categories', @@ -12,29 +12,29 @@ exports.frontmatter_fields = [ // Time in ms to wait between requesting image files. Increase this if you see timeouts or // server errors. -exports.image_file_request_delay = 500; +export const image_file_request_delay = 500; // Time in ms to wait between saving Markdown files. Increase this if your file system becomes // overloaded. -exports.markdown_file_write_delay = 25; +export const markdown_file_write_delay = 25; // Enable this to include time with post dates. For example, "2020-12-25" would become // "2020-12-25T11:20:35.000Z". -exports.include_time_with_date = false; +export const include_time_with_date = false; // Override post date formatting with a custom formatting string (for example: 'yyyy LLL dd'). // Tokens are documented here: https://moment.github.io/luxon/#/parsing?id=table-of-tokens. If // set, this takes precedence over include_time_with_date. -exports.custom_date_formatting = ''; +export const custom_date_formatting = ''; // Specify the timezone used for post dates. See available zone values and examples here: // https://moment.github.io/luxon/#/zones?id=specifying-a-zone. -exports.custom_date_timezone = 'utc'; +export const custom_date_timezone = 'utc'; // Categories to be excluded from post frontmatter. This does not filter out posts themselves, // just the categories listed in their frontmatter. -exports.filter_categories = ['uncategorized']; +export const filter_categories = ['uncategorized']; // Strict SSL is enabled as the safe default when downloading images, but will not work with // self-signed servers. You can disable it if you're getting a "self-signed certificate" error. -exports.strict_ssl = true; +export const strict_ssl = true; diff --git a/src/shared.js b/src/shared.js index 786f3ac..0853bfe 100644 --- a/src/shared.js +++ b/src/shared.js @@ -1,4 +1,4 @@ -function getFilenameFromUrl(url) { +export function getFilenameFromUrl(url) { let filename = url.split('/').slice(-1)[0]; try { filename = decodeURIComponent(filename) @@ -8,5 +8,3 @@ function getFilenameFromUrl(url) { } return filename; } - -exports.getFilenameFromUrl = getFilenameFromUrl; diff --git a/src/translator.js b/src/translator.js index c974599..2e43c35 100644 --- a/src/translator.js +++ b/src/translator.js @@ -1,7 +1,7 @@ -const turndown = require('turndown'); -const turndownPluginGfm = require('turndown-plugin-gfm'); +import turndown from 'turndown'; +import turndownPluginGfm from 'turndown-plugin-gfm'; -function initTurndownService() { +export function initTurndownService() { const turndownService = new turndown({ headingStyle: 'atx', bulletListMarker: '-', @@ -73,7 +73,7 @@ function initTurndownService() { // preserve
turndownService.addRule('figcaption', { filter: 'figcaption', - replacement: (content, node) => { + replacement: (content) => { // extra newlines are necessary for markdown and HTML to render correctly together return '\n\n
\n\n' + content + '\n\n
\n\n'; } @@ -94,7 +94,7 @@ function initTurndownService() { return turndownService; } -function getPostContent(postData, turndownService, config) { +export function getPostContent(postData, turndownService, config) { let content = postData.encoded[0]; // insert an empty div element between double line breaks @@ -124,6 +124,3 @@ function getPostContent(postData, turndownService, config) { return content; } - -exports.initTurndownService = initTurndownService; -exports.getPostContent = getPostContent; diff --git a/src/wizard.js b/src/wizard.js index 5c3a08c..2f95c16 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -1,10 +1,8 @@ -const camelcase = require('camelcase'); -const commander = require('commander'); -const fs = require('fs'); -const inquirer = require('inquirer'); -const path = require('path'); - -const package = require('../package.json'); +import camelcase from 'camelcase'; +import * as commander from 'commander'; +import fs from 'fs'; +import inquirer from 'inquirer'; +import path from 'path'; // all user options for command line and wizard are declared here const options = [ @@ -77,7 +75,7 @@ const options = [ } ]; -async function getConfig(argv) { +export async function getConfig(argv) { extendOptionsData(); const unaliasedArgv = replaceAliases(argv); const opts = parseCommandLine(unaliasedArgv); @@ -160,7 +158,6 @@ function parseCommandLine(argv) { // setup for help output commander.program .name('node index.js') - .version('v' + package.version, '-v, --version', 'Display version number') .helpOption('-h, --help', 'See the thing you\'re looking at right now') .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown'); @@ -180,7 +177,7 @@ function parseCommandLine(argv) { } function coerceBoolean(value) { - return !['false', 'no', '0'].includes(value.toLowerCase()); + return !['false', 'no', '0'].includes(value.toString().toLowerCase()); } function coercePath(value) { @@ -197,5 +194,3 @@ function validateFile(value) { return isValid ? true : 'Unable to find file: ' + path.resolve(value); } - -exports.getConfig = getConfig; diff --git a/src/writer.js b/src/writer.js index ed4f064..25bdb6e 100644 --- a/src/writer.js +++ b/src/writer.js @@ -1,15 +1,15 @@ -const axios = require('axios'); -const chalk = require('chalk'); -const fs = require('fs'); -const http = require('http'); -const https = require('https'); -const luxon = require('luxon'); -const path = require('path'); +import axios from 'axios'; +import chalk from 'chalk'; +import fs from 'fs'; +import http from 'http'; +import https from 'https'; +import * as luxon from 'luxon'; +import path from 'path'; -const shared = require('./shared'); -const settings = require('./settings'); +import * as shared from './shared.js'; +import * as settings from './settings.js'; -async function writeFilesPromise(posts, config) { +export async function writeFilesPromise(posts, config) { await writeMarkdownFilesPromise(posts, config); await writeImageFilesPromise(posts, config); } @@ -215,5 +215,3 @@ function getPostPath(post, config) { function checkFile(path) { return fs.existsSync(path); } - -exports.writeFilesPromise = writeFilesPromise; From 7101ae43dcd67c7baf644920a74bd726a4042a4f Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 18 Jan 2025 15:22:19 -0500 Subject: [PATCH 02/79] Bump inquirer down to v10 --- package-lock.json | 250 ++++++++++++++++++++++------------------------ package.json | 2 +- 2 files changed, 121 insertions(+), 131 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2a9e210..1f7a255 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "camelcase": "^8.0.0", "chalk": "^5.4.1", "commander": "^13.0.0", - "inquirer": "^12.3.2", + "inquirer": "^10.2.2", "luxon": "^3.5.0", "require-directory": "^2.1.1", "turndown": "^7.2.0", @@ -125,48 +125,45 @@ "dev": true }, "node_modules/@inquirer/checkbox": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.0.6.tgz", - "integrity": "sha512-PgP35JfmGjHU0LSXOyRew0zHuA9N6OJwOlos1fZ20b7j8ISeAdib3L+n0jIxBtX958UeEpte6xhG/gxJ5iUqMw==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-2.5.0.tgz", + "integrity": "sha512-sMgdETOfi2dUHT8r7TT1BTKOwNvdDGFDXYWtQ2J69SvlYNntk9I/gJe7r5yvMwwsuKnYbuRs3pNhx4tgNck5aA==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^9.1.0", + "@inquirer/figures": "^1.0.5", + "@inquirer/type": "^1.5.3", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/confirm": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.3.tgz", - "integrity": "sha512-fuF9laMmHoOgWapF9h9hv6opA5WvmGFHsTYGCmuFxcghIhEhb3dN0CdQR4BUMqa2H506NCj8cGX4jwMsE4t6dA==", + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.2.0.tgz", + "integrity": "sha512-oOIwPs0Dvq5220Z8lGL/6LHRTEr9TgLHmiI99Rj1PJ1p1czTys+olrgBqZk4E2qC0YTzeHprxSQmoHioVdJ7Lw==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2" + "@inquirer/core": "^9.1.0", + "@inquirer/type": "^1.5.3" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/core": { - "version": "10.1.4", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.4.tgz", - "integrity": "sha512-5y4/PUJVnRb4bwWY67KLdebWOhOc7xj5IP2J80oWXa64mVag24rwQ1VAdnj7/eDY/odhguW0zQ1Mp1pj6fO/2w==", + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.2.1.tgz", + "integrity": "sha512-F2VBt7W/mwqEU4bL0RnHNZmC/OxzNx9cOYxHqnXX3MP6ruYvZUZAW9imgN9+h/uBT/oP8Gh888J2OZSbjSeWcg==", "dependencies": { - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/figures": "^1.0.6", + "@inquirer/type": "^2.0.0", + "@types/mute-stream": "^0.0.4", + "@types/node": "^22.5.5", + "@types/wrap-ansi": "^3.0.0", "ansi-escapes": "^4.3.2", "cli-width": "^4.1.0", - "mute-stream": "^2.0.0", + "mute-stream": "^1.0.0", "signal-exit": "^4.1.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^6.2.0", @@ -176,36 +173,41 @@ "node": ">=18" } }, - "node_modules/@inquirer/editor": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.3.tgz", - "integrity": "sha512-S9KnIOJuTZpb9upeRSBBhoDZv7aSV3pG9TECrBj0f+ZsFwccz886hzKBrChGrXMJwd4NKY+pOA9Vy72uqnd6Eg==", + "node_modules/@inquirer/core/node_modules/@inquirer/type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-2.0.0.tgz", + "integrity": "sha512-XvJRx+2KR3YXyYtPUUy+qd9i7p+GO9Ko6VIIpWlBrpWwXDv8WLFeHTxz35CfQFUiBMLXlGHhGzys7lqit9gWag==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "mute-stream": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/editor": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-2.2.0.tgz", + "integrity": "sha512-9KHOpJ+dIL5SZli8lJ6xdaYLPPzB8xB9GZItg39MBybzhxA16vxmszmQFrRwbOA918WA2rvu8xhDEg/p6LXKbw==", + "dependencies": { + "@inquirer/core": "^9.1.0", + "@inquirer/type": "^1.5.3", "external-editor": "^3.1.0" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/expand": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.6.tgz", - "integrity": "sha512-TRTfi1mv1GeIZGyi9PQmvAaH65ZlG4/FACq6wSzs7Vvf1z5dnNWsAAXBjWMHt76l+1hUY8teIqJFrWBk5N6gsg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-2.3.0.tgz", + "integrity": "sha512-qnJsUcOGCSG1e5DTOErmv2BPQqrtT6uzqn1vI/aYGiPKq+FgslGZmtdnXbhuI7IlT7OByDoEEqdnhUnVR2hhLw==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^9.1.0", + "@inquirer/type": "^1.5.3", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/figures": { @@ -217,134 +219,113 @@ } }, "node_modules/@inquirer/input": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.3.tgz", - "integrity": "sha512-zeo++6f7hxaEe7OjtMzdGZPHiawsfmCZxWB9X1NpmYgbeoyerIbWemvlBxxl+sQIlHC0WuSAG19ibMq3gbhaqQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-2.3.0.tgz", + "integrity": "sha512-XfnpCStx2xgh1LIRqPXrTNEEByqQWoxsWYzNRSEUxJ5c6EQlhMogJ3vHKu8aXuTacebtaZzMAHwEL0kAflKOBw==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2" + "@inquirer/core": "^9.1.0", + "@inquirer/type": "^1.5.3" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/number": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.6.tgz", - "integrity": "sha512-xO07lftUHk1rs1gR0KbqB+LJPhkUNkyzV/KhH+937hdkMazmAYHLm1OIrNKpPelppeV1FgWrgFDjdUD8mM+XUg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-1.1.0.tgz", + "integrity": "sha512-ilUnia/GZUtfSZy3YEErXLJ2Sljo/mf9fiKc08n18DdwdmDbOzRcTv65H1jjDvlsAuvdFXf4Sa/aL7iw/NanVA==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2" + "@inquirer/core": "^9.1.0", + "@inquirer/type": "^1.5.3" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/password": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.6.tgz", - "integrity": "sha512-QLF0HmMpHZPPMp10WGXh6F+ZPvzWE7LX6rNoccdktv/Rov0B+0f+eyXkAcgqy5cH9V+WSpbLxu2lo3ysEVK91w==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-2.2.0.tgz", + "integrity": "sha512-5otqIpgsPYIshqhgtEwSspBQE40etouR8VIxzpJkv9i0dVHIpyhiivbkH9/dGiMLdyamT54YRdGJLfl8TFnLHg==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^9.1.0", + "@inquirer/type": "^1.5.3", "ansi-escapes": "^4.3.2" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/prompts": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.2.3.tgz", - "integrity": "sha512-hzfnm3uOoDySDXfDNOm9usOuYIaQvTgKp/13l1uJoe6UNY+Zpcn2RYt0jXz3yA+yemGHvDOxVzqWl3S5sQq53Q==", + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-5.5.0.tgz", + "integrity": "sha512-BHDeL0catgHdcHbSFFUddNzvx/imzJMft+tWDPwTm3hfu8/tApk1HrooNngB2Mb4qY+KaRWF+iZqoVUPeslEog==", "dependencies": { - "@inquirer/checkbox": "^4.0.6", - "@inquirer/confirm": "^5.1.3", - "@inquirer/editor": "^4.2.3", - "@inquirer/expand": "^4.0.6", - "@inquirer/input": "^4.1.3", - "@inquirer/number": "^3.0.6", - "@inquirer/password": "^4.0.6", - "@inquirer/rawlist": "^4.0.6", - "@inquirer/search": "^3.0.6", - "@inquirer/select": "^4.0.6" + "@inquirer/checkbox": "^2.5.0", + "@inquirer/confirm": "^3.2.0", + "@inquirer/editor": "^2.2.0", + "@inquirer/expand": "^2.3.0", + "@inquirer/input": "^2.3.0", + "@inquirer/number": "^1.1.0", + "@inquirer/password": "^2.2.0", + "@inquirer/rawlist": "^2.3.0", + "@inquirer/search": "^1.1.0", + "@inquirer/select": "^2.5.0" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/rawlist": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.6.tgz", - "integrity": "sha512-QoE4s1SsIPx27FO4L1b1mUjVcoHm1pWE/oCmm4z/Hl+V1Aw5IXl8FYYzGmfXaBT0l/sWr49XmNSiq7kg3Kd/Lg==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-2.3.0.tgz", + "integrity": "sha512-zzfNuINhFF7OLAtGHfhwOW2TlYJyli7lOUoJUXw/uyklcwalV6WRXBXtFIicN8rTRK1XTiPWB4UY+YuW8dsnLQ==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^9.1.0", + "@inquirer/type": "^1.5.3", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/search": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.6.tgz", - "integrity": "sha512-eFZ2hiAq0bZcFPuFFBmZEtXU1EarHLigE+ENCtpO+37NHCl4+Yokq1P/d09kUblObaikwfo97w+0FtG/EXl5Ng==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-1.1.0.tgz", + "integrity": "sha512-h+/5LSj51dx7hp5xOn4QFnUaKeARwUCLs6mIhtkJ0JYPBLmEYjdHSYh7I6GrLg9LwpJ3xeX0FZgAG1q0QdCpVQ==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^9.1.0", + "@inquirer/figures": "^1.0.5", + "@inquirer/type": "^1.5.3", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/select": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.0.6.tgz", - "integrity": "sha512-yANzIiNZ8fhMm4NORm+a74+KFYHmf7BZphSOBovIzYPVLquseTGEkU5l2UTnBOf5k0VLmTgPighNDLE9QtbViQ==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-2.5.0.tgz", + "integrity": "sha512-YmDobTItPP3WcEI86GvPo+T2sRHkxxOq/kXmsBjHS5BVXUgvgZ5AfJjkvQvZr03T81NnI3KrrRuMzeuYUQRFOA==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^9.1.0", + "@inquirer/figures": "^1.0.5", + "@inquirer/type": "^1.5.3", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@inquirer/type": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.2.tgz", - "integrity": "sha512-ZhQ4TvhwHZF+lGhQ2O/rsjo80XoZR5/5qhOY3t6FJuX5XBg5Be8YzYTvaUGJnc12AUGI2nr4QSUE4PhKSigx7g==", + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.5.tgz", + "integrity": "sha512-MzICLu4yS7V8AA61sANROZ9vT1H3ooca5dSmI1FjZkzq7o/koMsRfQSzRtFo+F3Ao4Sf1C0bpLKejpKB/+j6MA==", + "dependencies": { + "mute-stream": "^1.0.0" + }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/@mixmark-io/domino": { @@ -387,15 +368,27 @@ "node": ">= 8" } }, + "node_modules/@types/mute-stream": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", + "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/node": { "version": "22.10.7", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.7.tgz", "integrity": "sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==", - "peer": true, "dependencies": { "undici-types": "~6.20.0" } }, + "node_modules/@types/wrap-ansi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", + "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==" + }, "node_modules/@ungap/structured-clone": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.1.tgz", @@ -1093,23 +1086,21 @@ "dev": true }, "node_modules/inquirer": { - "version": "12.3.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-12.3.2.tgz", - "integrity": "sha512-YjQCIcDd3yyDuQrbII0FBtm/ZqNoWtvaC71yeCnd5Vbg4EgzsAGaemzfpzmqfvIZEp2roSwuZZKdM0C65hA43g==", + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-10.2.2.tgz", + "integrity": "sha512-tyao/4Vo36XnUItZ7DnUXX4f1jVao2mSrleV/5IPtW/XAEA26hRVsbc68nuTEKWcr5vMP/1mVoT2O7u8H4v1Vg==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/prompts": "^7.2.3", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^9.1.0", + "@inquirer/prompts": "^5.5.0", + "@inquirer/type": "^1.5.3", + "@types/mute-stream": "^0.0.4", "ansi-escapes": "^4.3.2", - "mute-stream": "^2.0.0", + "mute-stream": "^1.0.0", "run-async": "^3.0.0", "rxjs": "^7.8.1" }, "engines": { "node": ">=18" - }, - "peerDependencies": { - "@types/node": ">=18" } }, "node_modules/is-extglob": { @@ -1275,11 +1266,11 @@ "dev": true }, "node_modules/mute-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", - "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", "engines": { - "node": "^18.17.0 || >=20.5.0" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, "node_modules/natural-compare": { @@ -1668,8 +1659,7 @@ "node_modules/undici-types": { "version": "6.20.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "peer": true + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" }, "node_modules/uri-js": { "version": "4.4.1", diff --git a/package.json b/package.json index 4abd834..32248c1 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "camelcase": "^8.0.0", "chalk": "^5.4.1", "commander": "^13.0.0", - "inquirer": "^12.3.2", + "inquirer": "^10.2.2", "luxon": "^3.5.0", "require-directory": "^2.1.1", "turndown": "^7.2.0", From c6a6c1e1ccfa5397f5554b463b611411ecc6eab5 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 20 Jan 2025 11:49:26 -0500 Subject: [PATCH 03/79] Sort imports, delete old frontmatter getters --- index.js | 3 +-- src/frontmatter.js | 1 - src/frontmatter/author.js | 5 ----- src/frontmatter/categories.js | 14 -------------- src/frontmatter/coverImage.js | 5 ----- src/frontmatter/date.js | 17 ----------------- src/frontmatter/example.js | 19 ------------------- src/frontmatter/excerpt.js | 4 ---- src/frontmatter/id.js | 4 ---- src/frontmatter/slug.js | 4 ---- src/frontmatter/tags.js | 12 ------------ src/frontmatter/title.js | 4 ---- src/frontmatter/type.js | 5 ----- src/parser.js | 7 +++---- src/writer.js | 3 +-- 15 files changed, 5 insertions(+), 102 deletions(-) delete mode 100644 src/frontmatter/author.js delete mode 100644 src/frontmatter/categories.js delete mode 100644 src/frontmatter/coverImage.js delete mode 100644 src/frontmatter/date.js delete mode 100644 src/frontmatter/example.js delete mode 100644 src/frontmatter/excerpt.js delete mode 100644 src/frontmatter/id.js delete mode 100644 src/frontmatter/slug.js delete mode 100644 src/frontmatter/tags.js delete mode 100644 src/frontmatter/title.js delete mode 100644 src/frontmatter/type.js diff --git a/index.js b/index.js index 1e3bdf5..cd286c6 100644 --- a/index.js +++ b/index.js @@ -2,9 +2,8 @@ import path from 'path'; import process from 'process'; - -import * as wizard from './src/wizard.js'; import * as parser from './src/parser.js'; +import * as wizard from './src/wizard.js'; import * as writer from './src/writer.js'; (async () => { diff --git a/src/frontmatter.js b/src/frontmatter.js index 43eb6dd..390815a 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -1,5 +1,4 @@ import * as luxon from 'luxon'; - import * as settings from './settings.js'; // get author, without decoding diff --git a/src/frontmatter/author.js b/src/frontmatter/author.js deleted file mode 100644 index e6a9c4f..0000000 --- a/src/frontmatter/author.js +++ /dev/null @@ -1,5 +0,0 @@ -// get author, without decoding -// WordPress doesn't allow funky characters in usernames anyway -module.exports = (post) => { - return post.data.creator[0]; -} diff --git a/src/frontmatter/categories.js b/src/frontmatter/categories.js deleted file mode 100644 index 0328e0f..0000000 --- a/src/frontmatter/categories.js +++ /dev/null @@ -1,14 +0,0 @@ -const settings = require('../settings'); - -// get array of decoded category names, filtered as specified in settings -module.exports = (post) => { - if (!post.data.category) { - return []; - } - - const categories = post.data.category - .filter(category => category.$.domain === 'category') - .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); - - return categories.filter(category => !settings.filter_categories.includes(category)); -}; diff --git a/src/frontmatter/coverImage.js b/src/frontmatter/coverImage.js deleted file mode 100644 index ea63c6b..0000000 --- a/src/frontmatter/coverImage.js +++ /dev/null @@ -1,5 +0,0 @@ -// get cover image filename, previously decoded and set on post.meta -// this one is unique as it relies on special logic executed by the parser -module.exports = (post) => { - return post.meta.coverImage; -}; diff --git a/src/frontmatter/date.js b/src/frontmatter/date.js deleted file mode 100644 index 0cbe1bb..0000000 --- a/src/frontmatter/date.js +++ /dev/null @@ -1,17 +0,0 @@ -const luxon = require('luxon'); - -const settings = require('../settings'); - -// get post date, optionally formatted as specified in settings -// this value is also used for year/month folders, date prefixes, etc. as needed -module.exports = (post) => { - const dateTime = luxon.DateTime.fromRFC2822(post.data.pubDate[0], { zone: settings.custom_date_timezone }); - - if (settings.custom_date_formatting) { - return dateTime.toFormat(settings.custom_date_formatting); - } else if (settings.include_time_with_date) { - return dateTime.toISO(); - } else { - return dateTime.toISODate(); - } -}; diff --git a/src/frontmatter/example.js b/src/frontmatter/example.js deleted file mode 100644 index 2159d87..0000000 --- a/src/frontmatter/example.js +++ /dev/null @@ -1,19 +0,0 @@ -/* - 1. Copy this file, rename to the frontmatter field name you want, camelcased - 2. Edit frontmatter_fields in settings.js to include your new field name - 3. Run the script to see post data dumps, to see what you can work with - 4. Write your code to get and return what you want - 5. Update "get whatever" comment to describe what you're getting - 6. Remove your field name from frontmatter_fields in settings.js - 7. Remove this comment block and the debug console code - 8. Make that pull request! -*/ - -// get whatever -module.exports = (post) => { - console.log('\nBEGIN POST DATA DUMP ===========================================================\n'); - console.dir(post, { depth: null }); - console.log('\nEND POST DATA DUMP =============================================================\n'); - - return 'EXAMPLE: ' + post.data.title[0]; -}; diff --git a/src/frontmatter/excerpt.js b/src/frontmatter/excerpt.js deleted file mode 100644 index bc6310d..0000000 --- a/src/frontmatter/excerpt.js +++ /dev/null @@ -1,4 +0,0 @@ -// get excerpt, not decoded, newlines collapsed -module.exports = (post) => { - return post.data.encoded[1].replace(/[\r\n]+/gm, ' '); -}; diff --git a/src/frontmatter/id.js b/src/frontmatter/id.js deleted file mode 100644 index 5403fc0..0000000 --- a/src/frontmatter/id.js +++ /dev/null @@ -1,4 +0,0 @@ -// get ID -module.exports = (post) => { - return post.data.post_id[0]; -} diff --git a/src/frontmatter/slug.js b/src/frontmatter/slug.js deleted file mode 100644 index 7664b22..0000000 --- a/src/frontmatter/slug.js +++ /dev/null @@ -1,4 +0,0 @@ -// get slug, previously decoded and set on post.meta -module.exports = (post) => { - return post.meta.slug; -}; diff --git a/src/frontmatter/tags.js b/src/frontmatter/tags.js deleted file mode 100644 index 0b53fd8..0000000 --- a/src/frontmatter/tags.js +++ /dev/null @@ -1,12 +0,0 @@ -// get array of decoded tag names -module.exports = (post) => { - if (!post.data.category) { - return []; - } - - const categories = post.data.category - .filter(category => category.$.domain === 'post_tag') - .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); - - return categories; -}; diff --git a/src/frontmatter/title.js b/src/frontmatter/title.js deleted file mode 100644 index b10bd29..0000000 --- a/src/frontmatter/title.js +++ /dev/null @@ -1,4 +0,0 @@ -// get simple post title, but not decoded like other frontmatter string fields -module.exports = (post) => { - return post.data.title[0]; -}; diff --git a/src/frontmatter/type.js b/src/frontmatter/type.js deleted file mode 100644 index 25bfab4..0000000 --- a/src/frontmatter/type.js +++ /dev/null @@ -1,5 +0,0 @@ -// get type, often this will always be "post" -// but can also be "page" or other custom types -module.exports = (post) => { - return post.data.post_type[0]; -} diff --git a/src/parser.js b/src/parser.js index 0e751db..4083dd0 100644 --- a/src/parser.js +++ b/src/parser.js @@ -1,10 +1,9 @@ import fs from 'fs'; import xml2js from 'xml2js'; - -import * as shared from './shared.js'; -import * as settings from './settings.js'; -import * as translator from './translator.js'; import * as frontmatter from './frontmatter.js'; +import * as settings from './settings.js'; +import * as shared from './shared.js'; +import * as translator from './translator.js'; export async function parseFilePromise(config) { console.log('\nParsing...'); diff --git a/src/writer.js b/src/writer.js index 25bdb6e..510eaf3 100644 --- a/src/writer.js +++ b/src/writer.js @@ -5,9 +5,8 @@ import http from 'http'; import https from 'https'; import * as luxon from 'luxon'; import path from 'path'; - -import * as shared from './shared.js'; import * as settings from './settings.js'; +import * as shared from './shared.js'; export async function writeFilesPromise(posts, config) { await writeMarkdownFilesPromise(posts, config); From efd1423bde3ff4492dd7bc399f219e4a4edfd59c Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 20 Jan 2025 11:56:58 -0500 Subject: [PATCH 04/79] Update documentation for frontmatter.js change --- CONTRIBUTING.md | 4 +--- src/settings.js | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbfa64b..bdf4c6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,8 +16,6 @@ Keeping the wizard as short as possible is a priority. Pull requests that add op ## Adding Frontmatter Fields -Similarly, default frontmatter output is limited to just a few widely used fields to avoid bloat. However, you may add new optional frontmatter fields. - -To do so, follow the instructions in [/src/frontmatter/example.js](https://github.com/lonekorean/wordpress-export-to-markdown/blob/master/src/frontmatter/example.js). +Similarly, default frontmatter output is limited to just a few widely used fields to avoid bloat. However, you may add new optional frontmatter fields to [/src/frontmatter.js](https://github.com/lonekorean/wordpress-export-to-markdown/blob/master/src/frontmatter.js). Users will be able to include your new frontmatter field by editing `frontmatter_fields` in [settings.js](https://github.com/lonekorean/wordpress-export-to-markdown/blob/master/src/settings.js). diff --git a/src/settings.js b/src/settings.js index 195b8f8..111ed0e 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1,4 +1,4 @@ -// Which fields to include in frontmatter. Look in /src/frontmatter to see available fields. +// Which fields to include in frontmatter. Look in /src/frontmatter.js to see available fields. // Order is preserved. If a field has an empty value, it will not be included. You can rename a // field by providing an alias after a ':'. For example, 'date:created' will include 'date' in // frontmatter, but renamed to 'created'. From 8f2b387f6f3e28969b04467c01278fc4da3b220d Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 20 Jan 2025 12:19:31 -0500 Subject: [PATCH 05/79] Remove require-directory --- package-lock.json | 9 --------- package.json | 1 - 2 files changed, 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1f7a255..70998ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,6 @@ "commander": "^13.0.0", "inquirer": "^10.2.2", "luxon": "^3.5.0", - "require-directory": "^2.1.1", "turndown": "^7.2.0", "turndown-plugin-gfm": "^1.0.2", "xml2js": "^0.6.2" @@ -1425,14 +1424,6 @@ } ] }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", diff --git a/package.json b/package.json index 32248c1..cb12728 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,6 @@ "commander": "^13.0.0", "inquirer": "^10.2.2", "luxon": "^3.5.0", - "require-directory": "^2.1.1", "turndown": "^7.2.0", "turndown-plugin-gfm": "^1.0.2", "xml2js": "^0.6.2" From ed9aad04e29dc77147362f1dd835c8fb36995616 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 20 Jan 2025 15:30:19 -0500 Subject: [PATCH 06/79] Remove aliases --- src/wizard.js | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/src/wizard.js b/src/wizard.js index 2f95c16..e5d4b1a 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -27,42 +27,36 @@ const options = [ }, { name: 'year-folders', - aliases: ['yearfolders', 'yearmonthfolders'], type: 'boolean', description: 'Create year folders', default: false }, { name: 'month-folders', - aliases: ['yearmonthfolders'], type: 'boolean', description: 'Create month folders', default: false }, { name: 'post-folders', - aliases: ['postfolders'], type: 'boolean', description: 'Create a folder for each post', default: true }, { name: 'prefix-date', - aliases: ['prefixdate'], type: 'boolean', description: 'Prefix post folders/files with date', default: false }, { name: 'save-attached-images', - aliases: ['saveimages'], type: 'boolean', description: 'Save images attached to posts', default: true }, { name: 'save-scraped-images', - aliases: ['addcontentimages'], type: 'boolean', description: 'Save images scraped from post body content', default: true @@ -77,8 +71,7 @@ const options = [ export async function getConfig(argv) { extendOptionsData(); - const unaliasedArgv = replaceAliases(argv); - const opts = parseCommandLine(unaliasedArgv); + const opts = parseCommandLine(argv); let answers; if (opts.wizard) { @@ -127,33 +120,6 @@ function extendOptionsData() { }); } -function replaceAliases(argv) { - let paths = argv.slice(0, 2); - let replaced = []; - let unmodified = []; - - argv.slice(2).forEach(arg => { - let aliasFound = false; - - // this loop does not short circuit because an alias can map to multiple options - options.forEach(option => { - const aliases = option.aliases || []; - aliases.forEach(alias => { - if (arg.includes('--' + alias)) { - replaced.push(arg.replace('--' + alias, '--' + option.name)); - aliasFound = true; - } - }); - }); - - if (!aliasFound) { - unmodified.push(arg); - } - }); - - return [...paths, ...replaced, ...unmodified]; -} - function parseCommandLine(argv) { // setup for help output commander.program From 2add3804061f85365458203f49f135a94f38d90e Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 20 Jan 2025 16:09:27 -0500 Subject: [PATCH 07/79] Remove include-other-types option --- src/parser.js | 20 +++++++------------- src/settings.js | 11 +++++++++++ src/wizard.js | 6 ------ src/writer.js | 11 +++-------- 4 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/parser.js b/src/parser.js index 4083dd0..a4cbadc 100644 --- a/src/parser.js +++ b/src/parser.js @@ -14,7 +14,7 @@ export async function parseFilePromise(config) { }); const channelData = allData.rss.channel[0].item; - const postTypes = getPostTypes(channelData, config); + const postTypes = getPostTypes(channelData); const posts = collectPosts(channelData, postTypes, config); const images = []; @@ -31,18 +31,12 @@ export async function parseFilePromise(config) { return posts; } -function getPostTypes(channelData, config) { - if (config.includeOtherTypes) { - // search export file for all post types minus some default types we don't want - // effectively this will be 'post', 'page', and custom post types - const types = channelData - .map(item => item.post_type[0]) - .filter(type => !['attachment', 'revision', 'nav_menu_item', 'custom_css', 'customize_changeset'].includes(type)); - return [...new Set(types)]; // remove duplicates - } else { - // just plain old vanilla "post" posts - return ['post']; - } +function getPostTypes(channelData) { + // search export file for all post types minus some specific types we don't want + const types = channelData + .map(item => item.post_type[0]) + .filter(type => !settings.filter_post_types.includes(type)); + return [...new Set(types)]; // remove duplicates } function getItemsOfType(channelData, type) { diff --git a/src/settings.js b/src/settings.js index 111ed0e..58e67b3 100644 --- a/src/settings.js +++ b/src/settings.js @@ -38,3 +38,14 @@ export const filter_categories = ['uncategorized']; // Strict SSL is enabled as the safe default when downloading images, but will not work with // self-signed servers. You can disable it if you're getting a "self-signed certificate" error. export const strict_ssl = true; + +// Post types to exclude from output. +export const filter_post_types = [ + 'attachment', + 'revision', + 'nav_menu_item', + 'custom_css', + 'customize_changeset', + 'wp_global_styles', + 'wp_navigation' +]; diff --git a/src/wizard.js b/src/wizard.js index e5d4b1a..e625a47 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -60,12 +60,6 @@ const options = [ type: 'boolean', description: 'Save images scraped from post body content', default: true - }, - { - name: 'include-other-types', - type: 'boolean', - description: 'Include custom post types and pages', - default: false } ]; diff --git a/src/writer.js b/src/writer.js index 510eaf3..7df7520 100644 --- a/src/writer.js +++ b/src/writer.js @@ -55,7 +55,7 @@ async function writeMarkdownFilesPromise(posts, config) { } else { const payload = { item: post, - name: (config.includeOtherTypes ? post.meta.type + ' - ' : '') + post.meta.slug, + name: post.meta.type + ' - ' + post.meta.slug, destinationPath, delay }; @@ -179,13 +179,8 @@ function getPostPath(post, config) { dt = luxon.DateTime.fromISO(post.frontmatter.date); } - // start with base output dir - const pathSegments = [config.output]; - - // create segment for post type if we're dealing with more than just "post" - if (config.includeOtherTypes) { - pathSegments.push(post.meta.type); - } + // start with base output dir and post type + const pathSegments = [config.output, post.meta.type]; if (config.yearFolders) { pathSegments.push(dt.toFormat('yyyy')); From bafe8692213d778190bdcf4bce783d986e57e4f7 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 20 Jan 2025 16:28:36 -0500 Subject: [PATCH 08/79] Remove output option --- index.js | 3 ++- src/settings.js | 3 +++ src/wizard.js | 6 ------ src/writer.js | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index cd286c6..f1a9298 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ import path from 'path'; import process from 'process'; import * as parser from './src/parser.js'; +import * as settings from './src/settings.js'; import * as wizard from './src/wizard.js'; import * as writer from './src/writer.js'; @@ -18,7 +19,7 @@ import * as writer from './src/writer.js'; // happy goodbye console.log('\nAll done!'); - console.log('Look for your output files in: ' + path.resolve(config.output)); + console.log('Look for your output files in: ' + path.resolve(settings.output_directory)); })().catch(ex => { // sad goodbye console.log('\nSomething went wrong, execution halted early.'); diff --git a/src/settings.js b/src/settings.js index 58e67b3..28ef891 100644 --- a/src/settings.js +++ b/src/settings.js @@ -49,3 +49,6 @@ export const filter_post_types = [ 'wp_global_styles', 'wp_navigation' ]; + +// Output directory. +export const output_directory = 'output'; diff --git a/src/wizard.js b/src/wizard.js index e625a47..2436439 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -19,12 +19,6 @@ const options = [ description: 'Path to WordPress export file', default: 'export.xml' }, - { - name: 'output', - type: 'folder', - description: 'Path to output folder', - default: 'output' - }, { name: 'year-folders', type: 'boolean', diff --git a/src/writer.js b/src/writer.js index 7df7520..0b4ea54 100644 --- a/src/writer.js +++ b/src/writer.js @@ -180,7 +180,7 @@ function getPostPath(post, config) { } // start with base output dir and post type - const pathSegments = [config.output, post.meta.type]; + const pathSegments = [settings.output_directory, post.meta.type]; if (config.yearFolders) { pathSegments.push(dt.toFormat('yyyy')); From 3f4b2449350a493e3f64995e4db192ade0db74ed Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Tue, 21 Jan 2025 10:32:28 -0500 Subject: [PATCH 09/79] Update inquirer and fix code --- package-lock.json | 305 +++++++++++++++++++++------------------------- package.json | 2 +- src/wizard.js | 32 ++--- 3 files changed, 155 insertions(+), 184 deletions(-) diff --git a/package-lock.json b/package-lock.json index 70998ba..73a51c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,11 @@ "version": "2.4.2", "license": "MIT", "dependencies": { + "@inquirer/prompts": "^7.2.3", "axios": "^1.7.9", "camelcase": "^8.0.0", "chalk": "^5.4.1", "commander": "^13.0.0", - "inquirer": "^10.2.2", "luxon": "^3.5.0", "turndown": "^7.2.0", "turndown-plugin-gfm": "^1.0.2", @@ -124,45 +124,48 @@ "dev": true }, "node_modules/@inquirer/checkbox": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-2.5.0.tgz", - "integrity": "sha512-sMgdETOfi2dUHT8r7TT1BTKOwNvdDGFDXYWtQ2J69SvlYNntk9I/gJe7r5yvMwwsuKnYbuRs3pNhx4tgNck5aA==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.0.6.tgz", + "integrity": "sha512-PgP35JfmGjHU0LSXOyRew0zHuA9N6OJwOlos1fZ20b7j8ISeAdib3L+n0jIxBtX958UeEpte6xhG/gxJ5iUqMw==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/figures": "^1.0.5", - "@inquirer/type": "^1.5.3", + "@inquirer/core": "^10.1.4", + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/confirm": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-3.2.0.tgz", - "integrity": "sha512-oOIwPs0Dvq5220Z8lGL/6LHRTEr9TgLHmiI99Rj1PJ1p1czTys+olrgBqZk4E2qC0YTzeHprxSQmoHioVdJ7Lw==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.3.tgz", + "integrity": "sha512-fuF9laMmHoOgWapF9h9hv6opA5WvmGFHsTYGCmuFxcghIhEhb3dN0CdQR4BUMqa2H506NCj8cGX4jwMsE4t6dA==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/type": "^1.5.3" + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/core": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.2.1.tgz", - "integrity": "sha512-F2VBt7W/mwqEU4bL0RnHNZmC/OxzNx9cOYxHqnXX3MP6ruYvZUZAW9imgN9+h/uBT/oP8Gh888J2OZSbjSeWcg==", + "version": "10.1.4", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.4.tgz", + "integrity": "sha512-5y4/PUJVnRb4bwWY67KLdebWOhOc7xj5IP2J80oWXa64mVag24rwQ1VAdnj7/eDY/odhguW0zQ1Mp1pj6fO/2w==", "dependencies": { - "@inquirer/figures": "^1.0.6", - "@inquirer/type": "^2.0.0", - "@types/mute-stream": "^0.0.4", - "@types/node": "^22.5.5", - "@types/wrap-ansi": "^3.0.0", + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", "ansi-escapes": "^4.3.2", "cli-width": "^4.1.0", - "mute-stream": "^1.0.0", + "mute-stream": "^2.0.0", "signal-exit": "^4.1.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^6.2.0", @@ -172,41 +175,36 @@ "node": ">=18" } }, - "node_modules/@inquirer/core/node_modules/@inquirer/type": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-2.0.0.tgz", - "integrity": "sha512-XvJRx+2KR3YXyYtPUUy+qd9i7p+GO9Ko6VIIpWlBrpWwXDv8WLFeHTxz35CfQFUiBMLXlGHhGzys7lqit9gWag==", - "dependencies": { - "mute-stream": "^1.0.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/@inquirer/editor": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-2.2.0.tgz", - "integrity": "sha512-9KHOpJ+dIL5SZli8lJ6xdaYLPPzB8xB9GZItg39MBybzhxA16vxmszmQFrRwbOA918WA2rvu8xhDEg/p6LXKbw==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.3.tgz", + "integrity": "sha512-S9KnIOJuTZpb9upeRSBBhoDZv7aSV3pG9TECrBj0f+ZsFwccz886hzKBrChGrXMJwd4NKY+pOA9Vy72uqnd6Eg==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/type": "^1.5.3", + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", "external-editor": "^3.1.0" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/expand": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-2.3.0.tgz", - "integrity": "sha512-qnJsUcOGCSG1e5DTOErmv2BPQqrtT6uzqn1vI/aYGiPKq+FgslGZmtdnXbhuI7IlT7OByDoEEqdnhUnVR2hhLw==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.6.tgz", + "integrity": "sha512-TRTfi1mv1GeIZGyi9PQmvAaH65ZlG4/FACq6wSzs7Vvf1z5dnNWsAAXBjWMHt76l+1hUY8teIqJFrWBk5N6gsg==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/type": "^1.5.3", + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/figures": { @@ -218,113 +216,134 @@ } }, "node_modules/@inquirer/input": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-2.3.0.tgz", - "integrity": "sha512-XfnpCStx2xgh1LIRqPXrTNEEByqQWoxsWYzNRSEUxJ5c6EQlhMogJ3vHKu8aXuTacebtaZzMAHwEL0kAflKOBw==", + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.3.tgz", + "integrity": "sha512-zeo++6f7hxaEe7OjtMzdGZPHiawsfmCZxWB9X1NpmYgbeoyerIbWemvlBxxl+sQIlHC0WuSAG19ibMq3gbhaqQ==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/type": "^1.5.3" + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/number": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-1.1.0.tgz", - "integrity": "sha512-ilUnia/GZUtfSZy3YEErXLJ2Sljo/mf9fiKc08n18DdwdmDbOzRcTv65H1jjDvlsAuvdFXf4Sa/aL7iw/NanVA==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.6.tgz", + "integrity": "sha512-xO07lftUHk1rs1gR0KbqB+LJPhkUNkyzV/KhH+937hdkMazmAYHLm1OIrNKpPelppeV1FgWrgFDjdUD8mM+XUg==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/type": "^1.5.3" + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/password": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-2.2.0.tgz", - "integrity": "sha512-5otqIpgsPYIshqhgtEwSspBQE40etouR8VIxzpJkv9i0dVHIpyhiivbkH9/dGiMLdyamT54YRdGJLfl8TFnLHg==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.6.tgz", + "integrity": "sha512-QLF0HmMpHZPPMp10WGXh6F+ZPvzWE7LX6rNoccdktv/Rov0B+0f+eyXkAcgqy5cH9V+WSpbLxu2lo3ysEVK91w==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/type": "^1.5.3", + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", "ansi-escapes": "^4.3.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/prompts": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-5.5.0.tgz", - "integrity": "sha512-BHDeL0catgHdcHbSFFUddNzvx/imzJMft+tWDPwTm3hfu8/tApk1HrooNngB2Mb4qY+KaRWF+iZqoVUPeslEog==", + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.2.3.tgz", + "integrity": "sha512-hzfnm3uOoDySDXfDNOm9usOuYIaQvTgKp/13l1uJoe6UNY+Zpcn2RYt0jXz3yA+yemGHvDOxVzqWl3S5sQq53Q==", "dependencies": { - "@inquirer/checkbox": "^2.5.0", - "@inquirer/confirm": "^3.2.0", - "@inquirer/editor": "^2.2.0", - "@inquirer/expand": "^2.3.0", - "@inquirer/input": "^2.3.0", - "@inquirer/number": "^1.1.0", - "@inquirer/password": "^2.2.0", - "@inquirer/rawlist": "^2.3.0", - "@inquirer/search": "^1.1.0", - "@inquirer/select": "^2.5.0" + "@inquirer/checkbox": "^4.0.6", + "@inquirer/confirm": "^5.1.3", + "@inquirer/editor": "^4.2.3", + "@inquirer/expand": "^4.0.6", + "@inquirer/input": "^4.1.3", + "@inquirer/number": "^3.0.6", + "@inquirer/password": "^4.0.6", + "@inquirer/rawlist": "^4.0.6", + "@inquirer/search": "^3.0.6", + "@inquirer/select": "^4.0.6" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/rawlist": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-2.3.0.tgz", - "integrity": "sha512-zzfNuINhFF7OLAtGHfhwOW2TlYJyli7lOUoJUXw/uyklcwalV6WRXBXtFIicN8rTRK1XTiPWB4UY+YuW8dsnLQ==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.6.tgz", + "integrity": "sha512-QoE4s1SsIPx27FO4L1b1mUjVcoHm1pWE/oCmm4z/Hl+V1Aw5IXl8FYYzGmfXaBT0l/sWr49XmNSiq7kg3Kd/Lg==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/type": "^1.5.3", + "@inquirer/core": "^10.1.4", + "@inquirer/type": "^3.0.2", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/search": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-1.1.0.tgz", - "integrity": "sha512-h+/5LSj51dx7hp5xOn4QFnUaKeARwUCLs6mIhtkJ0JYPBLmEYjdHSYh7I6GrLg9LwpJ3xeX0FZgAG1q0QdCpVQ==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.6.tgz", + "integrity": "sha512-eFZ2hiAq0bZcFPuFFBmZEtXU1EarHLigE+ENCtpO+37NHCl4+Yokq1P/d09kUblObaikwfo97w+0FtG/EXl5Ng==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/figures": "^1.0.5", - "@inquirer/type": "^1.5.3", + "@inquirer/core": "^10.1.4", + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/select": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-2.5.0.tgz", - "integrity": "sha512-YmDobTItPP3WcEI86GvPo+T2sRHkxxOq/kXmsBjHS5BVXUgvgZ5AfJjkvQvZr03T81NnI3KrrRuMzeuYUQRFOA==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.0.6.tgz", + "integrity": "sha512-yANzIiNZ8fhMm4NORm+a74+KFYHmf7BZphSOBovIzYPVLquseTGEkU5l2UTnBOf5k0VLmTgPighNDLE9QtbViQ==", "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/figures": "^1.0.5", - "@inquirer/type": "^1.5.3", + "@inquirer/core": "^10.1.4", + "@inquirer/figures": "^1.0.9", + "@inquirer/type": "^3.0.2", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@inquirer/type": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-1.5.5.tgz", - "integrity": "sha512-MzICLu4yS7V8AA61sANROZ9vT1H3ooca5dSmI1FjZkzq7o/koMsRfQSzRtFo+F3Ao4Sf1C0bpLKejpKB/+j6MA==", - "dependencies": { - "mute-stream": "^1.0.0" - }, + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.2.tgz", + "integrity": "sha512-ZhQ4TvhwHZF+lGhQ2O/rsjo80XoZR5/5qhOY3t6FJuX5XBg5Be8YzYTvaUGJnc12AUGI2nr4QSUE4PhKSigx7g==", "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" } }, "node_modules/@mixmark-io/domino": { @@ -367,27 +386,15 @@ "node": ">= 8" } }, - "node_modules/@types/mute-stream": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", - "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/node": { "version": "22.10.7", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.7.tgz", "integrity": "sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==", + "peer": true, "dependencies": { "undici-types": "~6.20.0" } }, - "node_modules/@types/wrap-ansi": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", - "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==" - }, "node_modules/@ungap/structured-clone": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.1.tgz", @@ -445,17 +452,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/ansi-escapes/node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -587,9 +583,9 @@ } }, "node_modules/commander": { - "version": "13.0.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-13.0.0.tgz", - "integrity": "sha512-oPYleIY8wmTVzkvQq10AEok6YcTC4sRUBl8F9gVuwchGVUCTbl/vhLTaQqutuuySYOsu8YTgV+OxKc/8Yvx+mQ==", + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", + "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==", "engines": { "node": ">=18" } @@ -1007,6 +1003,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/globals/node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/graphemer": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", @@ -1084,24 +1092,6 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, - "node_modules/inquirer": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-10.2.2.tgz", - "integrity": "sha512-tyao/4Vo36XnUItZ7DnUXX4f1jVao2mSrleV/5IPtW/XAEA26hRVsbc68nuTEKWcr5vMP/1mVoT2O7u8H4v1Vg==", - "dependencies": { - "@inquirer/core": "^9.1.0", - "@inquirer/prompts": "^5.5.0", - "@inquirer/type": "^1.5.3", - "@types/mute-stream": "^0.0.4", - "ansi-escapes": "^4.3.2", - "mute-stream": "^1.0.0", - "run-async": "^3.0.0", - "rxjs": "^7.8.1" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1265,11 +1255,11 @@ "dev": true }, "node_modules/mute-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", - "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", + "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", "engines": { - "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + "node": "^18.17.0 || >=20.5.0" } }, "node_modules/natural-compare": { @@ -1459,14 +1449,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/run-async": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", - "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -1490,14 +1472,6 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/rxjs": { - "version": "7.8.1", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", - "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", - "dependencies": { - "tslib": "^2.1.0" - } - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1605,11 +1579,6 @@ "node": ">=0.6.0" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" - }, "node_modules/turndown": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.0.tgz", @@ -1636,10 +1605,9 @@ } }, "node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", "engines": { "node": ">=10" }, @@ -1650,7 +1618,8 @@ "node_modules/undici-types": { "version": "6.20.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "peer": true }, "node_modules/uri-js": { "version": "4.4.1", diff --git a/package.json b/package.json index cb12728..55c4509 100644 --- a/package.json +++ b/package.json @@ -21,11 +21,11 @@ }, "type": "module", "dependencies": { + "@inquirer/prompts": "^7.2.3", "axios": "^1.7.9", "camelcase": "^8.0.0", "chalk": "^5.4.1", "commander": "^13.0.0", - "inquirer": "^10.2.2", "luxon": "^3.5.0", "turndown": "^7.2.0", "turndown-plugin-gfm": "^1.0.2", diff --git a/src/wizard.js b/src/wizard.js index 2436439..da550c4 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -1,7 +1,8 @@ +import * as inquirer from '@inquirer/prompts'; import camelcase from 'camelcase'; +import chalk from 'chalk'; import * as commander from 'commander'; import fs from 'fs'; -import inquirer from 'inquirer'; import path from 'path'; // all user options for command line and wizard are declared here @@ -61,24 +62,25 @@ export async function getConfig(argv) { extendOptionsData(); const opts = parseCommandLine(argv); - let answers; + const answers = {}; if (opts.wizard) { console.log('\nStarting wizard...'); - const questions = options.map(option => ({ - when: option.name !== 'wizard' && !option.isProvided, - name: camelcase(option.name), - type: option.prompt, - message: option.description + '?', - default: option.default, - - // these are not used for all option types and that's fine - filter: option.coerce, - validate: option.validate - })); - answers = await inquirer.prompt(questions); + const questions = options.filter(option => (option.name !== 'wizard' && !option.isProvided)); + for (const question of questions) { + answers[camelcase(question.name)] = await inquirer[question.prompt]({ + message: question.description + '?', + default: question.default, + validate: question.validate, // not all questions have this, which is fine + theme: { + prefix: { + idle: chalk.cyan('?'), + done: chalk.green('✓') + } + } + }); + } } else { console.log('\nSkipping wizard...'); - answers = {}; } const config = { ...opts, ...answers }; From 1f890b407912d3b53f62730f3c579a0d9a0bd9e5 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 25 Jan 2025 10:27:24 -0500 Subject: [PATCH 10/79] Just a whole lot of stuff in progress --- index.js | 5 +- src/wizard.js | 247 +++++++++++++++++++++++++++++++++----------------- 2 files changed, 166 insertions(+), 86 deletions(-) diff --git a/index.js b/index.js index f1a9298..9745d45 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,6 @@ #!/usr/bin/env node import path from 'path'; -import process from 'process'; import * as parser from './src/parser.js'; import * as settings from './src/settings.js'; import * as wizard from './src/wizard.js'; @@ -9,7 +8,7 @@ import * as writer from './src/writer.js'; (async () => { // parse any command line arguments and run wizard - const config = await wizard.getConfig(process.argv); + const config = await wizard.getConfig(); // parse data from XML and do Markdown translations const posts = await parser.parseFilePromise(config) @@ -20,7 +19,7 @@ import * as writer from './src/writer.js'; // happy goodbye console.log('\nAll done!'); console.log('Look for your output files in: ' + path.resolve(settings.output_directory)); -})().catch(ex => { +})().catch((ex) => { // sad goodbye console.log('\nSomething went wrong, execution halted early.'); console.error(ex); diff --git a/src/wizard.js b/src/wizard.js index da550c4..b2d47a8 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -7,7 +7,6 @@ import path from 'path'; // all user options for command line and wizard are declared here const options = [ - // wizard must always be first { name: 'wizard', type: 'boolean', @@ -16,50 +15,132 @@ const options = [ }, { name: 'input', - type: 'file', + type: 'file-path', description: 'Path to WordPress export file', - default: 'export.xml' - }, - { - name: 'year-folders', - type: 'boolean', - description: 'Create year folders', - default: false - }, - { - name: 'month-folders', - type: 'boolean', - description: 'Create month folders', - default: false + default: 'export.xml', + prompt: inquirer.input }, { name: 'post-folders', type: 'boolean', - description: 'Create a folder for each post', - default: true + description: 'Put each post into its own folder', + default: true, + choices: [ + { + name: 'Yes', + value: true, + description: '/my-post/index.md' + }, + { + name: 'No', + value: false, + description: '/my-post.md' + } + ], + prompt: inquirer.select }, { name: 'prefix-date', type: 'boolean', - description: 'Prefix post folders/files with date', - default: false + description: 'Prefix with date', + default: false, + choices: [ + { + name: 'Yes', + value: true, + description: '' + }, + { + name: 'No', + value: false, + description: '' + } + ], + prompt: inquirer.select }, { - name: 'save-attached-images', - type: 'boolean', - description: 'Save images attached to posts', - default: true + name: 'date-folders', + type: 'choice', + description: 'Organize into folders based on date', + default: 'none', + choices: [ + { + name: 'Year folders', + value: 'year', + description: '' + }, + { + name: 'Year and month folders', + value: 'year-month', + description: '' + }, + { + name: 'No', + value: 'none', + description: '' + } + ], + prompt: inquirer.select }, { - name: 'save-scraped-images', - type: 'boolean', - description: 'Save images scraped from post body content', - default: true + name: 'save-images', + type: 'choice', + description: 'Save images', + default: 'all', + choices: [ + { + name: 'Images attached to posts', + value: 'attached' + }, + { + name: 'Images scraped from post body content', + value: 'scraped' + }, + { + name: 'Both', + value: 'all' + }, + { + name: 'No', + value: 'none' + } + ], + prompt: inquirer.select } ]; +const validators = { + 'boolean': (value) => { + if (typeof value === 'boolean') { + return value; + } else if (value === 'true') { + return true; + } else if (value === 'false') { + return false; + } + + throw 'Must be true or false.'; + }, + 'file-path': (value) => { + const unwrapped = value.replace(/"(.*?)"/, '$1'); + const absolute = path.resolve(unwrapped); + + let fileExists; + try { + fileExists = fs.existsSync(absolute) && fs.statSync(absolute).isFile(); + } catch (ex) { + fileExists = false; + } + + if (fileExists) { + return absolute; + } else { + throw 'File not found at ' + absolute + '.'; + } + } +}; + export async function getConfig(argv) { - extendOptionsData(); const opts = parseCommandLine(argv); const answers = {}; @@ -67,17 +148,36 @@ export async function getConfig(argv) { console.log('\nStarting wizard...'); const questions = options.filter(option => (option.name !== 'wizard' && !option.isProvided)); for (const question of questions) { - answers[camelcase(question.name)] = await inquirer[question.prompt]({ + let answer = await question.prompt({ message: question.description + '?', + choices: question.choices, + loop: false, default: question.default, validate: question.validate, // not all questions have this, which is fine + theme: { prefix: { - idle: chalk.cyan('?'), + idle: chalk.cyan('\n?'), done: chalk.green('✓') + }, + style: { + description: (text) => chalk.gray('example: ' + text) } } + }).catch((ex) => { + if (ex instanceof Error && ex.name === 'ExitPromptError') { + console.log('\nUser quit wizard early.'); + process.exit(0); + } else { + throw ex; + } }); + + if (question.normalize) { + answer = question.normalize(answer); + } + + answers[camelcase(question.name)] = answer; } } else { console.log('\nSkipping wizard...'); @@ -87,66 +187,47 @@ export async function getConfig(argv) { return config; } -function extendOptionsData() { - // add more data to each option based on its type - const map = { - boolean: { - prompt: 'confirm', - coerce: coerceBoolean, - }, - file: { - prompt: 'input', - coerce: coercePath, - validate: validateFile - }, - folder: { - prompt: 'input', - coerce: coercePath - } - }; - - options.forEach(option => { - Object.assign(option, map[option.type]); - }); -} - -function parseCommandLine(argv) { - // setup for help output +function parseCommandLine() { commander.program .name('node index.js') .helpOption('-h, --help', 'See the thing you\'re looking at right now') - .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown'); + .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown') + .configureOutput({ + outputError: (str, write) => write(chalk.red(str)) + }); + options.forEach(input => { const flag = '--' + input.name + ' <' + input.type + '>'; - const coerce = (value) => { - // commander only calls coerce when an input is provided on the command line, which - // makes for an easy way to flag (for later) if it should be excluded from the wizard - input.isProvided = true; - return input.coerce(value); - }; - commander.program.option(flag, input.description, coerce, input.default); + const option = new commander.Option(flag, input.description); + option.default(input.default); + + if (input.choices && input.type !== 'boolean') { + option.choices(input.choices.map((choice) => choice.value)); + } else { + option.argParser((value) => { + const validator = validators[input.type]; + if (!validator) { + return value; + } + + try { + return validator(value); + } catch (ex) { + commander.program.error(`error: option '${flag}' argument '${value}' is invalid. ${ex.toString()}`); + } + }); + } + + commander.program.addOption(option); + }); + + commander.program.parse(); + + options.forEach((option) => { + const opt = camelcase(option.name); + option.isProvided = commander.program.getOptionValueSource(opt) === 'cli'; }); - commander.program.parse(argv); return commander.program.opts(); } - -function coerceBoolean(value) { - return !['false', 'no', '0'].includes(value.toString().toLowerCase()); -} - -function coercePath(value) { - return path.normalize(value); -} - -function validateFile(value) { - let isValid; - try { - isValid = fs.existsSync(value) && fs.statSync(value).isFile(); - } catch (ex) { - isValid = false; - } - - return isValid ? true : 'Unable to find file: ' + path.resolve(value); -} From 6a6e5074f0a59b25bbaad43b8188f133dd0bac9d Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 25 Jan 2025 13:46:47 -0500 Subject: [PATCH 11/79] Fixed validation/normalization for wizard --- src/wizard.js | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/wizard.js b/src/wizard.js index b2d47a8..9e4cfd4 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -148,23 +148,41 @@ export async function getConfig(argv) { console.log('\nStarting wizard...'); const questions = options.filter(option => (option.name !== 'wizard' && !option.isProvided)); for (const question of questions) { - let answer = await question.prompt({ + const promptConfig = { message: question.description + '?', - choices: question.choices, - loop: false, default: question.default, - validate: question.validate, // not all questions have this, which is fine theme: { prefix: { - idle: chalk.cyan('\n?'), + idle: chalk.gray('\n?'), done: chalk.green('✓') }, style: { description: (text) => chalk.gray('example: ' + text) } } - }).catch((ex) => { + }; + + if (question.choices) { + promptConfig.choices = question.choices; + promptConfig.loop = false; + } else { + const validator = validators[question.type]; + if (validator) { + promptConfig.validate = (value) => { + try { + normalized = validator(value); + } catch (ex) { + return ex.toString(); + } + + return true; + } + } + } + + let normalized = undefined; + let answer = await question.prompt(promptConfig).catch((ex) => { if (ex instanceof Error && ex.name === 'ExitPromptError') { console.log('\nUser quit wizard early.'); process.exit(0); @@ -173,11 +191,9 @@ export async function getConfig(argv) { } }); - if (question.normalize) { - answer = question.normalize(answer); - } + console.log(normalized, answer); - answers[camelcase(question.name)] = answer; + answers[camelcase(question.name)] = normalized ?? answer; } } else { console.log('\nSkipping wizard...'); From 8841c607e7cacdf3751a4212614c285bc332f2f4 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 25 Jan 2025 16:23:45 -0500 Subject: [PATCH 12/79] Separate normalizers out, fix config flag checks --- src/normalizers.js | 32 ++++++++++++++++++++++++++++ src/parser.js | 4 ++-- src/wizard.js | 53 +++++++++------------------------------------- src/writer.js | 4 ++-- 4 files changed, 46 insertions(+), 47 deletions(-) create mode 100644 src/normalizers.js diff --git a/src/normalizers.js b/src/normalizers.js new file mode 100644 index 0000000..44f4121 --- /dev/null +++ b/src/normalizers.js @@ -0,0 +1,32 @@ +import fs from 'fs'; +import path from 'path'; + +export function boolean(value) { + if (typeof value === 'boolean') { + return value; + } else if (value === 'true') { + return true; + } else if (value === 'false') { + return false; + } + + throw 'Must be true or false.'; +} + +export function filePath(value) { + const unwrapped = value.replace(/"(.*?)"/, '$1'); + const absolute = path.resolve(unwrapped); + + let fileExists; + try { + fileExists = fs.existsSync(absolute) && fs.statSync(absolute).isFile(); + } catch (ex) { + fileExists = false; + } + + if (fileExists) { + return absolute; + } else { + throw 'File not found at ' + absolute + '.'; + } +} diff --git a/src/parser.js b/src/parser.js index a4cbadc..08775ad 100644 --- a/src/parser.js +++ b/src/parser.js @@ -18,10 +18,10 @@ export async function parseFilePromise(config) { const posts = collectPosts(channelData, postTypes, config); const images = []; - if (config.saveAttachedImages) { + if (config.saveImages === 'attached' || config.saveImages === 'all') { images.push(...collectAttachedImages(channelData)); } - if (config.saveScrapedImages) { + if (config.saveImages === 'scraped' || config.saveImages === 'all') { images.push(...collectScrapedImages(channelData, postTypes)); } diff --git a/src/wizard.js b/src/wizard.js index 9e4cfd4..eb24b54 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -2,8 +2,7 @@ import * as inquirer from '@inquirer/prompts'; import camelcase from 'camelcase'; import chalk from 'chalk'; import * as commander from 'commander'; -import fs from 'fs'; -import path from 'path'; +import * as normalizers from './normalizers.js'; // all user options for command line and wizard are declared here const options = [ @@ -109,37 +108,6 @@ const options = [ } ]; -const validators = { - 'boolean': (value) => { - if (typeof value === 'boolean') { - return value; - } else if (value === 'true') { - return true; - } else if (value === 'false') { - return false; - } - - throw 'Must be true or false.'; - }, - 'file-path': (value) => { - const unwrapped = value.replace(/"(.*?)"/, '$1'); - const absolute = path.resolve(unwrapped); - - let fileExists; - try { - fileExists = fs.existsSync(absolute) && fs.statSync(absolute).isFile(); - } catch (ex) { - fileExists = false; - } - - if (fileExists) { - return absolute; - } else { - throw 'File not found at ' + absolute + '.'; - } - } -}; - export async function getConfig(argv) { const opts = parseCommandLine(argv); @@ -148,6 +116,8 @@ export async function getConfig(argv) { console.log('\nStarting wizard...'); const questions = options.filter(option => (option.name !== 'wizard' && !option.isProvided)); for (const question of questions) { + let normalizedAnswer = undefined; + const promptConfig = { message: question.description + '?', default: question.default, @@ -167,11 +137,11 @@ export async function getConfig(argv) { promptConfig.choices = question.choices; promptConfig.loop = false; } else { - const validator = validators[question.type]; - if (validator) { + const normalizer = normalizers[camelcase(question.type)]; + if (normalizer) { promptConfig.validate = (value) => { try { - normalized = validator(value); + normalizedAnswer = normalizer(value); } catch (ex) { return ex.toString(); } @@ -181,7 +151,6 @@ export async function getConfig(argv) { } } - let normalized = undefined; let answer = await question.prompt(promptConfig).catch((ex) => { if (ex instanceof Error && ex.name === 'ExitPromptError') { console.log('\nUser quit wizard early.'); @@ -191,9 +160,7 @@ export async function getConfig(argv) { } }); - console.log(normalized, answer); - - answers[camelcase(question.name)] = normalized ?? answer; + answers[camelcase(question.name)] = normalizedAnswer ?? answer; } } else { console.log('\nSkipping wizard...'); @@ -222,13 +189,13 @@ function parseCommandLine() { option.choices(input.choices.map((choice) => choice.value)); } else { option.argParser((value) => { - const validator = validators[input.type]; - if (!validator) { + const normalizer = normalizers[camelcase(input.type)]; + if (!normalizer) { return value; } try { - return validator(value); + return normalizer(value); } catch (ex) { commander.program.error(`error: option '${flag}' argument '${value}' is invalid. ${ex.toString()}`); } diff --git a/src/writer.js b/src/writer.js index 0b4ea54..61d4fcf 100644 --- a/src/writer.js +++ b/src/writer.js @@ -182,11 +182,11 @@ function getPostPath(post, config) { // start with base output dir and post type const pathSegments = [settings.output_directory, post.meta.type]; - if (config.yearFolders) { + if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { pathSegments.push(dt.toFormat('yyyy')); } - if (config.monthFolders) { + if (config.dateFolders === 'year-month') { pathSegments.push(dt.toFormat('LL')); } From 07fbdc31e77632c6093709b3c63e511f8e92b131 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 26 Jan 2025 19:22:59 -0500 Subject: [PATCH 13/79] Option validation and edge cases --- src/wizard.js | 56 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/src/wizard.js b/src/wizard.js index eb24b54..a58c211 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -114,7 +114,9 @@ export async function getConfig(argv) { const answers = {}; if (opts.wizard) { console.log('\nStarting wizard...'); - const questions = options.filter(option => (option.name !== 'wizard' && !option.isProvided)); + const questions = options + .filter((option) => option.name !== 'wizard') + .filter((option) => !opts[camelcase(option.name)]); for (const question of questions) { let normalizedAnswer = undefined; @@ -180,37 +182,47 @@ function parseCommandLine() { }); - options.forEach(input => { - const flag = '--' + input.name + ' <' + input.type + '>'; - const option = new commander.Option(flag, input.description); + options.forEach((input) => { + const option = new commander.Option('--' + input.name + ' <' + input.type + '>', input.description); option.default(input.default); if (input.choices && input.type !== 'boolean') { option.choices(input.choices.map((choice) => choice.value)); } else { - option.argParser((value) => { - const normalizer = normalizers[camelcase(input.type)]; - if (!normalizer) { - return value; - } - - try { - return normalizer(value); - } catch (ex) { - commander.program.error(`error: option '${flag}' argument '${value}' is invalid. ${ex.toString()}`); - } - }); + option.argParser((value) => normalizeCommandLineArg(input.type, input.name, value)); } commander.program.addOption(option); }); - commander.program.parse(); + const opts = commander.program.parse().opts(); - options.forEach((option) => { - const opt = camelcase(option.name); - option.isProvided = commander.program.getOptionValueSource(opt) === 'cli'; - }); + for (const [key, value] of Object.entries(opts)) { + if (key === 'wizard' || commander.program.getOptionValueSource(key) !== 'default') { + continue; + } - return commander.program.opts(); + if (opts.wizard) { + delete opts[key]; + } else { + const option = options.find((option) => camelcase(option.name) === key); + opts[key] = normalizeCommandLineArg(option.type, option.name, value); + } + } + + return opts; +} + +function normalizeCommandLineArg(type, name, value) { + const normalizer = normalizers[camelcase(type)]; + if (!normalizer) { + return value; + } + + try { + return normalizer(value); + } catch (ex) { + commander.program.error(`error: option '--${name} <${type}>' argument '${value}' is invalid. ${ex.toString()}`); + // throw new commander.InvalidArgumentError('potato'); + } } From 7cd5722195eab34199fff2d68cd51415542269ad Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 08:37:37 -0500 Subject: [PATCH 14/79] Throw Error instead of string --- src/writer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/writer.js b/src/writer.js index 61d4fcf..7a642c9 100644 --- a/src/writer.js +++ b/src/writer.js @@ -22,7 +22,7 @@ async function processPayloadsPromise(payloads, loadFunc) { console.log(chalk.green('[OK]') + ' ' + payload.name); resolve(); } catch (ex) { - console.log(chalk.red('[FAILED]') + ' ' + payload.name + ' ' + chalk.red('(' + ex.toString() + ')')); + console.log(chalk.red('[FAILED]') + ' ' + payload.name + ' ' + chalk.red('(' + ex.message + ')')); reject(); } }, payload.delay); @@ -162,7 +162,7 @@ async function loadImageFilePromise(imageUrl) { } catch (ex) { if (ex.response) { // request was made, but server responded with an error status code - throw 'StatusCodeError: ' + ex.response.status; + throw new Error('StatusCodeError: ' + ex.response.status); } else { // something else went wrong, rethrow throw ex; From 19c65b21e485ac9b64d8a050f6a980f12171acc6 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 08:42:08 -0500 Subject: [PATCH 15/79] Common normalize logic --- src/normalizers.js | 4 ++-- src/wizard.js | 52 +++++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/normalizers.js b/src/normalizers.js index 44f4121..5234646 100644 --- a/src/normalizers.js +++ b/src/normalizers.js @@ -10,7 +10,7 @@ export function boolean(value) { return false; } - throw 'Must be true or false.'; + throw new Error('Must be true or false.'); } export function filePath(value) { @@ -27,6 +27,6 @@ export function filePath(value) { if (fileExists) { return absolute; } else { - throw 'File not found at ' + absolute + '.'; + throw new Error('File not found at ' + absolute + '.'); } } diff --git a/src/wizard.js b/src/wizard.js index a58c211..24a9e26 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -4,6 +4,16 @@ import chalk from 'chalk'; import * as commander from 'commander'; import * as normalizers from './normalizers.js'; +const promptTheme = { + prefix: { + idle: chalk.gray('\n?'), + done: chalk.green('✓') + }, + style: { + description: (text) => chalk.gray('example: ' + text) + } +}; + // all user options for command line and wizard are declared here const options = [ { @@ -121,35 +131,21 @@ export async function getConfig(argv) { let normalizedAnswer = undefined; const promptConfig = { + theme: promptTheme, message: question.description + '?', default: question.default, - - theme: { - prefix: { - idle: chalk.gray('\n?'), - done: chalk.green('✓') - }, - style: { - description: (text) => chalk.gray('example: ' + text) - } - } }; if (question.choices) { promptConfig.choices = question.choices; promptConfig.loop = false; } else { - const normalizer = normalizers[camelcase(question.type)]; - if (normalizer) { - promptConfig.validate = (value) => { - try { - normalizedAnswer = normalizer(value); - } catch (ex) { - return ex.toString(); - } - - return true; - } + promptConfig.validate = (value) => { + let validationResult; + normalizedAnswer = normalize(value, question.type, (errorMessage) => { + validationResult = errorMessage; + }); + return validationResult ?? true; } } @@ -189,7 +185,9 @@ function parseCommandLine() { if (input.choices && input.type !== 'boolean') { option.choices(input.choices.map((choice) => choice.value)); } else { - option.argParser((value) => normalizeCommandLineArg(input.type, input.name, value)); + option.argParser((value) => normalize(value, input.type, (errorMessage) => { + throw new commander.InvalidArgumentError(errorMessage); + })); } commander.program.addOption(option); @@ -198,6 +196,7 @@ function parseCommandLine() { const opts = commander.program.parse().opts(); for (const [key, value] of Object.entries(opts)) { + console.log(key, value); if (key === 'wizard' || commander.program.getOptionValueSource(key) !== 'default') { continue; } @@ -206,14 +205,16 @@ function parseCommandLine() { delete opts[key]; } else { const option = options.find((option) => camelcase(option.name) === key); - opts[key] = normalizeCommandLineArg(option.type, option.name, value); + opts[key] = normalize(value, option.type, (errorMessage) => { + commander.program.error(`error: option '--${option.name} <${option.type}>' argument '${value}' is invalid. ${errorMessage}`); + }); } } return opts; } -function normalizeCommandLineArg(type, name, value) { +function normalize(value, type, onError) { const normalizer = normalizers[camelcase(type)]; if (!normalizer) { return value; @@ -222,7 +223,6 @@ function normalizeCommandLineArg(type, name, value) { try { return normalizer(value); } catch (ex) { - commander.program.error(`error: option '--${name} <${type}>' argument '${value}' is invalid. ${ex.toString()}`); - // throw new commander.InvalidArgumentError('potato'); + onError(ex.message); } } From a61e2be9e3220b67b765c9fe1eb4c2103e90a61e Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 09:30:28 -0500 Subject: [PATCH 16/79] Move options out, refactor getConfig() --- src/options.js | 104 ++++++++++++++++++++++++++ src/wizard.js | 196 +++++++++++++------------------------------------ 2 files changed, 154 insertions(+), 146 deletions(-) create mode 100644 src/options.js diff --git a/src/options.js b/src/options.js new file mode 100644 index 0000000..1eb50df --- /dev/null +++ b/src/options.js @@ -0,0 +1,104 @@ +import * as inquirer from '@inquirer/prompts'; + +export const all = [ + { + name: 'wizard', + type: 'boolean', + description: 'Use wizard', + default: true + }, + { + name: 'input', + type: 'file-path', + description: 'Path to WordPress export file', + default: 'export.xml', + prompt: inquirer.input + }, + { + name: 'post-folders', + type: 'boolean', + description: 'Put each post into its own folder', + default: true, + choices: [ + { + name: 'Yes', + value: true, + description: '/my-post/index.md' + }, + { + name: 'No', + value: false, + description: '/my-post.md' + } + ], + prompt: inquirer.select + }, + { + name: 'prefix-date', + type: 'boolean', + description: 'Prefix with date', + default: false, + choices: [ + { + name: 'Yes', + value: true, + description: '' + }, + { + name: 'No', + value: false, + description: '' + } + ], + prompt: inquirer.select + }, + { + name: 'date-folders', + type: 'choice', + description: 'Organize into folders based on date', + default: 'none', + choices: [ + { + name: 'Year folders', + value: 'year', + description: '' + }, + { + name: 'Year and month folders', + value: 'year-month', + description: '' + }, + { + name: 'No', + value: 'none', + description: '' + } + ], + prompt: inquirer.select + }, + { + name: 'save-images', + type: 'choice', + description: 'Save images', + default: 'all', + choices: [ + { + name: 'Images attached to posts', + value: 'attached' + }, + { + name: 'Images scraped from post body content', + value: 'scraped' + }, + { + name: 'Both', + value: 'all' + }, + { + name: 'No', + value: 'none' + } + ], + prompt: inquirer.select + } +]; diff --git a/src/wizard.js b/src/wizard.js index 24a9e26..b996cf8 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -1,8 +1,8 @@ -import * as inquirer from '@inquirer/prompts'; import camelcase from 'camelcase'; import chalk from 'chalk'; import * as commander from 'commander'; import * as normalizers from './normalizers.js'; +import * as options from './options.js'; const promptTheme = { prefix: { @@ -14,161 +14,26 @@ const promptTheme = { } }; -// all user options for command line and wizard are declared here -const options = [ - { - name: 'wizard', - type: 'boolean', - description: 'Use wizard', - default: true - }, - { - name: 'input', - type: 'file-path', - description: 'Path to WordPress export file', - default: 'export.xml', - prompt: inquirer.input - }, - { - name: 'post-folders', - type: 'boolean', - description: 'Put each post into its own folder', - default: true, - choices: [ - { - name: 'Yes', - value: true, - description: '/my-post/index.md' - }, - { - name: 'No', - value: false, - description: '/my-post.md' - } - ], - prompt: inquirer.select - }, - { - name: 'prefix-date', - type: 'boolean', - description: 'Prefix with date', - default: false, - choices: [ - { - name: 'Yes', - value: true, - description: '' - }, - { - name: 'No', - value: false, - description: '' - } - ], - prompt: inquirer.select - }, - { - name: 'date-folders', - type: 'choice', - description: 'Organize into folders based on date', - default: 'none', - choices: [ - { - name: 'Year folders', - value: 'year', - description: '' - }, - { - name: 'Year and month folders', - value: 'year-month', - description: '' - }, - { - name: 'No', - value: 'none', - description: '' - } - ], - prompt: inquirer.select - }, - { - name: 'save-images', - type: 'choice', - description: 'Save images', - default: 'all', - choices: [ - { - name: 'Images attached to posts', - value: 'attached' - }, - { - name: 'Images scraped from post body content', - value: 'scraped' - }, - { - name: 'Both', - value: 'all' - }, - { - name: 'No', - value: 'none' - } - ], - prompt: inquirer.select - } -]; +export async function getConfig() { + const config = {}; -export async function getConfig(argv) { - const opts = parseCommandLine(argv); + const commandLineOptions = options.all; + Object.assign(config, getCommandLineAnswers(commandLineOptions)); + console.log(1, config); - const answers = {}; - if (opts.wizard) { + if (config.wizard) { console.log('\nStarting wizard...'); - const questions = options - .filter((option) => option.name !== 'wizard') - .filter((option) => !opts[camelcase(option.name)]); - for (const question of questions) { - let normalizedAnswer = undefined; - - const promptConfig = { - theme: promptTheme, - message: question.description + '?', - default: question.default, - }; - - if (question.choices) { - promptConfig.choices = question.choices; - promptConfig.loop = false; - } else { - promptConfig.validate = (value) => { - let validationResult; - normalizedAnswer = normalize(value, question.type, (errorMessage) => { - validationResult = errorMessage; - }); - return validationResult ?? true; - } - } - - let answer = await question.prompt(promptConfig).catch((ex) => { - if (ex instanceof Error && ex.name === 'ExitPromptError') { - console.log('\nUser quit wizard early.'); - process.exit(0); - } else { - throw ex; - } - }); - - answers[camelcase(question.name)] = normalizedAnswer ?? answer; - } + const wizardOptions = options.all.filter((option) => option.name !== 'wizard' && !(camelcase(option.name) in config)); + Object.assign(config, await getWizardAnswers(wizardOptions)); } else { console.log('\nSkipping wizard...'); } - const config = { ...opts, ...answers }; + console.log(2, config); return config; } -function parseCommandLine() { +function getCommandLineAnswers(options) { commander.program .name('node index.js') .helpOption('-h, --help', 'See the thing you\'re looking at right now') @@ -214,6 +79,45 @@ function parseCommandLine() { return opts; } +export async function getWizardAnswers(options) { + const answers = {}; + for (const question of options) { + let normalizedAnswer = undefined; + + const promptConfig = { + theme: promptTheme, + message: question.description + '?', + default: question.default, + }; + + if (question.choices) { + promptConfig.choices = question.choices; + promptConfig.loop = false; + } else { + promptConfig.validate = (value) => { + let validationResult; + normalizedAnswer = normalize(value, question.type, (errorMessage) => { + validationResult = errorMessage; + }); + return validationResult ?? true; + } + } + + let answer = await question.prompt(promptConfig).catch((ex) => { + if (ex instanceof Error && ex.name === 'ExitPromptError') { + console.log('\nUser quit wizard early.'); + process.exit(0); + } else { + throw ex; + } + }); + + answers[camelcase(question.name)] = normalizedAnswer ?? answer; + } + + return answers; +} + function normalize(value, type, onError) { const normalizer = normalizers[camelcase(type)]; if (!normalizer) { From d68c3feaff4097e67530eb9ca7cc5e3f392a5a1f Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 09:33:42 -0500 Subject: [PATCH 17/79] Clean up console logs --- src/wizard.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/wizard.js b/src/wizard.js index b996cf8..7653b51 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -19,7 +19,6 @@ export async function getConfig() { const commandLineOptions = options.all; Object.assign(config, getCommandLineAnswers(commandLineOptions)); - console.log(1, config); if (config.wizard) { console.log('\nStarting wizard...'); @@ -29,7 +28,6 @@ export async function getConfig() { console.log('\nSkipping wizard...'); } - console.log(2, config); return config; } @@ -61,7 +59,6 @@ function getCommandLineAnswers(options) { const opts = commander.program.parse().opts(); for (const [key, value] of Object.entries(opts)) { - console.log(key, value); if (key === 'wizard' || commander.program.getOptionValueSource(key) !== 'default') { continue; } From 80eb589d4b162325484011b0136d31af713b2fd6 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 12:59:21 -0500 Subject: [PATCH 18/79] Rename wizard/questions things --- index.js | 14 +++++++-- src/{options.js => questions.js} | 0 src/wizard.js | 51 +++++++++++++------------------- 3 files changed, 33 insertions(+), 32 deletions(-) rename src/{options.js => questions.js} (100%) diff --git a/index.js b/index.js index 9745d45..a6715ec 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,7 @@ #!/usr/bin/env node +import chalk from 'chalk'; +import * as commander from 'commander'; import path from 'path'; import * as parser from './src/parser.js'; import * as settings from './src/settings.js'; @@ -7,13 +9,21 @@ import * as wizard from './src/wizard.js'; import * as writer from './src/writer.js'; (async () => { - // parse any command line arguments and run wizard + commander.program + .name('node index.js') + .helpOption('-h, --help', 'See the thing you\'re looking at right now') + .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown') + .configureOutput({ + outputError: (str, write) => write(chalk.red(str)) + }); + + // gather config options from command line and wizard const config = await wizard.getConfig(); // parse data from XML and do Markdown translations const posts = await parser.parseFilePromise(config) - // write files, downloading images as needed + // write files and download images await writer.writeFilesPromise(posts, config); // happy goodbye diff --git a/src/options.js b/src/questions.js similarity index 100% rename from src/options.js rename to src/questions.js diff --git a/src/wizard.js b/src/wizard.js index 7653b51..8835834 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -2,7 +2,7 @@ import camelcase from 'camelcase'; import chalk from 'chalk'; import * as commander from 'commander'; import * as normalizers from './normalizers.js'; -import * as options from './options.js'; +import * as questions from './questions.js'; const promptTheme = { prefix: { @@ -17,13 +17,13 @@ const promptTheme = { export async function getConfig() { const config = {}; - const commandLineOptions = options.all; - Object.assign(config, getCommandLineAnswers(commandLineOptions)); + const commandLineQuestions = questions.all; + Object.assign(config, getCommandLineAnswers(commandLineQuestions)); if (config.wizard) { console.log('\nStarting wizard...'); - const wizardOptions = options.all.filter((option) => option.name !== 'wizard' && !(camelcase(option.name) in config)); - Object.assign(config, await getWizardAnswers(wizardOptions)); + const wizardQuestions = questions.all.filter((question) => question.name !== 'wizard' && !(camelcase(question.name) in config)); + Object.assign(config, await getWizardAnswers(wizardQuestions)); } else { console.log('\nSkipping wizard...'); } @@ -31,24 +31,15 @@ export async function getConfig() { return config; } -function getCommandLineAnswers(options) { - commander.program - .name('node index.js') - .helpOption('-h, --help', 'See the thing you\'re looking at right now') - .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown') - .configureOutput({ - outputError: (str, write) => write(chalk.red(str)) - }); +function getCommandLineAnswers(questions) { + questions.forEach((question) => { + const option = new commander.Option('--' + question.name + ' <' + question.type + '>', question.description); + option.default(question.default); - - options.forEach((input) => { - const option = new commander.Option('--' + input.name + ' <' + input.type + '>', input.description); - option.default(input.default); - - if (input.choices && input.type !== 'boolean') { - option.choices(input.choices.map((choice) => choice.value)); + if (question.choices && question.type !== 'boolean') { + option.choices(question.choices.map((choice) => choice.value)); } else { - option.argParser((value) => normalize(value, input.type, (errorMessage) => { + option.argParser((value) => normalize(value, question.type, (errorMessage) => { throw new commander.InvalidArgumentError(errorMessage); })); } @@ -56,29 +47,29 @@ function getCommandLineAnswers(options) { commander.program.addOption(option); }); - const opts = commander.program.parse().opts(); + const answers = commander.program.parse().opts(); - for (const [key, value] of Object.entries(opts)) { + for (const [key, value] of Object.entries(answers)) { if (key === 'wizard' || commander.program.getOptionValueSource(key) !== 'default') { continue; } - if (opts.wizard) { - delete opts[key]; + if (answers.wizard) { + delete answers[key]; } else { - const option = options.find((option) => camelcase(option.name) === key); - opts[key] = normalize(value, option.type, (errorMessage) => { + const option = questions.find((option) => camelcase(option.name) === key); + answers[key] = normalize(value, option.type, (errorMessage) => { commander.program.error(`error: option '--${option.name} <${option.type}>' argument '${value}' is invalid. ${errorMessage}`); }); } } - return opts; + return answers; } -export async function getWizardAnswers(options) { +export async function getWizardAnswers(questions) { const answers = {}; - for (const question of options) { + for (const question of questions) { let normalizedAnswer = undefined; const promptConfig = { From fad9d974888dd7f789437f2b38dddb0defb5cddc Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 14:11:31 -0500 Subject: [PATCH 19/79] Some wizard comments and renamed variables --- index.js | 5 +---- src/wizard.js | 37 +++++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/index.js b/index.js index a6715ec..2c04919 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,5 @@ #!/usr/bin/env node -import chalk from 'chalk'; import * as commander from 'commander'; import path from 'path'; import * as parser from './src/parser.js'; @@ -9,13 +8,11 @@ import * as wizard from './src/wizard.js'; import * as writer from './src/writer.js'; (async () => { + // configure command line help output commander.program .name('node index.js') .helpOption('-h, --help', 'See the thing you\'re looking at right now') .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown') - .configureOutput({ - outputError: (str, write) => write(chalk.red(str)) - }); // gather config options from command line and wizard const config = await wizard.getConfig(); diff --git a/src/wizard.js b/src/wizard.js index 8835834..3e7ce28 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -4,6 +4,7 @@ import * as commander from 'commander'; import * as normalizers from './normalizers.js'; import * as questions from './questions.js'; +// visual formatting for wizard const promptTheme = { prefix: { idle: chalk.gray('\n?'), @@ -17,12 +18,15 @@ const promptTheme = { export async function getConfig() { const config = {}; + // check command line for any config options const commandLineQuestions = questions.all; Object.assign(config, getCommandLineAnswers(commandLineQuestions)); if (config.wizard) { console.log('\nStarting wizard...'); - const wizardQuestions = questions.all.filter((question) => question.name !== 'wizard' && !(camelcase(question.name) in config)); + + // run wizard for remaining config options + const wizardQuestions = questions.all.filter((question) => !(camelcase(question.name) in config)); Object.assign(config, await getWizardAnswers(wizardQuestions)); } else { console.log('\nSkipping wizard...'); @@ -32,11 +36,17 @@ export async function getConfig() { } function getCommandLineAnswers(questions) { + // show errors in red + commander.program.configureOutput({ + outputError: (str, write) => write(chalk.red(str)) + }); + questions.forEach((question) => { const option = new commander.Option('--' + question.name + ' <' + question.type + '>', question.description); option.default(question.default); if (question.choices && question.type !== 'boolean') { + // let commander handle non-boolean multiple choice validation option.choices(question.choices.map((choice) => choice.value)); } else { option.argParser((value) => normalize(value, question.type, (errorMessage) => { @@ -49,17 +59,21 @@ function getCommandLineAnswers(questions) { const answers = commander.program.parse().opts(); + // do some post-processing on the answers for (const [key, value] of Object.entries(answers)) { + // the "wizard" answer and any user-provided (not defaulted) answers are left alone if (key === 'wizard' || commander.program.getOptionValueSource(key) !== 'default') { continue; } if (answers.wizard) { + // remove this default answer so the wizard will ask about it later delete answers[key]; } else { - const option = questions.find((option) => camelcase(option.name) === key); - answers[key] = normalize(value, option.type, (errorMessage) => { - commander.program.error(`error: option '--${option.name} <${option.type}>' argument '${value}' is invalid. ${errorMessage}`); + // normalize and validate default answer + const question = questions.find((question) => camelcase(question.name) === key); + answers[key] = normalize(value, question.type, (errorMessage) => { + commander.program.error(`error: option '--${question.name} <${question.type}>' argument '${value}' is invalid. ${errorMessage}`); }); } } @@ -70,7 +84,8 @@ function getCommandLineAnswers(questions) { export async function getWizardAnswers(questions) { const answers = {}; for (const question of questions) { - let normalizedAnswer = undefined; + // this will be set to the normalized answer during validation + let normalizedAnswer; const promptConfig = { theme: promptTheme, @@ -83,15 +98,17 @@ export async function getWizardAnswers(questions) { promptConfig.loop = false; } else { promptConfig.validate = (value) => { - let validationResult; + let validationErrorMessage; normalizedAnswer = normalize(value, question.type, (errorMessage) => { - validationResult = errorMessage; + validationErrorMessage = errorMessage; }); - return validationResult ?? true; + return validationErrorMessage ?? true; } } - let answer = await question.prompt(promptConfig).catch((ex) => { + // don't care about the return value of prompt() because normalizedAnswer will be used + await question.prompt(promptConfig).catch((ex) => { + // exit gracefully if user hits ctrl + c during wizard if (ex instanceof Error && ex.name === 'ExitPromptError') { console.log('\nUser quit wizard early.'); process.exit(0); @@ -100,7 +117,7 @@ export async function getWizardAnswers(questions) { } }); - answers[camelcase(question.name)] = normalizedAnswer ?? answer; + answers[camelcase(question.name)] = normalizedAnswer; } return answers; From 91073742faa81a14bc890777fe3dc048f7916c4b Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 14:57:04 -0500 Subject: [PATCH 20/79] Fix undefined answer bug, start on example path --- src/questions.js | 21 +++++++-------------- src/wizard.js | 31 +++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/questions.js b/src/questions.js index 1eb50df..8c96b69 100644 --- a/src/questions.js +++ b/src/questions.js @@ -22,13 +22,11 @@ export const all = [ choices: [ { name: 'Yes', - value: true, - description: '/my-post/index.md' + value: true }, { name: 'No', - value: false, - description: '/my-post.md' + value: false } ], prompt: inquirer.select @@ -41,13 +39,11 @@ export const all = [ choices: [ { name: 'Yes', - value: true, - description: '' + value: true }, { name: 'No', - value: false, - description: '' + value: false } ], prompt: inquirer.select @@ -60,18 +56,15 @@ export const all = [ choices: [ { name: 'Year folders', - value: 'year', - description: '' + value: 'year' }, { name: 'Year and month folders', - value: 'year-month', - description: '' + value: 'year-month' }, { name: 'No', - value: 'none', - description: '' + value: 'none' } ], prompt: inquirer.select diff --git a/src/wizard.js b/src/wizard.js index 3e7ce28..e2654a0 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -84,7 +84,6 @@ function getCommandLineAnswers(questions) { export async function getWizardAnswers(questions) { const answers = {}; for (const question of questions) { - // this will be set to the normalized answer during validation let normalizedAnswer; const promptConfig = { @@ -106,8 +105,7 @@ export async function getWizardAnswers(questions) { } } - // don't care about the return value of prompt() because normalizedAnswer will be used - await question.prompt(promptConfig).catch((ex) => { + const answer = await question.prompt(promptConfig).catch((ex) => { // exit gracefully if user hits ctrl + c during wizard if (ex instanceof Error && ex.name === 'ExitPromptError') { console.log('\nUser quit wizard early.'); @@ -117,7 +115,7 @@ export async function getWizardAnswers(questions) { } }); - answers[camelcase(question.name)] = normalizedAnswer; + answers[camelcase(question.name)] = normalizedAnswer ?? answer; } return answers; @@ -135,3 +133,28 @@ function normalize(value, type, onError) { onError(ex.message); } } + +function buildSamplePath(config) { + const pathSegments = []; + + if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { + pathSegments.push('2025'); + } + + if (config.dateFolders === 'year-month') { + pathSegments.push('01'); + } + + let slugFragment = 'my-post'; + if (config.prefixDate) { + slugFragment = '2025-01-31-' + slugFragment; + } + + if (config.postFolders) { + pathSegments.push(slugFragment, 'index.md'); + } else { + pathSegments.push(slugFragment + '.md'); + } + + return '/' + pathSegments.join('/'); +} From 8d4e7a03fc992c55c2698161cedd000103070f6c Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 27 Jan 2025 17:12:17 -0500 Subject: [PATCH 21/79] Show example paths during wizard --- src/questions.js | 4 ++++ src/wizard.js | 31 +++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/questions.js b/src/questions.js index 8c96b69..86da5b1 100644 --- a/src/questions.js +++ b/src/questions.js @@ -29,6 +29,7 @@ export const all = [ value: false } ], + isPathQuestion: true, prompt: inquirer.select }, { @@ -46,6 +47,7 @@ export const all = [ value: false } ], + isPathQuestion: true, prompt: inquirer.select }, { @@ -67,6 +69,7 @@ export const all = [ value: 'none' } ], + isPathQuestion: true, prompt: inquirer.select }, { @@ -92,6 +95,7 @@ export const all = [ value: 'none' } ], + isPathQuestion: true, prompt: inquirer.select } ]; diff --git a/src/wizard.js b/src/wizard.js index e2654a0..de22e55 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -16,23 +16,22 @@ const promptTheme = { }; export async function getConfig() { - const config = {}; - // check command line for any config options const commandLineQuestions = questions.all; - Object.assign(config, getCommandLineAnswers(commandLineQuestions)); + const commandLineAnswers = getCommandLineAnswers(commandLineQuestions); - if (config.wizard) { + let wizardAnswers; + if (commandLineAnswers.wizard) { console.log('\nStarting wizard...'); // run wizard for remaining config options - const wizardQuestions = questions.all.filter((question) => !(camelcase(question.name) in config)); - Object.assign(config, await getWizardAnswers(wizardQuestions)); + const wizardQuestions = questions.all.filter((question) => !(camelcase(question.name) in commandLineAnswers)); + wizardAnswers = await getWizardAnswers(wizardQuestions, commandLineAnswers); } else { console.log('\nSkipping wizard...'); } - return config; + return { ...commandLineAnswers, ...wizardAnswers }; } function getCommandLineAnswers(questions) { @@ -81,9 +80,10 @@ function getCommandLineAnswers(questions) { return answers; } -export async function getWizardAnswers(questions) { +export async function getWizardAnswers(questions, commandLineAnswers) { const answers = {}; for (const question of questions) { + let answerKey = camelcase(question.name); let normalizedAnswer; const promptConfig = { @@ -95,6 +95,17 @@ export async function getWizardAnswers(questions) { if (question.choices) { promptConfig.choices = question.choices; promptConfig.loop = false; + + if (question.isPathQuestion) { + // create a snapshot config of command line answers and wizard answers so far + const config = { ...commandLineAnswers, ...answers }; + + promptConfig.choices.forEach((choice) => { + // show example path if this choice is selected + config[answerKey] = choice.value; + choice.description = buildExamplePath(config); + }); + } } else { promptConfig.validate = (value) => { let validationErrorMessage; @@ -115,7 +126,7 @@ export async function getWizardAnswers(questions) { } }); - answers[camelcase(question.name)] = normalizedAnswer ?? answer; + answers[answerKey] = normalizedAnswer ?? answer; } return answers; @@ -134,7 +145,7 @@ function normalize(value, type, onError) { } } -function buildSamplePath(config) { +function buildExamplePath(config) { const pathSegments = []; if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { From 1877e43611b47697d65eef9d3d4ffe9a61527873 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Tue, 28 Jan 2025 08:14:15 -0500 Subject: [PATCH 22/79] buildSamplePostPath() --- src/questions.js | 1 - src/shared.js | 39 +++++++++++++++++++++++++++++++++++++++ src/wizard.js | 35 +++++++++++------------------------ 3 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/questions.js b/src/questions.js index 86da5b1..44f475d 100644 --- a/src/questions.js +++ b/src/questions.js @@ -95,7 +95,6 @@ export const all = [ value: 'none' } ], - isPathQuestion: true, prompt: inquirer.select } ]; diff --git a/src/shared.js b/src/shared.js index 0853bfe..6557462 100644 --- a/src/shared.js +++ b/src/shared.js @@ -1,3 +1,42 @@ +import * as luxon from 'luxon'; +import path from 'path'; +import * as settings from './settings.js'; + +export function buildPostPath(outputDir, type, date, slug, config) { + let dt; + if (settings.custom_date_formatting) { + dt = luxon.DateTime.fromFormat(date, settings.custom_date_formatting); + } else { + dt = luxon.DateTime.fromISO(date); + } + + // start with base output dir and post type + const pathSegments = [outputDir, type]; + + if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { + pathSegments.push(dt.toFormat('yyyy')); + } + + if (config.dateFolders === 'year-month') { + pathSegments.push(dt.toFormat('LL')); + } + + // create slug fragment, possibly date prefixed + let slugFragment = slug; + if (config.prefixDate) { + slugFragment = dt.toFormat('yyyy-LL-dd') + '-' + slugFragment; + } + + // use slug fragment as folder or filename as specified + if (config.postFolders) { + pathSegments.push(slugFragment, 'index.md'); + } else { + pathSegments.push(slugFragment + '.md'); + } + + return path.join(...pathSegments); +} + export function getFilenameFromUrl(url) { let filename = url.split('/').slice(-1)[0]; try { diff --git a/src/wizard.js b/src/wizard.js index de22e55..e8ef5f1 100644 --- a/src/wizard.js +++ b/src/wizard.js @@ -1,8 +1,11 @@ import camelcase from 'camelcase'; import chalk from 'chalk'; import * as commander from 'commander'; +import * as luxon from 'luxon'; +import path from 'path'; import * as normalizers from './normalizers.js'; import * as questions from './questions.js'; +import * as shared from './shared.js'; // visual formatting for wizard const promptTheme = { @@ -84,7 +87,7 @@ export async function getWizardAnswers(questions, commandLineAnswers) { const answers = {}; for (const question of questions) { let answerKey = camelcase(question.name); - let normalizedAnswer; + let normalizedAnswer; // holds normalized answer value potentially returned during validation const promptConfig = { theme: promptTheme, @@ -103,7 +106,7 @@ export async function getWizardAnswers(questions, commandLineAnswers) { promptConfig.choices.forEach((choice) => { // show example path if this choice is selected config[answerKey] = choice.value; - choice.description = buildExamplePath(config); + choice.description = buildSamplePostPath(config); }); } } else { @@ -145,27 +148,11 @@ function normalize(value, type, onError) { } } -function buildExamplePath(config) { - const pathSegments = []; +export function buildSamplePostPath(config) { + const outputDir = path.sep; + const type = '' + const date = luxon.DateTime.now().toFormat('yyyy-LL-dd'); + const slug = 'my-post'; - if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { - pathSegments.push('2025'); - } - - if (config.dateFolders === 'year-month') { - pathSegments.push('01'); - } - - let slugFragment = 'my-post'; - if (config.prefixDate) { - slugFragment = '2025-01-31-' + slugFragment; - } - - if (config.postFolders) { - pathSegments.push(slugFragment, 'index.md'); - } else { - pathSegments.push(slugFragment + '.md'); - } - - return '/' + pathSegments.join('/'); + return shared.buildPostPath(outputDir, type, date, slug, config); } From afc22e7ba5c0688337ae5da61fdc1cf4c8576b29 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Tue, 28 Jan 2025 16:38:29 -0500 Subject: [PATCH 23/79] Rename wizard.js to intake.js --- index.js | 4 ++-- src/{wizard.js => intake.js} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename src/{wizard.js => intake.js} (100%) diff --git a/index.js b/index.js index 2c04919..70349a6 100644 --- a/index.js +++ b/index.js @@ -4,7 +4,7 @@ import * as commander from 'commander'; import path from 'path'; import * as parser from './src/parser.js'; import * as settings from './src/settings.js'; -import * as wizard from './src/wizard.js'; +import * as intake from './src/intake.js'; import * as writer from './src/writer.js'; (async () => { @@ -15,7 +15,7 @@ import * as writer from './src/writer.js'; .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown') // gather config options from command line and wizard - const config = await wizard.getConfig(); + const config = await intake.getConfig(); // parse data from XML and do Markdown translations const posts = await parser.parseFilePromise(config) diff --git a/src/wizard.js b/src/intake.js similarity index 100% rename from src/wizard.js rename to src/intake.js From 896eaadb4fc55607a47180b4924314ed86627c40 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Tue, 28 Jan 2025 16:54:27 -0500 Subject: [PATCH 24/79] Have writer use shared.buildPostPath() --- src/intake.js | 1 + src/writer.js | 43 ++++++++----------------------------------- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/src/intake.js b/src/intake.js index e8ef5f1..16c15c9 100644 --- a/src/intake.js +++ b/src/intake.js @@ -75,6 +75,7 @@ function getCommandLineAnswers(questions) { // normalize and validate default answer const question = questions.find((question) => camelcase(question.name) === key); answers[key] = normalize(value, question.type, (errorMessage) => { + // this is formatted to match how commander displays other errors commander.program.error(`error: option '--${question.name} <${question.type}>' argument '${value}' is invalid. ${errorMessage}`); }); } diff --git a/src/writer.js b/src/writer.js index 7a642c9..52af1de 100644 --- a/src/writer.js +++ b/src/writer.js @@ -3,7 +3,6 @@ import chalk from 'chalk'; import fs from 'fs'; import http from 'http'; import https from 'https'; -import * as luxon from 'luxon'; import path from 'path'; import * as settings from './settings.js'; import * as shared from './shared.js'; @@ -47,7 +46,7 @@ async function writeMarkdownFilesPromise(posts, config) { let skipCount = 0; let delay = 0; const payloads = posts.flatMap(post => { - const destinationPath = getPostPath(post, config); + const destinationPath = buildPostPath(post, config); if (checkFile(destinationPath)) { // already exists, don't need to save again skipCount++; @@ -105,7 +104,7 @@ async function writeImageFilesPromise(posts, config) { let skipCount = 0; let delay = 0; const payloads = posts.flatMap(post => { - const postPath = getPostPath(post, config); + const postPath = buildPostPath(post, config); const imagesDir = path.join(path.dirname(postPath), 'images'); return post.meta.imageUrls.flatMap(imageUrl => { const filename = shared.getFilenameFromUrl(imageUrl); @@ -171,39 +170,13 @@ async function loadImageFilePromise(imageUrl) { return buffer; } -function getPostPath(post, config) { - let dt; - if (settings.custom_date_formatting) { - dt = luxon.DateTime.fromFormat(post.frontmatter.date, settings.custom_date_formatting); - } else { - dt = luxon.DateTime.fromISO(post.frontmatter.date); - } +function buildPostPath(post, config) { + const outputDir = settings.output_directory; + const type = post.meta.type; + const date = post.frontmatter.date; + const slug = post.meta.slug; - // start with base output dir and post type - const pathSegments = [settings.output_directory, post.meta.type]; - - if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { - pathSegments.push(dt.toFormat('yyyy')); - } - - if (config.dateFolders === 'year-month') { - pathSegments.push(dt.toFormat('LL')); - } - - // create slug fragment, possibly date prefixed - let slugFragment = post.meta.slug; - if (config.prefixDate) { - slugFragment = dt.toFormat('yyyy-LL-dd') + '-' + slugFragment; - } - - // use slug fragment as folder or filename as specified - if (config.postFolders) { - pathSegments.push(slugFragment, 'index.md'); - } else { - pathSegments.push(slugFragment + '.md'); - } - - return path.join(...pathSegments); + return shared.buildPostPath(outputDir, type, date, slug, config); } function checkFile(path) { From fbd36cf3a26153aff02364232e676c955084b575 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Tue, 28 Jan 2025 16:56:37 -0500 Subject: [PATCH 25/79] Semicolon kinda life --- src/intake.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intake.js b/src/intake.js index 16c15c9..aedf47e 100644 --- a/src/intake.js +++ b/src/intake.js @@ -151,7 +151,7 @@ function normalize(value, type, onError) { export function buildSamplePostPath(config) { const outputDir = path.sep; - const type = '' + const type = ''; const date = luxon.DateTime.now().toFormat('yyyy-LL-dd'); const slug = 'my-post'; From 865795cf11f80b3eee12f1f8865906646976dfc2 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Wed, 29 Jan 2025 15:55:36 -0500 Subject: [PATCH 26/79] Fix malformed tables from causing error --- package-lock.json | 182 +++++++++++++++++++++++----------------------- package.json | 2 +- src/translator.js | 2 +- 3 files changed, 94 insertions(+), 92 deletions(-) diff --git a/package-lock.json b/package-lock.json index 73a51c9..b848d25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "2.4.2", "license": "MIT", "dependencies": { + "@guyplusplus/turndown-plugin-gfm": "^1.0.7", "@inquirer/prompts": "^7.2.3", "axios": "^1.7.9", "camelcase": "^8.0.0", @@ -16,7 +17,6 @@ "commander": "^13.0.0", "luxon": "^3.5.0", "turndown": "^7.2.0", - "turndown-plugin-gfm": "^1.0.2", "xml2js": "^0.6.2" }, "bin": { @@ -88,6 +88,14 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@guyplusplus/turndown-plugin-gfm": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@guyplusplus/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.7.tgz", + "integrity": "sha512-k2KATk491JIeq1KNsjOwaD88cdknQeTpKVXh9OuZfHdjFuSEtEVVDCfCbxzmRvJxeKDPbZQl8DGP5o7SaIvWBw==", + "dependencies": { + "turndown": "^7.1.1" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.13.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", @@ -124,13 +132,13 @@ "dev": true }, "node_modules/@inquirer/checkbox": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.0.6.tgz", - "integrity": "sha512-PgP35JfmGjHU0LSXOyRew0zHuA9N6OJwOlos1fZ20b7j8ISeAdib3L+n0jIxBtX958UeEpte6xhG/gxJ5iUqMw==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.0.7.tgz", + "integrity": "sha512-lyoF4uYdBBTnqeB1gjPdYkiQ++fz/iYKaP9DON1ZGlldkvAEJsjaOBRdbl5UW1pOSslBRd701jxhAG0MlhHd2w==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^10.1.5", + "@inquirer/figures": "^1.0.10", + "@inquirer/type": "^3.0.3", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, @@ -142,12 +150,12 @@ } }, "node_modules/@inquirer/confirm": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.3.tgz", - "integrity": "sha512-fuF9laMmHoOgWapF9h9hv6opA5WvmGFHsTYGCmuFxcghIhEhb3dN0CdQR4BUMqa2H506NCj8cGX4jwMsE4t6dA==", + "version": "5.1.4", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.4.tgz", + "integrity": "sha512-EsiT7K4beM5fN5Mz6j866EFA9+v9d5o9VUra3hrg8zY4GHmCS8b616FErbdo5eyKoVotBQkHzMIeeKYsKDStDw==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2" + "@inquirer/core": "^10.1.5", + "@inquirer/type": "^3.0.3" }, "engines": { "node": ">=18" @@ -157,17 +165,16 @@ } }, "node_modules/@inquirer/core": { - "version": "10.1.4", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.4.tgz", - "integrity": "sha512-5y4/PUJVnRb4bwWY67KLdebWOhOc7xj5IP2J80oWXa64mVag24rwQ1VAdnj7/eDY/odhguW0zQ1Mp1pj6fO/2w==", + "version": "10.1.5", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.5.tgz", + "integrity": "sha512-/vyCWhET0ktav/mUeBqJRYTwmjFPIKPRYb3COAw7qORULgipGSUO2vL32lQKki3UxDKJ8BvuEbokaoyCA6YlWw==", "dependencies": { - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/figures": "^1.0.10", + "@inquirer/type": "^3.0.3", "ansi-escapes": "^4.3.2", "cli-width": "^4.1.0", "mute-stream": "^2.0.0", "signal-exit": "^4.1.0", - "strip-ansi": "^6.0.1", "wrap-ansi": "^6.2.0", "yoctocolors-cjs": "^2.1.2" }, @@ -176,12 +183,12 @@ } }, "node_modules/@inquirer/editor": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.3.tgz", - "integrity": "sha512-S9KnIOJuTZpb9upeRSBBhoDZv7aSV3pG9TECrBj0f+ZsFwccz886hzKBrChGrXMJwd4NKY+pOA9Vy72uqnd6Eg==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.4.tgz", + "integrity": "sha512-S8b6+K9PLzxiFGGc02m4syhEu5JsH0BukzRsuZ+tpjJ5aDsDX1WfNfOil2fmsO36Y1RMcpJGxlfQ1yh4WfU28Q==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^10.1.5", + "@inquirer/type": "^3.0.3", "external-editor": "^3.1.0" }, "engines": { @@ -192,12 +199,12 @@ } }, "node_modules/@inquirer/expand": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.6.tgz", - "integrity": "sha512-TRTfi1mv1GeIZGyi9PQmvAaH65ZlG4/FACq6wSzs7Vvf1z5dnNWsAAXBjWMHt76l+1hUY8teIqJFrWBk5N6gsg==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.7.tgz", + "integrity": "sha512-PsUQ5t7r+DPjW0VVEHzssOTBM2UPHnvBNse7hzuki7f6ekRL94drjjfBLrGEDe7cgj3pguufy/cuFwMeWUWHXw==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^10.1.5", + "@inquirer/type": "^3.0.3", "yoctocolors-cjs": "^2.1.2" }, "engines": { @@ -208,20 +215,20 @@ } }, "node_modules/@inquirer/figures": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.9.tgz", - "integrity": "sha512-BXvGj0ehzrngHTPTDqUoDT3NXL8U0RxUk2zJm2A66RhCEIWdtU1v6GuUqNAgArW4PQ9CinqIWyHdQgdwOj06zQ==", + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.10.tgz", + "integrity": "sha512-Ey6176gZmeqZuY/W/nZiUyvmb1/qInjcpiZjXWi6nON+nxJpD1bxtSoBxNliGISae32n6OwbY+TSXPZ1CfS4bw==", "engines": { "node": ">=18" } }, "node_modules/@inquirer/input": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.3.tgz", - "integrity": "sha512-zeo++6f7hxaEe7OjtMzdGZPHiawsfmCZxWB9X1NpmYgbeoyerIbWemvlBxxl+sQIlHC0WuSAG19ibMq3gbhaqQ==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.4.tgz", + "integrity": "sha512-CKKF8otRBdIaVnRxkFLs00VNA9HWlEh3x4SqUfC3A8819TeOZpTYG/p+4Nqu3hh97G+A0lxkOZNYE7KISgU8BA==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2" + "@inquirer/core": "^10.1.5", + "@inquirer/type": "^3.0.3" }, "engines": { "node": ">=18" @@ -231,12 +238,12 @@ } }, "node_modules/@inquirer/number": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.6.tgz", - "integrity": "sha512-xO07lftUHk1rs1gR0KbqB+LJPhkUNkyzV/KhH+937hdkMazmAYHLm1OIrNKpPelppeV1FgWrgFDjdUD8mM+XUg==", + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.7.tgz", + "integrity": "sha512-uU2nmXGC0kD8+BLgwZqcgBD1jcw2XFww2GmtP6b4504DkOp+fFAhydt7JzRR1TAI2dmj175p4SZB0lxVssNreA==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2" + "@inquirer/core": "^10.1.5", + "@inquirer/type": "^3.0.3" }, "engines": { "node": ">=18" @@ -246,12 +253,12 @@ } }, "node_modules/@inquirer/password": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.6.tgz", - "integrity": "sha512-QLF0HmMpHZPPMp10WGXh6F+ZPvzWE7LX6rNoccdktv/Rov0B+0f+eyXkAcgqy5cH9V+WSpbLxu2lo3ysEVK91w==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.7.tgz", + "integrity": "sha512-DFpqWLx+C5GV5zeFWuxwDYaeYnTWYphO07pQ2VnP403RIqRIpwBG0ATWf7pF+3IDbaXEtWatCJWxyDrJ+rkj2A==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^10.1.5", + "@inquirer/type": "^3.0.3", "ansi-escapes": "^4.3.2" }, "engines": { @@ -262,20 +269,20 @@ } }, "node_modules/@inquirer/prompts": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.2.3.tgz", - "integrity": "sha512-hzfnm3uOoDySDXfDNOm9usOuYIaQvTgKp/13l1uJoe6UNY+Zpcn2RYt0jXz3yA+yemGHvDOxVzqWl3S5sQq53Q==", + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.2.4.tgz", + "integrity": "sha512-Zn2XZL2VZl76pllUjeDnS6Poz2Oiv9kmAZdSZw1oFya985+/JXZ3GZ2JUWDokAPDhvuhkv9qz0Z7z/U80G8ztA==", "dependencies": { - "@inquirer/checkbox": "^4.0.6", - "@inquirer/confirm": "^5.1.3", - "@inquirer/editor": "^4.2.3", - "@inquirer/expand": "^4.0.6", - "@inquirer/input": "^4.1.3", - "@inquirer/number": "^3.0.6", - "@inquirer/password": "^4.0.6", - "@inquirer/rawlist": "^4.0.6", - "@inquirer/search": "^3.0.6", - "@inquirer/select": "^4.0.6" + "@inquirer/checkbox": "^4.0.7", + "@inquirer/confirm": "^5.1.4", + "@inquirer/editor": "^4.2.4", + "@inquirer/expand": "^4.0.7", + "@inquirer/input": "^4.1.4", + "@inquirer/number": "^3.0.7", + "@inquirer/password": "^4.0.7", + "@inquirer/rawlist": "^4.0.7", + "@inquirer/search": "^3.0.7", + "@inquirer/select": "^4.0.7" }, "engines": { "node": ">=18" @@ -285,12 +292,12 @@ } }, "node_modules/@inquirer/rawlist": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.6.tgz", - "integrity": "sha512-QoE4s1SsIPx27FO4L1b1mUjVcoHm1pWE/oCmm4z/Hl+V1Aw5IXl8FYYzGmfXaBT0l/sWr49XmNSiq7kg3Kd/Lg==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.7.tgz", + "integrity": "sha512-ZeBca+JCCtEIwQMvhuROT6rgFQWWvAImdQmIIP3XoyDFjrp2E0gZlEn65sWIoR6pP2EatYK96pvx0887OATWQQ==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^10.1.5", + "@inquirer/type": "^3.0.3", "yoctocolors-cjs": "^2.1.2" }, "engines": { @@ -301,13 +308,13 @@ } }, "node_modules/@inquirer/search": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.6.tgz", - "integrity": "sha512-eFZ2hiAq0bZcFPuFFBmZEtXU1EarHLigE+ENCtpO+37NHCl4+Yokq1P/d09kUblObaikwfo97w+0FtG/EXl5Ng==", + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.7.tgz", + "integrity": "sha512-Krq925SDoLh9AWSNee8mbSIysgyWtcPnSAp5YtPBGCQ+OCO+5KGC8FwLpyxl8wZ2YAov/8Tp21stTRK/fw5SGg==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^10.1.5", + "@inquirer/figures": "^1.0.10", + "@inquirer/type": "^3.0.3", "yoctocolors-cjs": "^2.1.2" }, "engines": { @@ -318,13 +325,13 @@ } }, "node_modules/@inquirer/select": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.0.6.tgz", - "integrity": "sha512-yANzIiNZ8fhMm4NORm+a74+KFYHmf7BZphSOBovIzYPVLquseTGEkU5l2UTnBOf5k0VLmTgPighNDLE9QtbViQ==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.0.7.tgz", + "integrity": "sha512-ejGBMDSD+Iqk60u5t0Zf2UQhGlJWDM78Ep70XpNufIfc+f4VOTeybYKXu9pDjz87FkRzLiVsGpQG2SzuGlhaJw==", "dependencies": { - "@inquirer/core": "^10.1.4", - "@inquirer/figures": "^1.0.9", - "@inquirer/type": "^3.0.2", + "@inquirer/core": "^10.1.5", + "@inquirer/figures": "^1.0.10", + "@inquirer/type": "^3.0.3", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, @@ -336,9 +343,9 @@ } }, "node_modules/@inquirer/type": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.2.tgz", - "integrity": "sha512-ZhQ4TvhwHZF+lGhQ2O/rsjo80XoZR5/5qhOY3t6FJuX5XBg5Be8YzYTvaUGJnc12AUGI2nr4QSUE4PhKSigx7g==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.3.tgz", + "integrity": "sha512-I4VIHFxUuY1bshGbXZTxCmhwaaEst9s/lll3ekok+o1Z26/ZUKdx8y1b7lsoG6rtsBDwEGfiBJ2SfirjoISLpg==", "engines": { "node": ">=18" }, @@ -387,18 +394,18 @@ } }, "node_modules/@types/node": { - "version": "22.10.7", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.7.tgz", - "integrity": "sha512-V09KvXxFiutGp6B7XkpaDXlNadZxrzajcY50EuoLIpQ6WWYCSvf19lVIazzfIzQvhUN2HjX12spLojTnhuKlGg==", + "version": "22.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.12.0.tgz", + "integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==", "peer": true, "dependencies": { "undici-types": "~6.20.0" } }, "node_modules/@ungap/structured-clone": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.1.tgz", - "integrity": "sha512-fEzPV3hSkSMltkw152tJKNARhOupqbH96MZWyRjNaYZOMIzbrTeQDG+MTc6Mr2pgzFQzFxAfmhGDNP5QK++2ZA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", "dev": true }, "node_modules/acorn": { @@ -1587,11 +1594,6 @@ "@mixmark-io/domino": "^2.2.0" } }, - "node_modules/turndown-plugin-gfm": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz", - "integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==" - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 55c4509..5f6ab0c 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ }, "type": "module", "dependencies": { + "@guyplusplus/turndown-plugin-gfm": "^1.0.7", "@inquirer/prompts": "^7.2.3", "axios": "^1.7.9", "camelcase": "^8.0.0", @@ -28,7 +29,6 @@ "commander": "^13.0.0", "luxon": "^3.5.0", "turndown": "^7.2.0", - "turndown-plugin-gfm": "^1.0.2", "xml2js": "^0.6.2" }, "devDependencies": { diff --git a/src/translator.js b/src/translator.js index 2e43c35..163c602 100644 --- a/src/translator.js +++ b/src/translator.js @@ -1,5 +1,5 @@ import turndown from 'turndown'; -import turndownPluginGfm from 'turndown-plugin-gfm'; +import turndownPluginGfm from '@guyplusplus/turndown-plugin-gfm'; export function initTurndownService() { const turndownService = new turndown({ From cf338813f1a555de15dd8d9a4b82406972f4e245 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Fri, 31 Jan 2025 13:11:07 -0500 Subject: [PATCH 27/79] Move output from settings to questions --- index.js | 3 +-- src/intake.js | 16 +++++++++++----- src/questions.js | 18 ++++++++++++------ src/settings.js | 3 --- src/writer.js | 2 +- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/index.js b/index.js index 70349a6..7a5c7a0 100644 --- a/index.js +++ b/index.js @@ -3,7 +3,6 @@ import * as commander from 'commander'; import path from 'path'; import * as parser from './src/parser.js'; -import * as settings from './src/settings.js'; import * as intake from './src/intake.js'; import * as writer from './src/writer.js'; @@ -25,7 +24,7 @@ import * as writer from './src/writer.js'; // happy goodbye console.log('\nAll done!'); - console.log('Look for your output files in: ' + path.resolve(settings.output_directory)); + console.log('Look for your output files in: ' + path.resolve(config.output)); })().catch((ex) => { // sad goodbye console.log('\nSomething went wrong, execution halted early.'); diff --git a/src/intake.js b/src/intake.js index aedf47e..63763e2 100644 --- a/src/intake.js +++ b/src/intake.js @@ -27,8 +27,10 @@ export async function getConfig() { if (commandLineAnswers.wizard) { console.log('\nStarting wizard...'); - // run wizard for remaining config options - const wizardQuestions = questions.all.filter((question) => !(camelcase(question.name) in commandLineAnswers)); + // run wizard for questions with prompts that were not answered via the command line + const wizardQuestions = questions.all.filter((question) => { + return question.prompt && !(camelcase(question.name) in commandLineAnswers); + }); wizardAnswers = await getWizardAnswers(wizardQuestions, commandLineAnswers); } else { console.log('\nSkipping wizard...'); @@ -47,6 +49,10 @@ function getCommandLineAnswers(questions) { const option = new commander.Option('--' + question.name + ' <' + question.type + '>', question.description); option.default(question.default); + if (!question.description) { + option.hideHelp(); + } + if (question.choices && question.type !== 'boolean') { // let commander handle non-boolean multiple choice validation option.choices(question.choices.map((choice) => choice.value)); @@ -68,12 +74,12 @@ function getCommandLineAnswers(questions) { continue; } - if (answers.wizard) { - // remove this default answer so the wizard will ask about it later + const question = questions.find((question) => camelcase(question.name) === key); + if (answers.wizard && question.prompt) { + // remove this default answer, allowing the wizard to ask about it later delete answers[key]; } else { // normalize and validate default answer - const question = questions.find((question) => camelcase(question.name) === key); answers[key] = normalize(value, question.type, (errorMessage) => { // this is formatted to match how commander displays other errors commander.program.error(`error: option '--${question.name} <${question.type}>' argument '${value}' is invalid. ${errorMessage}`); diff --git a/src/questions.js b/src/questions.js index 44f475d..6269338 100644 --- a/src/questions.js +++ b/src/questions.js @@ -1,12 +1,6 @@ import * as inquirer from '@inquirer/prompts'; export const all = [ - { - name: 'wizard', - type: 'boolean', - description: 'Use wizard', - default: true - }, { name: 'input', type: 'file-path', @@ -96,5 +90,17 @@ export const all = [ } ], prompt: inquirer.select + }, + { + name: 'wizard', + type: 'boolean', + description: 'Use wizard', + default: true + }, + { + name: 'output', + type: 'folder-path', + description: 'Path to output folder', + default: 'output' } ]; diff --git a/src/settings.js b/src/settings.js index 28ef891..58e67b3 100644 --- a/src/settings.js +++ b/src/settings.js @@ -49,6 +49,3 @@ export const filter_post_types = [ 'wp_global_styles', 'wp_navigation' ]; - -// Output directory. -export const output_directory = 'output'; diff --git a/src/writer.js b/src/writer.js index 52af1de..6a902eb 100644 --- a/src/writer.js +++ b/src/writer.js @@ -171,7 +171,7 @@ async function loadImageFilePromise(imageUrl) { } function buildPostPath(post, config) { - const outputDir = settings.output_directory; + const outputDir = config.output; const type = post.meta.type; const date = post.frontmatter.date; const slug = post.meta.slug; From f6197d1d70cf8e1e1399a67c21f01c1189197f76 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 1 Feb 2025 08:27:58 -0500 Subject: [PATCH 28/79] First list and integer questions, spaces to tabs --- src/frontmatter.js | 30 +++++++++++------------ src/normalizers.js | 59 +++++++++++++++++++++++++++++----------------- src/parser.js | 6 ++--- src/questions.js | 26 ++++++++++++++------ src/settings.js | 16 ++++++------- src/writer.js | 2 +- 6 files changed, 84 insertions(+), 55 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index 390815a..029832b 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -9,15 +9,15 @@ export function getAuthor(post) { // get array of decoded category names, filtered as specified in settings export function getCategories(post) { - if (!post.data.category) { - return []; - } + if (!post.data.category) { + return []; + } - const categories = post.data.category - .filter(category => category.$.domain === 'category') - .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); + const categories = post.data.category + .filter(category => category.$.domain === 'category') + .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); - return categories.filter(category => !settings.filter_categories.includes(category)); + return categories.filter(category => !settings.filter_categories.includes(category)); } // get cover image filename, previously decoded and set on post.meta @@ -29,15 +29,15 @@ export function getCoverImage(post) { // get post date, optionally formatted as specified in settings // this value is also used for year/month folders, date prefixes, etc. as needed export function getDate(post) { - const dateTime = luxon.DateTime.fromRFC2822(post.data.pubDate[0], { zone: settings.custom_date_timezone }); + const dateTime = luxon.DateTime.fromRFC2822(post.data.pubDate[0], { zone: settings.custom_date_timezone }); - if (settings.custom_date_formatting) { - return dateTime.toFormat(settings.custom_date_formatting); - } else if (settings.include_time_with_date) { - return dateTime.toISO(); - } else { - return dateTime.toISODate(); - } + if (settings.custom_date_formatting) { + return dateTime.toFormat(settings.custom_date_formatting); + } else if (settings.include_time_with_date) { + return dateTime.toISO(); + } else { + return dateTime.toISODate(); + } } // get excerpt, not decoded, newlines collapsed diff --git a/src/normalizers.js b/src/normalizers.js index 5234646..b6674e0 100644 --- a/src/normalizers.js +++ b/src/normalizers.js @@ -2,31 +2,48 @@ import fs from 'fs'; import path from 'path'; export function boolean(value) { - if (typeof value === 'boolean') { - return value; - } else if (value === 'true') { - return true; - } else if (value === 'false') { - return false; - } + if (typeof value === 'boolean') { + return value; + } else if (value === 'true') { + return true; + } else if (value === 'false') { + return false; + } - throw new Error('Must be true or false.'); + throw new Error('Must be true or false.'); } export function filePath(value) { - const unwrapped = value.replace(/"(.*?)"/, '$1'); - const absolute = path.resolve(unwrapped); + const unwrapped = value.replace(/"(.*?)"/, '$1'); + const absolute = path.resolve(unwrapped); - let fileExists; - try { - fileExists = fs.existsSync(absolute) && fs.statSync(absolute).isFile(); - } catch (ex) { - fileExists = false; - } + let fileExists; + try { + fileExists = fs.existsSync(absolute) && fs.statSync(absolute).isFile(); + } catch (ex) { + fileExists = false; + } - if (fileExists) { - return absolute; - } else { - throw new Error('File not found at ' + absolute + '.'); - } + if (fileExists) { + return absolute; + } + + throw new Error('File not found at ' + absolute + '.'); +} + +export function list(value) { + if (Array.isArray(value)) { + return value; + } else { + return value.trim().split(/\s*,\s*/); + } +} + +export function integer(value) { + const int = parseInt(value); + if (!Number.isNaN(int) && int >= 0) { + return int; + } + + throw new Error('Must be an integer >= 0.'); } diff --git a/src/parser.js b/src/parser.js index 08775ad..0109dda 100644 --- a/src/parser.js +++ b/src/parser.js @@ -26,7 +26,7 @@ export async function parseFilePromise(config) { } mergeImagesIntoPosts(images, posts); - populateFrontmatter(posts); + populateFrontmatter(posts, config); return posts; } @@ -162,10 +162,10 @@ function mergeImagesIntoPosts(images, posts) { }); } -function populateFrontmatter(posts) { +function populateFrontmatter(posts, config) { posts.forEach(post => { post.frontmatter = {}; - settings.frontmatter_fields.forEach(field => { + config.frontmatterFields.forEach(field => { const [key, alias] = field.split(':'); let frontmatterGetter = frontmatter['get' + key.replace(/^./, (match) => match.toUpperCase())]; diff --git a/src/questions.js b/src/questions.js index 6269338..4a7f1de 100644 --- a/src/questions.js +++ b/src/questions.js @@ -1,6 +1,14 @@ import * as inquirer from '@inquirer/prompts'; +// questions with a description are displayed in command line help +// questions with a prompt are included in the wizard (if not set on the command line) export const all = [ + { + name: 'wizard', + type: 'boolean', + description: 'Use wizard', + default: true + }, { name: 'input', type: 'file-path', @@ -91,16 +99,20 @@ export const all = [ ], prompt: inquirer.select }, - { - name: 'wizard', - type: 'boolean', - description: 'Use wizard', - default: true - }, { name: 'output', type: 'folder-path', description: 'Path to output folder', default: 'output' - } + }, + { + name: 'frontmatter-fields', + type: 'list', + default: ['title', 'date', 'categories', 'tags', 'coverImage'] + }, + { + name: 'image-file-request-delay', + type: 'integer', + default: 500 + }, ]; diff --git a/src/settings.js b/src/settings.js index 58e67b3..df0247c 100644 --- a/src/settings.js +++ b/src/settings.js @@ -2,17 +2,17 @@ // Order is preserved. If a field has an empty value, it will not be included. You can rename a // field by providing an alias after a ':'. For example, 'date:created' will include 'date' in // frontmatter, but renamed to 'created'. -export const frontmatter_fields = [ - 'title', - 'date', - 'categories', - 'tags', - 'coverImage' -]; +// export const frontmatter_fields = [ +// 'title', +// 'date', +// 'categories', +// 'tags', +// 'coverImage' +// ]; // Time in ms to wait between requesting image files. Increase this if you see timeouts or // server errors. -export const image_file_request_delay = 500; +// export const image_file_request_delay = 500; // Time in ms to wait between saving Markdown files. Increase this if your file system becomes // overloaded. diff --git a/src/writer.js b/src/writer.js index 6a902eb..e54903e 100644 --- a/src/writer.js +++ b/src/writer.js @@ -120,7 +120,7 @@ async function writeImageFilesPromise(posts, config) { destinationPath, delay }; - delay += settings.image_file_request_delay; + delay += config.imageFileRequestDelay; return [payload]; } }); From 3bbf0274cec5b1b1b7d148cfcbd619027cf2479d Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 1 Feb 2025 15:10:45 -0500 Subject: [PATCH 29/79] Small frontmatter getter refactor, more questions --- src/frontmatter.js | 22 +++++++++++----------- src/parser.js | 4 ++-- src/questions.js | 10 ++++++++++ src/settings.js | 4 ++-- src/writer.js | 2 +- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index 029832b..ce4522b 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -3,12 +3,12 @@ import * as settings from './settings.js'; // get author, without decoding // WordPress doesn't allow funky characters in usernames anyway -export function getAuthor(post) { +export function author(post) { return post.data.creator[0]; } // get array of decoded category names, filtered as specified in settings -export function getCategories(post) { +export function categories(post) { if (!post.data.category) { return []; } @@ -22,18 +22,18 @@ export function getCategories(post) { // get cover image filename, previously decoded and set on post.meta // this one is unique as it relies on special logic executed by the parser -export function getCoverImage(post) { +export function coverImage(post) { return post.meta.coverImage; } // get post date, optionally formatted as specified in settings // this value is also used for year/month folders, date prefixes, etc. as needed -export function getDate(post) { +export function date(post, config) { const dateTime = luxon.DateTime.fromRFC2822(post.data.pubDate[0], { zone: settings.custom_date_timezone }); if (settings.custom_date_formatting) { return dateTime.toFormat(settings.custom_date_formatting); - } else if (settings.include_time_with_date) { + } else if (config.includeTimeWithDate) { return dateTime.toISO(); } else { return dateTime.toISODate(); @@ -41,22 +41,22 @@ export function getDate(post) { } // get excerpt, not decoded, newlines collapsed -export function getExcerpt(post) { +export function excerpt(post) { return post.data.encoded[1].replace(/[\r\n]+/gm, ' '); } // get ID -export function getId(post) { +export function id(post) { return post.data.post_id[0]; } // get slug, previously decoded and set on post.meta -export function getSlug(post) { +export function slug(post) { return post.meta.slug; } // get array of decoded tag names -export function getTags(post) { +export function tags(post) { if (!post.data.category) { return []; } @@ -69,12 +69,12 @@ export function getTags(post) { } // get simple post title, but not decoded like other frontmatter string fields -export function getTitle(post) { +export function title(post) { return post.data.title[0]; } // get type, often this will always be "post" // but can also be "page" or other custom types -export function getType(post) { +export function type(post) { return post.data.post_type[0]; } diff --git a/src/parser.js b/src/parser.js index 0109dda..50bec3e 100644 --- a/src/parser.js +++ b/src/parser.js @@ -168,12 +168,12 @@ function populateFrontmatter(posts, config) { config.frontmatterFields.forEach(field => { const [key, alias] = field.split(':'); - let frontmatterGetter = frontmatter['get' + key.replace(/^./, (match) => match.toUpperCase())]; + let frontmatterGetter = frontmatter[key]; if (!frontmatterGetter) { throw `Could not find a frontmatter getter named "${key}".`; } - post.frontmatter[alias || key] = frontmatterGetter(post); + post.frontmatter[alias || key] = frontmatterGetter(post, config); }); }); } diff --git a/src/questions.js b/src/questions.js index 4a7f1de..a2db567 100644 --- a/src/questions.js +++ b/src/questions.js @@ -115,4 +115,14 @@ export const all = [ type: 'integer', default: 500 }, + { + name: 'markdown-file-write-delay', + type: 'integer', + default: 25 + }, + { + name: 'include-time-with-date', + type: 'boolean', + default: false + }, ]; diff --git a/src/settings.js b/src/settings.js index df0247c..a072e1e 100644 --- a/src/settings.js +++ b/src/settings.js @@ -16,11 +16,11 @@ // Time in ms to wait between saving Markdown files. Increase this if your file system becomes // overloaded. -export const markdown_file_write_delay = 25; +// export const markdown_file_write_delay = 25; // Enable this to include time with post dates. For example, "2020-12-25" would become // "2020-12-25T11:20:35.000Z". -export const include_time_with_date = false; +// export const include_time_with_date = false; // Override post date formatting with a custom formatting string (for example: 'yyyy LLL dd'). // Tokens are documented here: https://moment.github.io/luxon/#/parsing?id=table-of-tokens. If diff --git a/src/writer.js b/src/writer.js index e54903e..ba9fe7d 100644 --- a/src/writer.js +++ b/src/writer.js @@ -58,7 +58,7 @@ async function writeMarkdownFilesPromise(posts, config) { destinationPath, delay }; - delay += settings.markdown_file_write_delay; + delay += config.markdownFileWriteDelay; return [payload]; } }); From faff0ec8562bc3d0f7f7e1296a05ad1b02dda3a3 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 2 Feb 2025 12:37:49 -0500 Subject: [PATCH 30/79] filter-categories and strict-ssl questions --- src/frontmatter.js | 4 ++-- src/questions.js | 10 ++++++++++ src/writer.js | 21 ++++++++++----------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index ce4522b..6b62c3b 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -8,7 +8,7 @@ export function author(post) { } // get array of decoded category names, filtered as specified in settings -export function categories(post) { +export function categories(post, config) { if (!post.data.category) { return []; } @@ -17,7 +17,7 @@ export function categories(post) { .filter(category => category.$.domain === 'category') .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); - return categories.filter(category => !settings.filter_categories.includes(category)); + return categories.filter((category) => !config.filterCategories.includes(category)); } // get cover image filename, previously decoded and set on post.meta diff --git a/src/questions.js b/src/questions.js index a2db567..b08d2df 100644 --- a/src/questions.js +++ b/src/questions.js @@ -125,4 +125,14 @@ export const all = [ type: 'boolean', default: false }, + { + name: 'filter-categories', + type: 'list', + default: ['uncategorized'] + }, + { + name: 'strict-ssl', + type: 'boolean', + default: true + } ]; diff --git a/src/writer.js b/src/writer.js index ba9fe7d..d8f8701 100644 --- a/src/writer.js +++ b/src/writer.js @@ -4,7 +4,6 @@ import fs from 'fs'; import http from 'http'; import https from 'https'; import path from 'path'; -import * as settings from './settings.js'; import * as shared from './shared.js'; export async function writeFilesPromise(posts, config) { @@ -12,11 +11,11 @@ export async function writeFilesPromise(posts, config) { await writeImageFilesPromise(posts, config); } -async function processPayloadsPromise(payloads, loadFunc) { +async function processPayloadsPromise(payloads, loadFunc, config) { const promises = payloads.map(payload => new Promise((resolve, reject) => { setTimeout(async () => { try { - const data = await loadFunc(payload.item); + const data = await loadFunc(payload.item, config); await writeFile(payload.destinationPath, data); console.log(chalk.green('[OK]') + ' ' + payload.name); resolve(); @@ -68,7 +67,7 @@ async function writeMarkdownFilesPromise(posts, config) { console.log('\nNo posts to save...'); } else { console.log(`\nSaving ${remainingCount} posts (${skipCount} already exist)...`); - await processPayloadsPromise(payloads, loadMarkdownFilePromise); + await processPayloadsPromise(payloads, loadMarkdownFilePromise, config); } } @@ -131,15 +130,15 @@ async function writeImageFilesPromise(posts, config) { console.log('\nNo images to download and save...'); } else { console.log(`\nDownloading and saving ${remainingCount} images (${skipCount} already exist)...`); - await processPayloadsPromise(payloads, loadImageFilePromise); + await processPayloadsPromise(payloads, loadImageFilePromise, config); } } -async function loadImageFilePromise(imageUrl) { +async function loadImageFilePromise(imageUrl, config) { // only encode the URL if it doesn't already have encoded characters const url = (/%[\da-f]{2}/i).test(imageUrl) ? imageUrl : encodeURI(imageUrl); - const config = { + const requestConfig = { method: 'get', url, headers: { @@ -148,15 +147,15 @@ async function loadImageFilePromise(imageUrl) { responseType: 'arraybuffer' }; - if (!settings.strict_ssl) { + if (!config.strictSsl) { // custom agents to disable SSL errors (adding both http and https, just in case) - config.httpAgent = new http.Agent({ rejectUnauthorized: false }); - config.httpsAgent = new https.Agent({ rejectUnauthorized: false }); + requestConfig.httpAgent = new http.Agent({ rejectUnauthorized: false }); + requestConfig.httpsAgent = new https.Agent({ rejectUnauthorized: false }); } let buffer; try { - const response = await axios(config); + const response = await axios(requestConfig); buffer = Buffer.from(response.data, 'binary'); } catch (ex) { if (ex.response) { From da4af6608a7949dc8562934b43523bd3f54baa27 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 2 Feb 2025 15:32:37 -0500 Subject: [PATCH 31/79] Hardcode filter-categories and filter-post-types --- src/frontmatter.js | 6 +++--- src/parser.js | 11 +++++++++-- src/settings.js | 22 +++++++++++----------- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index 6b62c3b..f2c74e1 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -7,8 +7,8 @@ export function author(post) { return post.data.creator[0]; } -// get array of decoded category names, filtered as specified in settings -export function categories(post, config) { +// get array of decoded category names, excluding 'uncategorized' +export function categories(post) { if (!post.data.category) { return []; } @@ -17,7 +17,7 @@ export function categories(post, config) { .filter(category => category.$.domain === 'category') .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); - return categories.filter((category) => !config.filterCategories.includes(category)); + return categories.filter((category) => category !== 'uncategorized'); } // get cover image filename, previously decoded and set on post.meta diff --git a/src/parser.js b/src/parser.js index 50bec3e..0537bf2 100644 --- a/src/parser.js +++ b/src/parser.js @@ -1,7 +1,6 @@ import fs from 'fs'; import xml2js from 'xml2js'; import * as frontmatter from './frontmatter.js'; -import * as settings from './settings.js'; import * as shared from './shared.js'; import * as translator from './translator.js'; @@ -35,7 +34,15 @@ function getPostTypes(channelData) { // search export file for all post types minus some specific types we don't want const types = channelData .map(item => item.post_type[0]) - .filter(type => !settings.filter_post_types.includes(type)); + .filter(type => ![ + 'attachment', + 'revision', + 'nav_menu_item', + 'custom_css', + 'customize_changeset', + 'wp_global_styles', + 'wp_navigation' + ].includes(type)); return [...new Set(types)]; // remove duplicates } diff --git a/src/settings.js b/src/settings.js index a072e1e..97f5e9e 100644 --- a/src/settings.js +++ b/src/settings.js @@ -33,19 +33,19 @@ export const custom_date_timezone = 'utc'; // Categories to be excluded from post frontmatter. This does not filter out posts themselves, // just the categories listed in their frontmatter. -export const filter_categories = ['uncategorized']; +// export const filter_categories = ['uncategorized']; // Strict SSL is enabled as the safe default when downloading images, but will not work with // self-signed servers. You can disable it if you're getting a "self-signed certificate" error. -export const strict_ssl = true; +// export const strict_ssl = true; // Post types to exclude from output. -export const filter_post_types = [ - 'attachment', - 'revision', - 'nav_menu_item', - 'custom_css', - 'customize_changeset', - 'wp_global_styles', - 'wp_navigation' -]; +// export const filter_post_types = [ +// 'attachment', +// 'revision', +// 'nav_menu_item', +// 'custom_css', +// 'customize_changeset', +// 'wp_global_styles', +// 'wp_navigation' +// ]; From b86185a12141d50d1c383638fd52f017aeee2729 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 2 Feb 2025 15:52:36 -0500 Subject: [PATCH 32/79] Filter out the WordPress sample page --- src/parser.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/parser.js b/src/parser.js index 0537bf2..b30c655 100644 --- a/src/parser.js +++ b/src/parser.js @@ -58,6 +58,7 @@ function collectPosts(channelData, postTypes, config) { postTypes.forEach(postType => { const postsForType = getItemsOfType(channelData, postType) .filter(postData => postData.status[0] !== 'trash' && postData.status[0] !== 'draft') + .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) .map(postData => ({ // raw post data, used by frontmatter getters data: postData, From 67bf6488c12563efabdef08ec266f760a2404ee1 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 2 Feb 2025 18:08:09 -0500 Subject: [PATCH 33/79] Unused filter-categories question --- src/questions.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/questions.js b/src/questions.js index b08d2df..0e09b86 100644 --- a/src/questions.js +++ b/src/questions.js @@ -125,11 +125,6 @@ export const all = [ type: 'boolean', default: false }, - { - name: 'filter-categories', - type: 'list', - default: ['uncategorized'] - }, { name: 'strict-ssl', type: 'boolean', From 8a25175c733dfbfdd353392fa745281f9d42afa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20N=C3=BDvlt?= Date: Mon, 17 Jun 2024 10:13:05 +0200 Subject: [PATCH 34/79] Fix datetimes being incorrectly encoded in frontmatter as strings instead of YAML timestamps --- src/writer.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/writer.js b/src/writer.js index d8f8701..c81d229 100644 --- a/src/writer.js +++ b/src/writer.js @@ -81,6 +81,8 @@ async function loadMarkdownFilePromise(post) { // array of one or more strings outputValue = value.reduce((list, item) => `${list}\n - "${item}"`, ''); } + } else if (value instanceof luxon.DateTime) { + outputValue = encodeDate(value); } else { // single string value const escapedValue = (value || '').replace(/"/g, '\\"'); @@ -98,6 +100,16 @@ async function loadMarkdownFilePromise(post) { return output; } +function encodeDate(dateTime) { + if (settings.custom_date_formatting) { + return dateTime.toFormat(settings.custom_date_formatting); + } else if (settings.include_time_with_date) { + return dateTime.toISO(); + } else { + return dateTime.toISODate(); + } +} + async function writeImageFilesPromise(posts, config) { // collect image data from all posts into a single flattened array of payloads let skipCount = 0; From 7934f4e4b44c42bcf45759b3fc55770752b3b7f4 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 3 Feb 2025 14:27:37 -0500 Subject: [PATCH 35/79] Change when date is formatted --- src/frontmatter.js | 17 +++-------------- src/parser.js | 14 +++++++++++--- src/questions.js | 10 ++++++++++ src/writer.js | 19 +++++++------------ 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index f2c74e1..1926037 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -1,6 +1,3 @@ -import * as luxon from 'luxon'; -import * as settings from './settings.js'; - // get author, without decoding // WordPress doesn't allow funky characters in usernames anyway export function author(post) { @@ -26,18 +23,10 @@ export function coverImage(post) { return post.meta.coverImage; } -// get post date, optionally formatted as specified in settings +// get post date, previously saved as a luxon datetime object on post.meta // this value is also used for year/month folders, date prefixes, etc. as needed -export function date(post, config) { - const dateTime = luxon.DateTime.fromRFC2822(post.data.pubDate[0], { zone: settings.custom_date_timezone }); - - if (settings.custom_date_formatting) { - return dateTime.toFormat(settings.custom_date_formatting); - } else if (config.includeTimeWithDate) { - return dateTime.toISO(); - } else { - return dateTime.toISODate(); - } +export function date(post) { + return post.meta.date; } // get excerpt, not decoded, newlines collapsed diff --git a/src/parser.js b/src/parser.js index b30c655..6819488 100644 --- a/src/parser.js +++ b/src/parser.js @@ -1,4 +1,5 @@ import fs from 'fs'; +import * as luxon from 'luxon'; import xml2js from 'xml2js'; import * as frontmatter from './frontmatter.js'; import * as shared from './shared.js'; @@ -65,12 +66,15 @@ function collectPosts(channelData, postTypes, config) { // meta data isn't written to file, but is used to help with other things meta: { + type: postType, id: getPostId(postData), slug: getPostSlug(postData), + date: getPostDate(postData, config), coverImageId: getPostCoverImageId(postData), - coverImage: undefined, // possibly set later in mergeImagesIntoPosts() - type: postType, - imageUrls: [] // possibly set later in mergeImagesIntoPosts() + + // these are possibly set later in mergeImagesIntoPosts() + coverImage: undefined, + imageUrls: [] }, // contents of the post in markdown @@ -98,6 +102,10 @@ function getPostSlug(postData) { return decodeURIComponent(postData.post_name[0]); } +function getPostDate(postData, config) { + return luxon.DateTime.fromRFC2822(postData.pubDate[0], { zone: config.customDateTimezone }); +} + function getPostCoverImageId(postData) { if (postData.postmeta === undefined) { return undefined; diff --git a/src/questions.js b/src/questions.js index 0e09b86..bf31cbe 100644 --- a/src/questions.js +++ b/src/questions.js @@ -125,6 +125,16 @@ export const all = [ type: 'boolean', default: false }, + { + name: 'custom-date-formatting', + type: 'string', + default: '' + }, + { + name: 'custom-date-timezone', + type: 'string', + default: 'utc' + }, { name: 'strict-ssl', type: 'boolean', diff --git a/src/writer.js b/src/writer.js index c81d229..239803b 100644 --- a/src/writer.js +++ b/src/writer.js @@ -3,6 +3,7 @@ import chalk from 'chalk'; import fs from 'fs'; import http from 'http'; import https from 'https'; +import * as luxon from 'luxon'; import path from 'path'; import * as shared from './shared.js'; @@ -71,7 +72,7 @@ async function writeMarkdownFilesPromise(posts, config) { } } -async function loadMarkdownFilePromise(post) { +async function loadMarkdownFilePromise(post, config) { let output = '---\n'; Object.entries(post.frontmatter).forEach(([key, value]) => { @@ -82,7 +83,11 @@ async function loadMarkdownFilePromise(post) { outputValue = value.reduce((list, item) => `${list}\n - "${item}"`, ''); } } else if (value instanceof luxon.DateTime) { - outputValue = encodeDate(value); + if (config.customDateFormatting) { + outputValue = value.toFormat(config.customDateFormatting); + } else { + outputValue = config.includeTimeWithDate ? value.toISO() : value.toISODate(); + } } else { // single string value const escapedValue = (value || '').replace(/"/g, '\\"'); @@ -100,16 +105,6 @@ async function loadMarkdownFilePromise(post) { return output; } -function encodeDate(dateTime) { - if (settings.custom_date_formatting) { - return dateTime.toFormat(settings.custom_date_formatting); - } else if (settings.include_time_with_date) { - return dateTime.toISO(); - } else { - return dateTime.toISODate(); - } -} - async function writeImageFilesPromise(posts, config) { // collect image data from all posts into a single flattened array of payloads let skipCount = 0; From 71028a272b4629cbf94d09b1d2de9ccff208f7b7 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 3 Feb 2025 14:47:43 -0500 Subject: [PATCH 36/79] Fix date stuff in path building --- src/intake.js | 2 +- src/shared.js | 15 +++------------ src/writer.js | 2 +- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/intake.js b/src/intake.js index 63763e2..ea2dfa8 100644 --- a/src/intake.js +++ b/src/intake.js @@ -158,7 +158,7 @@ function normalize(value, type, onError) { export function buildSamplePostPath(config) { const outputDir = path.sep; const type = ''; - const date = luxon.DateTime.now().toFormat('yyyy-LL-dd'); + const date = luxon.DateTime.now(); const slug = 'my-post'; return shared.buildPostPath(outputDir, type, date, slug, config); diff --git a/src/shared.js b/src/shared.js index 6557462..ecade32 100644 --- a/src/shared.js +++ b/src/shared.js @@ -1,30 +1,21 @@ -import * as luxon from 'luxon'; import path from 'path'; -import * as settings from './settings.js'; export function buildPostPath(outputDir, type, date, slug, config) { - let dt; - if (settings.custom_date_formatting) { - dt = luxon.DateTime.fromFormat(date, settings.custom_date_formatting); - } else { - dt = luxon.DateTime.fromISO(date); - } - // start with base output dir and post type const pathSegments = [outputDir, type]; if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { - pathSegments.push(dt.toFormat('yyyy')); + pathSegments.push(date.toFormat('yyyy')); } if (config.dateFolders === 'year-month') { - pathSegments.push(dt.toFormat('LL')); + pathSegments.push(date.toFormat('LL')); } // create slug fragment, possibly date prefixed let slugFragment = slug; if (config.prefixDate) { - slugFragment = dt.toFormat('yyyy-LL-dd') + '-' + slugFragment; + slugFragment = date.toFormat('yyyy-LL-dd') + '-' + slugFragment; } // use slug fragment as folder or filename as specified diff --git a/src/writer.js b/src/writer.js index 239803b..1d82afa 100644 --- a/src/writer.js +++ b/src/writer.js @@ -179,7 +179,7 @@ async function loadImageFilePromise(imageUrl, config) { function buildPostPath(post, config) { const outputDir = config.output; const type = post.meta.type; - const date = post.frontmatter.date; + const date = post.meta.date; const slug = post.meta.slug; return shared.buildPostPath(outputDir, type, date, slug, config); From 032131818346686974f7a2d813ae216ee0133c0a Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Tue, 4 Feb 2025 13:51:24 -0500 Subject: [PATCH 37/79] Remove settings.js --- src/settings.js | 51 ------------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 src/settings.js diff --git a/src/settings.js b/src/settings.js deleted file mode 100644 index 97f5e9e..0000000 --- a/src/settings.js +++ /dev/null @@ -1,51 +0,0 @@ -// Which fields to include in frontmatter. Look in /src/frontmatter.js to see available fields. -// Order is preserved. If a field has an empty value, it will not be included. You can rename a -// field by providing an alias after a ':'. For example, 'date:created' will include 'date' in -// frontmatter, but renamed to 'created'. -// export const frontmatter_fields = [ -// 'title', -// 'date', -// 'categories', -// 'tags', -// 'coverImage' -// ]; - -// Time in ms to wait between requesting image files. Increase this if you see timeouts or -// server errors. -// export const image_file_request_delay = 500; - -// Time in ms to wait between saving Markdown files. Increase this if your file system becomes -// overloaded. -// export const markdown_file_write_delay = 25; - -// Enable this to include time with post dates. For example, "2020-12-25" would become -// "2020-12-25T11:20:35.000Z". -// export const include_time_with_date = false; - -// Override post date formatting with a custom formatting string (for example: 'yyyy LLL dd'). -// Tokens are documented here: https://moment.github.io/luxon/#/parsing?id=table-of-tokens. If -// set, this takes precedence over include_time_with_date. -export const custom_date_formatting = ''; - -// Specify the timezone used for post dates. See available zone values and examples here: -// https://moment.github.io/luxon/#/zones?id=specifying-a-zone. -export const custom_date_timezone = 'utc'; - -// Categories to be excluded from post frontmatter. This does not filter out posts themselves, -// just the categories listed in their frontmatter. -// export const filter_categories = ['uncategorized']; - -// Strict SSL is enabled as the safe default when downloading images, but will not work with -// self-signed servers. You can disable it if you're getting a "self-signed certificate" error. -// export const strict_ssl = true; - -// Post types to exclude from output. -// export const filter_post_types = [ -// 'attachment', -// 'revision', -// 'nav_menu_item', -// 'custom_css', -// 'customize_changeset', -// 'wp_global_styles', -// 'wp_navigation' -// ]; From 5f5ab6f2bd63fa48be96aa5a7b04c9fae665aecb Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Tue, 4 Feb 2025 14:01:07 -0500 Subject: [PATCH 38/79] Fix incorrect config name to saveImages --- src/translator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/translator.js b/src/translator.js index 163c602..1fc3f75 100644 --- a/src/translator.js +++ b/src/translator.js @@ -102,7 +102,7 @@ export function getPostContent(postData, turndownService, config) { // without mucking up content inside of other elements (like blocks) content = content.replace(/(\r?\n){2}/g, '\n
\n'); - if (config.saveScrapedImages) { + if (config.saveImages === 'scraped' || config.saveImages === 'all') { // writeImageFile() will save all content images to a relative /images // folder so update references in post content to match content = content.replace(/(]*src=").*?([^/"]+\.(?:gif|jpe?g|png|webp))("[^>]*>)/gi, '$1images/$2$3'); From 9fb7ae47c5d86cc4b0f052de23ff8bc698232371 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Wed, 5 Feb 2025 15:58:26 -0500 Subject: [PATCH 39/79] Refactor how post meta is parsed --- src/frontmatter.js | 12 ++++---- src/parser.js | 71 ++++++++++++++++++---------------------------- src/writer.js | 11 ++----- 3 files changed, 37 insertions(+), 57 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index 1926037..821104c 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -17,16 +17,16 @@ export function categories(post) { return categories.filter((category) => category !== 'uncategorized'); } -// get cover image filename, previously decoded and set on post.meta +// get cover image filename, previously decoded and set on post // this one is unique as it relies on special logic executed by the parser export function coverImage(post) { - return post.meta.coverImage; + return post.coverImage; } -// get post date, previously saved as a luxon datetime object on post.meta +// get post date, previously saved as a luxon datetime object on post // this value is also used for year/month folders, date prefixes, etc. as needed export function date(post) { - return post.meta.date; + return post.date; } // get excerpt, not decoded, newlines collapsed @@ -39,9 +39,9 @@ export function id(post) { return post.data.post_id[0]; } -// get slug, previously decoded and set on post.meta +// get slug, previously decoded and set on post export function slug(post) { - return post.meta.slug; + return post.slug; } // get array of decoded tag names diff --git a/src/parser.js b/src/parser.js index 6819488..b143db0 100644 --- a/src/parser.js +++ b/src/parser.js @@ -60,26 +60,7 @@ function collectPosts(channelData, postTypes, config) { const postsForType = getItemsOfType(channelData, postType) .filter(postData => postData.status[0] !== 'trash' && postData.status[0] !== 'draft') .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) - .map(postData => ({ - // raw post data, used by frontmatter getters - data: postData, - - // meta data isn't written to file, but is used to help with other things - meta: { - type: postType, - id: getPostId(postData), - slug: getPostSlug(postData), - date: getPostDate(postData, config), - coverImageId: getPostCoverImageId(postData), - - // these are possibly set later in mergeImagesIntoPosts() - coverImage: undefined, - imageUrls: [] - }, - - // contents of the post in markdown - content: translator.getPostContent(postData, turndownService, config) - })); + .map(postData => buildPost(postData, turndownService, config)); if (postTypes.length > 1) { console.log(`${postsForType.length} "${postType}" posts found.`); @@ -94,26 +75,30 @@ function collectPosts(channelData, postTypes, config) { return allPosts; } -function getPostId(postData) { - return postData.post_id[0]; +function buildPost(data, turndownService, config) { + return { + // full raw post data, used by some frontmatter getters + data, + + // contents of the post in markdown + content: translator.getPostContent(data, turndownService, config), + + // these are not written to file, but help with other things + type: data.post_type[0], + id: data.post_id[0], + slug: decodeURIComponent(data.post_name[0]), + date: luxon.DateTime.fromRFC2822(data.pubDate[0], { zone: config.customDateTimezone }), + coverImageId: getPostMetaValue(data.postmeta, '_thumbnail_id'), + + // these are possibly set later in mergeImagesIntoPosts() + coverImage: undefined, + imageUrls: [] + }; } -function getPostSlug(postData) { - return decodeURIComponent(postData.post_name[0]); -} - -function getPostDate(postData, config) { - return luxon.DateTime.fromRFC2822(postData.pubDate[0], { zone: config.customDateTimezone }); -} - -function getPostCoverImageId(postData) { - if (postData.postmeta === undefined) { - return undefined; - } - - const postmeta = postData.postmeta.find(postmeta => postmeta.meta_key[0] === '_thumbnail_id'); - const id = postmeta ? postmeta.meta_value[0] : undefined; - return id; +function getPostMetaValue(metas, key) { + const meta = metas && metas.find((meta) => meta.meta_key[0] === key); + return meta ? meta.meta_value[0] : undefined; } function collectAttachedImages(channelData) { @@ -161,18 +146,18 @@ function mergeImagesIntoPosts(images, posts) { let shouldAttach = false; // this image was uploaded as an attachment to this post - if (image.postId === post.meta.id) { + if (image.postId === post.id) { shouldAttach = true; } // this image was set as the featured image for this post - if (image.id === post.meta.coverImageId) { + if (image.id === post.coverImageId) { shouldAttach = true; - post.meta.coverImage = shared.getFilenameFromUrl(image.url); + post.coverImage = shared.getFilenameFromUrl(image.url); } - if (shouldAttach && !post.meta.imageUrls.includes(image.url)) { - post.meta.imageUrls.push(image.url); + if (shouldAttach && !post.imageUrls.includes(image.url)) { + post.imageUrls.push(image.url); } }); }); diff --git a/src/writer.js b/src/writer.js index 1d82afa..4b0df4e 100644 --- a/src/writer.js +++ b/src/writer.js @@ -54,7 +54,7 @@ async function writeMarkdownFilesPromise(posts, config) { } else { const payload = { item: post, - name: post.meta.type + ' - ' + post.meta.slug, + name: post.type + ' - ' + post.slug, destinationPath, delay }; @@ -112,7 +112,7 @@ async function writeImageFilesPromise(posts, config) { const payloads = posts.flatMap(post => { const postPath = buildPostPath(post, config); const imagesDir = path.join(path.dirname(postPath), 'images'); - return post.meta.imageUrls.flatMap(imageUrl => { + return post.imageUrls.flatMap(imageUrl => { const filename = shared.getFilenameFromUrl(imageUrl); const destinationPath = path.join(imagesDir, filename); if (checkFile(destinationPath)) { @@ -177,12 +177,7 @@ async function loadImageFilePromise(imageUrl, config) { } function buildPostPath(post, config) { - const outputDir = config.output; - const type = post.meta.type; - const date = post.meta.date; - const slug = post.meta.slug; - - return shared.buildPostPath(outputDir, type, date, slug, config); + return shared.buildPostPath(config.output, post.type, post.date, post.slug, config); } function checkFile(path) { From df5731985df287ee40ac648916a51777288145b5 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Thu, 6 Feb 2025 13:56:27 -0500 Subject: [PATCH 40/79] Logging refactor, quote-date, type count fix --- src/parser.js | 7 ++----- src/questions.js | 5 +++++ src/writer.js | 25 ++++++++++++++++++++++--- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/src/parser.js b/src/parser.js index b143db0..2a3c95a 100644 --- a/src/parser.js +++ b/src/parser.js @@ -62,16 +62,13 @@ function collectPosts(channelData, postTypes, config) { .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) .map(postData => buildPost(postData, turndownService, config)); - if (postTypes.length > 1) { - console.log(`${postsForType.length} "${postType}" posts found.`); + if (postsForType.length > 0) { + console.log(`${postsForType.length} posts of type "${postType}" found.`); } allPosts.push(...postsForType); }); - if (postTypes.length === 1) { - console.log(allPosts.length + ' posts found.'); - } return allPosts; } diff --git a/src/questions.js b/src/questions.js index bf31cbe..0ae9f62 100644 --- a/src/questions.js +++ b/src/questions.js @@ -135,6 +135,11 @@ export const all = [ type: 'string', default: 'utc' }, + { + name: 'quote-date', + type: 'boolean', + default: false + }, { name: 'strict-ssl', type: 'boolean', diff --git a/src/writer.js b/src/writer.js index 4b0df4e..17a28a1 100644 --- a/src/writer.js +++ b/src/writer.js @@ -18,10 +18,10 @@ async function processPayloadsPromise(payloads, loadFunc, config) { try { const data = await loadFunc(payload.item, config); await writeFile(payload.destinationPath, data); - console.log(chalk.green('[OK]') + ' ' + payload.name); + logPayloadResult(payload); resolve(); } catch (ex) { - console.log(chalk.red('[FAILED]') + ' ' + payload.name + ' ' + chalk.red('(' + ex.message + ')')); + logPayloadResult(payload, ex.message); reject(); } }, payload.delay); @@ -54,7 +54,8 @@ async function writeMarkdownFilesPromise(posts, config) { } else { const payload = { item: post, - name: post.type + ' - ' + post.slug, + type: post.type, + name: post.slug, destinationPath, delay }; @@ -88,6 +89,10 @@ async function loadMarkdownFilePromise(post, config) { } else { outputValue = config.includeTimeWithDate ? value.toISO() : value.toISODate(); } + + if (config.quoteDate) { + outputValue = `"${outputValue}"`; + } } else { // single string value const escapedValue = (value || '').replace(/"/g, '\\"'); @@ -122,6 +127,7 @@ async function writeImageFilesPromise(posts, config) { } else { const payload = { item: imageUrl, + type: 'image', name: filename, destinationPath, delay @@ -183,3 +189,16 @@ function buildPostPath(post, config) { function checkFile(path) { return fs.existsSync(path); } + +function logPayloadResult(payload, errorMessage) { + const messageBits = [ + errorMessage ? chalk.red('✗') : chalk.green('✓'), + chalk.gray(`[${payload.type}]`), + payload.name + ]; + if (errorMessage) { + messageBits.push(chalk.red(`(${errorMessage})`)); + } + + console.log(messageBits.join(' ')); +} From 66798a044f06d17a3ea89d1c7f27579db2b7f7bb Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Thu, 6 Feb 2025 14:12:47 -0500 Subject: [PATCH 41/79] Remove camelcase dep, roll my own --- package-lock.json | 247 +++++++++++++++++++++++++++------------------- package.json | 1 - src/intake.js | 9 +- src/shared.js | 4 + 4 files changed, 152 insertions(+), 109 deletions(-) diff --git a/package-lock.json b/package-lock.json index b848d25..83cc499 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@guyplusplus/turndown-plugin-gfm": "^1.0.7", "@inquirer/prompts": "^7.2.3", "axios": "^1.7.9", - "camelcase": "^8.0.0", "chalk": "^5.4.1", "commander": "^13.0.0", "luxon": "^3.5.0", @@ -132,13 +131,13 @@ "dev": true }, "node_modules/@inquirer/checkbox": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.0.7.tgz", - "integrity": "sha512-lyoF4uYdBBTnqeB1gjPdYkiQ++fz/iYKaP9DON1ZGlldkvAEJsjaOBRdbl5UW1pOSslBRd701jxhAG0MlhHd2w==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.1.1.tgz", + "integrity": "sha512-os5kFd/52gZTl/W6xqMfhaKVJHQM8V/U1P8jcSaQJ/C4Qhdrf2jEXdA/HaxfQs9iiUA/0yzYhk5d3oRHTxGDDQ==", "dependencies": { - "@inquirer/core": "^10.1.5", + "@inquirer/core": "^10.1.6", "@inquirer/figures": "^1.0.10", - "@inquirer/type": "^3.0.3", + "@inquirer/type": "^3.0.4", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, @@ -147,30 +146,40 @@ }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/confirm": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.4.tgz", - "integrity": "sha512-EsiT7K4beM5fN5Mz6j866EFA9+v9d5o9VUra3hrg8zY4GHmCS8b616FErbdo5eyKoVotBQkHzMIeeKYsKDStDw==", + "version": "5.1.5", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.5.tgz", + "integrity": "sha512-ZB2Cz8KeMINUvoeDi7IrvghaVkYT2RB0Zb31EaLWOE87u276w4wnApv0SH2qWaJ3r0VSUa3BIuz7qAV2ZvsZlg==", "dependencies": { - "@inquirer/core": "^10.1.5", - "@inquirer/type": "^3.0.3" + "@inquirer/core": "^10.1.6", + "@inquirer/type": "^3.0.4" }, "engines": { "node": ">=18" }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/core": { - "version": "10.1.5", - "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.5.tgz", - "integrity": "sha512-/vyCWhET0ktav/mUeBqJRYTwmjFPIKPRYb3COAw7qORULgipGSUO2vL32lQKki3UxDKJ8BvuEbokaoyCA6YlWw==", + "version": "10.1.6", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.1.6.tgz", + "integrity": "sha512-Bwh/Zk6URrHwZnSSzAZAKH7YgGYi0xICIBDFOqBQoXNNAzBHw/bgXgLmChfp+GyR3PnChcTbiCTZGC6YJNJkMA==", "dependencies": { "@inquirer/figures": "^1.0.10", - "@inquirer/type": "^3.0.3", + "@inquirer/type": "^3.0.4", "ansi-escapes": "^4.3.2", "cli-width": "^4.1.0", "mute-stream": "^2.0.0", @@ -180,15 +189,23 @@ }, "engines": { "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/editor": { - "version": "4.2.4", - "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.4.tgz", - "integrity": "sha512-S8b6+K9PLzxiFGGc02m4syhEu5JsH0BukzRsuZ+tpjJ5aDsDX1WfNfOil2fmsO36Y1RMcpJGxlfQ1yh4WfU28Q==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.6.tgz", + "integrity": "sha512-l0smvr8g/KAVdXx4I92sFxZiaTG4kFc06cFZw+qqwTirwdUHMFLnouXBB9OafWhpO3cfEkEz2CdPoCmor3059A==", "dependencies": { - "@inquirer/core": "^10.1.5", - "@inquirer/type": "^3.0.3", + "@inquirer/core": "^10.1.6", + "@inquirer/type": "^3.0.4", "external-editor": "^3.1.0" }, "engines": { @@ -196,15 +213,20 @@ }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/expand": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.7.tgz", - "integrity": "sha512-PsUQ5t7r+DPjW0VVEHzssOTBM2UPHnvBNse7hzuki7f6ekRL94drjjfBLrGEDe7cgj3pguufy/cuFwMeWUWHXw==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.8.tgz", + "integrity": "sha512-k0ouAC6L+0Yoj/j0ys2bat0fYcyFVtItDB7h+pDFKaDDSFJey/C/YY1rmIOqkmFVZ5rZySeAQuS8zLcKkKRLmg==", "dependencies": { - "@inquirer/core": "^10.1.5", - "@inquirer/type": "^3.0.3", + "@inquirer/core": "^10.1.6", + "@inquirer/type": "^3.0.4", "yoctocolors-cjs": "^2.1.2" }, "engines": { @@ -212,6 +234,11 @@ }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/figures": { @@ -223,42 +250,52 @@ } }, "node_modules/@inquirer/input": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.4.tgz", - "integrity": "sha512-CKKF8otRBdIaVnRxkFLs00VNA9HWlEh3x4SqUfC3A8819TeOZpTYG/p+4Nqu3hh97G+A0lxkOZNYE7KISgU8BA==", + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.1.5.tgz", + "integrity": "sha512-bB6wR5wBCz5zbIVBPnhp94BHv/G4eKbUEjlpCw676pI2chcvzTx1MuwZSCZ/fgNOdqDlAxkhQ4wagL8BI1D3Zg==", "dependencies": { - "@inquirer/core": "^10.1.5", - "@inquirer/type": "^3.0.3" + "@inquirer/core": "^10.1.6", + "@inquirer/type": "^3.0.4" }, "engines": { "node": ">=18" }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/number": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.7.tgz", - "integrity": "sha512-uU2nmXGC0kD8+BLgwZqcgBD1jcw2XFww2GmtP6b4504DkOp+fFAhydt7JzRR1TAI2dmj175p4SZB0lxVssNreA==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.8.tgz", + "integrity": "sha512-CTKs+dT1gw8dILVWATn8Ugik1OHLkkfY82J+Musb57KpmF6EKyskv8zmMiEJPzOnLTZLo05X/QdMd8VH9oulXw==", "dependencies": { - "@inquirer/core": "^10.1.5", - "@inquirer/type": "^3.0.3" + "@inquirer/core": "^10.1.6", + "@inquirer/type": "^3.0.4" }, "engines": { "node": ">=18" }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/password": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.7.tgz", - "integrity": "sha512-DFpqWLx+C5GV5zeFWuxwDYaeYnTWYphO07pQ2VnP403RIqRIpwBG0ATWf7pF+3IDbaXEtWatCJWxyDrJ+rkj2A==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.8.tgz", + "integrity": "sha512-MgA+Z7o3K1df2lGY649fyOBowHGfrKRz64dx3+b6c1w+h2W7AwBoOkHhhF/vfhbs5S4vsKNCuDzS3s9r5DpK1g==", "dependencies": { - "@inquirer/core": "^10.1.5", - "@inquirer/type": "^3.0.3", + "@inquirer/core": "^10.1.6", + "@inquirer/type": "^3.0.4", "ansi-escapes": "^4.3.2" }, "engines": { @@ -266,38 +303,48 @@ }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/prompts": { - "version": "7.2.4", - "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.2.4.tgz", - "integrity": "sha512-Zn2XZL2VZl76pllUjeDnS6Poz2Oiv9kmAZdSZw1oFya985+/JXZ3GZ2JUWDokAPDhvuhkv9qz0Z7z/U80G8ztA==", + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.3.1.tgz", + "integrity": "sha512-r1CiKuDV86BDpvj9DRFR+V+nIjsVBOsa2++dqdPqLYAef8kgHYvmQ8ySdP/ZeAIOWa27YGJZRkENdP3dK0H3gg==", "dependencies": { - "@inquirer/checkbox": "^4.0.7", - "@inquirer/confirm": "^5.1.4", - "@inquirer/editor": "^4.2.4", - "@inquirer/expand": "^4.0.7", - "@inquirer/input": "^4.1.4", - "@inquirer/number": "^3.0.7", - "@inquirer/password": "^4.0.7", - "@inquirer/rawlist": "^4.0.7", - "@inquirer/search": "^3.0.7", - "@inquirer/select": "^4.0.7" + "@inquirer/checkbox": "^4.1.1", + "@inquirer/confirm": "^5.1.5", + "@inquirer/editor": "^4.2.6", + "@inquirer/expand": "^4.0.8", + "@inquirer/input": "^4.1.5", + "@inquirer/number": "^3.0.8", + "@inquirer/password": "^4.0.8", + "@inquirer/rawlist": "^4.0.8", + "@inquirer/search": "^3.0.8", + "@inquirer/select": "^4.0.8" }, "engines": { "node": ">=18" }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/rawlist": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.7.tgz", - "integrity": "sha512-ZeBca+JCCtEIwQMvhuROT6rgFQWWvAImdQmIIP3XoyDFjrp2E0gZlEn65sWIoR6pP2EatYK96pvx0887OATWQQ==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.0.8.tgz", + "integrity": "sha512-hl7rvYW7Xl4un8uohQRUgO6uc2hpn7PKqfcGkCOWC0AA4waBxAv6MpGOFCEDrUaBCP+pXPVqp4LmnpWmn1E1+g==", "dependencies": { - "@inquirer/core": "^10.1.5", - "@inquirer/type": "^3.0.3", + "@inquirer/core": "^10.1.6", + "@inquirer/type": "^3.0.4", "yoctocolors-cjs": "^2.1.2" }, "engines": { @@ -305,16 +352,21 @@ }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/search": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.7.tgz", - "integrity": "sha512-Krq925SDoLh9AWSNee8mbSIysgyWtcPnSAp5YtPBGCQ+OCO+5KGC8FwLpyxl8wZ2YAov/8Tp21stTRK/fw5SGg==", + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.0.8.tgz", + "integrity": "sha512-ihSE9D3xQAupNg/aGDZaukqoUSXG2KfstWosVmFCG7jbMQPaj2ivxWtsB+CnYY/T4D6LX1GHKixwJLunNCffww==", "dependencies": { - "@inquirer/core": "^10.1.5", + "@inquirer/core": "^10.1.6", "@inquirer/figures": "^1.0.10", - "@inquirer/type": "^3.0.3", + "@inquirer/type": "^3.0.4", "yoctocolors-cjs": "^2.1.2" }, "engines": { @@ -322,16 +374,21 @@ }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/select": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.0.7.tgz", - "integrity": "sha512-ejGBMDSD+Iqk60u5t0Zf2UQhGlJWDM78Ep70XpNufIfc+f4VOTeybYKXu9pDjz87FkRzLiVsGpQG2SzuGlhaJw==", + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.0.8.tgz", + "integrity": "sha512-Io2prxFyN2jOCcu4qJbVoilo19caiD3kqkD3WR0q3yDA5HUCo83v4LrRtg55ZwniYACW64z36eV7gyVbOfORjA==", "dependencies": { - "@inquirer/core": "^10.1.5", + "@inquirer/core": "^10.1.6", "@inquirer/figures": "^1.0.10", - "@inquirer/type": "^3.0.3", + "@inquirer/type": "^3.0.4", "ansi-escapes": "^4.3.2", "yoctocolors-cjs": "^2.1.2" }, @@ -340,17 +397,27 @@ }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@inquirer/type": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.3.tgz", - "integrity": "sha512-I4VIHFxUuY1bshGbXZTxCmhwaaEst9s/lll3ekok+o1Z26/ZUKdx8y1b7lsoG6rtsBDwEGfiBJ2SfirjoISLpg==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.4.tgz", + "integrity": "sha512-2MNFrDY8jkFYc9Il9DgLsHhMzuHnOYM1+CUYVWbzu9oT0hC7V7EcYvdCKeoll/Fcci04A+ERZ9wcc7cQ8lTkIA==", "engines": { "node": ">=18" }, "peerDependencies": { "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } } }, "node_modules/@mixmark-io/domino": { @@ -393,15 +460,6 @@ "node": ">= 8" } }, - "node_modules/@types/node": { - "version": "22.12.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.12.0.tgz", - "integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==", - "peer": true, - "dependencies": { - "undici-types": "~6.20.0" - } - }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", @@ -527,17 +585,6 @@ "node": ">=6" } }, - "node_modules/camelcase": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-8.0.0.tgz", - "integrity": "sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/chalk": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", @@ -868,9 +915,9 @@ "dev": true }, "node_modules/fastq": { - "version": "1.18.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.18.0.tgz", - "integrity": "sha512-QKHXPW0hD8g4UET03SdOdunzSouc9N4AuHdsX8XNcTsuz+yYFILVNIX4l9yHABMhiEI9Db0JTTIpu0wB+Y1QQw==", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.19.0.tgz", + "integrity": "sha512-7SFSRCNjBQIZH/xZR3iy5iQYR8aGBE0h3VG6/cwlbrpdciNYBMotQav8c1XI3HjHH+NikUpP53nPdlZSdWmFzA==", "dev": true, "dependencies": { "reusify": "^1.0.4" @@ -1058,9 +1105,9 @@ } }, "node_modules/import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", "dev": true, "dependencies": { "parent-module": "^1.0.0", @@ -1617,12 +1664,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "peer": true - }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", diff --git a/package.json b/package.json index 5f6ab0c..f498857 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "@guyplusplus/turndown-plugin-gfm": "^1.0.7", "@inquirer/prompts": "^7.2.3", "axios": "^1.7.9", - "camelcase": "^8.0.0", "chalk": "^5.4.1", "commander": "^13.0.0", "luxon": "^3.5.0", diff --git a/src/intake.js b/src/intake.js index ea2dfa8..aec05f0 100644 --- a/src/intake.js +++ b/src/intake.js @@ -1,4 +1,3 @@ -import camelcase from 'camelcase'; import chalk from 'chalk'; import * as commander from 'commander'; import * as luxon from 'luxon'; @@ -29,7 +28,7 @@ export async function getConfig() { // run wizard for questions with prompts that were not answered via the command line const wizardQuestions = questions.all.filter((question) => { - return question.prompt && !(camelcase(question.name) in commandLineAnswers); + return question.prompt && !(shared.camelCase(question.name) in commandLineAnswers); }); wizardAnswers = await getWizardAnswers(wizardQuestions, commandLineAnswers); } else { @@ -74,7 +73,7 @@ function getCommandLineAnswers(questions) { continue; } - const question = questions.find((question) => camelcase(question.name) === key); + const question = questions.find((question) => shared.camelCase(question.name) === key); if (answers.wizard && question.prompt) { // remove this default answer, allowing the wizard to ask about it later delete answers[key]; @@ -93,7 +92,7 @@ function getCommandLineAnswers(questions) { export async function getWizardAnswers(questions, commandLineAnswers) { const answers = {}; for (const question of questions) { - let answerKey = camelcase(question.name); + let answerKey = shared.camelCase(question.name); let normalizedAnswer; // holds normalized answer value potentially returned during validation const promptConfig = { @@ -143,7 +142,7 @@ export async function getWizardAnswers(questions, commandLineAnswers) { } function normalize(value, type, onError) { - const normalizer = normalizers[camelcase(type)]; + const normalizer = normalizers[shared.camelCase(type)]; if (!normalizer) { return value; } diff --git a/src/shared.js b/src/shared.js index ecade32..af467f0 100644 --- a/src/shared.js +++ b/src/shared.js @@ -1,5 +1,9 @@ import path from 'path'; +export function camelCase(str) { + return str.replace(/-(.)/g, (match) => match[1].toUpperCase()); +} + export function buildPostPath(outputDir, type, date, slug, config) { // start with base output dir and post type const pathSegments = [outputDir, type]; From f8d214c362d61075f73e443e5d4e4c5d4152ae69 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Fri, 7 Feb 2025 17:49:31 -0500 Subject: [PATCH 42/79] questions.load(), config refactor --- index.js | 11 +- src/intake.js | 28 ++--- src/parser.js | 28 ++--- src/questions.js | 294 +++++++++++++++++++++++----------------------- src/shared.js | 19 +-- src/translator.js | 7 +- src/writer.js | 44 +++---- 7 files changed, 218 insertions(+), 213 deletions(-) diff --git a/index.js b/index.js index 7a5c7a0..63fd3b9 100644 --- a/index.js +++ b/index.js @@ -2,8 +2,9 @@ import * as commander from 'commander'; import path from 'path'; -import * as parser from './src/parser.js'; import * as intake from './src/intake.js'; +import * as parser from './src/parser.js'; +import * as shared from './src/shared.js'; import * as writer from './src/writer.js'; (async () => { @@ -14,17 +15,17 @@ import * as writer from './src/writer.js'; .addHelpText('after', '\nMore documentation is at https://github.com/lonekorean/wordpress-export-to-markdown') // gather config options from command line and wizard - const config = await intake.getConfig(); + await intake.getConfig(); // parse data from XML and do Markdown translations - const posts = await parser.parseFilePromise(config) + const posts = await parser.parseFilePromise() // write files and download images - await writer.writeFilesPromise(posts, config); + await writer.writeFilesPromise(posts); // happy goodbye console.log('\nAll done!'); - console.log('Look for your output files in: ' + path.resolve(config.output)); + console.log('Look for your output files in: ' + path.resolve(shared.config.output)); })().catch((ex) => { // sad goodbye console.log('\nSomething went wrong, execution halted early.'); diff --git a/src/intake.js b/src/intake.js index aec05f0..f43b5a0 100644 --- a/src/intake.js +++ b/src/intake.js @@ -19,7 +19,7 @@ const promptTheme = { export async function getConfig() { // check command line for any config options - const commandLineQuestions = questions.all; + const commandLineQuestions = questions.load(); const commandLineAnswers = getCommandLineAnswers(commandLineQuestions); let wizardAnswers; @@ -27,15 +27,15 @@ export async function getConfig() { console.log('\nStarting wizard...'); // run wizard for questions with prompts that were not answered via the command line - const wizardQuestions = questions.all.filter((question) => { + const wizardQuestions = questions.load().filter((question) => { return question.prompt && !(shared.camelCase(question.name) in commandLineAnswers); }); wizardAnswers = await getWizardAnswers(wizardQuestions, commandLineAnswers); } else { - console.log('\nSkipping wizard...'); + console.dir('\nSkipping wizard...'); } - return { ...commandLineAnswers, ...wizardAnswers }; + Object.assign(shared.config, commandLineAnswers, wizardAnswers); } function getCommandLineAnswers(questions) { @@ -106,13 +106,14 @@ export async function getWizardAnswers(questions, commandLineAnswers) { promptConfig.loop = false; if (question.isPathQuestion) { - // create a snapshot config of command line answers and wizard answers so far - const config = { ...commandLineAnswers, ...answers }; - promptConfig.choices.forEach((choice) => { // show example path if this choice is selected - config[answerKey] = choice.value; - choice.description = buildSamplePostPath(config); + choice.description = buildSamplePostPath({ + ...commandLineAnswers, // with command line answers + ...answers, // and wizard answers so far + output: path.sep, // and a simplified output folder + [answerKey]: choice.value // and this choice selected + }); }); } } else { @@ -154,11 +155,6 @@ function normalize(value, type, onError) { } } -export function buildSamplePostPath(config) { - const outputDir = path.sep; - const type = ''; - const date = luxon.DateTime.now(); - const slug = 'my-post'; - - return shared.buildPostPath(outputDir, type, date, slug, config); +export function buildSamplePostPath(overrideConfig) { + return shared.buildPostPath('', luxon.DateTime.now(), 'my-post', overrideConfig); } diff --git a/src/parser.js b/src/parser.js index 2a3c95a..f8c06b6 100644 --- a/src/parser.js +++ b/src/parser.js @@ -5,9 +5,9 @@ import * as frontmatter from './frontmatter.js'; import * as shared from './shared.js'; import * as translator from './translator.js'; -export async function parseFilePromise(config) { +export async function parseFilePromise() { console.log('\nParsing...'); - const content = await fs.promises.readFile(config.input, 'utf8'); + const content = await fs.promises.readFile(shared.config.input, 'utf8'); const allData = await xml2js.parseStringPromise(content, { trim: true, tagNameProcessors: [xml2js.processors.stripPrefix] @@ -15,18 +15,18 @@ export async function parseFilePromise(config) { const channelData = allData.rss.channel[0].item; const postTypes = getPostTypes(channelData); - const posts = collectPosts(channelData, postTypes, config); + const posts = collectPosts(channelData, postTypes); const images = []; - if (config.saveImages === 'attached' || config.saveImages === 'all') { + if (shared.config.saveImages === 'attached' || shared.config.saveImages === 'all') { images.push(...collectAttachedImages(channelData)); } - if (config.saveImages === 'scraped' || config.saveImages === 'all') { + if (shared.config.saveImages === 'scraped' || shared.config.saveImages === 'all') { images.push(...collectScrapedImages(channelData, postTypes)); } mergeImagesIntoPosts(images, posts); - populateFrontmatter(posts, config); + populateFrontmatter(posts); return posts; } @@ -51,7 +51,7 @@ function getItemsOfType(channelData, type) { return channelData.filter(item => item.post_type[0] === type); } -function collectPosts(channelData, postTypes, config) { +function collectPosts(channelData, postTypes) { // this is passed into getPostContent() for the markdown conversion const turndownService = translator.initTurndownService(); @@ -60,7 +60,7 @@ function collectPosts(channelData, postTypes, config) { const postsForType = getItemsOfType(channelData, postType) .filter(postData => postData.status[0] !== 'trash' && postData.status[0] !== 'draft') .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) - .map(postData => buildPost(postData, turndownService, config)); + .map(postData => buildPost(postData, turndownService)); if (postsForType.length > 0) { console.log(`${postsForType.length} posts of type "${postType}" found.`); @@ -72,19 +72,19 @@ function collectPosts(channelData, postTypes, config) { return allPosts; } -function buildPost(data, turndownService, config) { +function buildPost(data, turndownService) { return { // full raw post data, used by some frontmatter getters data, // contents of the post in markdown - content: translator.getPostContent(data, turndownService, config), + content: translator.getPostContent(data, turndownService), // these are not written to file, but help with other things type: data.post_type[0], id: data.post_id[0], slug: decodeURIComponent(data.post_name[0]), - date: luxon.DateTime.fromRFC2822(data.pubDate[0], { zone: config.customDateTimezone }), + date: luxon.DateTime.fromRFC2822(data.pubDate[0], { zone: shared.config.customDateTimezone }), coverImageId: getPostMetaValue(data.postmeta, '_thumbnail_id'), // these are possibly set later in mergeImagesIntoPosts() @@ -160,10 +160,10 @@ function mergeImagesIntoPosts(images, posts) { }); } -function populateFrontmatter(posts, config) { +function populateFrontmatter(posts) { posts.forEach(post => { post.frontmatter = {}; - config.frontmatterFields.forEach(field => { + shared.config.frontmatterFields.forEach(field => { const [key, alias] = field.split(':'); let frontmatterGetter = frontmatter[key]; @@ -171,7 +171,7 @@ function populateFrontmatter(posts, config) { throw `Could not find a frontmatter getter named "${key}".`; } - post.frontmatter[alias || key] = frontmatterGetter(post, config); + post.frontmatter[alias || key] = frontmatterGetter(post); }); }); } diff --git a/src/questions.js b/src/questions.js index 0ae9f62..f2783eb 100644 --- a/src/questions.js +++ b/src/questions.js @@ -1,148 +1,150 @@ import * as inquirer from '@inquirer/prompts'; -// questions with a description are displayed in command line help -// questions with a prompt are included in the wizard (if not set on the command line) -export const all = [ - { - name: 'wizard', - type: 'boolean', - description: 'Use wizard', - default: true - }, - { - name: 'input', - type: 'file-path', - description: 'Path to WordPress export file', - default: 'export.xml', - prompt: inquirer.input - }, - { - name: 'post-folders', - type: 'boolean', - description: 'Put each post into its own folder', - default: true, - choices: [ - { - name: 'Yes', - value: true - }, - { - name: 'No', - value: false - } - ], - isPathQuestion: true, - prompt: inquirer.select - }, - { - name: 'prefix-date', - type: 'boolean', - description: 'Prefix with date', - default: false, - choices: [ - { - name: 'Yes', - value: true - }, - { - name: 'No', - value: false - } - ], - isPathQuestion: true, - prompt: inquirer.select - }, - { - name: 'date-folders', - type: 'choice', - description: 'Organize into folders based on date', - default: 'none', - choices: [ - { - name: 'Year folders', - value: 'year' - }, - { - name: 'Year and month folders', - value: 'year-month' - }, - { - name: 'No', - value: 'none' - } - ], - isPathQuestion: true, - prompt: inquirer.select - }, - { - name: 'save-images', - type: 'choice', - description: 'Save images', - default: 'all', - choices: [ - { - name: 'Images attached to posts', - value: 'attached' - }, - { - name: 'Images scraped from post body content', - value: 'scraped' - }, - { - name: 'Both', - value: 'all' - }, - { - name: 'No', - value: 'none' - } - ], - prompt: inquirer.select - }, - { - name: 'output', - type: 'folder-path', - description: 'Path to output folder', - default: 'output' - }, - { - name: 'frontmatter-fields', - type: 'list', - default: ['title', 'date', 'categories', 'tags', 'coverImage'] - }, - { - name: 'image-file-request-delay', - type: 'integer', - default: 500 - }, - { - name: 'markdown-file-write-delay', - type: 'integer', - default: 25 - }, - { - name: 'include-time-with-date', - type: 'boolean', - default: false - }, - { - name: 'custom-date-formatting', - type: 'string', - default: '' - }, - { - name: 'custom-date-timezone', - type: 'string', - default: 'utc' - }, - { - name: 'quote-date', - type: 'boolean', - default: false - }, - { - name: 'strict-ssl', - type: 'boolean', - default: true - } -]; +export function load() { + // questions with a description are displayed in command line help + // questions with a prompt are included in the wizard (if not set on the command line) + return [ + { + name: 'wizard', + type: 'boolean', + description: 'Use wizard', + default: true + }, + { + name: 'input', + type: 'file-path', + description: 'Path to WordPress export file', + default: 'export.xml', + prompt: inquirer.input + }, + { + name: 'post-folders', + type: 'boolean', + description: 'Put each post into its own folder', + default: true, + choices: [ + { + name: 'Yes', + value: true + }, + { + name: 'No', + value: false + } + ], + isPathQuestion: true, + prompt: inquirer.select + }, + { + name: 'prefix-date', + type: 'boolean', + description: 'Prefix with date', + default: false, + choices: [ + { + name: 'Yes', + value: true + }, + { + name: 'No', + value: false + } + ], + isPathQuestion: true, + prompt: inquirer.select + }, + { + name: 'date-folders', + type: 'choice', + description: 'Organize into folders based on date', + default: 'none', + choices: [ + { + name: 'Year folders', + value: 'year' + }, + { + name: 'Year and month folders', + value: 'year-month' + }, + { + name: 'No', + value: 'none' + } + ], + isPathQuestion: true, + prompt: inquirer.select + }, + { + name: 'save-images', + type: 'choice', + description: 'Save images', + default: 'all', + choices: [ + { + name: 'Images attached to posts', + value: 'attached' + }, + { + name: 'Images scraped from post body content', + value: 'scraped' + }, + { + name: 'Both', + value: 'all' + }, + { + name: 'No', + value: 'none' + } + ], + prompt: inquirer.select + }, + { + name: 'output', + type: 'folder-path', + description: 'Path to output folder', + default: 'output' + }, + { + name: 'frontmatter-fields', + type: 'list', + default: ['title', 'date', 'categories', 'tags', 'coverImage'] + }, + { + name: 'image-file-request-delay', + type: 'integer', + default: 500 + }, + { + name: 'markdown-file-write-delay', + type: 'integer', + default: 25 + }, + { + name: 'include-time-with-date', + type: 'boolean', + default: false + }, + { + name: 'custom-date-formatting', + type: 'string', + default: '' + }, + { + name: 'custom-date-timezone', + type: 'string', + default: 'utc' + }, + { + name: 'quote-date', + type: 'boolean', + default: false + }, + { + name: 'strict-ssl', + type: 'boolean', + default: true + } + ]; +} diff --git a/src/shared.js b/src/shared.js index af467f0..3c6dc04 100644 --- a/src/shared.js +++ b/src/shared.js @@ -1,29 +1,34 @@ import path from 'path'; +// simple data store, populated via intake, used everywhere +export const config = {}; + export function camelCase(str) { return str.replace(/-(.)/g, (match) => match[1].toUpperCase()); } -export function buildPostPath(outputDir, type, date, slug, config) { - // start with base output dir and post type - const pathSegments = [outputDir, type]; +export function buildPostPath(type, date, slug, overrideConfig) { + const pathConfig = overrideConfig ?? config; - if (config.dateFolders === 'year' || config.dateFolders === 'year-month') { + // start with base output dir and post type + const pathSegments = [pathConfig.output, type]; + + if (pathConfig.dateFolders === 'year' || pathConfig.dateFolders === 'year-month') { pathSegments.push(date.toFormat('yyyy')); } - if (config.dateFolders === 'year-month') { + if (pathConfig.dateFolders === 'year-month') { pathSegments.push(date.toFormat('LL')); } // create slug fragment, possibly date prefixed let slugFragment = slug; - if (config.prefixDate) { + if (pathConfig.prefixDate) { slugFragment = date.toFormat('yyyy-LL-dd') + '-' + slugFragment; } // use slug fragment as folder or filename as specified - if (config.postFolders) { + if (pathConfig.postFolders) { pathSegments.push(slugFragment, 'index.md'); } else { pathSegments.push(slugFragment + '.md'); diff --git a/src/translator.js b/src/translator.js index 1fc3f75..f1b8df0 100644 --- a/src/translator.js +++ b/src/translator.js @@ -1,5 +1,6 @@ -import turndown from 'turndown'; import turndownPluginGfm from '@guyplusplus/turndown-plugin-gfm'; +import turndown from 'turndown'; +import * as shared from './shared.js'; export function initTurndownService() { const turndownService = new turndown({ @@ -94,7 +95,7 @@ export function initTurndownService() { return turndownService; } -export function getPostContent(postData, turndownService, config) { +export function getPostContent(postData, turndownService) { let content = postData.encoded[0]; // insert an empty div element between double line breaks @@ -102,7 +103,7 @@ export function getPostContent(postData, turndownService, config) { // without mucking up content inside of other elements (like blocks) content = content.replace(/(\r?\n){2}/g, '\n
\n'); - if (config.saveImages === 'scraped' || config.saveImages === 'all') { + if (shared.config.saveImages === 'scraped' || shared.config.saveImages === 'all') { // writeImageFile() will save all content images to a relative /images // folder so update references in post content to match content = content.replace(/(]*src=").*?([^/"]+\.(?:gif|jpe?g|png|webp))("[^>]*>)/gi, '$1images/$2$3'); diff --git a/src/writer.js b/src/writer.js index 17a28a1..a201fcf 100644 --- a/src/writer.js +++ b/src/writer.js @@ -7,16 +7,16 @@ import * as luxon from 'luxon'; import path from 'path'; import * as shared from './shared.js'; -export async function writeFilesPromise(posts, config) { - await writeMarkdownFilesPromise(posts, config); - await writeImageFilesPromise(posts, config); +export async function writeFilesPromise(posts) { + await writeMarkdownFilesPromise(posts); + await writeImageFilesPromise(posts); } -async function processPayloadsPromise(payloads, loadFunc, config) { +async function processPayloadsPromise(payloads, loadFunc) { const promises = payloads.map(payload => new Promise((resolve, reject) => { setTimeout(async () => { try { - const data = await loadFunc(payload.item, config); + const data = await loadFunc(payload.item); await writeFile(payload.destinationPath, data); logPayloadResult(payload); resolve(); @@ -41,12 +41,12 @@ async function writeFile(destinationPath, data) { await fs.promises.writeFile(destinationPath, data); } -async function writeMarkdownFilesPromise(posts, config) { +async function writeMarkdownFilesPromise(posts) { // package up posts into payloads let skipCount = 0; let delay = 0; const payloads = posts.flatMap(post => { - const destinationPath = buildPostPath(post, config); + const destinationPath = buildPostPath(post); if (checkFile(destinationPath)) { // already exists, don't need to save again skipCount++; @@ -59,7 +59,7 @@ async function writeMarkdownFilesPromise(posts, config) { destinationPath, delay }; - delay += config.markdownFileWriteDelay; + delay += shared.config.markdownFileWriteDelay; return [payload]; } }); @@ -69,11 +69,11 @@ async function writeMarkdownFilesPromise(posts, config) { console.log('\nNo posts to save...'); } else { console.log(`\nSaving ${remainingCount} posts (${skipCount} already exist)...`); - await processPayloadsPromise(payloads, loadMarkdownFilePromise, config); + await processPayloadsPromise(payloads, loadMarkdownFilePromise); } } -async function loadMarkdownFilePromise(post, config) { +async function loadMarkdownFilePromise(post) { let output = '---\n'; Object.entries(post.frontmatter).forEach(([key, value]) => { @@ -84,13 +84,13 @@ async function loadMarkdownFilePromise(post, config) { outputValue = value.reduce((list, item) => `${list}\n - "${item}"`, ''); } } else if (value instanceof luxon.DateTime) { - if (config.customDateFormatting) { - outputValue = value.toFormat(config.customDateFormatting); + if (shared.config.customDateFormatting) { + outputValue = value.toFormat(shared.config.customDateFormatting); } else { - outputValue = config.includeTimeWithDate ? value.toISO() : value.toISODate(); + outputValue = shared.config.includeTimeWithDate ? value.toISO() : value.toISODate(); } - if (config.quoteDate) { + if (shared.config.quoteDate) { outputValue = `"${outputValue}"`; } } else { @@ -110,12 +110,12 @@ async function loadMarkdownFilePromise(post, config) { return output; } -async function writeImageFilesPromise(posts, config) { +async function writeImageFilesPromise(posts) { // collect image data from all posts into a single flattened array of payloads let skipCount = 0; let delay = 0; const payloads = posts.flatMap(post => { - const postPath = buildPostPath(post, config); + const postPath = buildPostPath(post); const imagesDir = path.join(path.dirname(postPath), 'images'); return post.imageUrls.flatMap(imageUrl => { const filename = shared.getFilenameFromUrl(imageUrl); @@ -132,7 +132,7 @@ async function writeImageFilesPromise(posts, config) { destinationPath, delay }; - delay += config.imageFileRequestDelay; + delay += shared.config.imageFileRequestDelay; return [payload]; } }); @@ -143,11 +143,11 @@ async function writeImageFilesPromise(posts, config) { console.log('\nNo images to download and save...'); } else { console.log(`\nDownloading and saving ${remainingCount} images (${skipCount} already exist)...`); - await processPayloadsPromise(payloads, loadImageFilePromise, config); + await processPayloadsPromise(payloads, loadImageFilePromise); } } -async function loadImageFilePromise(imageUrl, config) { +async function loadImageFilePromise(imageUrl) { // only encode the URL if it doesn't already have encoded characters const url = (/%[\da-f]{2}/i).test(imageUrl) ? imageUrl : encodeURI(imageUrl); @@ -160,7 +160,7 @@ async function loadImageFilePromise(imageUrl, config) { responseType: 'arraybuffer' }; - if (!config.strictSsl) { + if (!shared.config.strictSsl) { // custom agents to disable SSL errors (adding both http and https, just in case) requestConfig.httpAgent = new http.Agent({ rejectUnauthorized: false }); requestConfig.httpsAgent = new https.Agent({ rejectUnauthorized: false }); @@ -182,8 +182,8 @@ async function loadImageFilePromise(imageUrl, config) { return buffer; } -function buildPostPath(post, config) { - return shared.buildPostPath(config.output, post.type, post.date, post.slug, config); +function buildPostPath(post) { + return shared.buildPostPath(post.type, post.date, post.slug); } function checkFile(path) { From 2367998e667f2f1e674537522445639815ebc387 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Fri, 7 Feb 2025 17:57:28 -0500 Subject: [PATCH 43/79] console fix --- src/intake.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/intake.js b/src/intake.js index f43b5a0..e16e946 100644 --- a/src/intake.js +++ b/src/intake.js @@ -32,7 +32,7 @@ export async function getConfig() { }); wizardAnswers = await getWizardAnswers(wizardQuestions, commandLineAnswers); } else { - console.dir('\nSkipping wizard...'); + console.log('\nSkipping wizard...'); } Object.assign(shared.config, commandLineAnswers, wizardAnswers); From a645b2bfdccb9f8a36b3fcb6a033ae2490664075 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 8 Feb 2025 12:36:08 -0500 Subject: [PATCH 44/79] Output frontmatter id as integer --- src/frontmatter.js | 6 +++--- src/parser.js | 2 +- src/writer.js | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index 821104c..d513079 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -34,9 +34,9 @@ export function excerpt(post) { return post.data.encoded[1].replace(/[\r\n]+/gm, ' '); } -// get ID +// get ID, as an integer export function id(post) { - return post.data.post_id[0]; + return parseInt(post.id); } // get slug, previously decoded and set on post @@ -65,5 +65,5 @@ export function title(post) { // get type, often this will always be "post" // but can also be "page" or other custom types export function type(post) { - return post.data.post_type[0]; + return post.type; } diff --git a/src/parser.js b/src/parser.js index f8c06b6..212774f 100644 --- a/src/parser.js +++ b/src/parser.js @@ -74,7 +74,7 @@ function collectPosts(channelData, postTypes) { function buildPost(data, turndownService) { return { - // full raw post data, used by some frontmatter getters + // full raw post data data, // contents of the post in markdown diff --git a/src/writer.js b/src/writer.js index a201fcf..f289591 100644 --- a/src/writer.js +++ b/src/writer.js @@ -83,6 +83,9 @@ async function loadMarkdownFilePromise(post) { // array of one or more strings outputValue = value.reduce((list, item) => `${list}\n - "${item}"`, ''); } + } else if (Number.isInteger(value)) { + // output unquoted + outputValue = value.toString(); } else if (value instanceof luxon.DateTime) { if (shared.config.customDateFormatting) { outputValue = value.toFormat(shared.config.customDateFormatting); From 42d0688654183c63c79e3482a54d39198ab4b6dc Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 9 Feb 2025 12:21:13 -0500 Subject: [PATCH 45/79] Stuff for drafts, handling missing slugs and dates --- src/frontmatter.js | 7 ++++++- src/intake.js | 7 ++++++- src/parser.js | 7 ++++--- src/questions.js | 2 +- src/shared.js | 43 +++++++++++++++++++++++++++++-------------- src/translator.js | 2 +- src/writer.js | 13 ++++++------- 7 files changed, 53 insertions(+), 28 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index d513079..3e34b9e 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -24,11 +24,16 @@ export function coverImage(post) { } // get post date, previously saved as a luxon datetime object on post -// this value is also used for year/month folders, date prefixes, etc. as needed export function date(post) { return post.date; } +// get boolean indicating if post is a draft +// this will only be included if true, otherwise it's left off +export function draft(post) { + return post.isDraft ? true : undefined; +} + // get excerpt, not decoded, newlines collapsed export function excerpt(post) { return post.data.encoded[1].replace(/[\r\n]+/gm, ' '); diff --git a/src/intake.js b/src/intake.js index e16e946..676e648 100644 --- a/src/intake.js +++ b/src/intake.js @@ -156,5 +156,10 @@ function normalize(value, type, onError) { } export function buildSamplePostPath(overrideConfig) { - return shared.buildPostPath('', luxon.DateTime.now(), 'my-post', overrideConfig); + const samplePost = { + date: luxon.DateTime.now(), + slug: 'my-post' + }; + + return shared.buildPostPath(samplePost, overrideConfig); } diff --git a/src/parser.js b/src/parser.js index 212774f..f255ec4 100644 --- a/src/parser.js +++ b/src/parser.js @@ -58,7 +58,7 @@ function collectPosts(channelData, postTypes) { let allPosts = []; postTypes.forEach(postType => { const postsForType = getItemsOfType(channelData, postType) - .filter(postData => postData.status[0] !== 'trash' && postData.status[0] !== 'draft') + .filter(postData => postData.status[0] !== 'trash') .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) .map(postData => buildPost(postData, turndownService)); @@ -83,8 +83,9 @@ function buildPost(data, turndownService) { // these are not written to file, but help with other things type: data.post_type[0], id: data.post_id[0], + isDraft: data.status[0] === 'draft', slug: decodeURIComponent(data.post_name[0]), - date: luxon.DateTime.fromRFC2822(data.pubDate[0], { zone: shared.config.customDateTimezone }), + date: data.pubDate[0] ? luxon.DateTime.fromRFC2822(data.pubDate[0], { zone: shared.config.customDateTimezone }) : undefined, coverImageId: getPostMetaValue(data.postmeta, '_thumbnail_id'), // these are possibly set later in mergeImagesIntoPosts() @@ -171,7 +172,7 @@ function populateFrontmatter(posts) { throw `Could not find a frontmatter getter named "${key}".`; } - post.frontmatter[alias || key] = frontmatterGetter(post); + post.frontmatter[alias ?? key] = frontmatterGetter(post); }); }); } diff --git a/src/questions.js b/src/questions.js index f2783eb..088f886 100644 --- a/src/questions.js +++ b/src/questions.js @@ -109,7 +109,7 @@ export function load() { { name: 'frontmatter-fields', type: 'list', - default: ['title', 'date', 'categories', 'tags', 'coverImage'] + default: ['title', 'date', 'categories', 'tags', 'coverImage', 'draft'] }, { name: 'image-file-request-delay', diff --git a/src/shared.js b/src/shared.js index 3c6dc04..4e0ae75 100644 --- a/src/shared.js +++ b/src/shared.js @@ -7,31 +7,46 @@ export function camelCase(str) { return str.replace(/-(.)/g, (match) => match[1].toUpperCase()); } -export function buildPostPath(type, date, slug, overrideConfig) { +export function buildPostPath(post, overrideConfig) { const pathConfig = overrideConfig ?? config; - // start with base output dir and post type - const pathSegments = [pathConfig.output, type]; + // start with output folder + const pathSegments = [pathConfig.output]; - if (pathConfig.dateFolders === 'year' || pathConfig.dateFolders === 'year-month') { - pathSegments.push(date.toFormat('yyyy')); + // add folder for post type if exists + if (post.type) { + pathSegments.push(post.type); } - if (pathConfig.dateFolders === 'year-month') { - pathSegments.push(date.toFormat('LL')); + // add drafts folder if this is a draft post + if (post.isDraft) { + pathSegments.push('_drafts'); } - // create slug fragment, possibly date prefixed - let slugFragment = slug; - if (pathConfig.prefixDate) { - slugFragment = date.toFormat('yyyy-LL-dd') + '-' + slugFragment; + // add folders for date year/month as appropriate + if (post.date) { + if (pathConfig.dateFolders === 'year' || pathConfig.dateFolders === 'year-month') { + pathSegments.push(post.date.toFormat('yyyy')); + } + + if (pathConfig.dateFolders === 'year-month') { + pathSegments.push(post.date.toFormat('LL')); + } } - // use slug fragment as folder or filename as specified + // get slug with fallback + let slug = post.slug ? post.slug : 'id-' + post.id; + + // prepend date to slug as appropriate + if (pathConfig.prefixDate && post.date) { + slug = post.date.toFormat('yyyy-LL-dd') + '-' + slug; + } + + // use slug as folder or filename as specified if (pathConfig.postFolders) { - pathSegments.push(slugFragment, 'index.md'); + pathSegments.push(slug, 'index.md'); } else { - pathSegments.push(slugFragment + '.md'); + pathSegments.push(slug + '.md'); } return path.join(...pathSegments); diff --git a/src/translator.js b/src/translator.js index f1b8df0..3b727ea 100644 --- a/src/translator.js +++ b/src/translator.js @@ -87,7 +87,7 @@ export function initTurndownService() { return node.nodeName === 'PRE' && !node.querySelector('code'); }, replacement: (content, node) => { - const language = node.getAttribute('data-wetm-language') || ''; + const language = node.getAttribute('data-wetm-language') ?? ''; return '\n\n```' + language + '\n' + node.textContent + '\n```\n\n'; } }); diff --git a/src/writer.js b/src/writer.js index f289591..0c16f46 100644 --- a/src/writer.js +++ b/src/writer.js @@ -46,7 +46,7 @@ async function writeMarkdownFilesPromise(posts) { let skipCount = 0; let delay = 0; const payloads = posts.flatMap(post => { - const destinationPath = buildPostPath(post); + const destinationPath = shared.buildPostPath(post); if (checkFile(destinationPath)) { // already exists, don't need to save again skipCount++; @@ -96,9 +96,12 @@ async function loadMarkdownFilePromise(post) { if (shared.config.quoteDate) { outputValue = `"${outputValue}"`; } + } else if (typeof value === 'boolean') { + // output unquoted + outputValue = value.toString(); } else { // single string value - const escapedValue = (value || '').replace(/"/g, '\\"'); + const escapedValue = (value ?? '').replace(/"/g, '\\"'); if (escapedValue.length > 0) { outputValue = `"${escapedValue}"`; } @@ -118,7 +121,7 @@ async function writeImageFilesPromise(posts) { let skipCount = 0; let delay = 0; const payloads = posts.flatMap(post => { - const postPath = buildPostPath(post); + const postPath = shared.buildPostPath(post); const imagesDir = path.join(path.dirname(postPath), 'images'); return post.imageUrls.flatMap(imageUrl => { const filename = shared.getFilenameFromUrl(imageUrl); @@ -185,10 +188,6 @@ async function loadImageFilePromise(imageUrl) { return buffer; } -function buildPostPath(post) { - return shared.buildPostPath(post.type, post.date, post.slug); -} - function checkFile(path) { return fs.existsSync(path); } From cc09a41744533ee07372e79db88fa6b1600b511d Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 10 Feb 2025 16:06:04 -0500 Subject: [PATCH 46/79] Refactor turndown service init --- src/parser.js | 9 +++------ src/translator.js | 9 +++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/parser.js b/src/parser.js index f255ec4..f1291ab 100644 --- a/src/parser.js +++ b/src/parser.js @@ -52,15 +52,12 @@ function getItemsOfType(channelData, type) { } function collectPosts(channelData, postTypes) { - // this is passed into getPostContent() for the markdown conversion - const turndownService = translator.initTurndownService(); - let allPosts = []; postTypes.forEach(postType => { const postsForType = getItemsOfType(channelData, postType) .filter(postData => postData.status[0] !== 'trash') .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) - .map(postData => buildPost(postData, turndownService)); + .map(postData => buildPost(postData)); if (postsForType.length > 0) { console.log(`${postsForType.length} posts of type "${postType}" found.`); @@ -72,13 +69,13 @@ function collectPosts(channelData, postTypes) { return allPosts; } -function buildPost(data, turndownService) { +function buildPost(data) { return { // full raw post data data, // contents of the post in markdown - content: translator.getPostContent(data, turndownService), + content: translator.getPostContent(data.encoded[0]), // these are not written to file, but help with other things type: data.post_type[0], diff --git a/src/translator.js b/src/translator.js index 3b727ea..a4f3bb1 100644 --- a/src/translator.js +++ b/src/translator.js @@ -2,7 +2,10 @@ import turndownPluginGfm from '@guyplusplus/turndown-plugin-gfm'; import turndown from 'turndown'; import * as shared from './shared.js'; -export function initTurndownService() { +// init single reusable turndown service object upon import +const turndownService = initTurndownService(); + +function initTurndownService() { const turndownService = new turndown({ headingStyle: 'atx', bulletListMarker: '-', @@ -95,9 +98,7 @@ export function initTurndownService() { return turndownService; } -export function getPostContent(postData, turndownService) { - let content = postData.encoded[0]; - +export function getPostContent(content) { // insert an empty div element between double line breaks // this nifty trick causes turndown to keep adjacent paragraphs separated // without mucking up content inside of other elements (like blocks) From cb9dd9255e49aa691f1b4b5e45724a4ac02653d9 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 10 Feb 2025 16:33:24 -0500 Subject: [PATCH 47/79] Gracefully handle invalid dates --- src/parser.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/parser.js b/src/parser.js index f1291ab..5fa53a6 100644 --- a/src/parser.js +++ b/src/parser.js @@ -74,15 +74,15 @@ function buildPost(data) { // full raw post data data, - // contents of the post in markdown + // body content converted to markdown content: translator.getPostContent(data.encoded[0]), - // these are not written to file, but help with other things + // particularly useful values for all sorts of things type: data.post_type[0], id: data.post_id[0], isDraft: data.status[0] === 'draft', slug: decodeURIComponent(data.post_name[0]), - date: data.pubDate[0] ? luxon.DateTime.fromRFC2822(data.pubDate[0], { zone: shared.config.customDateTimezone }) : undefined, + date: getPostDate(data), coverImageId: getPostMetaValue(data.postmeta, '_thumbnail_id'), // these are possibly set later in mergeImagesIntoPosts() @@ -91,6 +91,11 @@ function buildPost(data) { }; } +function getPostDate(data) { + const date = luxon.DateTime.fromRFC2822(data.pubDate[0] ?? '', { zone: shared.config.customDateTimezone }); + return date.isValid ? date : undefined; +} + function getPostMetaValue(metas, key) { const meta = metas && metas.find((meta) => meta.meta_key[0] === key); return meta ? meta.meta_value[0] : undefined; From e8852a2900e035784f39550b0f015a6c1c2dc6c6 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 15 Feb 2025 10:22:13 -0500 Subject: [PATCH 48/79] Ignore more reserved post types --- src/parser.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/parser.js b/src/parser.js index 5fa53a6..cd5cda9 100644 --- a/src/parser.js +++ b/src/parser.js @@ -41,8 +41,13 @@ function getPostTypes(channelData) { 'nav_menu_item', 'custom_css', 'customize_changeset', + 'oembed_cache', + 'user_request', + 'wp_block', 'wp_global_styles', - 'wp_navigation' + 'wp_navigation', + 'wp_template', + 'wp_template_part' ].includes(type)); return [...new Set(types)]; // remove duplicates } From c546cd47caf19debce3721fbc0382b9aac977cfa Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sat, 15 Feb 2025 10:23:18 -0500 Subject: [PATCH 49/79] Show slug fallback when writing --- src/shared.js | 6 +++++- src/writer.js | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/shared.js b/src/shared.js index 4e0ae75..38316af 100644 --- a/src/shared.js +++ b/src/shared.js @@ -7,6 +7,10 @@ export function camelCase(str) { return str.replace(/-(.)/g, (match) => match[1].toUpperCase()); } +export function getSlugWithFallback(post) { + return post.slug ? post.slug : 'id-' + post.id; +} + export function buildPostPath(post, overrideConfig) { const pathConfig = overrideConfig ?? config; @@ -35,7 +39,7 @@ export function buildPostPath(post, overrideConfig) { } // get slug with fallback - let slug = post.slug ? post.slug : 'id-' + post.id; + let slug = getSlugWithFallback(post); // prepend date to slug as appropriate if (pathConfig.prefixDate && post.date) { diff --git a/src/writer.js b/src/writer.js index 0c16f46..796a445 100644 --- a/src/writer.js +++ b/src/writer.js @@ -55,7 +55,7 @@ async function writeMarkdownFilesPromise(posts) { const payload = { item: post, type: post.type, - name: post.slug, + name: shared.getSlugWithFallback(post), destinationPath, delay }; From 470ba2dc00926ed4437096266e6bf7721cea8ee3 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 16 Feb 2025 14:43:22 -0500 Subject: [PATCH 50/79] Catch initial parsing errors, shared.getValue() --- src/parser.js | 44 ++++++++++++++++++++++++++++---------------- src/shared.js | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 16 deletions(-) diff --git a/src/parser.js b/src/parser.js index cd5cda9..053432c 100644 --- a/src/parser.js +++ b/src/parser.js @@ -8,21 +8,33 @@ import * as translator from './translator.js'; export async function parseFilePromise() { console.log('\nParsing...'); const content = await fs.promises.readFile(shared.config.input, 'utf8'); - const allData = await xml2js.parseStringPromise(content, { + + const rootData = await xml2js.parseStringPromise(content, { trim: true, tagNameProcessors: [xml2js.processors.stripPrefix] + }).catch((ex) => { + ex.message = 'Could not parse XML. This likely means your import file is malformed.\n\n' + ex.message; + throw ex; }); - const channelData = allData.rss.channel[0].item; - const postTypes = getPostTypes(channelData); - const posts = collectPosts(channelData, postTypes); + const rssData = rootData.rss; + if (rssData === undefined) { + throw new Error('Could not find root node. This likely means your import file is malformed.') + } + rssData['wetm-expression'] = 'rss'; + + const channelData = shared.getValue(rssData, 'channel', 0); + const allPostData = shared.getValue(channelData, 'item'); + + const postTypes = getPostTypes(allPostData); + const posts = collectPosts(allPostData, postTypes); const images = []; if (shared.config.saveImages === 'attached' || shared.config.saveImages === 'all') { - images.push(...collectAttachedImages(channelData)); + images.push(...collectAttachedImages(allPostData)); } if (shared.config.saveImages === 'scraped' || shared.config.saveImages === 'all') { - images.push(...collectScrapedImages(channelData, postTypes)); + images.push(...collectScrapedImages(allPostData, postTypes)); } mergeImagesIntoPosts(images, posts); @@ -31,9 +43,9 @@ export async function parseFilePromise() { return posts; } -function getPostTypes(channelData) { +function getPostTypes(allPostData) { // search export file for all post types minus some specific types we don't want - const types = channelData + const types = allPostData .map(item => item.post_type[0]) .filter(type => ![ 'attachment', @@ -52,14 +64,14 @@ function getPostTypes(channelData) { return [...new Set(types)]; // remove duplicates } -function getItemsOfType(channelData, type) { - return channelData.filter(item => item.post_type[0] === type); +function getItemsOfType(allPostData, type) { + return allPostData.filter(item => item.post_type[0] === type); } -function collectPosts(channelData, postTypes) { +function collectPosts(allPostData, postTypes) { let allPosts = []; postTypes.forEach(postType => { - const postsForType = getItemsOfType(channelData, postType) + const postsForType = getItemsOfType(allPostData, postType) .filter(postData => postData.status[0] !== 'trash') .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) .map(postData => buildPost(postData)); @@ -106,8 +118,8 @@ function getPostMetaValue(metas, key) { return meta ? meta.meta_value[0] : undefined; } -function collectAttachedImages(channelData) { - const images = getItemsOfType(channelData, 'attachment') +function collectAttachedImages(allPostData) { + const images = getItemsOfType(allPostData, 'attachment') // filter to certain image file types .filter(attachment => attachment.attachment_url && (/\.(gif|jpe?g|png|webp)$/i).test(attachment.attachment_url[0])) .map(attachment => ({ @@ -120,10 +132,10 @@ function collectAttachedImages(channelData) { return images; } -function collectScrapedImages(channelData, postTypes) { +function collectScrapedImages(allPostData, postTypes) { const images = []; postTypes.forEach(postType => { - getItemsOfType(channelData, postType).forEach(postData => { + getItemsOfType(allPostData, postType).forEach(postData => { const postId = postData.post_id[0]; const postContent = postData.encoded[0]; const postLink = postData.link[0]; diff --git a/src/shared.js b/src/shared.js index 38316af..1a2216d 100644 --- a/src/shared.js +++ b/src/shared.js @@ -7,6 +7,44 @@ export function camelCase(str) { return str.replace(/-(.)/g, (match) => match[1].toUpperCase()); } +export function getValue(obj, propName, index) { + if (obj === undefined) { + throw new Error(`Could not find undefined.${propName}.`) + } + + let expression = `${obj['wetm-expression'] ?? 'object'}.${propName}`; + + const values = obj[propName]; + if (values === undefined) { + throw new Error(`Could not find ${expression}.`) + } + + if (index === undefined) { + values.forEach((value, index) => { + value['wetm-expression'] = `${expression}[${index}]`; + }); + return values; + } else { + expression += `[${index}]`; + + const value = values[index]; + if (value === undefined) { + throw new Error(`Could not find ${expression}.`) + } + + value['wetm-expression'] = expression; + return value; + } +} + +export function getOptionalValue(obj, propName, index) { + try { + return getValue(obj, propName, index); + } catch (ex) { + return undefined; + } +} + export function getSlugWithFallback(post) { return post.slug ? post.slug : 'id-' + post.id; } From 922515ec23261f10c26654f43a8802b8853a597b Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 17 Feb 2025 16:24:24 -0500 Subject: [PATCH 51/79] Use shared.getValue() wherever relevant --- src/frontmatter.js | 60 +++++++++++++++++++--------------------------- src/parser.js | 48 ++++++++++++++++++++----------------- src/shared.js | 6 ++++- 3 files changed, 55 insertions(+), 59 deletions(-) diff --git a/src/frontmatter.js b/src/frontmatter.js index 3e34b9e..bae2b64 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -1,74 +1,62 @@ -// get author, without decoding -// WordPress doesn't allow funky characters in usernames anyway +import * as shared from './shared.js'; + export function author(post) { - return post.data.creator[0]; + // not decoded, WordPress doesn't allow funky characters in usernames anyway + return shared.getValue(post.data, 'creator', 0); } -// get array of decoded category names, excluding 'uncategorized' export function categories(post) { - if (!post.data.category) { - return []; - } - - const categories = post.data.category - .filter(category => category.$.domain === 'category') - .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); - - return categories.filter((category) => category !== 'uncategorized'); + // array of decoded category names, excluding 'uncategorized' + const categories = shared.getOptionalValue(post.data, 'category') ?? []; + return categories + .filter((category) => category.$.domain === 'category' && category.$.nicename !== 'uncategorized') + .map((category) => decodeURIComponent(category.$.nicename)); } -// get cover image filename, previously decoded and set on post -// this one is unique as it relies on special logic executed by the parser export function coverImage(post) { + // cover image filename, previously parsed and decoded return post.coverImage; } -// get post date, previously saved as a luxon datetime object on post export function date(post) { + // a luxon datetime object, previously parsed return post.date; } -// get boolean indicating if post is a draft -// this will only be included if true, otherwise it's left off export function draft(post) { + // boolean representing the previously parsed draft status, only included when true return post.isDraft ? true : undefined; } -// get excerpt, not decoded, newlines collapsed export function excerpt(post) { - return post.data.encoded[1].replace(/[\r\n]+/gm, ' '); + // not decoded, newlines collapsed + return shared.getValue(post.data, 'encoded', 1).replace(/[\r\n]+/gm, ' '); } -// get ID, as an integer export function id(post) { + // previously parsed as a string, converted to integer here return parseInt(post.id); } -// get slug, previously decoded and set on post export function slug(post) { + // previously parsed and decoded return post.slug; } -// get array of decoded tag names export function tags(post) { - if (!post.data.category) { - return []; - } - - const categories = post.data.category - .filter(category => category.$.domain === 'post_tag') - .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); - - return categories; + // array of decoded tag names (yes, they come from nodes, not a typo) + const categories = shared.getOptionalValue(post.data, 'category') ?? []; + return categories + .filter((category) => category.$.domain === 'post_tag') + .map((category) => decodeURIComponent(category.$.nicename)); } -// get simple post title, but not decoded like other frontmatter string fields export function title(post) { - return post.data.title[0]; + // not decoded + return shared.getValue(post.data, 'title', 0); } -// get type, often this will always be "post" -// but can also be "page" or other custom types export function type(post) { + // previously parsed but not decoded, can be "post", "page", or other custom types return post.type; } diff --git a/src/parser.js b/src/parser.js index 053432c..daed8a6 100644 --- a/src/parser.js +++ b/src/parser.js @@ -21,7 +21,7 @@ export async function parseFilePromise() { if (rssData === undefined) { throw new Error('Could not find root node. This likely means your import file is malformed.') } - rssData['wetm-expression'] = 'rss'; + rssData['wetm-expression'] = 'rss'; const channelData = shared.getValue(rssData, 'channel', 0); const allPostData = shared.getValue(channelData, 'item'); @@ -46,7 +46,7 @@ export async function parseFilePromise() { function getPostTypes(allPostData) { // search export file for all post types minus some specific types we don't want const types = allPostData - .map(item => item.post_type[0]) + .map(item => shared.getValue(item, 'post_type', 0)) .filter(type => ![ 'attachment', 'revision', @@ -65,15 +65,15 @@ function getPostTypes(allPostData) { } function getItemsOfType(allPostData, type) { - return allPostData.filter(item => item.post_type[0] === type); + return allPostData.filter(item => shared.getValue(item, 'post_type', 0) === type); } function collectPosts(allPostData, postTypes) { let allPosts = []; postTypes.forEach(postType => { const postsForType = getItemsOfType(allPostData, postType) - .filter(postData => postData.status[0] !== 'trash') - .filter(postData => !(postType === 'page' && postData.post_name[0] === 'sample-page')) + .filter(postData => shared.getValue(postData, 'status', 0) !== 'trash') + .filter(postData => !(postType === 'page' && shared.getValue(postData, 'post_name', 0) === 'sample-page')) .map(postData => buildPost(postData)); if (postsForType.length > 0) { @@ -92,15 +92,15 @@ function buildPost(data) { data, // body content converted to markdown - content: translator.getPostContent(data.encoded[0]), + content: translator.getPostContent(shared.getValue(data, 'encoded', 0)), // particularly useful values for all sorts of things - type: data.post_type[0], - id: data.post_id[0], - isDraft: data.status[0] === 'draft', - slug: decodeURIComponent(data.post_name[0]), + type: shared.getValue(data, 'post_type', 0), + id: shared.getValue(data, 'post_id', 0), + isDraft: shared.getValue(data, 'status', 0) === 'draft', + slug: decodeURIComponent(shared.getValue(data, 'post_name', 0)), date: getPostDate(data), - coverImageId: getPostMetaValue(data.postmeta, '_thumbnail_id'), + coverImageId: getPostMetaValue(data, '_thumbnail_id'), // these are possibly set later in mergeImagesIntoPosts() coverImage: undefined, @@ -109,23 +109,27 @@ function buildPost(data) { } function getPostDate(data) { - const date = luxon.DateTime.fromRFC2822(data.pubDate[0] ?? '', { zone: shared.config.customDateTimezone }); + const date = luxon.DateTime.fromRFC2822(shared.getValue(data, 'pubDate', 0) ?? '', { zone: shared.config.customDateTimezone }); return date.isValid ? date : undefined; } -function getPostMetaValue(metas, key) { - const meta = metas && metas.find((meta) => meta.meta_key[0] === key); - return meta ? meta.meta_value[0] : undefined; +function getPostMetaValue(data, key) { + const metas = shared.getOptionalValue(data, 'postmeta'); + const meta = metas && metas.find((meta) => shared.getValue(meta, 'meta_key', 0) === key); + return meta ? shared.getValue(meta, 'meta_value', 0) : undefined; } function collectAttachedImages(allPostData) { const images = getItemsOfType(allPostData, 'attachment') // filter to certain image file types - .filter(attachment => attachment.attachment_url && (/\.(gif|jpe?g|png|webp)$/i).test(attachment.attachment_url[0])) + .filter(attachment => { + const url = shared.getOptionalValue(attachment, 'attachment_url', 0); + return url && (/\.(gif|jpe?g|png|webp)$/i).test(url); + }) .map(attachment => ({ - id: attachment.post_id[0], - postId: attachment.post_parent[0], - url: attachment.attachment_url[0] + id: shared.getValue(attachment, 'post_id', 0), + postId: shared.getValue(attachment, 'post_parent', 0), + url: shared.getValue(attachment, 'attachment_url', 0) })); console.log(images.length + ' attached images found.'); @@ -136,9 +140,9 @@ function collectScrapedImages(allPostData, postTypes) { const images = []; postTypes.forEach(postType => { getItemsOfType(allPostData, postType).forEach(postData => { - const postId = postData.post_id[0]; - const postContent = postData.encoded[0]; - const postLink = postData.link[0]; + const postId = shared.getValue(postData, 'post_id', 0); + const postContent = shared.getValue(postData, 'encoded', 0); + const postLink = shared.getValue(postData, 'link', 0); const matches = [...postContent.matchAll(/]*src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)]; matches.forEach(match => { diff --git a/src/shared.js b/src/shared.js index 1a2216d..e15cb02 100644 --- a/src/shared.js +++ b/src/shared.js @@ -22,6 +22,7 @@ export function getValue(obj, propName, index) { if (index === undefined) { values.forEach((value, index) => { value['wetm-expression'] = `${expression}[${index}]`; + // console.log('>>>', value['wetm-expression']); }); return values; } else { @@ -32,7 +33,10 @@ export function getValue(obj, propName, index) { throw new Error(`Could not find ${expression}.`) } - value['wetm-expression'] = expression; + if (typeof value === 'object') { + value['wetm-expression'] = expression; + // console.log('>>>', value['wetm-expression']); + } return value; } } From aaafd6bd07e9e473a49b821196ea77e6c2b28cbb Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Fri, 21 Feb 2025 14:54:50 -0500 Subject: [PATCH 52/79] Data wrapper class --- src/data.js | 79 ++++++++++++++++++++++++++++++++++++++++++++++ src/frontmatter.js | 20 ++++++------ src/parser.js | 69 ++++++++++++++++------------------------ src/shared.js | 42 ------------------------ 4 files changed, 116 insertions(+), 94 deletions(-) create mode 100644 src/data.js diff --git a/src/data.js b/src/data.js new file mode 100644 index 0000000..44ce3f5 --- /dev/null +++ b/src/data.js @@ -0,0 +1,79 @@ +import xml2js from 'xml2js'; + +class Data { + #obj; + #expression; + + constructor(obj, expression) { + this.#obj = typeof obj === 'string' ? { _: obj } : obj; + this.#expression = expression; + } + + get value() { + const value = this.#obj._; + if (value === undefined) { + throw new Error(`Could not get value from ${this.#expression}.`); + } + + return value; + } + + #buildExpression(propName, index) { + let expression = `${this.#expression}.${propName}`; + if (index !== undefined) { + expression += `[${index}]`; + } + + return expression; + } + + #getPropArray(propName, isRequired) { + const propArray = this.#obj[propName]; + if (propArray === undefined && isRequired) { + throw new Error(`Could not find ${this.#buildExpression(propName)}.`); + } + + return propArray; + } + + getAll(propName, isRequired = true) { + const propArray = this.#getPropArray(propName, isRequired); + return propArray !== undefined ? propArray.map((value, index) => new Data(value, this.#buildExpression(propName, index))) : undefined; + } + + getSingle(propName, index, isRequired = true) { + const prop = (this.#getPropArray(propName, isRequired) ?? [])[index]; + + if (prop === undefined && isRequired) { + throw new Error(`Could not find ${this.#buildExpression(propName, index)}.`) + } + + return prop !== undefined ? new Data(prop, this.#buildExpression(propName, index)) : undefined; + } + + getAttribute(attrName) { + const attribute = this.#obj.$?.[attrName]; + if (attribute === undefined) { + throw new Error(`Could not get attribute ${attrName} from ${this.#expression}.`); + } + + return attribute; + } +} + +export async function load(content) { + const rootData = await xml2js.parseStringPromise(content, { + tagNameProcessors: [xml2js.processors.stripPrefix], + trim: true + }).catch((ex) => { + ex.message = 'Could not parse XML. This likely means your import file is malformed.\n\n' + ex.message; + throw ex; + }); + + const rssData = rootData.rss; + if (rssData === undefined) { + throw new Error('Could not find root node. This likely means your import file is malformed.') + } + + return new Data(rssData, 'rss'); +} diff --git a/src/frontmatter.js b/src/frontmatter.js index bae2b64..e600198 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -1,16 +1,14 @@ -import * as shared from './shared.js'; - export function author(post) { // not decoded, WordPress doesn't allow funky characters in usernames anyway - return shared.getValue(post.data, 'creator', 0); + return post.data.getSingle('creator', 0).value; } export function categories(post) { // array of decoded category names, excluding 'uncategorized' - const categories = shared.getOptionalValue(post.data, 'category') ?? []; + const categories = post.data.getAll('category', false) ?? []; return categories - .filter((category) => category.$.domain === 'category' && category.$.nicename !== 'uncategorized') - .map((category) => decodeURIComponent(category.$.nicename)); + .filter((category) => category.getAttribute('domain') === 'category' && category.getAttribute('nicename') !== 'uncategorized') + .map((category) => decodeURIComponent(category.getAttribute('nicename'))); } export function coverImage(post) { @@ -30,7 +28,7 @@ export function draft(post) { export function excerpt(post) { // not decoded, newlines collapsed - return shared.getValue(post.data, 'encoded', 1).replace(/[\r\n]+/gm, ' '); + return post.data.getSingle('encoded', 1).value.replace(/[\r\n]+/gm, ' '); } export function id(post) { @@ -45,15 +43,15 @@ export function slug(post) { export function tags(post) { // array of decoded tag names (yes, they come from nodes, not a typo) - const categories = shared.getOptionalValue(post.data, 'category') ?? []; + const categories = post.data.getAll('category', false) ?? []; return categories - .filter((category) => category.$.domain === 'post_tag') - .map((category) => decodeURIComponent(category.$.nicename)); + .filter((category) => category.getAttribute('domain') === 'post_tag') + .map((category) => decodeURIComponent(category.getAttribute('nicename'))); } export function title(post) { // not decoded - return shared.getValue(post.data, 'title', 0); + return post.data.getSingle('title', 0).value; } export function type(post) { diff --git a/src/parser.js b/src/parser.js index daed8a6..924cc2c 100644 --- a/src/parser.js +++ b/src/parser.js @@ -1,6 +1,6 @@ import fs from 'fs'; import * as luxon from 'luxon'; -import xml2js from 'xml2js'; +import * as data from './data.js'; import * as frontmatter from './frontmatter.js'; import * as shared from './shared.js'; import * as translator from './translator.js'; @@ -8,23 +8,10 @@ import * as translator from './translator.js'; export async function parseFilePromise() { console.log('\nParsing...'); const content = await fs.promises.readFile(shared.config.input, 'utf8'); + const rssData = await data.load(content); - const rootData = await xml2js.parseStringPromise(content, { - trim: true, - tagNameProcessors: [xml2js.processors.stripPrefix] - }).catch((ex) => { - ex.message = 'Could not parse XML. This likely means your import file is malformed.\n\n' + ex.message; - throw ex; - }); - - const rssData = rootData.rss; - if (rssData === undefined) { - throw new Error('Could not find root node. This likely means your import file is malformed.') - } - rssData['wetm-expression'] = 'rss'; - - const channelData = shared.getValue(rssData, 'channel', 0); - const allPostData = shared.getValue(channelData, 'item'); + const channelData = rssData.getSingle('channel', 0); + const allPostData = channelData.getAll('item'); const postTypes = getPostTypes(allPostData); const posts = collectPosts(allPostData, postTypes); @@ -45,9 +32,9 @@ export async function parseFilePromise() { function getPostTypes(allPostData) { // search export file for all post types minus some specific types we don't want - const types = allPostData - .map(item => shared.getValue(item, 'post_type', 0)) - .filter(type => ![ + const postTypes = allPostData + .map((postData) => postData.getSingle('post_type', 0).value) + .filter((postType) => ![ 'attachment', 'revision', 'nav_menu_item', @@ -60,20 +47,20 @@ function getPostTypes(allPostData) { 'wp_navigation', 'wp_template', 'wp_template_part' - ].includes(type)); - return [...new Set(types)]; // remove duplicates + ].includes(postType)); + return [...new Set(postTypes)]; // remove duplicates } function getItemsOfType(allPostData, type) { - return allPostData.filter(item => shared.getValue(item, 'post_type', 0) === type); + return allPostData.filter(item => item.getSingle('post_type', 0).value === type); } function collectPosts(allPostData, postTypes) { let allPosts = []; postTypes.forEach(postType => { const postsForType = getItemsOfType(allPostData, postType) - .filter(postData => shared.getValue(postData, 'status', 0) !== 'trash') - .filter(postData => !(postType === 'page' && shared.getValue(postData, 'post_name', 0) === 'sample-page')) + .filter(postData => postData.getSingle('status', 0).value !== 'trash') + .filter(postData => !(postType === 'page' && postData.getSingle('post_name', 0).value === 'sample-page')) .map(postData => buildPost(postData)); if (postsForType.length > 0) { @@ -92,13 +79,13 @@ function buildPost(data) { data, // body content converted to markdown - content: translator.getPostContent(shared.getValue(data, 'encoded', 0)), + content: translator.getPostContent(data.getSingle('encoded', 0).value), // particularly useful values for all sorts of things - type: shared.getValue(data, 'post_type', 0), - id: shared.getValue(data, 'post_id', 0), - isDraft: shared.getValue(data, 'status', 0) === 'draft', - slug: decodeURIComponent(shared.getValue(data, 'post_name', 0)), + type: data.getSingle('post_type', 0).value, + id: data.getSingle('post_id', 0).value, + isDraft: data.getSingle('status', 0).value === 'draft', + slug: decodeURIComponent(data.getSingle('post_name', 0).value), date: getPostDate(data), coverImageId: getPostMetaValue(data, '_thumbnail_id'), @@ -109,27 +96,27 @@ function buildPost(data) { } function getPostDate(data) { - const date = luxon.DateTime.fromRFC2822(shared.getValue(data, 'pubDate', 0) ?? '', { zone: shared.config.customDateTimezone }); + const date = luxon.DateTime.fromRFC2822(data.getSingle('pubDate', 0).value ?? '', { zone: shared.config.customDateTimezone }); return date.isValid ? date : undefined; } function getPostMetaValue(data, key) { - const metas = shared.getOptionalValue(data, 'postmeta'); - const meta = metas && metas.find((meta) => shared.getValue(meta, 'meta_key', 0) === key); - return meta ? shared.getValue(meta, 'meta_value', 0) : undefined; + const metas = data.getAll('postmeta', false) ?? []; + const meta = metas.find((meta) => meta.getSingle('meta_key', 0).value === key); + return meta ? meta.getSingle('meta_value', 0).value : undefined; } function collectAttachedImages(allPostData) { const images = getItemsOfType(allPostData, 'attachment') // filter to certain image file types .filter(attachment => { - const url = shared.getOptionalValue(attachment, 'attachment_url', 0); + const url = attachment.getSingle('attachment_url', 0).value; return url && (/\.(gif|jpe?g|png|webp)$/i).test(url); }) .map(attachment => ({ - id: shared.getValue(attachment, 'post_id', 0), - postId: shared.getValue(attachment, 'post_parent', 0), - url: shared.getValue(attachment, 'attachment_url', 0) + id: attachment.getSingle('post_id', 0).value, + postId: attachment.getSingle('post_parent', 0).value, + url: attachment.getSingle('attachment_url', 0).value })); console.log(images.length + ' attached images found.'); @@ -140,9 +127,9 @@ function collectScrapedImages(allPostData, postTypes) { const images = []; postTypes.forEach(postType => { getItemsOfType(allPostData, postType).forEach(postData => { - const postId = shared.getValue(postData, 'post_id', 0); - const postContent = shared.getValue(postData, 'encoded', 0); - const postLink = shared.getValue(postData, 'link', 0); + const postId = postData.getSingle('post_id', 0).value; + const postContent = postData.getSingle('encoded', 0).value; + const postLink = postData.getSingle('link', 0).value; const matches = [...postContent.matchAll(/]*src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)]; matches.forEach(match => { diff --git a/src/shared.js b/src/shared.js index e15cb02..38316af 100644 --- a/src/shared.js +++ b/src/shared.js @@ -7,48 +7,6 @@ export function camelCase(str) { return str.replace(/-(.)/g, (match) => match[1].toUpperCase()); } -export function getValue(obj, propName, index) { - if (obj === undefined) { - throw new Error(`Could not find undefined.${propName}.`) - } - - let expression = `${obj['wetm-expression'] ?? 'object'}.${propName}`; - - const values = obj[propName]; - if (values === undefined) { - throw new Error(`Could not find ${expression}.`) - } - - if (index === undefined) { - values.forEach((value, index) => { - value['wetm-expression'] = `${expression}[${index}]`; - // console.log('>>>', value['wetm-expression']); - }); - return values; - } else { - expression += `[${index}]`; - - const value = values[index]; - if (value === undefined) { - throw new Error(`Could not find ${expression}.`) - } - - if (typeof value === 'object') { - value['wetm-expression'] = expression; - // console.log('>>>', value['wetm-expression']); - } - return value; - } -} - -export function getOptionalValue(obj, propName, index) { - try { - return getValue(obj, propName, index); - } catch (ex) { - return undefined; - } -} - export function getSlugWithFallback(post) { return post.slug ? post.slug : 'id-' + post.id; } From 841111f85061ff41e81e7cf0830671009469963a Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Fri, 21 Feb 2025 18:54:19 -0500 Subject: [PATCH 53/79] Data refactoring and renaming --- src/data.js | 35 +++++++++++++++++------------------ src/frontmatter.js | 18 +++++++++--------- src/parser.js | 44 ++++++++++++++++++++++---------------------- 3 files changed, 48 insertions(+), 49 deletions(-) diff --git a/src/data.js b/src/data.js index 44ce3f5..2c1d4b6 100644 --- a/src/data.js +++ b/src/data.js @@ -27,31 +27,30 @@ class Data { return expression; } - #getPropArray(propName, isRequired) { - const propArray = this.#obj[propName]; - if (propArray === undefined && isRequired) { + children(propName) { + const nodes = this.#obj[propName] ?? []; + return nodes.map((value, index) => new Data(value, this.#buildExpression(propName, index))); + } + + child(propName, index = 0) { + const nodes = this.#obj[propName]; + if (nodes === undefined) { throw new Error(`Could not find ${this.#buildExpression(propName)}.`); } - return propArray; - } - - getAll(propName, isRequired = true) { - const propArray = this.#getPropArray(propName, isRequired); - return propArray !== undefined ? propArray.map((value, index) => new Data(value, this.#buildExpression(propName, index))) : undefined; - } - - getSingle(propName, index, isRequired = true) { - const prop = (this.#getPropArray(propName, isRequired) ?? [])[index]; - - if (prop === undefined && isRequired) { - throw new Error(`Could not find ${this.#buildExpression(propName, index)}.`) + const node = nodes[index]; + if (node === undefined) { + throw new Error(`Could not find ${this.#buildExpression(propName, index)}.`); } - return prop !== undefined ? new Data(prop, this.#buildExpression(propName, index)) : undefined; + return new Data(node, this.#buildExpression(propName, index)); } - getAttribute(attrName) { + childValue(propName, index = 0) { + return this.child(propName, index).value; + } + + attribute(attrName) { const attribute = this.#obj.$?.[attrName]; if (attribute === undefined) { throw new Error(`Could not get attribute ${attrName} from ${this.#expression}.`); diff --git a/src/frontmatter.js b/src/frontmatter.js index e600198..74aede8 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -1,14 +1,14 @@ export function author(post) { // not decoded, WordPress doesn't allow funky characters in usernames anyway - return post.data.getSingle('creator', 0).value; + return post.data.childValue('creator'); } export function categories(post) { // array of decoded category names, excluding 'uncategorized' - const categories = post.data.getAll('category', false) ?? []; + const categories = post.data.children('category'); return categories - .filter((category) => category.getAttribute('domain') === 'category' && category.getAttribute('nicename') !== 'uncategorized') - .map((category) => decodeURIComponent(category.getAttribute('nicename'))); + .filter((category) => category.attribute('domain') === 'category' && category.attribute('nicename') !== 'uncategorized') + .map((category) => decodeURIComponent(category.attribute('nicename'))); } export function coverImage(post) { @@ -28,7 +28,7 @@ export function draft(post) { export function excerpt(post) { // not decoded, newlines collapsed - return post.data.getSingle('encoded', 1).value.replace(/[\r\n]+/gm, ' '); + return post.data.childValue('encoded', 1).replace(/[\r\n]+/gm, ' '); } export function id(post) { @@ -43,15 +43,15 @@ export function slug(post) { export function tags(post) { // array of decoded tag names (yes, they come from nodes, not a typo) - const categories = post.data.getAll('category', false) ?? []; + const categories = post.data.children('category'); return categories - .filter((category) => category.getAttribute('domain') === 'post_tag') - .map((category) => decodeURIComponent(category.getAttribute('nicename'))); + .filter((category) => category.attribute('domain') === 'post_tag') + .map((category) => decodeURIComponent(category.attribute('nicename'))); } export function title(post) { // not decoded - return post.data.getSingle('title', 0).value; + return post.data.childValue('title'); } export function type(post) { diff --git a/src/parser.js b/src/parser.js index 924cc2c..023b4da 100644 --- a/src/parser.js +++ b/src/parser.js @@ -10,8 +10,8 @@ export async function parseFilePromise() { const content = await fs.promises.readFile(shared.config.input, 'utf8'); const rssData = await data.load(content); - const channelData = rssData.getSingle('channel', 0); - const allPostData = channelData.getAll('item'); + const channelData = rssData.child('channel'); + const allPostData = channelData.children('item'); const postTypes = getPostTypes(allPostData); const posts = collectPosts(allPostData, postTypes); @@ -33,7 +33,7 @@ export async function parseFilePromise() { function getPostTypes(allPostData) { // search export file for all post types minus some specific types we don't want const postTypes = allPostData - .map((postData) => postData.getSingle('post_type', 0).value) + .map((postData) => postData.childValue('post_type')) .filter((postType) => ![ 'attachment', 'revision', @@ -52,15 +52,15 @@ function getPostTypes(allPostData) { } function getItemsOfType(allPostData, type) { - return allPostData.filter(item => item.getSingle('post_type', 0).value === type); + return allPostData.filter(item => item.childValue('post_type') === type); } function collectPosts(allPostData, postTypes) { let allPosts = []; postTypes.forEach(postType => { const postsForType = getItemsOfType(allPostData, postType) - .filter(postData => postData.getSingle('status', 0).value !== 'trash') - .filter(postData => !(postType === 'page' && postData.getSingle('post_name', 0).value === 'sample-page')) + .filter(postData => postData.childValue('status') !== 'trash') + .filter(postData => !(postType === 'page' && postData.childValue('post_name') === 'sample-page')) .map(postData => buildPost(postData)); if (postsForType.length > 0) { @@ -79,13 +79,13 @@ function buildPost(data) { data, // body content converted to markdown - content: translator.getPostContent(data.getSingle('encoded', 0).value), + content: translator.getPostContent(data.childValue('encoded')), // particularly useful values for all sorts of things - type: data.getSingle('post_type', 0).value, - id: data.getSingle('post_id', 0).value, - isDraft: data.getSingle('status', 0).value === 'draft', - slug: decodeURIComponent(data.getSingle('post_name', 0).value), + type: data.childValue('post_type'), + id: data.childValue('post_id'), + isDraft: data.childValue('status') === 'draft', + slug: decodeURIComponent(data.childValue('post_name')), date: getPostDate(data), coverImageId: getPostMetaValue(data, '_thumbnail_id'), @@ -96,27 +96,27 @@ function buildPost(data) { } function getPostDate(data) { - const date = luxon.DateTime.fromRFC2822(data.getSingle('pubDate', 0).value ?? '', { zone: shared.config.customDateTimezone }); + const date = luxon.DateTime.fromRFC2822(data.childValue('pubDate'), { zone: shared.config.customDateTimezone }); return date.isValid ? date : undefined; } function getPostMetaValue(data, key) { - const metas = data.getAll('postmeta', false) ?? []; - const meta = metas.find((meta) => meta.getSingle('meta_key', 0).value === key); - return meta ? meta.getSingle('meta_value', 0).value : undefined; + const metas = data.children('postmeta'); + const meta = metas.find((meta) => meta.childValue('meta_key') === key); + return meta ? meta.childValue('meta_value') : undefined; } function collectAttachedImages(allPostData) { const images = getItemsOfType(allPostData, 'attachment') // filter to certain image file types .filter(attachment => { - const url = attachment.getSingle('attachment_url', 0).value; + const url = attachment.childValue('attachment_url'); return url && (/\.(gif|jpe?g|png|webp)$/i).test(url); }) .map(attachment => ({ - id: attachment.getSingle('post_id', 0).value, - postId: attachment.getSingle('post_parent', 0).value, - url: attachment.getSingle('attachment_url', 0).value + id: attachment.childValue('post_id'), + postId: attachment.childValue('post_parent'), + url: attachment.childValue('attachment_url') })); console.log(images.length + ' attached images found.'); @@ -127,9 +127,9 @@ function collectScrapedImages(allPostData, postTypes) { const images = []; postTypes.forEach(postType => { getItemsOfType(allPostData, postType).forEach(postData => { - const postId = postData.getSingle('post_id', 0).value; - const postContent = postData.getSingle('encoded', 0).value; - const postLink = postData.getSingle('link', 0).value; + const postId = postData.childValue('post_id'); + const postContent = postData.childValue('encoded'); + const postLink = postData.childValue('link'); const matches = [...postContent.matchAll(/]*src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)]; matches.forEach(match => { From f0e8400ccd5e4de4a60d8b8c6b08ae227c68612b Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Fri, 21 Feb 2025 18:54:41 -0500 Subject: [PATCH 54/79] Decrease default write delay --- src/questions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/questions.js b/src/questions.js index 088f886..63d9b7a 100644 --- a/src/questions.js +++ b/src/questions.js @@ -119,7 +119,7 @@ export function load() { { name: 'markdown-file-write-delay', type: 'integer', - default: 25 + default: 10 }, { name: 'include-time-with-date', From db5430117ef5214011e837625c2fc94430f72e83 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 23 Feb 2025 13:22:32 -0500 Subject: [PATCH 55/79] Data fixes, fix for with 2+ src attributes --- src/data.js | 52 ++++++++++++++++++++++++++++++++++++---------- src/frontmatter.js | 9 +++++--- src/parser.js | 29 +++++++++++++++++++------- src/translator.js | 2 +- 4 files changed, 69 insertions(+), 23 deletions(-) diff --git a/src/data.js b/src/data.js index 2c1d4b6..55db05a 100644 --- a/src/data.js +++ b/src/data.js @@ -5,20 +5,15 @@ class Data { #expression; constructor(obj, expression) { + // xml2js returns leaf nodes as strings, turn those into consistent objects + // I found this to be safer and more efficient than using the explicitCharkey option this.#obj = typeof obj === 'string' ? { _: obj } : obj; + + // this identifies how the object was referenced, helps a ton with debugging this.#expression = expression; } - get value() { - const value = this.#obj._; - if (value === undefined) { - throw new Error(`Could not get value from ${this.#expression}.`); - } - - return value; - } - - #buildExpression(propName, index) { + #buildExpression(propName, index = undefined) { let expression = `${this.#expression}.${propName}`; if (index !== undefined) { expression += `[${index}]`; @@ -27,11 +22,22 @@ class Data { return expression; } + // used by "optional" functions to return undefined instead of throwing an error + #optional(func) { + try { + return func(); + } catch (ex) { + return undefined; + } + } + + // will not throw an error if property doesn't exist, defaults to empty array children(propName) { const nodes = this.#obj[propName] ?? []; return nodes.map((value, index) => new Data(value, this.#buildExpression(propName, index))); } + // throws an error if property (or index on property) doesn't exist child(propName, index = 0) { const nodes = this.#obj[propName]; if (nodes === undefined) { @@ -46,10 +52,22 @@ class Data { return new Data(node, this.#buildExpression(propName, index)); } + // convenience function, since it's very common to want the value of a child childValue(propName, index = 0) { - return this.child(propName, index).value; + return this.child(propName, index).value(); + } + + // throws an error if this object doesn't have a value string + value() { + const value = this.#obj._; + if (value === undefined) { + throw new Error(`Could not get value from ${this.#expression}.`); + } + + return value; } + // throws an error if attribute does not exist attribute(attrName) { const attribute = this.#obj.$?.[attrName]; if (attribute === undefined) { @@ -58,6 +76,18 @@ class Data { return attribute; } + + optionalChild(propName, index = 0) { + return this.#optional(() => this.child(propName, index)); + } + + optionalChildValue(propName, index = 0) { + return this.#optional(() => this.childValue(propName, index)); + } + + optionalValue() { + return this.#optional(() => this.value()); + } } export async function load(content) { diff --git a/src/frontmatter.js b/src/frontmatter.js index 74aede8..22f37c9 100644 --- a/src/frontmatter.js +++ b/src/frontmatter.js @@ -1,6 +1,7 @@ export function author(post) { - // not decoded, WordPress doesn't allow funky characters in usernames anyway - return post.data.childValue('creator'); + // not decoded (WordPress doesn't allow funky characters in usernames anyway) + // surprisingly, does not always exist (squarespace exports, for example) + return post.data.optionalChildValue('creator'); } export function categories(post) { @@ -28,7 +29,9 @@ export function draft(post) { export function excerpt(post) { // not decoded, newlines collapsed - return post.data.childValue('encoded', 1).replace(/[\r\n]+/gm, ' '); + // does not always exist (squarespace exports, for example) + const encoded = post.data.optionalChildValue('encoded', 1); + return encoded ? encoded.replace(/[\r\n]+/gm, ' ') : undefined; } export function id(post) { diff --git a/src/parser.js b/src/parser.js index 023b4da..915c97b 100644 --- a/src/parser.js +++ b/src/parser.js @@ -115,7 +115,7 @@ function collectAttachedImages(allPostData) { }) .map(attachment => ({ id: attachment.childValue('post_id'), - postId: attachment.childValue('post_parent'), + postId: attachment.optionalChildValue('post_parent') ?? 'nope', // may not exist (cover image in a squarespace export, for example) url: attachment.childValue('attachment_url') })); @@ -128,16 +128,25 @@ function collectScrapedImages(allPostData, postTypes) { postTypes.forEach(postType => { getItemsOfType(allPostData, postType).forEach(postData => { const postId = postData.childValue('post_id'); + const postContent = postData.childValue('encoded'); - const postLink = postData.childValue('link'); + const scrapedUrls = [...postContent.matchAll(/]*?src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)].map((match) => match[1]); + scrapedUrls.forEach((scrapedUrl) => { + let url; + if (isAbsoluteUrl(scrapedUrl)) { + url = scrapedUrl; + } else { + const postLink = postData.childValue('link'); + if (isAbsoluteUrl(postLink)) { + url = new URL(scrapedUrl, postLink).href; + } else { + throw new Error(`Unable to determine absolute URL from scraped image URL '${scrapedUrl}' and post link URL '${postLink}'.`); + } + } - const matches = [...postContent.matchAll(/]*src="(.+?\.(?:gif|jpe?g|png|webp))"[^>]*>/gi)]; - matches.forEach(match => { - // base the matched image URL relative to the post URL - const url = new URL(match[1], postLink).href; images.push({ - id: -1, - postId: postId, + id: 'nope', // scraped images don't have an id + postId, url }); }); @@ -187,3 +196,7 @@ function populateFrontmatter(posts) { }); } +function isAbsoluteUrl(url) { + return (/^https?:\/\//i).test(url); +} + diff --git a/src/translator.js b/src/translator.js index a4f3bb1..a317f4b 100644 --- a/src/translator.js +++ b/src/translator.js @@ -107,7 +107,7 @@ export function getPostContent(content) { if (shared.config.saveImages === 'scraped' || shared.config.saveImages === 'all') { // writeImageFile() will save all content images to a relative /images // folder so update references in post content to match - content = content.replace(/(]*src=").*?([^/"]+\.(?:gif|jpe?g|png|webp))("[^>]*>)/gi, '$1images/$2$3'); + content = content.replace(/(]*?src=").*?([^/"]+\.(?:gif|jpe?g|png|webp))("[^>]*>)/gi, '$1images/$2$3'); } // preserve "more" separator, max one per post, optionally with custom label From 94c93d045c6d2ca01c6b0c458059ed0a148a7a47 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Sun, 23 Feb 2025 16:18:40 -0500 Subject: [PATCH 56/79] Fix for awful markdown links with
--- src/translator.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/translator.js b/src/translator.js index a317f4b..2642106 100644 --- a/src/translator.js +++ b/src/translator.js @@ -34,6 +34,14 @@ function initTurndownService() { replacement: (content, node) => '\n\n' + node.outerHTML }); + //
within can cause extra whitespace that wreck markdown links, so this removes them + turndownService.addRule('a', { + filter: 'a', + replacement: (content) => { + return content.replace(/<\/?div[^>]*>/gi, ''); + } + }); + // preserve embedded scripts (for tweets, codepens, gists, etc.) turndownService.addRule('script', { filter: 'script', From 5aef591c3d304392e09f9e38739c4005d5836c55 Mon Sep 17 00:00:00 2001 From: Will Boyd Date: Mon, 24 Feb 2025 16:53:43 -0500 Subject: [PATCH 57/79] Remove style contents, excessive line breaks --- src/translator.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/translator.js b/src/translator.js index 2642106..6c31b81 100644 --- a/src/translator.js +++ b/src/translator.js @@ -14,6 +14,8 @@ function initTurndownService() { turndownService.use(turndownPluginGfm.tables); + turndownService.remove(['style']); //