From f4ae769f13751d1d2a97f50c9d228ae5203b661f Mon Sep 17 00:00:00 2001
From: Will Boyd <will@codersblock.com>
Date: Tue, 17 Dec 2019 13:52:09 -0500
Subject: [PATCH] Split out code for writer and translator

---
 index.js          | 112 +-------------------------------------------
 src/parser.js     |  94 ++-----------------------------------
 src/shared.js     |   2 +-
 src/translator.js |  90 ++++++++++++++++++++++++++++++++++++
 src/wizard.js     |   2 +-
 src/writer.js     | 115 ++++++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 213 insertions(+), 202 deletions(-)
 create mode 100644 src/translator.js

diff --git a/index.js b/index.js
index 6cb75b3..8807aa5 100644
--- a/index.js
+++ b/index.js
@@ -1,11 +1,6 @@
-const fs = require('fs');
-const luxon = require('luxon');
-const path = require('path');
-const request = require('request');
-
-const shared = require('./src/shared');
 const wizard = require('./src/wizard');
 const parser = require('./src/parser');
+const writer = require('./src/writer');
 
 // global so various functions can access arguments
 let config;
@@ -14,115 +9,12 @@ async function init() {
 	try {
 		config = wizard.getConfig();
 		let posts = await parser.parseFilePromise(config)
-		writeFiles(posts);
+		writer.writeFiles(posts, config);
 	} catch (ex) {
 		// appease the UnhandledPromiseRejectionWarning
 		console.error(ex);
 	}
 }
 
-function writeFiles(posts) {
-	let delay = 0;
-	posts.forEach(post => {
-		const postDir = getPostDir(post);
-		createDir(postDir);
-		writeMarkdownFile(post, postDir);
-
-		if (config.saveimages && post.meta.imageUrls) {
-			post.meta.imageUrls.forEach(imageUrl => {
-				const imageDir = path.join(postDir, 'images');
-				createDir(imageDir);
-				writeImageFile(imageUrl, imageDir, delay);
-				delay += 25;
-			});
-		}
-	});
-}
-
-function writeMarkdownFile(post, postDir) {
-	const frontmatter = Object.entries(post.frontmatter)
-		.reduce((accumulator, pair) => {
-			return accumulator + pair[0] + ': "' + pair[1] + '"\n'
-		}, '');
-	const data = '---\n' + frontmatter + '---\n\n' + post.content + '\n';
-	
-	const postPath = path.join(postDir, getPostFilename(post));
-	fs.writeFile(postPath, data, (err) => {
-		if (err) {
-			console.log('Unable to write file.')
-			console.log(err);
-		} else {
-			console.log('Wrote ' + postPath + '.');
-		}
-	});
-}
-
-function writeImageFile(imageUrl, imageDir, delay) {
-	let imagePath = path.join(imageDir, shared.getFilenameFromUrl(imageUrl));
-	let stream = fs.createWriteStream(imagePath);
-	stream.on('finish', () => {
-		console.log('Saved ' + imagePath + '.');
-	});
-
-	// stagger image requests so we don't piss off hosts
-	setTimeout(() => {
-		request
-			.get(imageUrl)
-			.on('response', response => {
-				if (response.statusCode !== 200) {
-					console.log('Response status code ' + response.statusCode + ' received for ' + imageUrl + '.');
-				}
-			})
-			.on('error', err => {
-				console.log('Unable to download image.');
-				console.log(err);
-			})
-			.pipe(stream);
-	}, delay);
-}
-
-function createDir(dir) {
-	try {
-		fs.accessSync(dir, fs.constants.F_OK);
-	} catch (ex) {
-		fs.mkdirSync(dir, { recursive: true });
-	}
-}
-
-function getPostDir(post) {
-	let dir = config.output;
-	let dt = luxon.DateTime.fromISO(post.frontmatter.date);
-
-	if (config.yearmonthfolders) {
-		dir = path.join(dir, dt.toFormat('yyyy'), dt.toFormat('LL'));
-	} else if (config.yearfolders) {
-		dir = path.join(dir, dt.toFormat('yyyy'));
-	}
-
-	if (config.postfolders) {
-		let folder = post.meta.slug;
-		if (config.prefixdate) {
-			folder = dt.toFormat('yyyy-LL-dd') + '-' + folder;
-		}
-		dir = path.join(dir, folder);
-	}
-
-	return dir;
-}
-
-function getPostFilename(post) {
-	if (config.postfolders) {
-		// the containing folder name will be unique, just use index.md here
-		return 'index.md';
-	} else {
-		let filename = post.meta.slug + '.md';
-		if (config.prefixdate) {
-			let dt = luxon.DateTime.fromISO(post.frontmatter.date);
-			filename = dt.toFormat('yyyy-LL-dd') + '-' + filename;
-		}
-		return filename;
-	}
-}
-
 // it's go time!
 init();
diff --git a/src/parser.js b/src/parser.js
index c022877..e7c0c4b 100644
--- a/src/parser.js
+++ b/src/parser.js
@@ -1,9 +1,9 @@
 const fs = require('fs');
 const luxon = require('luxon');
-const turndown = require('turndown');
 const xml2js = require('xml2js');
 
 const shared = require('./shared');
+const translator = require('./translator');
 
 let config;
 
@@ -76,7 +76,7 @@ function addContentImages(data, images) {
 
 function collectPosts(data) {
 	// this is passed into getPostContent() for the markdown conversion
-	turndownService = initTurndownService();
+	turndownService = translator.initTurndownService();
 
 	return getItemsOfType(data, 'post')
 		.map(post => ({
@@ -90,65 +90,10 @@ function collectPosts(data) {
 				title: getPostTitle(post),
 				date: getPostDate(post)
 			},
-			content: getPostContent(post, turndownService)
+			content: translator.getPostContent(post, turndownService, config)
 		}));
 }
 
-function initTurndownService() {
-	let turndownService = new turndown({
-		headingStyle: 'atx',
-		bulletListMarker: '-',
-		codeBlockStyle: 'fenced'
-	});
-
-	// preserve embedded tweets
-	turndownService.addRule('tweet', {
-		filter: node => node.nodeName === 'BLOCKQUOTE' && node.getAttribute('class') === 'twitter-tweet',
-		replacement: (content, node) => '\n\n' + node.outerHTML
-	});
-
-	// preserve embedded codepens
-	turndownService.addRule('codepen', {
-		filter: node => {
-			// codepen embed snippets have changed over the years
-			// but this series of checks should find the commonalities
-			return (
-				['P', 'DIV'].includes(node.nodeName) &&
-				node.attributes['data-slug-hash'] && 
-				node.getAttribute('class') === 'codepen'
-			);
-		},
-		replacement: (content, node) => '\n\n' + node.outerHTML
-	});
-		
-	// preserve embedded scripts (for tweets, codepens, gists, etc.)
-	turndownService.addRule('script', {
-		filter: 'script',
-		replacement: (content, node) => {
-			let before = '\n\n';
-			let src = node.getAttribute('src');
-			if (node.previousSibling && node.previousSibling.nodeName !== '#text') {
-				// keep twitter and codepen <script> tags snug with the element above them
-				before = '\n';
-			}
-			let html = node.outerHTML.replace('async=""', 'async');
-			return before + html + '\n\n';
-		}
-	});
-
-	// preserve iframes (common for embedded audio/video)
-	turndownService.addRule('iframe', {
-		filter: 'iframe',
-		replacement: (content, node) => {
-			let html = node.outerHTML
-				.replace('allowfullscreen=""', 'allowfullscreen');
-			return '\n\n' + html + '\n\n';
-		}
-	});
-
-	return turndownService;
-}
-
 function getItemsOfType(data, type) {
 	return data.rss.channel[0].item.filter(item => item.post_type[0] === type);
 }
@@ -176,37 +121,6 @@ function getPostDate(post) {
 	return luxon.DateTime.fromRFC2822(post.pubDate[0], { zone: 'utc' }).toISODate();
 }
 
-function getPostContent(post, turndownService) {
-	let content = post.encoded[0].trim();
-
-	// insert an empty div element between double line breaks
-	// this nifty trick causes turndown to keep adjacent paragraphs separated
-	// without mucking up content inside of other elemnts (like <code> blocks)
-	content = content.replace(/(\r?\n){2}/g, '\n<div></div>\n');
-
-	if (config.addcontentimages) {
-		// writeImageFile() will save all content images to a relative /images
-		// folder so update references in post content to match
-		content = content.replace(/(<img[^>]*src=").*?([^\/"]+\.(?:gif|jpg|png))("[^>]*>)/gi, '$1images/$2$3');
-	}
-
-	// this is a hack to make <iframe> nodes non-empty by inserting a "." which
-	// allows the iframe rule declared in initTurndownService() to take effect
-	// (using turndown's blankRule() and keep() solution did not work for me)
-	content = content.replace(/(<\/iframe>)/gi, '.$1');
-
-	// use turndown to convert HTML to Markdown
-	content = turndownService.turndown(content);
-
-	// clean up extra spaces in list items
-	content = content.replace(/(-|\d+\.) +/g, '$1 ');
-
-	// clean up the "." from the iframe hack above
-	content = content.replace(/\.(<\/iframe>)/gi, '$1');
-
-	return content;
-}
-
 function mergeImagesIntoPosts(images, posts) {
 	// create lookup table for quicker traversal
 	let postsLookup = posts.reduce((lookup, post) => {
@@ -229,4 +143,4 @@ function mergeImagesIntoPosts(images, posts) {
 	});
 }
 
-exports.parseFilePromise = parseFilePromise;
\ No newline at end of file
+exports.parseFilePromise = parseFilePromise;
diff --git a/src/shared.js b/src/shared.js
index 5398675..b3e6165 100644
--- a/src/shared.js
+++ b/src/shared.js
@@ -2,4 +2,4 @@ function getFilenameFromUrl(url) {
 	return url.split('/').slice(-1)[0];
 }
 
-exports.getFilenameFromUrl = getFilenameFromUrl;
\ No newline at end of file
+exports.getFilenameFromUrl = getFilenameFromUrl;
diff --git a/src/translator.js b/src/translator.js
new file mode 100644
index 0000000..44dd2f3
--- /dev/null
+++ b/src/translator.js
@@ -0,0 +1,90 @@
+const turndown = require('turndown');
+
+function initTurndownService() {
+	let turndownService = new turndown({
+		headingStyle: 'atx',
+		bulletListMarker: '-',
+		codeBlockStyle: 'fenced'
+	});
+
+	// preserve embedded tweets
+	turndownService.addRule('tweet', {
+		filter: node => node.nodeName === 'BLOCKQUOTE' && node.getAttribute('class') === 'twitter-tweet',
+		replacement: (content, node) => '\n\n' + node.outerHTML
+	});
+
+	// preserve embedded codepens
+	turndownService.addRule('codepen', {
+		filter: node => {
+			// codepen embed snippets have changed over the years
+			// but this series of checks should find the commonalities
+			return (
+				['P', 'DIV'].includes(node.nodeName) &&
+				node.attributes['data-slug-hash'] && 
+				node.getAttribute('class') === 'codepen'
+			);
+		},
+		replacement: (content, node) => '\n\n' + node.outerHTML
+	});
+		
+	// preserve embedded scripts (for tweets, codepens, gists, etc.)
+	turndownService.addRule('script', {
+		filter: 'script',
+		replacement: (content, node) => {
+			let before = '\n\n';
+			let src = node.getAttribute('src');
+			if (node.previousSibling && node.previousSibling.nodeName !== '#text') {
+				// keep twitter and codepen <script> tags snug with the element above them
+				before = '\n';
+			}
+			let html = node.outerHTML.replace('async=""', 'async');
+			return before + html + '\n\n';
+		}
+	});
+
+	// preserve iframes (common for embedded audio/video)
+	turndownService.addRule('iframe', {
+		filter: 'iframe',
+		replacement: (content, node) => {
+			let html = node.outerHTML
+				.replace('allowfullscreen=""', 'allowfullscreen');
+			return '\n\n' + html + '\n\n';
+		}
+	});
+
+	return turndownService;
+}
+
+function getPostContent(post, turndownService, config) {
+	let content = post.encoded[0].trim();
+
+	// insert an empty div element between double line breaks
+	// this nifty trick causes turndown to keep adjacent paragraphs separated
+	// without mucking up content inside of other elemnts (like <code> blocks)
+	content = content.replace(/(\r?\n){2}/g, '\n<div></div>\n');
+
+	if (config.addcontentimages) {
+		// writeImageFile() will save all content images to a relative /images
+		// folder so update references in post content to match
+		content = content.replace(/(<img[^>]*src=").*?([^\/"]+\.(?:gif|jpg|png))("[^>]*>)/gi, '$1images/$2$3');
+	}
+
+	// this is a hack to make <iframe> nodes non-empty by inserting a "." which
+	// allows the iframe rule declared in initTurndownService() to take effect
+	// (using turndown's blankRule() and keep() solution did not work for me)
+	content = content.replace(/(<\/iframe>)/gi, '.$1');
+
+	// use turndown to convert HTML to Markdown
+	content = turndownService.turndown(content);
+
+	// clean up extra spaces in list items
+	content = content.replace(/(-|\d+\.) +/g, '$1 ');
+
+	// clean up the "." from the iframe hack above
+	content = content.replace(/\.(<\/iframe>)/gi, '$1');
+
+	return content;
+}
+
+exports.initTurndownService = initTurndownService;
+exports.getPostContent = getPostContent;
diff --git a/src/wizard.js b/src/wizard.js
index d601880..45bde86 100644
--- a/src/wizard.js
+++ b/src/wizard.js
@@ -45,4 +45,4 @@ function checkFileExists(path) {
 	}
 }
 
-exports.getConfig = getConfig;
\ No newline at end of file
+exports.getConfig = getConfig;
diff --git a/src/writer.js b/src/writer.js
index e69de29..5a5b519 100644
--- a/src/writer.js
+++ b/src/writer.js
@@ -0,0 +1,115 @@
+const fs = require('fs');
+const luxon = require('luxon');
+const path = require('path');
+const request = require('request');
+
+const shared = require('./shared');
+
+let config;
+
+function writeFiles(posts, configIn) {
+    config = configIn;
+    
+    let delay = 0;
+	posts.forEach(post => {
+		const postDir = getPostDir(post);
+		createDir(postDir);
+		writeMarkdownFile(post, postDir);
+
+		if (config.saveimages && post.meta.imageUrls) {
+			post.meta.imageUrls.forEach(imageUrl => {
+				const imageDir = path.join(postDir, 'images');
+				createDir(imageDir);
+				writeImageFile(imageUrl, imageDir, delay);
+				delay += 25;
+			});
+		}
+	});
+}
+
+function writeMarkdownFile(post, postDir) {
+	const frontmatter = Object.entries(post.frontmatter)
+		.reduce((accumulator, pair) => {
+			return accumulator + pair[0] + ': "' + pair[1] + '"\n'
+		}, '');
+	const data = '---\n' + frontmatter + '---\n\n' + post.content + '\n';
+	
+	const postPath = path.join(postDir, getPostFilename(post));
+	fs.writeFile(postPath, data, (err) => {
+		if (err) {
+			console.log('Unable to write file.')
+			console.log(err);
+		} else {
+			console.log('Wrote ' + postPath + '.');
+		}
+	});
+}
+
+function writeImageFile(imageUrl, imageDir, delay) {
+	let imagePath = path.join(imageDir, shared.getFilenameFromUrl(imageUrl));
+	let stream = fs.createWriteStream(imagePath);
+	stream.on('finish', () => {
+		console.log('Saved ' + imagePath + '.');
+	});
+
+	// stagger image requests so we don't piss off hosts
+	setTimeout(() => {
+		request
+			.get(imageUrl)
+			.on('response', response => {
+				if (response.statusCode !== 200) {
+					console.log('Response status code ' + response.statusCode + ' received for ' + imageUrl + '.');
+				}
+			})
+			.on('error', err => {
+				console.log('Unable to download image.');
+				console.log(err);
+			})
+			.pipe(stream);
+	}, delay);
+}
+
+function createDir(dir) {
+	try {
+		fs.accessSync(dir, fs.constants.F_OK);
+	} catch (ex) {
+		fs.mkdirSync(dir, { recursive: true });
+	}
+}
+
+function getPostDir(post) {
+	let dir = config.output;
+	let dt = luxon.DateTime.fromISO(post.frontmatter.date);
+
+	if (config.yearmonthfolders) {
+		dir = path.join(dir, dt.toFormat('yyyy'), dt.toFormat('LL'));
+	} else if (config.yearfolders) {
+		dir = path.join(dir, dt.toFormat('yyyy'));
+	}
+
+	if (config.postfolders) {
+		let folder = post.meta.slug;
+		if (config.prefixdate) {
+			folder = dt.toFormat('yyyy-LL-dd') + '-' + folder;
+		}
+		dir = path.join(dir, folder);
+	}
+
+	return dir;
+}
+
+function getPostFilename(post) {
+	if (config.postfolders) {
+		// the containing folder name will be unique, just use index.md here
+		return 'index.md';
+	} else {
+		let filename = post.meta.slug + '.md';
+		if (config.prefixdate) {
+			let dt = luxon.DateTime.fromISO(post.frontmatter.date);
+			filename = dt.toFormat('yyyy-LL-dd') + '-' + filename;
+		}
+		return filename;
+	}
+}
+
+exports.writeFiles = writeFiles;