2020-04-16 00:11:42 +02:00
|
|
|
// Fetch Patreon posts to Markdown so that we have them in a more versatile format
|
|
|
|
// and to add them to the "News" notifications later on.
|
|
|
|
|
|
|
|
const fetch = require('node-fetch');
|
|
|
|
const fs = require('fs-extra');
|
|
|
|
const { patreonOauthToken } = require('./tool-utils');
|
2021-04-11 19:01:06 +02:00
|
|
|
const HtmlToMd = require('@joplin/lib/HtmlToMd').default;
|
2020-11-07 17:59:37 +02:00
|
|
|
const { dirname, filename, basename } = require('@joplin/lib/path-utils');
|
|
|
|
const markdownUtils = require('@joplin/lib/markdownUtils').default;
|
|
|
|
const mimeUtils = require('@joplin/lib/mime-utils.js').mime;
|
|
|
|
const { netUtils } = require('@joplin/lib/net-utils');
|
|
|
|
const shim = require('@joplin/lib/shim').default;
|
2020-04-16 00:11:42 +02:00
|
|
|
const moment = require('moment');
|
2020-11-07 17:59:37 +02:00
|
|
|
const { pregQuote } = require('@joplin/lib/string-utils');
|
|
|
|
const { shimInit } = require('@joplin/lib/shim-init-node.js');
|
2020-04-16 00:11:42 +02:00
|
|
|
|
|
|
|
shimInit();
|
|
|
|
|
|
|
|
const blogDir = `${dirname(__dirname)}/readme/blog`;
|
|
|
|
const tempDir = `${__dirname}/temp`;
|
|
|
|
const imageDir = `${dirname(__dirname)}/readme/blog/images`;
|
|
|
|
|
|
|
|
const htmlToMd = new HtmlToMd();
|
|
|
|
|
|
|
|
async function fetchPosts(url) {
|
|
|
|
const token = await patreonOauthToken();
|
|
|
|
|
|
|
|
const response = await fetch(url, {
|
|
|
|
headers: {
|
|
|
|
'Authorization': `Bearer ${token}`,
|
|
|
|
},
|
|
|
|
});
|
|
|
|
|
|
|
|
const responseJson = await response.json();
|
|
|
|
|
2020-05-21 10:14:33 +02:00
|
|
|
const posts = responseJson.data.map(p => {
|
2020-04-16 00:11:42 +02:00
|
|
|
return {
|
|
|
|
id: p.id,
|
|
|
|
title: p.attributes.title,
|
|
|
|
content: p.attributes.content,
|
|
|
|
published_at: p.attributes.published_at,
|
|
|
|
url: p.attributes.url,
|
|
|
|
};
|
|
|
|
});
|
|
|
|
|
|
|
|
return {
|
|
|
|
data: posts,
|
|
|
|
nextUrl: responseJson.links && responseJson.links.next ? responseJson.links.next : null,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
async function createPostFile(post, filePath) {
|
|
|
|
let contentMd = htmlToMd.parse(post.content, { preserveImageTagsWithSize: true });
|
|
|
|
|
|
|
|
const imageUrls = markdownUtils.extractImageUrls(contentMd);
|
|
|
|
|
|
|
|
const imageUrlsToFiles = {};
|
|
|
|
|
|
|
|
for (let i = 0; i < imageUrls.length; i++) {
|
|
|
|
const imageUrl = imageUrls[i];
|
|
|
|
const imageFilename = `${filename(filePath)}_${i}`;
|
|
|
|
const imagePath = `${tempDir}/${imageFilename}`;
|
|
|
|
const response = await shim.fetchBlob(imageUrl, { path: imagePath, maxRetry: 1 });
|
|
|
|
|
|
|
|
const mimeType = netUtils.mimeTypeFromHeaders(response.headers);
|
|
|
|
let ext = 'jpg';
|
|
|
|
if (mimeType) {
|
|
|
|
const newExt = mimeUtils.toFileExtension(mimeType);
|
|
|
|
if (newExt) ext = newExt;
|
|
|
|
}
|
|
|
|
|
|
|
|
const destFile = `${imageDir}/${imageFilename}.${ext}`;
|
|
|
|
await fs.move(imagePath, destFile, { overwrite: true });
|
|
|
|
|
|
|
|
imageUrlsToFiles[imageUrl] = destFile;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (const imageUrl in imageUrlsToFiles) {
|
|
|
|
const r = `images/${basename(imageUrlsToFiles[imageUrl])}`;
|
|
|
|
contentMd = contentMd.replace(new RegExp(pregQuote(imageUrl), 'g'), r);
|
|
|
|
}
|
|
|
|
|
|
|
|
const fileMd = [`# ${post.title}`];
|
|
|
|
fileMd.push('');
|
|
|
|
fileMd.push(contentMd);
|
|
|
|
fileMd.push('');
|
|
|
|
fileMd.push('* * *');
|
|
|
|
fileMd.push('');
|
|
|
|
fileMd.push(`url: https://www.patreon.com${post.url}`);
|
|
|
|
fileMd.push(`published_at: ${post.published_at}`);
|
|
|
|
await fs.writeFile(filePath, fileMd.join('\n'));
|
|
|
|
}
|
|
|
|
|
|
|
|
async function createPostFiles(posts) {
|
|
|
|
for (const post of posts) {
|
|
|
|
const filename = `${moment(post.published_at).format('YYYYMMDD-HHmmss')}.md`;
|
|
|
|
await createPostFile(post, `${blogDir}/${filename}`);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
async function main() {
|
|
|
|
await fs.mkdirp(blogDir);
|
|
|
|
await fs.mkdirp(imageDir);
|
|
|
|
await fs.mkdirp(tempDir);
|
|
|
|
|
|
|
|
const fields = [
|
|
|
|
'title',
|
|
|
|
'content',
|
|
|
|
'published_at',
|
|
|
|
'url',
|
|
|
|
];
|
|
|
|
|
|
|
|
let url = `https://www.patreon.com/api/oauth2/v2/campaigns/1818121/posts?fields%5Bpost%5D=${fields.join(',')}`;
|
|
|
|
|
|
|
|
while (url) {
|
|
|
|
console.info('Fetching ', url);
|
|
|
|
const result = await fetchPosts(url);
|
|
|
|
console.info(`Found ${result.data.length} posts`);
|
|
|
|
await createPostFiles(result.data);
|
|
|
|
url = result.nextUrl;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
main().catch((error) => {
|
|
|
|
console.error(error);
|
|
|
|
process.exit(1);
|
|
|
|
}).then(() => {
|
|
|
|
return fs.remove(tempDir);
|
|
|
|
});
|