1
0
mirror of https://github.com/laurent22/joplin.git synced 2025-01-20 18:48:28 +02:00

Tools: Build website to separate repository

This commit is contained in:
Laurent Cozic 2021-12-19 08:53:19 +01:00
parent 250e97247f
commit b14956afbd
10 changed files with 187 additions and 92 deletions

View File

@ -1974,9 +1974,15 @@ packages/tools/update-readme-download.js.map
packages/tools/update-readme-sponsors.d.ts
packages/tools/update-readme-sponsors.js
packages/tools/update-readme-sponsors.js.map
packages/tools/updateMarkdownDoc.d.ts
packages/tools/updateMarkdownDoc.js
packages/tools/updateMarkdownDoc.js.map
packages/tools/website/build.d.ts
packages/tools/website/build.js
packages/tools/website/build.js.map
packages/tools/website/updateDownloadPage.d.ts
packages/tools/website/updateDownloadPage.js
packages/tools/website/updateDownloadPage.js.map
packages/tools/website/utils/frontMatter.d.ts
packages/tools/website/utils/frontMatter.js
packages/tools/website/utils/frontMatter.js.map

6
.gitignore vendored
View File

@ -1957,9 +1957,15 @@ packages/tools/update-readme-download.js.map
packages/tools/update-readme-sponsors.d.ts
packages/tools/update-readme-sponsors.js
packages/tools/update-readme-sponsors.js.map
packages/tools/updateMarkdownDoc.d.ts
packages/tools/updateMarkdownDoc.js
packages/tools/updateMarkdownDoc.js.map
packages/tools/website/build.d.ts
packages/tools/website/build.js
packages/tools/website/build.js.map
packages/tools/website/updateDownloadPage.d.ts
packages/tools/website/updateDownloadPage.js
packages/tools/website/updateDownloadPage.js.map
packages/tools/website/utils/frontMatter.d.ts
packages/tools/website/utils/frontMatter.js
packages/tools/website/utils/frontMatter.js.map

View File

@ -12,12 +12,12 @@
"build": "lerna run build && npm run tsc",
"buildApiDoc": "npm start --prefix=packages/app-cli -- apidoc ../../readme/api/references/rest_api.md",
"buildCommandIndex": "gulp buildCommandIndex",
"buildDoc": "./packages/tools/build-all.sh",
"buildPluginDoc": "typedoc --name 'Joplin Plugin API Documentation' --mode file -theme './Assets/PluginDocTheme/' --readme './Assets/PluginDocTheme/index.md' --excludeNotExported --excludeExternals --excludePrivate --excludeProtected --out docs/api/references/plugin_api packages/lib/services/plugins/api/",
"buildSettingJsonSchema": "npm start --prefix=packages/app-cli -- settingschema ../../docs/schema/settings.json",
"updateMarkdownDoc": "node ./packages/tools/updateMarkdownDoc",
"buildPluginDoc": "typedoc --name 'Joplin Plugin API Documentation' --mode file -theme './Assets/PluginDocTheme/' --readme './Assets/PluginDocTheme/index.md' --excludeNotExported --excludeExternals --excludePrivate --excludeProtected --out ../joplin-website/docs/api/references/plugin_api packages/lib/services/plugins/api/",
"buildSettingJsonSchema": "npm start --prefix=packages/app-cli -- settingschema ../../../joplin-website/docs/schema/settings.json",
"buildTranslations": "npm run tsc && node packages/tools/build-translation.js",
"buildTranslationsNoTsc": "node packages/tools/build-translation.js",
"buildWebsite": "npm run buildApiDoc && node ./packages/tools/website/build.js && npm run buildPluginDoc && npm run buildSettingJsonSchema",
"buildWebsite": "node ./packages/tools/website/build.js && npm run buildPluginDoc && npm run buildSettingJsonSchema",
"circularDependencyCheck": "madge --warning --circular --extensions js ./",
"clean": "lerna clean -y && lerna run clean",
"dependencyTree": "madge",
@ -45,7 +45,7 @@
"updateIgnored": "gulp updateIgnoredTypeScriptBuild",
"updatePluginTypes": "./packages/generator-joplin/updateTypes.sh",
"watch": "lerna run watch --stream --parallel",
"watchWebsite": "nodemon --verbose --watch Assets/WebsiteAssets --watch packages/tools/website/build.js --ext md,ts,js,mustache,css,tsx,gif,png,svg --exec \"node packages/tools/website/build.js && http-server --port 8077 docs -a localhost\"",
"watchWebsite": "nodemon --verbose --watch Assets/WebsiteAssets --watch packages/tools/website/build.js --ext md,ts,js,mustache,css,tsx,gif,png,svg --exec \"node packages/tools/website/build.js && http-server --port 8077 ../joplin-website/docs -a localhost\"",
"i": "node packages/tools/lernaInstall"
},
"husky": {

View File

@ -1,37 +0,0 @@
#!/bin/bash
set -e
ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
git pull
echo "---------------------------------------------------"
echo "$ROOT_DIR/update-readme-download.js..."
echo "---------------------------------------------------"
node "$ROOT_DIR/update-readme-download.js"
echo "---------------------------------------------------"
echo "$ROOT_DIR/build-release-stats.js..."
echo "---------------------------------------------------"
node "$ROOT_DIR/build-release-stats.js"
echo "---------------------------------------------------"
echo "$ROOT_DIR/build-welcome.js..."
echo "---------------------------------------------------"
node "$ROOT_DIR/build-welcome.js"
echo "---------------------------------------------------"
echo "$ROOT_DIR/update-readme-sponsors.js..."
echo "---------------------------------------------------"
node "$ROOT_DIR/update-readme-sponsors.js"
cd "$ROOT_DIR/.."
echo "---------------------------------------------------"
echo "npm run buildWebsite..."
echo "---------------------------------------------------"
npm run buildWebsite
echo "---------------------------------------------------"
echo "Commit changes..."
echo "---------------------------------------------------"
git add -A && git commit -m "Update website" && git pull && git push

View File

@ -4,6 +4,7 @@ const fetch = require('node-fetch');
const fs = require('fs-extra');
const { dirname } = require('@joplin/lib/path-utils');
const markdownUtils = require('@joplin/lib/markdownUtils').default;
const yargParser = require('yargs-parser');
const rootDir = dirname(dirname(__dirname));
@ -55,6 +56,11 @@ function createChangeLog(releases) {
}
async function main() {
const argv = yargParser(process.argv);
const types = argv.types ? argv.types.split(',') : ['stats', 'changelog'];
console.info(`Building docs: ${types.join(', ')}`);
const rows = [];
const totals = {
@ -99,8 +105,15 @@ async function main() {
pageNum++;
}
const changelogText = createChangeLog(rows);
await fs.writeFile(`${rootDir}/readme/changelog.md`, changelogText);
if (types.includes('changelog')) {
console.info('Build stats: Updating changelog...');
const changelogText = createChangeLog(rows);
await fs.writeFile(`${rootDir}/readme/changelog.md`, changelogText);
}
if (!types.includes('stats')) return;
console.info('Build stats: Updating stats...');
const grandTotal = totals.windows_count + totals.mac_count + totals.linux_count;
totals.windows_percent = totals.windows_count / grandTotal;

View File

@ -0,0 +1,60 @@
#!/bin/bash
set -e
# ------------------------------------------------------------------------------
# Setup environment
# ------------------------------------------------------------------------------
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SCRIPT_NAME=`basename "$0"`
JOPLIN_ROOT_DIR="$SCRIPT_DIR/../.."
JOPLIN_WEBSITE_ROOT_DIR="$JOPLIN_ROOT_DIR/../joplin-website"
# ------------------------------------------------------------------------------
# Update the Markdown files inside the Joplin directory. This is for example the
# download links README.md or the desktop app changelog.
# ------------------------------------------------------------------------------
cd "$JOPLIN_ROOT_DIR"
# Will fail if there's any local change in the repo, which is what we want
git pull --rebase
npm install
# Clean up npm's mess
git reset --hard
npm run updateMarkdownDoc
# We commit and push the change. It will be a noop if nothing was actually
# changed
git add -A
git commit -m "Doc: Updated Markdown files
Auto-updated using $SCRIPT_NAME"
git pull --rebase
git push
# ------------------------------------------------------------------------------
# Build and deploy the website
# ------------------------------------------------------------------------------
cd "$JOPLIN_WEBSITE_ROOT_DIR"
git pull --rebase
cd "$JOPLIN_ROOT_DIR"
npm run buildWebsite
cd "$JOPLIN_WEBSITE_ROOT_DIR"
git add -A
git commit -m "Updated website
Auto-updated using $SCRIPT_NAME"
git pull --rebase
git push

View File

@ -1,9 +1,8 @@
import { readFile } from 'fs-extra';
import { insertContentIntoFile, rootDir } from './tool-utils';
import markdownUtils, { MarkdownTableHeader, MarkdownTableJustify, MarkdownTableRow } from '@joplin/lib/markdownUtils';
const ArrayUtils = require('@joplin/lib/ArrayUtils');
const { escapeHtml } = require('@joplin/lib/string-utils');
import { GithubSponsor, OrgSponsor, Sponsors } from './website/utils/types';
const { escapeHtml } = require('@joplin/lib/string-utils');
const readmePath = `${rootDir}/README.md`;
const sponsorsPath = `${rootDir}/packages/tools/sponsors.json`;
@ -51,7 +50,7 @@ async function createGitHubSponsorTable(sponsors: GithubSponsor[]): Promise<stri
}
async function createOrgSponsorTable(sponsors: OrgSponsor[]): Promise<string> {
sponsors = ArrayUtils.shuffle(sponsors);
// sponsors = ArrayUtils.shuffle(sponsors);
const output: string[] = [];

View File

@ -0,0 +1,32 @@
import { chdir } from 'process';
import { execCommand2, rootDir, gitRepoCleanTry } from './tool-utils';
import updateDownloadPage from './website/updateDownloadPage';
async function main() {
const doGitOperations = false;
if (doGitOperations) {
await gitRepoCleanTry();
await execCommand2(['git', 'pull', '--rebase']);
}
await execCommand2(['node', `${rootDir}/packages/tools/update-readme-download.js`]);
await execCommand2(['node', `${rootDir}/packages/tools/build-release-stats.js`, '--types=changelog']);
await execCommand2(['node', `${rootDir}/packages/tools/update-readme-sponsors.js`]);
await execCommand2(['node', `${rootDir}/packages/tools/build-welcome.js`]);
chdir(rootDir);
await execCommand2(['npm', 'run', 'buildApiDoc']);
await updateDownloadPage();
if (doGitOperations) {
await execCommand2(['git', 'add', '-A']);
await execCommand2(['git', 'commit', '-m', 'Update Markdown doc']);
await execCommand2(['git', 'pull', '--rebase']);
await execCommand2(['git', 'push']);
}
}
main().catch((error) => {
console.error('Fatal error', error);
process.exit(1);
});

View File

@ -1,20 +1,24 @@
import { readFileSync, readFile, mkdirpSync, writeFileSync, remove, copy } from 'fs-extra';
import { insertContentIntoFile, rootDir } from '../tool-utils';
import { readFileSync, readFile, mkdirpSync, writeFileSync, remove, copy, pathExistsSync } from 'fs-extra';
import { rootDir } from '../tool-utils';
import { pressCarouselItems } from './utils/pressCarousel';
import { getMarkdownIt, loadMustachePartials, markdownToPageHtml, renderMustache } from './utils/render';
import { AssetUrls, Env, OrgSponsor, PlanPageParams, Sponsors, TemplateParams } from './utils/types';
import { getPlans, loadStripeConfig } from '@joplin/lib/utils/joplinCloud';
import { shuffle } from '@joplin/lib/array';
import { stripOffFrontMatter } from './utils/frontMatter';
import { dirname, basename } from 'path';
const moment = require('moment');
const dirname = require('path').dirname;
const glob = require('glob');
const path = require('path');
const md5File = require('md5-file/promise');
const env = Env.Prod;
const docDir = `${dirname(dirname(dirname(dirname(__dirname))))}/joplin-website/docs`;
if (!pathExistsSync(docDir)) throw new Error(`Doc directory does not exist: ${docDir}`);
const websiteAssetDir = `${rootDir}/Assets/WebsiteAssets`;
const mainTemplateHtml = readFileSync(`${websiteAssetDir}/templates/main-new.mustache`, 'utf8');
const frontTemplateHtml = readFileSync(`${websiteAssetDir}/templates/front.mustache`, 'utf8');
@ -165,37 +169,6 @@ function makeHomePageMd() {
return md;
}
async function createDownloadButtonsHtml(readmeMd: string): Promise<Record<string, string>> {
const output: Record<string, string> = {};
output['windows'] = readmeMd.match(/(<a href=.*?Joplin-Setup-.*?<\/a>)/)[0];
output['macOs'] = readmeMd.match(/(<a href=.*?Joplin-.*\.dmg.*?<\/a>)/)[0];
output['linux'] = readmeMd.match(/(<a href=.*?Joplin-.*\.AppImage.*?<\/a>)/)[0];
output['android'] = readmeMd.match(/(<a href='https:\/\/play.google.com\/store\/apps\/details\?id=net\.cozic\.joplin.*?<\/a>)/)[0];
output['ios'] = readmeMd.match(/(<a href='https:\/\/itunes\.apple\.com\/us\/app\/joplin\/id1315599797.*?<\/a>)/)[0];
for (const [k, v] of Object.entries(output)) {
if (!v) throw new Error(`Could not get download element for: ${k}`);
}
return output;
}
async function updateDownloadPage(downloadButtonsHtml: Record<string, string>) {
const desktopButtonsHtml = [
downloadButtonsHtml['windows'],
downloadButtonsHtml['macOs'],
downloadButtonsHtml['linux'],
];
const mobileButtonsHtml = [
downloadButtonsHtml['android'],
downloadButtonsHtml['ios'],
];
await insertContentIntoFile(`${rootDir}/readme/download.md`, '<!-- DESKTOP-DOWNLOAD-LINKS -->', '<!-- DESKTOP-DOWNLOAD-LINKS -->', desktopButtonsHtml.join(' '));
await insertContentIntoFile(`${rootDir}/readme/download.md`, '<!-- MOBILE-DOWNLOAD-LINKS -->', '<!-- MOBILE-DOWNLOAD-LINKS -->', mobileButtonsHtml.join(' '));
}
async function loadSponsors(): Promise<Sponsors> {
const sponsorsPath = `${rootDir}/packages/tools/sponsors.json`;
const output: Sponsors = JSON.parse(await readFile(sponsorsPath, 'utf8'));
@ -231,8 +204,8 @@ const isNewsFile = (filePath: string): boolean => {
};
async function main() {
await remove(`${rootDir}/docs`);
await copy(websiteAssetDir, `${rootDir}/docs`);
await remove(`${docDir}`);
await copy(websiteAssetDir, `${docDir}`);
const sponsors = await loadSponsors();
const partials = await loadMustachePartials(partialDir);
@ -240,20 +213,19 @@ async function main() {
const readmeMd = makeHomePageMd();
const downloadButtonsHtml = await createDownloadButtonsHtml(readmeMd);
await updateDownloadPage(downloadButtonsHtml);
// await updateDownloadPage(readmeMd);
// =============================================================
// HELP PAGE
// =============================================================
renderPageToHtml(readmeMd, `${rootDir}/docs/help/index.html`, { sourceMarkdownFile: 'README.md', partials, sponsors, assetUrls });
renderPageToHtml(readmeMd, `${docDir}/help/index.html`, { sourceMarkdownFile: 'README.md', partials, sponsors, assetUrls });
// =============================================================
// FRONT PAGE
// =============================================================
renderPageToHtml('', `${rootDir}/docs/index.html`, {
renderPageToHtml('', `${docDir}/index.html`, {
templateHtml: frontTemplateHtml,
partials,
pressCarouselRegular: {
@ -290,7 +262,7 @@ async function main() {
const planPageContentHtml = renderMustache('', planPageParams);
renderPageToHtml('', `${rootDir}/docs/plans/index.html`, {
renderPageToHtml('', `${docDir}/plans/index.html`, {
...defaultTemplateParams(assetUrls),
pageName: 'plans',
partials,
@ -310,10 +282,12 @@ async function main() {
const donateLinksMd = await getDonateLinks();
const makeTargetFilePath = (input: string): string => {
const filenameNoExt = basename(input, '.md');
if (isNewsFile(input)) {
return `${input.replace(/\.md/, '').replace(/readme\/news\//, 'docs/news/')}/index.html`;
return `${docDir}/news/${filenameNoExt}/index.html`; // `${input.replace(/\.md/, '').replace(/readme\/news\//, 'docs/news/')}/index.html`;
} else {
return `${input.replace(/\.md/, '').replace(/readme\//, 'docs/')}/index.html`;
return `${docDir}/${filenameNoExt}/index.html`;
}
};
@ -338,9 +312,10 @@ async function main() {
source[2].sourceMarkdownName = path.basename(source[0], path.extname(source[0]));
const sourceFilePath = `${rootDir}/${source[0]}`;
const targetFilePath = source[1];
const isNews = isNewsFile(sourceFilePath);
renderFileToHtml(sourceFilePath, `${rootDir}/${source[1]}`, {
renderFileToHtml(sourceFilePath, targetFilePath, {
...source[2],
templateHtml: mainTemplateHtml,
pageName: isNews ? 'news-item' : '',
@ -355,7 +330,7 @@ async function main() {
return a.toLowerCase() > b.toLowerCase() ? -1 : +1;
});
await makeNewsFrontPage(newsFilePaths, `${rootDir}/docs/news/index.html`, {
await makeNewsFrontPage(newsFilePaths, `${docDir}/news/index.html`, {
...defaultTemplateParams(assetUrls),
pageName: 'news',
partials,

View File

@ -0,0 +1,41 @@
import { readFile } from 'fs-extra';
import { rootDir, insertContentIntoFile } from '../tool-utils';
async function getReadmeMd() {
return readFile(`${rootDir}/README.md`, 'utf8');
}
async function createDownloadButtonsHtml(readmeMd: string): Promise<Record<string, string>> {
const output: Record<string, string> = {};
output['windows'] = readmeMd.match(/(<a href=.*?Joplin-Setup-.*?<\/a>)/)[0];
output['macOs'] = readmeMd.match(/(<a href=.*?Joplin-.*\.dmg.*?<\/a>)/)[0];
output['linux'] = readmeMd.match(/(<a href=.*?Joplin-.*\.AppImage.*?<\/a>)/)[0];
output['android'] = readmeMd.match(/(<a href='https:\/\/play.google.com\/store\/apps\/details\?id=net\.cozic\.joplin.*?<\/a>)/)[0];
output['ios'] = readmeMd.match(/(<a href='https:\/\/itunes\.apple\.com\/us\/app\/joplin\/id1315599797.*?<\/a>)/)[0];
for (const [k, v] of Object.entries(output)) {
if (!v) throw new Error(`Could not get download element for: ${k}`);
}
return output;
}
export default async function updateDownloadPage(readmeMd: string = null) {
readmeMd = readmeMd === null ? await getReadmeMd() : readmeMd;
const downloadButtonsHtml = await createDownloadButtonsHtml(readmeMd);
const desktopButtonsHtml = [
downloadButtonsHtml['windows'],
downloadButtonsHtml['macOs'],
downloadButtonsHtml['linux'],
];
const mobileButtonsHtml = [
downloadButtonsHtml['android'],
downloadButtonsHtml['ios'],
];
await insertContentIntoFile(`${rootDir}/readme/download.md`, '<!-- DESKTOP-DOWNLOAD-LINKS -->', '<!-- DESKTOP-DOWNLOAD-LINKS -->', desktopButtonsHtml.join(' '));
await insertContentIntoFile(`${rootDir}/readme/download.md`, '<!-- MOBILE-DOWNLOAD-LINKS -->', '<!-- MOBILE-DOWNLOAD-LINKS -->', mobileButtonsHtml.join(' '));
}