games/factorio-sites
1
0
Fork 0
mirror of https://github.com/barthuijgen/factorio-sites.git synced 2025-02-09 14:33:12 +02:00
Code Issues Releases Activity

Refactor image generation, new method of using pupeteer on the PBE app

Browse Source
This commit is contained in:
Bart Huijgen 2020-10-25 23:51:36 +01:00
parent c49af943d8
commit c595054a3b
20 changed files with 214 additions and 422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.vscode/extensions.json vendored
View File

@ -1,7 +1,3 @@
{
"recommendations": [
"ms-vscode.vscode-typescript-tslint-plugin",
"esbenp.prettier-vscode",
"firsttris.vscode-jest-runner"
]
"recommendations": ["ms-vscode.vscode-typescript-tslint-plugin", "esbenp.prettier-vscode"]
}

50
apps/blueprint-image-function/src/image-optimiser.ts Normal file
View File

@ -0,0 +1,50 @@
import * as sharp from "sharp";
const RESIZE_ENABLED = false;
// const calculateImageSizeMod = (pixels: number) =>
// Math.min(Math.max((-pixels + 500) / 20500 + 1, 0.3), 1);
const calculateImageSizeMod = (pixels: number) =>
Math.min(Math.max((-pixels + 3000) / 33000 + 1, 0.3), 1);
export const optimise = async (image: Buffer): Promise<Buffer> => {
let sharp_image = sharp(image);
if (RESIZE_ENABLED) {
const MAX_IMAGE_DIMENTION = 5000;
sharp_image = await sharp_image
.metadata()
.then((meta) => {
if (
meta.width &&
meta.height &&
(meta.width > MAX_IMAGE_DIMENTION || meta.height > MAX_IMAGE_DIMENTION)
) {
const mod = calculateImageSizeMod(Math.max(meta.width, meta.height));
console.log({
width: meta.width,
height: meta.height,
mod,
size_mb: image.byteLength / 1024_000,
});
return sharp_image.resize({
width: Math.round(meta.width * mod),
height: Math.round(meta.height * mod),
});
}
return sharp_image;
})
.then((image) => image.webp({ lossless: true }));
} else {
sharp_image = sharp_image.webp({ lossless: true });
}
const min_image = await sharp_image.toBuffer();
console.log({
input_size_mb: image.byteLength / 1024_000,
output_size_mb: min_image.byteLength / 1024_000,
});
return min_image;
};

60
apps/blueprint-image-function/src/image-renderer.ts Normal file
View File

@ -0,0 +1,60 @@
import { timeLogger } from "@factorio-sites/common-utils";
import * as Puppeteer from "puppeteer";
let BROWSER: Puppeteer.Browser;
let PAGE: Puppeteer.Page;
async function getPage() {
if (PAGE) return PAGE;
const _browser = await Puppeteer.launch({
headless: false,
args: ["--no-sandbox"],
});
BROWSER = _browser;
const [_page] = await BROWSER.pages();
await _page.goto("https://storage.googleapis.com/factorio-blueprints-assets/fbe/index.html");
await _page.waitForFunction(`!!window.app_loaded`);
PAGE = _page;
return _page;
}
export async function renderImage(blueprint_string: string) {
const tl = timeLogger("localFbeRenderer");
const page = await getPage();
tl("Page loaded");
await page.evaluate((string) => {
return (window as any).pasteBPString(string);
}, blueprint_string);
tl("Page string pasted");
const string = await page.evaluate(
(): Promise<string> => {
return new Promise((resolve, reject) => {
(window as any)
.savePicture()
.then((blob: any) => {
const reader = new FileReader();
reader.readAsBinaryString(blob);
reader.onload = () => resolve(reader.result as string);
reader.onerror = () => reject("Error occurred while reading binary string");
})
.catch(reject);
});
}
);
const image = Buffer.from(string, "binary");
tl("Image downloaded from page");
return image;
}

54
apps/blueprint-image-function/src/localFileUpload.ts → apps/blueprint-image-function/src/local-file-upload.ts
View File

@ -1,22 +1,19 @@
import * as fs from "fs";
import * as path from "path";
import { promisify } from "util";
import * as sharp from "sharp";
import {
hasBlueprintImage,
getBlueprintByImageHash,
saveBlueprintImage,
} from "@factorio-sites/database";
import { optimise } from "./image-optimiser";
if (!process.env.DIR) throw Error("no 'DIR' environment variable")
if (!process.env.DIR) throw Error("no 'DIR' environment variable");
// const fsReaddir = promisify(fs.readdir);
const fsReadFile = promisify(fs.readFile);
const fsUnlink = promisify(fs.unlink);
const FILE_DIR = path.normalize(process.env.DIR);
const RESIZE_ENABLED = false;
const uploadFile = async (image_path: string) => {
const image_hash = path.basename(image_path, ".png");
@ -37,53 +34,8 @@ const uploadFile = async (image_path: string) => {
console.log(`Processing ${image_hash}...`);
const image = await fsReadFile(image_path);
const min_image = await optimise(image);
// const calculateImageSizeMod = (pixels: number) =>
// Math.min(Math.max((-pixels + 500) / 20500 + 1, 0.3), 1);
const calculateImageSizeMod = (pixels: number) =>
Math.min(Math.max((-pixels + 3000) / 33000 + 1, 0.3), 1);
let sharp_image = sharp(image);
if (RESIZE_ENABLED) {
const MAX_IMAGE_DIMENTION = 5000;
sharp_image = await sharp_image
.metadata()
.then((meta) => {
if (
meta.width &&
meta.height &&
(meta.width > MAX_IMAGE_DIMENTION || meta.height > MAX_IMAGE_DIMENTION)
) {
const mod = calculateImageSizeMod(Math.max(meta.width, meta.height));
console.log({
file: image_path,
width: meta.width,
height: meta.height,
mod,
size_mb: image.byteLength / 1024_000,
});
return sharp_image.resize({
width: Math.round(meta.width * mod),
height: Math.round(meta.height * mod),
});
}
return sharp_image;
})
.then((image) => image.webp({ lossless: true }));
} else {
sharp_image = sharp_image.webp({ lossless: true });
}
const min_image = await sharp_image.toBuffer();
console.log({
input_size_mb: image.byteLength / 1024_000,
output_size_mb: min_image.byteLength / 1024_000,
});
// console.log(`Image ${image_hash} processed, writing...`);
// fs.writeFileSync(`${image_path}.webp`, min_image);
console.log(`Image ${image_hash} processed, uploading...`);
await saveBlueprintImage(image_hash, min_image);
await fsUnlink(image_path);

149
apps/blueprint-image-function/src/main.ts
View File

@ -1,142 +1,11 @@
import { generateScreenshot } from "@factorio-sites/generate-bp-image";
import {
saveBlueprintImage,
hasBlueprintImage,
getBlueprintById,
BlueprintEntry,
getBlueprintImageRequestTopic,
PubSubMessage,
getPaginatedBlueprints,
} from "@factorio-sites/database";
import { environment } from "./environments/environment";
import { uploadLocalFiles } from "./localFileUpload";
import { subscribeToPubSub } from "./pubsub-render";
const generateImageForSource = async (blueprint: BlueprintEntry) => {
if (await hasBlueprintImage(blueprint.image_hash)) {
throw Error("Image already exists");
}
const imageBuffer = await generateScreenshot(
blueprint,
environment.production ? "/tmp" : undefined
);
if (!imageBuffer) return false;
await saveBlueprintImage(blueprint.image_hash, imageBuffer);
console.log(`[generateImageForSource] image hash ${blueprint.image_hash} successfully saved`);
return true;
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type Handler = (req: any, res: any) => void; // Don't want to install express types just for this
/**
* Handler method supports Google cloud functions
* @param req express request object
* @param res express response object
*/
export const handler: Handler = async (req, res) => {
if (!req.query.source) {
return res.status(400).end("No source string given");
}
// generateImageForSource((req.query.source as string).replace(/ /g, "+"))
// .then(() => {
// res.status(200).end("done");
// })
// .catch((reason) => {
// res.status(200).end(reason.message);
// });
};
async function subscribeToPubSub() {
// Wait to make sure puppeteer browser started
await new Promise((resolve) => setTimeout(resolve, 2000));
const topic = getBlueprintImageRequestTopic();
const [subscription] = await topic
.subscription("blueprint-image-function-app", {
flowControl: { allowExcessMessages: false, maxMessages: 1, maxExtension: 3600 },
})
.get();
console.log(`[pubsub] Listening to subscription`);
let handlerBusy = false;
const messageHandler = async (message: PubSubMessage) => {
if (!handlerBusy) handlerBusy = true;
else {
console.log(`nack'd message because handler is busy ${message.data.toString()}`);
return message.nack();
}
try {
const data = JSON.parse(message.data.toString());
if (!data.blueprintId) return console.log("blueprintId not found in message body");
console.log("------------------------------------------------");
console.log("[pubsub] generating image for", data.blueprintId);
const blueprint = await getBlueprintById(data.blueprintId);
if (!blueprint) return console.log("Blueprint not found");
const start_time = Date.now();
await generateImageForSource(blueprint);
const duration = Date.now() - start_time;
console.log(`[pubsub] image generated in ${duration}ms`);
message.ack();
if (duration > 30000) {
console.log("Process too slow, closing...");
subscription.off("message", messageHandler);
return setTimeout(() => process.exit(1), 1000);
}
} catch (reason) {
if (reason.message === "Image already exists") {
console.log(`Image already exists`);
message.ack();
} else if (reason.message === "Failed to parse blueprint string") {
console.log(`Blueprint editor could not handle string`);
message.ack();
} else {
console.error("[pubsub:error]", reason);
message.nack();
}
}
handlerBusy = false;
};
subscription.on("message", messageHandler);
// image hash = a99525f97c26c7242ecdd96679043b1a5e65dd0c
// SELECT * FROM BlueprintBook WHERE blueprint_ids CONTAINS Key(Blueprint, 4532736400293888)
// bp = Key(Blueprint, 4532736400293888)
// book = Key(BlueprintBook, 5034207050989568)
// page = 6225886932107264
subscription.on("error", (error) => {
console.error("[pubsub] Received error:", error);
});
}
async function rePublishAllBlueprints() {
const topic = getBlueprintImageRequestTopic();
const fetchPage = async (page = 1) => {
const blueprints = await getPaginatedBlueprints(page);
if (blueprints.length === 0) {
return console.log("No more blueprints found");
}
console.log(`Publishing page ${page} with ${blueprints.length} blueprints`);
await Promise.all(
blueprints.map((blueprint) => {
return topic.publishJSON({ blueprintId: blueprint.id });
})
);
fetchPage(page + 1);
};
await fetchPage();
}
uploadLocalFiles().catch((reason) => console.error("Fatal error:", reason));
// subscribeToPubSub().catch((reason) => console.error("Fatal error:", reason));
subscribeToPubSub().catch((reason) => console.error("Fatal error:", reason));
// uploadLocalFiles().catch((reason) => console.error("Fatal error:", reason));
// rePublishAllBlueprints().catch((reason) => console.error("Fatal error:", reason));
// image hash = a99525f97c26c7242ecdd96679043b1a5e65dd0c
// SELECT * FROM BlueprintBook WHERE blueprint_ids CONTAINS Key(Blueprint, 4532736400293888)
// bp = Key(Blueprint, 4532736400293888)
// book = Key(BlueprintBook, 5034207050989568)
// page = 6225886932107264

71
apps/blueprint-image-function/src/pubsub-render.ts Normal file
View File

@ -0,0 +1,71 @@
import {
saveBlueprintImage,
hasBlueprintImage,
getBlueprintById,
getBlueprintImageRequestTopic,
PubSubMessage,
getBlueprintStringByHash,
} from "@factorio-sites/database";
import { optimise } from "./image-optimiser";
import { renderImage } from "./image-renderer";
export async function subscribeToPubSub() {
const topic = getBlueprintImageRequestTopic();
const [subscription] = await topic
.subscription("blueprint-image-function-app", {
flowControl: { allowExcessMessages: false, maxMessages: 1, maxExtension: 3600 },
})
.get();
let handlerBusy = false;
const messageHandler = async (message: PubSubMessage) => {
if (handlerBusy) {
console.log(`nack'd message because handler is busy ${message.data.toString()}`);
message.nack();
return;
}
handlerBusy = true;
const ack = (log: string, ack: boolean) => {
if (log) console.log(log);
handlerBusy = false;
if (ack) message.ack();
else message.nack();
};
try {
const data = JSON.parse(message.data.toString());
if (!data.blueprintId) return ack("blueprintId not found in message body", false);
console.log("------------------------------------------------");
console.log("[pubsub] generating image for", data.blueprintId);
const blueprint = await getBlueprintById(data.blueprintId);
if (!blueprint) return ack("Blueprint not found", false);
if (await hasBlueprintImage(blueprint.image_hash)) {
return ack("Image already exists", true);
}
const blueprint_string = await getBlueprintStringByHash(blueprint.blueprint_hash);
if (!blueprint_string) return ack("Blueprint string not found", false);
const image = await renderImage(blueprint_string);
console.log(`[pubsub] image generated`);
const min_image = await optimise(image);
await saveBlueprintImage(blueprint.image_hash, min_image);
return ack("[pubsub] image saved", true);
} catch (reason) {
return ack(`[pubsub:error] ${reason}`, false);
}
};
console.log("[pubsub] Listening to messages...");
subscription.on("message", messageHandler);
subscription.on("error", (error) => {
console.error("[pubsub] Received error:", error);
});
}

20
apps/blueprint-image-function/src/republish-pubsub.ts Normal file
View File

@ -0,0 +1,20 @@
import { getBlueprintImageRequestTopic, getPaginatedBlueprints } from "@factorio-sites/database";
export async function rePublishAllBlueprints() {
const topic = getBlueprintImageRequestTopic();
const fetchPage = async (page = 1) => {
const blueprints = await getPaginatedBlueprints(page);
if (blueprints.length === 0) {
return console.log("No more blueprints found");
}
console.log(`Publishing page ${page} with ${blueprints.length} blueprints`);
await Promise.all(
blueprints.map((blueprint) => {
return topic.publishJSON({ blueprintId: blueprint.id });
})
);
fetchPage(page + 1);
};
await fetchPage();
}

1
jest.config.js
View File

@ -1,7 +1,6 @@
module.exports = {
projects: [
"<rootDir>/apps/blueprints",
"<rootDir>/libs/generate-bp-image",
"<rootDir>/apps/blueprint-image-function",
"<rootDir>/apps/factorioprints-scraper",
"<rootDir>/libs/database",

1
libs/generate-bp-image/.eslintrc.json
View File

@ -1 +0,0 @@
{ "extends": "../../.eslintrc.json", "ignorePatterns": ["!**/*"], "rules": {} }

7
libs/generate-bp-image/README.md
View File

@ -1,7 +0,0 @@
# generate-bp-image
This library was generated with [Nx](https://nx.dev).
## Running unit tests
Run `nx test generate-bp-image` to execute the unit tests via [Jest](https://jestjs.io).

14
libs/generate-bp-image/jest.config.js
View File

@ -1,14 +0,0 @@
module.exports = {
displayName: "generate-bp-image",
preset: "../../jest.preset.js",
globals: {
"ts-jest": {
tsConfig: "<rootDir>/tsconfig.spec.json",
},
},
transform: {
"^.+\\.[tj]sx?$": "ts-jest",
},
moduleFileExtensions: ["ts", "tsx", "js", "jsx"],
coverageDirectory: "../../coverage/libs/generate-bp-image",
};

1
libs/generate-bp-image/src/index.ts
View File

@ -1 +0,0 @@
export * from "./lib/generate-bp-image";

11
libs/generate-bp-image/src/lib/generate-bp-image.spec.ts
View File

@ -1,11 +0,0 @@
// import { generateBpImage } from './generate-bp-image';
/**
* tests can not be properly written because all methods have major side effects
*/
describe("generateBpImage", () => {
it("should work", () => {
expect(true);
});
});

130
libs/generate-bp-image/src/lib/generate-bp-image.ts
View File

@ -1,130 +0,0 @@
import * as path from "path";
import * as fs from "fs";
import { promisify } from "util";
import * as Puppeteer from "puppeteer";
import { BlueprintEntry } from "@factorio-sites/database";
import { timeLogger } from "@factorio-sites/common-utils";
const fsMkdir = promisify(fs.mkdir);
const fsStat = promisify(fs.stat);
const fsReadFile = promisify(fs.readFile);
let browser: Puppeteer.Browser;
Puppeteer.launch({
headless: true,
args: ["--no-sandbox"],
}).then((_browser) => (browser = _browser));
async function downloadScreenshot(
blueprint: BlueprintEntry,
dir: string,
on_complete: Promise<any>
) {
// const browser = await Puppeteer.launch({
// headless: true,
// args: ["--no-sandbox"],
// });
const tl = timeLogger("downloadScreenshot");
// const [page] = await browser.pages();
const page = await browser.newPage();
await (page as any)._client.send("Page.setDownloadBehavior", {
behavior: "allow",
downloadPath: dir,
});
let pageErroredOnBlueprint = false;
page.on("console", (message) => {
if (message.type() === "error" && message.text() === "JSHandle@array")
pageErroredOnBlueprint = true;
});
const bp_url = `https://factorio-blueprints-nleb5djksq-ey.a.run.app/api/string/${blueprint.blueprint_hash}`;
tl(`Opening web page with blueprint_hash ${blueprint.blueprint_hash}`);
await page.goto("https://teoxoy.github.io/factorio-blueprint-editor/?source=" + bp_url, {
waitUntil: "load",
});
tl(`page load complete`);
await page.waitForFunction(`!!document.querySelector('.toasts-text')`);
if (pageErroredOnBlueprint) throw Error("Failed to parse blueprint string");
tl(`app initialized`);
await page.focus("canvas");
await page.keyboard.down("Control");
await page.keyboard.press("S");
await page.keyboard.up("Control");
tl("save image command entered");
on_complete.finally(() => {
// browser.close();
page.close();
});
}
export async function generateScreenshot(blueprint: BlueprintEntry, _cache_dir?: string) {
const cache_dir = _cache_dir || path.join(process.cwd(), ".cache/image-downloads");
const dir = path.join(cache_dir, String(blueprint.id));
if (!blueprint.blueprint_hash) {
throw Error("Failed to generate screenshot, no blueprint hash found");
}
await fsMkdir(cache_dir).catch((error) => {
if (error.code !== "EEXIST") throw error;
});
await fsMkdir(dir).catch((error) => {
if (error.code !== "EEXIST") throw error;
});
const result = await _generateScreenshot(blueprint, dir);
// .finally(() => {
// fsRmdir(dir, { recursive: true, maxRetries: 3, retryDelay: 100 }).catch((reason) => {
// console.log(`clearing directory failed: ${reason.code}`);
// });
// });
return result;
}
async function _generateScreenshot(blueprint: BlueprintEntry, dir: string) {
const tl = timeLogger("generateScreenshot");
const promise = new Promise<string>((resolve, reject) => {
const watcher = fs.watch(dir, async (type, file) => {
if (type === "change" && !file.endsWith(".crdownload")) {
const file_path = path.join(dir, file);
fsStat(file_path)
.then(() => resolve(file_path))
.catch(reject)
.finally(() => {
console.log(`closing watcher ${dir}`);
watcher.close();
});
}
});
});
await downloadScreenshot(blueprint, dir, promise);
const file_path = await promise;
tl(`Downloaded image ${file_path}`);
const buffer = await fsReadFile(file_path);
const buffermin = buffer;
// const buffermin = await imagemin.buffer(buffer, {
// plugins: [imageminWebp({ quality: 50 })],
// });
tl("imageminWebp");
return buffermin;
}

16
libs/generate-bp-image/tsconfig.json
View File

@ -1,16 +0,0 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"lib": ["ES2018", "ES2019"]
},
"files": [],
"include": [],
"references": [
{
"path": "./tsconfig.lib.json"
},
{
"path": "./tsconfig.spec.json"
}
]
}

11
libs/generate-bp-image/tsconfig.lib.json
View File

@ -1,11 +0,0 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"module": "commonjs",
"outDir": "../../dist/out-tsc",
"declaration": true,
"types": ["node"]
},
"exclude": ["**/*.spec.ts"],
"include": ["**/*.ts"]
}

9
libs/generate-bp-image/tsconfig.spec.json
View File

@ -1,9 +0,0 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"outDir": "../../dist/out-tsc",
"module": "commonjs",
"types": ["jest", "node"]
},
"include": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.spec.js", "**/*.spec.jsx", "**/*.d.ts"]
}

3
nx.json
View File

@ -30,9 +30,6 @@
"tags": [],
"implicitDependencies": ["blueprints"]
},
"generate-bp-image": {
"tags": []
},
"blueprint-image-function": {
"tags": []
},

1
tsconfig.base.json
View File

@ -17,7 +17,6 @@
"strict": true,
"baseUrl": ".",
"paths": {
"@factorio-sites/generate-bp-image": ["libs/generate-bp-image/src/index.ts"],
"@factorio-sites/database": ["libs/database/src/index.ts"],
"@factorio-sites/node-utils": ["libs/node-utils/src/index.ts"],
"@factorio-sites/common-utils": ["libs/common-utils/src/index.ts"],

21
workspace.json
View File

@ -77,27 +77,6 @@
}
}
},
"generate-bp-image": {
"root": "libs/generate-bp-image",
"sourceRoot": "libs/generate-bp-image/src",
"projectType": "library",
"schematics": {},
"architect": {
"lint": {
"builder": "@nrwl/linter:eslint",
"options": {
"lintFilePatterns": ["libs/generate-bp-image/**/*.ts"]
}
},
"test": {
"builder": "@nrwl/jest:jest",
"options": {
"jestConfig": "libs/generate-bp-image/jest.config.js",
"passWithNoTests": true
}
}
}
},
"blueprint-image-function": {
"root": "apps/blueprint-image-function",
"sourceRoot": "apps/blueprint-image-function/src",