diff --git a/.dockerignore b/.dockerignore index f222b95..ab9ebc4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,6 @@ node_modules -*.Dockerfile \ No newline at end of file +*.Dockerfile +/credentials +/dist +/.vscode +/.cache \ No newline at end of file diff --git a/.gitignore b/.gitignore index ee5c9d8..902d678 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ testem.log # System Files .DS_Store Thumbs.db + +# custom +/.cache +/credentials \ No newline at end of file diff --git a/CREDITS.md b/CREDITS.md new file mode 100644 index 0000000..58e3b21 --- /dev/null +++ b/CREDITS.md @@ -0,0 +1,6 @@ +### Factorio blueprint editor + +https://teoxoy.github.io/factorio-blueprint-editor/ +LISENCE: MIT + +Used for generating images based on blueprint string diff --git a/README.md b/README.md index 7fa19f2..eb9eec1 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Run `nx serve my-app` for a dev server. Navigate to http://localhost:4200/. The Run `nx build my-app` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `--prod` flag for a production build. +### blueprints app + ## Running unit tests Run `nx test my-app` to execute the unit tests via [Jest](https://jestjs.io). diff --git a/apps/blueprint-image-function/src/localFileUpload.ts b/apps/blueprint-image-function/src/localFileUpload.ts new file mode 100644 index 0000000..10781d6 --- /dev/null +++ b/apps/blueprint-image-function/src/localFileUpload.ts @@ -0,0 +1,125 @@ +import * as fs from "fs"; +import * as path from "path"; +import { promisify } from "util"; +import * as sharp from "sharp"; +import { + hasBlueprintImage, + getBlueprintByImageHash, + saveBlueprintImage, +} from "@factorio-sites/database"; + +if (!process.env.DIR) throw Error("no 'DIR' environment variable") + +// const fsReaddir = promisify(fs.readdir); +const fsReadFile = promisify(fs.readFile); +const fsUnlink = promisify(fs.unlink); +const FILE_DIR = path.normalize(process.env.DIR); + +const RESIZE_ENABLED = false; + +const uploadFile = async (image_path: string) => { + const image_hash = path.basename(image_path, ".png"); + + if (!(await getBlueprintByImageHash(image_hash))) { + console.log(`Image ${image_hash} has no database record, skipping...`); + return; + } + if (image_hash.includes("(")) { + console.log(`Image ${image_hash} is a duplicate, deleting...`); + await fsUnlink(image_path); + return; + } + if (await hasBlueprintImage(image_hash)) { + console.log(`Image ${image_hash} already exists, deleting...`); + await fsUnlink(image_path); + return; + } + + console.log(`Processing ${image_hash}...`); + const image = await fsReadFile(image_path); + + // const calculateImageSizeMod = (pixels: number) => + // Math.min(Math.max((-pixels + 500) / 20500 + 1, 0.3), 1); + + const calculateImageSizeMod = (pixels: number) => + Math.min(Math.max((-pixels + 3000) / 33000 + 1, 0.3), 1); + + let sharp_image = sharp(image); + if (RESIZE_ENABLED) { + const MAX_IMAGE_DIMENTION = 5000; + sharp_image = await sharp_image + .metadata() + .then((meta) => { + if ( + meta.width && + meta.height && + (meta.width > MAX_IMAGE_DIMENTION || meta.height > MAX_IMAGE_DIMENTION) + ) { + const mod = calculateImageSizeMod(Math.max(meta.width, meta.height)); + console.log({ + file: image_path, + width: meta.width, + height: meta.height, + mod, + size_mb: image.byteLength / 1024_000, + }); + return sharp_image.resize({ + width: Math.round(meta.width * mod), + height: Math.round(meta.height * mod), + }); + } + return sharp_image; + }) + .then((image) => image.webp({ lossless: true })); + } else { + sharp_image = sharp_image.webp({ lossless: true }); + } + + const min_image = await sharp_image.toBuffer(); + + console.log({ + input_size_mb: image.byteLength / 1024_000, + output_size_mb: min_image.byteLength / 1024_000, + }); + + // console.log(`Image ${image_hash} processed, writing...`); + // fs.writeFileSync(`${image_path}.webp`, min_image); + console.log(`Image ${image_hash} processed, uploading...`); + await saveBlueprintImage(image_hash, min_image); + await fsUnlink(image_path); +}; + +export async function uploadLocalFiles() { + // console.log(`Reading directory`, FILE_DIR); + // const files = await fsReaddir(FILE_DIR); + // for (let i = 0; i < files.length; i++) { + // if (fs.statSync(path.join(FILE_DIR, files[i])).isDirectory()) continue; + // await uploadFile(path.join(FILE_DIR, files[i])); + // } + console.log(`Watching directory`, FILE_DIR); + const work_done_buffeer: string[] = []; + const work_buffer: string[] = []; + fs.watch(FILE_DIR, (type, file) => { + if (type === "change" && file && file.endsWith(".png")) { + const file_path = path.join(FILE_DIR, file); + if (work_buffer.includes(file_path) || work_done_buffeer.includes(file_path)) { + return; + } + work_buffer.push(file_path); + } + }); + + let working = false; + const doWork = async () => { + if (working || !work_buffer.length) return setTimeout(doWork, 1000); + working = true; + const file_path = work_buffer.shift(); + if (file_path) { + await uploadFile(file_path); + work_done_buffeer.push(file_path); + } + working = false; + doWork(); + }; + doWork(); +} diff --git a/apps/blueprint-image-function/src/main.ts b/apps/blueprint-image-function/src/main.ts index 2d19de8..e09e501 100644 --- a/apps/blueprint-image-function/src/main.ts +++ b/apps/blueprint-image-function/src/main.ts @@ -1,15 +1,142 @@ import { generateScreenshot } from "@factorio-sites/generate-bp-image"; +import { + saveBlueprintImage, + hasBlueprintImage, + getBlueprintById, + BlueprintEntry, + getBlueprintImageRequestTopic, + PubSubMessage, + getPaginatedBlueprints, +} from "@factorio-sites/database"; +import { environment } from "./environments/environment"; +import { uploadLocalFiles } from "./localFileUpload"; + +const generateImageForSource = async (blueprint: BlueprintEntry) => { + if (await hasBlueprintImage(blueprint.image_hash)) { + throw Error("Image already exists"); + } + + const imageBuffer = await generateScreenshot( + blueprint, + environment.production ? "/tmp" : undefined + ); + + if (!imageBuffer) return false; + + await saveBlueprintImage(blueprint.image_hash, imageBuffer); + + console.log(`[generateImageForSource] image hash ${blueprint.image_hash} successfully saved`); + + return true; +}; // eslint-disable-next-line @typescript-eslint/no-explicit-any type Handler = (req: any, res: any) => void; // Don't want to install express types just for this +/** + * Handler method supports Google cloud functions + * @param req express request object + * @param res express response object + */ export const handler: Handler = async (req, res) => { if (!req.query.source) { return res.status(400).end("No source string given"); } - const string = (req.query.source as string).replace(/ /g, "+"); - const stream = await generateScreenshot(string, "/tmp"); - res.status(200); - res.setHeader("content-type", "image/png"); - stream.pipe(res); + + // generateImageForSource((req.query.source as string).replace(/ /g, "+")) + // .then(() => { + // res.status(200).end("done"); + // }) + // .catch((reason) => { + // res.status(200).end(reason.message); + // }); }; + +async function subscribeToPubSub() { + // Wait to make sure puppeteer browser started + await new Promise((resolve) => setTimeout(resolve, 2000)); + + const topic = getBlueprintImageRequestTopic(); + const [subscription] = await topic + .subscription("blueprint-image-function-app", { + flowControl: { allowExcessMessages: false, maxMessages: 1, maxExtension: 3600 }, + }) + .get(); + + console.log(`[pubsub] Listening to subscription`); + + let handlerBusy = false; + const messageHandler = async (message: PubSubMessage) => { + if (!handlerBusy) handlerBusy = true; + else { + console.log(`nack'd message because handler is busy ${message.data.toString()}`); + return message.nack(); + } + + try { + const data = JSON.parse(message.data.toString()); + if (!data.blueprintId) return console.log("blueprintId not found in message body"); + console.log("------------------------------------------------"); + console.log("[pubsub] generating image for", data.blueprintId); + const blueprint = await getBlueprintById(data.blueprintId); + if (!blueprint) return console.log("Blueprint not found"); + const start_time = Date.now(); + await generateImageForSource(blueprint); + const duration = Date.now() - start_time; + console.log(`[pubsub] image generated in ${duration}ms`); + message.ack(); + + if (duration > 30000) { + console.log("Process too slow, closing..."); + subscription.off("message", messageHandler); + return setTimeout(() => process.exit(1), 1000); + } + } catch (reason) { + if (reason.message === "Image already exists") { + console.log(`Image already exists`); + message.ack(); + } else if (reason.message === "Failed to parse blueprint string") { + console.log(`Blueprint editor could not handle string`); + message.ack(); + } else { + console.error("[pubsub:error]", reason); + message.nack(); + } + } + handlerBusy = false; + }; + + subscription.on("message", messageHandler); + // image hash = a99525f97c26c7242ecdd96679043b1a5e65dd0c + // SELECT * FROM BlueprintBook WHERE blueprint_ids CONTAINS Key(Blueprint, 4532736400293888) + // bp = Key(Blueprint, 4532736400293888) + // book = Key(BlueprintBook, 5034207050989568) + // page = 6225886932107264 + + subscription.on("error", (error) => { + console.error("[pubsub] Received error:", error); + }); +} + +async function rePublishAllBlueprints() { + const topic = getBlueprintImageRequestTopic(); + const fetchPage = async (page = 1) => { + const blueprints = await getPaginatedBlueprints(page); + if (blueprints.length === 0) { + return console.log("No more blueprints found"); + } + console.log(`Publishing page ${page} with ${blueprints.length} blueprints`); + + await Promise.all( + blueprints.map((blueprint) => { + return topic.publishJSON({ blueprintId: blueprint.id }); + }) + ); + fetchPage(page + 1); + }; + await fetchPage(); +} + +uploadLocalFiles().catch((reason) => console.error("Fatal error:", reason)); +// subscribeToPubSub().catch((reason) => console.error("Fatal error:", reason)); +// rePublishAllBlueprints().catch((reason) => console.error("Fatal error:", reason)); diff --git a/apps/blueprints/.eslintrc.json b/apps/blueprints/.eslintrc.json index bd7fc5f..055943f 100644 --- a/apps/blueprints/.eslintrc.json +++ b/apps/blueprints/.eslintrc.json @@ -194,10 +194,7 @@ "jsx-a11y/accessible-emoji": "warn", "jsx-a11y/alt-text": "warn", "jsx-a11y/anchor-has-content": "warn", - "jsx-a11y/anchor-is-valid": [ - "warn", - { "aspects": ["noHref", "invalidHref"] } - ], + "jsx-a11y/anchor-is-valid": ["warn", { "aspects": ["noHref", "invalidHref"] }], "jsx-a11y/aria-activedescendant-has-tabindex": "warn", "jsx-a11y/aria-props": "warn", "jsx-a11y/aria-proptypes": "warn", @@ -231,10 +228,7 @@ } ], "no-unused-vars": "off", - "@typescript-eslint/no-unused-vars": [ - "warn", - { "args": "none", "ignoreRestSiblings": true } - ], + "@typescript-eslint/no-unused-vars": ["warn", { "args": "none", "ignoreRestSiblings": true }], "no-useless-constructor": "off", "@typescript-eslint/no-useless-constructor": "warn", "@typescript-eslint/no-unused-expressions": [ diff --git a/apps/blueprints/README.md b/apps/blueprints/README.md new file mode 100644 index 0000000..1de9bcb --- /dev/null +++ b/apps/blueprints/README.md @@ -0,0 +1,16 @@ +## Before deploying + +Make sure `prod.package.json` is up to date on packages from the root `package.json` + +## Deploying + +`docker build -t eu.gcr.io/factorio-sites/blueprints --file blueprints.Dockerfile .` +`docker push eu.gcr.io/factorio-sites/blueprints` + +### Testing deployment locally + +`docker run --rm -p 3000:3000 eu.gcr.io/factorio-sites/blueprints` + +### windows env + +`$env:GOOGLE_APPLICATION_CREDENTIALS="D:\git\factorio-sites\credentials\factorio-sites.json"` diff --git a/apps/blueprints/index.d.ts b/apps/blueprints/index.d.ts index 7ba08fa..bafcc24 100644 --- a/apps/blueprints/index.d.ts +++ b/apps/blueprints/index.d.ts @@ -1,6 +1,10 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -declare module '*.svg' { +declare module "*.svg" { const content: any; export const ReactComponent: any; export default content; } + +declare module "react-map-interaction" { + export const MapInteractionCSS: any; +} diff --git a/apps/blueprints/pages/[blueprint]/[blueprintId].tsx b/apps/blueprints/pages/[blueprint]/[blueprintId].tsx new file mode 100644 index 0000000..267abbe --- /dev/null +++ b/apps/blueprints/pages/[blueprint]/[blueprintId].tsx @@ -0,0 +1,331 @@ +/** @jsx jsx */ +import React, { ReactNode, useEffect, useState } from "react"; +import { jsx, css } from "@emotion/core"; +import { NextPage, NextPageContext } from "next"; +import BBCode from "bbcode-to-react"; +import { Button, Grid } from "@chakra-ui/core"; +import { + BlueprintBookEntry, + BlueprintEntry, + BlueprintPageEntry, + getBlueprintBookById, + getBlueprintById, + getBlueprintPageById, + hasBlueprintImage, +} from "@factorio-sites/database"; +import { BlueprintData, timeLogger } from "@factorio-sites/common-utils"; +import { chakraResponsive, parseBlueprintStringClient } from "@factorio-sites/web-utils"; +import { Panel } from "../../src/Panel"; +import { Markdown } from "../../src/Markdown"; +import { FullscreenImage } from "../../src/FullscreenImage"; +import { BookChildTree } from "../../src/BookChildTree"; +import { CopyButton } from "../../src/CopyButton"; + +const imageStyle = css` + display: flex; + justify-content: center; + &:hover { + cursor: pointer; + } +`; + +type Selected = + | { type: "blueprint"; data: Pick } + | { type: "blueprint_book"; data: Pick }; + +interface IndexProps { + image_exists: boolean; + selected: Selected; + blueprint: BlueprintEntry | null; + blueprint_book: BlueprintBookEntry | null; + blueprint_page: BlueprintPageEntry; +} + +export const Index: NextPage = ({ + image_exists, + selected, + blueprint, + blueprint_book, + blueprint_page, +}) => { + const [imageZoom, setImageZoom] = useState(false); + const [blueprintString, setBlueprintString] = useState(null); + const [data, setData] = useState(null); + const [showJson, setShowJson] = useState(false); + + const selectedHash = selected.data.blueprint_hash; + + useEffect(() => { + fetch(`/api/string/${selectedHash}`) + .then((res) => res.text()) + .then((string) => { + setShowJson(false); + setBlueprintString(string); + if (selected.type === "blueprint") { + const { data } = parseBlueprintStringClient(string); + setData(data); + } else { + setData(null); + } + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedHash]); + + useEffect(() => { + console.log({ + image_exists, + selected, + blueprint, + blueprint_book, + blueprint_page, + }); + }, []); + + const renderImage = () => { + let render: ReactNode; + if (selected.type === "blueprint_book") { + render =
Can't show image for a book, select a blueprint to the the image
; + } else if (!image_exists) { + render =
The image is not generated yet
; + } else if (imageZoom) { + render = ( + setImageZoom(false)} + alt="blueprint" + src={`https://storage.googleapis.com/blueprint-images/${selected.data.image_hash}.webp`} + /> + ); + } else { + render = ( +
setImageZoom(true)}> + blueprint +
+ ); + } + return
{render}
; + }; + + return ( + + + {blueprint_book ? ( + <> +
This string contains a blueprint book
+
+ + + ) : blueprint ? ( + <> +
This string contains one blueprint
+
tags: {blueprint.tags.join(", ")}
+ + ) : null} + + + {renderImage()} + + + + {blueprint_page.description_markdown} + + {selected.type === "blueprint" && data && ( + Entities for {BBCode.toReact(data.blueprint.label)}) as any} + gridColumn={chakraResponsive({ mobile: "1", desktop: "1 / span 2" })} + > + + + {Object.entries( + data.blueprint.entities.reduce>((entities, entity) => { + if (entities[entity.name]) { + entities[entity.name]++; + } else { + entities[entity.name] = 1; + } + return entities; + }, {}) + ) + .sort((a, b) => b[1] - a[1]) + .map(([entry_name, entry]) => ( + + + + + + ))} + +
+ {entry_name.replace(/-/g, + {entry_name}{entry}
+ + )} + + <> + {blueprintString && } +