Add FBE editor for in-browser image rendering, ported database to postgres, updated packages

.gitignore

@@ -42,3 +42,5 @@ Thumbs.db
 /.cache
 /credentials
 .env.local
+.local.env
+/.yalc

.yarnrc

@@ -1 +1 @@
-save-prefix=
+save-prefix ""

apps/blueprints/.babelrc

@@ -1,4 +1,14 @@
 {
-  "presets": ["next/babel"],
-  "plugins": []
+  "presets": [
+    [
+      "next/babel",
+      {
+        "preset-react": {
+          "runtime": "automatic",
+          "importSource": "@emotion/react"
+        }
+      }
+    ]
+  ],
+  "plugins": ["@emotion/babel-plugin"]
 }

apps/blueprints/.eslintrc.json

@@ -187,14 +187,21 @@
     "react/no-direct-mutation-state": "warn",
     "react/no-is-mounted": "warn",
     "react/no-typos": "error",
-    "react/react-in-jsx-scope": "error",
+    "react/react-in-jsx-scope": "off",
     "react/require-render-return": "error",
     "react/style-prop-object": "warn",
     "react/jsx-no-useless-fragment": "warn",
     "jsx-a11y/accessible-emoji": "warn",
     "jsx-a11y/alt-text": "warn",
     "jsx-a11y/anchor-has-content": "warn",
-    "jsx-a11y/anchor-is-valid": ["warn", { "aspects": ["noHref", "invalidHref"] }],
+    "jsx-a11y/anchor-is-valid": [
+      "warn",
+      {
+        "components": ["Link"],
+        "specialLink": ["hrefLeft", "hrefRight"],
+        "aspects": ["noHref", "invalidHref", "preferButton"]
+      }
+    ],
     "jsx-a11y/aria-activedescendant-has-tabindex": "warn",
     "jsx-a11y/aria-props": "warn",
     "jsx-a11y/aria-proptypes": "warn",

apps/blueprints/next-env.d.ts

@@ -1,2 +1,3 @@
 /// <reference types="next" />
 /// <reference types="next/types/global" />
+/// <reference types="@emotion/react/types/css-prop" />

apps/blueprints/src/components/BookChildTree.tsx

@@ -1,9 +1,7 @@
-/** @jsx jsx */
-/* eslint-disable jsx-a11y/anchor-is-valid */
-import { jsx, css } from "@emotion/react";
+import { css } from "@emotion/react";
 import Link from "next/link";
 import BBCode from "bbcode-to-react";
-import { BlueprintBookEntry } from "@factorio-sites/database";
+import { ChildTree } from "@factorio-sites/database";
 
 const componentStyles = css`
   .blueprint,
@@ -22,7 +20,7 @@ const componentStyles = css`
 `;
 
 interface BookChildTreeProps {
-  child_tree: BlueprintBookEntry["child_tree"];
+  child_tree: ChildTree;
   base_url: string;
   selected_id: string;
 }

apps/blueprints/src/components/FullscreenImage.tsx

@@ -1,7 +1,5 @@
-/** @jsx jsx */
-/* eslint-disable jsx-a11y/anchor-is-valid */
 import React, { useState } from "react";
-import { jsx, css } from "@emotion/react";
+import { css } from "@emotion/react";
 import { MapInteractionCSS } from "react-map-interaction";
 
 const elementStyle = css`

apps/blueprints/src/components/ImageEditor.tsx

@@ -0,0 +1,72 @@
+import { useEffect, useRef, useState } from "react";
+
+type FBE = typeof import("@fbe/editor");
+type Editor = InstanceType<FBE["Editor"]>;
+
+export const ImageEditor: React.FC<{ string: string }> = ({ string }) => {
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  // const [image, setImage] = useState<string | undefined>();
+  const FbeRef = useRef<FBE>();
+  const editorRef = useRef<Editor>();
+  const [editorLoaded, setEditorLoaded] = useState(false);
+
+  // Load editor async, it does not work server side
+  useEffect(() => {
+    (async () => {
+      const FBE = await import("@fbe/editor");
+      const editor = new FBE.Editor();
+      const canvas = canvasRef.current;
+      if (!canvas) return;
+      await editor.init(canvas);
+      canvas.style.width = "100%";
+      canvas.style.height = "auto";
+      editor.setRendererSize(canvas.offsetWidth, canvas.offsetHeight);
+      FbeRef.current = FBE;
+      editorRef.current = editor;
+      setEditorLoaded(true);
+    })();
+  }, []);
+
+  useEffect(() => {
+    (async () => {
+      const FBE = FbeRef.current;
+      const editor = editorRef.current;
+      if (!editorLoaded || !FBE || !editor) return;
+
+      const bpOrBook = await FBE.getBlueprintOrBookFromSource(string);
+      const blueprint = bpOrBook instanceof FBE.Book ? bpOrBook.getBlueprint() : bpOrBook;
+
+      console.log({ blueprint });
+
+      await editor.loadBlueprint(blueprint);
+      // await FBE.default.waitForLoader();
+
+      // Wait a little extra, sometimes even after textures are loaded it neeeds a moment to render
+      // await new Promise((resolve) => setTimeout(resolve, 10));
+
+      // const picture = await editor.getPicture();
+      // setImage(URL.createObjectURL(picture));
+    })();
+  }, [string, editorLoaded]);
+
+  window.addEventListener(
+    "resize",
+    () => {
+      if (!canvasRef.current || !editorRef.current) return;
+      canvasRef.current.style.width = "100%";
+      canvasRef.current.style.height = "auto";
+      editorRef.current.setRendererSize(
+        canvasRef.current.offsetWidth,
+        canvasRef.current.offsetHeight
+      );
+    },
+    false
+  );
+
+  return (
+    <div>
+      <canvas ref={canvasRef} style={{ width: "100%", height: "auto" }} />
+      {/* <img src={image} alt="blueprint" style={{ width: "500px" }}></img> */}
+    </div>
+  );
+};

apps/blueprints/src/components/Markdown.tsx

@@ -1,6 +1,4 @@
-/** @jsx jsx */
-/* eslint-disable jsx-a11y/anchor-is-valid */
-import { jsx, css } from "@emotion/react";
+import { css } from "@emotion/react";
 import ReactMarkdown from "react-markdown";
 
 const markdownStyle = css`

apps/blueprints/src/components/Pagination.tsx

@@ -1,5 +1,3 @@
-/** @jsx jsx */
-import { jsx } from "@emotion/react";
 import React from "react";
 import Link from "next/link";
 import { useRouter } from "next/router";

apps/blueprints/src/components/Panel.tsx

@@ -1,5 +1,4 @@
-/** @jsx jsx */
-import { jsx, css, SerializedStyles } from "@emotion/react";
+import { css, SerializedStyles } from "@emotion/react";
 import { Box, BoxProps } from "@chakra-ui/react";
 import { ReactNode } from "react";
 

apps/blueprints/src/pages/_app.tsx

@@ -1,11 +1,10 @@
-/** @jsx jsx */
 import { AppProps, NextWebVitalsMetric } from "next/app";
 import Head from "next/head";
 import Router from "next/router";
-import { jsx, css, Global } from "@emotion/react";
+import { css, Global } from "@emotion/react";
 import { ChakraProvider } from "@chakra-ui/react";
 import NProgress from "nprogress";
-import { Header } from "../src/Header";
+import { Header } from "../components/Header";
 
 const globalStyles = css`
   html {
@@ -60,7 +59,7 @@ const BlueprintsApp = ({ Component, pageProps }: AppProps) => {
 };
 
 export function reportWebVitals(metric: NextWebVitalsMetric) {
-  console.log(metric);
+  // console.log(metric);
 }
 
 export default BlueprintsApp;

apps/blueprints/src/pages/api/db.ts

@@ -0,0 +1,15 @@
+import { NextApiHandler } from "next";
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+import { init, BlueprintModel } from "@factorio-sites/database";
+
+const handler: NextApiHandler = async (_, res) => {
+  await init();
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  const bp = await BlueprintModel.findByPk("uuid").catch(() => {});
+  console.log({ bp });
+
+  res.status(200).end("db sync");
+};
+
+export default handler;

apps/blueprints/src/pages/api/fbe-proxy/[...proxy].ts

@@ -0,0 +1,32 @@
+import { NextApiHandler } from "next";
+
+const handler: NextApiHandler = async (req, res) => {
+  const path = req.query.proxy ? (req.query.proxy as string[]).join("/") : "";
+
+  console.log("[fbe-proxy]", path);
+
+  const result = await fetch(
+    "https://static-fbe.teoxoy.com/file/factorio-blueprint-editor/" + path
+  );
+
+  res.setHeader("Access-Control-Allow-Origin", "*");
+
+  result.headers.forEach((val, key) => {
+    if (
+      !res.hasHeader(key) &&
+      !["content-encoding", "connection", "server", "transfer-encoding", "vary"].includes(key)
+    ) {
+      res.setHeader(key, val);
+    }
+  });
+
+  if (result.headers.get("content-type") === "application/octet-stream") {
+    const output = Buffer.from(await result.arrayBuffer());
+    res.end(output);
+  } else {
+    const text = await result.text();
+    res.end(text);
+  }
+};
+
+export default handler;

apps/blueprints/src/pages/blueprint/[blueprintId].tsx

@@ -1,25 +1,25 @@
-/** @jsx jsx */
 import React, { ReactNode, useEffect, useState } from "react";
-import { jsx, css } from "@emotion/react";
+import { css } from "@emotion/react";
 import { NextPage, NextPageContext } from "next";
 import BBCode from "bbcode-to-react";
 import { Button, Grid, Image } from "@chakra-ui/react";
 import {
-  BlueprintBookEntry,
-  BlueprintEntry,
-  BlueprintPageEntry,
+  BlueprintBook,
+  Blueprint,
+  BlueprintPage,
   getBlueprintBookById,
   getBlueprintById,
   getBlueprintPageById,
   hasBlueprintImage,
 } from "@factorio-sites/database";
-import { BlueprintData, timeLogger } from "@factorio-sites/common-utils";
+import { BlueprintStringData, timeLogger } from "@factorio-sites/common-utils";
 import { chakraResponsive, parseBlueprintStringClient } from "@factorio-sites/web-utils";
-import { Panel } from "../../src/Panel";
-import { Markdown } from "../../src/Markdown";
-import { FullscreenImage } from "../../src/FullscreenImage";
-import { BookChildTree } from "../../src/BookChildTree";
-import { CopyButton } from "../../src/CopyButton";
+import { Panel } from "../../components/Panel";
+import { Markdown } from "../../components/Markdown";
+import { FullscreenImage } from "../../components/FullscreenImage";
+import { BookChildTree } from "../../components/BookChildTree";
+import { CopyButton } from "../../components/CopyButton";
+import { ImageEditor } from "../../components/ImageEditor";
 
 const imageStyle = css`
   display: flex;
@@ -30,15 +30,15 @@ const imageStyle = css`
 `;
 
 type Selected =
-  | { type: "blueprint"; data: Pick<BlueprintEntry, "id" | "blueprint_hash" | "image_hash"> }
-  | { type: "blueprint_book"; data: Pick<BlueprintBookEntry, "id" | "blueprint_hash"> };
+  | { type: "blueprint"; data: Pick<Blueprint, "id" | "blueprint_hash" | "image_hash"> }
+  | { type: "blueprint_book"; data: Pick<BlueprintBook, "id" | "blueprint_hash"> };
 
 interface IndexProps {
   image_exists: boolean;
   selected: Selected;
-  blueprint: BlueprintEntry | null;
-  blueprint_book: BlueprintBookEntry | null;
-  blueprint_page: BlueprintPageEntry;
+  blueprint: Blueprint | null;
+  blueprint_book: BlueprintBook | null;
+  blueprint_page: BlueprintPage;
 }
 
 export const Index: NextPage<IndexProps> = ({
@@ -50,7 +50,7 @@ export const Index: NextPage<IndexProps> = ({
 }) => {
   const [imageZoom, setImageZoom] = useState(false);
   const [blueprintString, setBlueprintString] = useState<string | null>(null);
-  const [data, setData] = useState<BlueprintData | null>(null);
+  const [data, setData] = useState<BlueprintStringData | null>(null);
   const [showJson, setShowJson] = useState(false);
 
   const selectedHash = selected.data.blueprint_hash;
@@ -145,7 +145,8 @@ export const Index: NextPage<IndexProps> = ({
         gridColumn={chakraResponsive({ mobile: "1", desktop: "2" })}
         gridRow={chakraResponsive({ mobile: "1", desktop: null })}
       >
-        {renderImage()}
+        {/* {renderImage()} */}
+        {blueprintString && <ImageEditor string={blueprintString}></ImageEditor>}
       </Panel>
 
       <Panel
@@ -277,8 +278,8 @@ export async function getServerSideProps(context: NextPageContext) {
   let blueprint: IndexProps["blueprint"] = null;
   let blueprint_book: IndexProps["blueprint_book"] = null;
   let selected!: IndexProps["selected"];
-  let selected_blueprint!: BlueprintEntry | null;
-  let selected_blueprint_book!: BlueprintBookEntry | null;
+  let selected_blueprint!: Blueprint | null;
+  let selected_blueprint_book!: BlueprintBook | null;
 
   if (blueprint_page.blueprint_id) {
     blueprint = await getBlueprintById(blueprint_page.blueprint_id);

apps/blueprints/src/pages/index.tsx

@@ -1,13 +1,11 @@
-/** @jsx jsx */
-/* eslint-disable jsx-a11y/anchor-is-valid */
 import React from "react";
 import { NextPage, NextPageContext } from "next";
 import Link from "next/link";
-import { jsx, css } from "@emotion/react";
-import { BlueprintPageEntry, getMostRecentBlueprintPages } from "@factorio-sites/database";
-import { Panel } from "../src/Panel";
+import { css } from "@emotion/react";
+import { BlueprintPage, getMostRecentBlueprintPages } from "@factorio-sites/database";
 import { SimpleGrid } from "@chakra-ui/react";
-import { Pagination } from "../src/Pagination";
+import { Panel } from "../components/Panel";
+import { Pagination } from "../components/Pagination";
 
 const linkStyles = css`
   width: 100%;
@@ -23,7 +21,7 @@ const linkStyles = css`
   }
 `;
 
-const BlueprintComponent: React.FC<{ blueprint: BlueprintPageEntry }> = ({ blueprint }) => (
+const BlueprintComponent: React.FC<{ blueprint: BlueprintPage }> = ({ blueprint }) => (
   <div css={linkStyles}>
     <Link href={`/blueprint/${blueprint.id}`} passHref>
       <a>{blueprint.title}</a>
@@ -33,7 +31,7 @@ const BlueprintComponent: React.FC<{ blueprint: BlueprintPageEntry }> = ({ bluep
 
 interface IndexProps {
   page: number;
-  blueprints: BlueprintPageEntry[];
+  blueprints: BlueprintPage[];
 }
 
 export const Index: NextPage<IndexProps> = ({ page, blueprints }) => {

apps/blueprints/tsconfig.json

@@ -9,7 +9,9 @@
     "forceConsistentCasingInFileNames": true,
     "noEmit": true,
     "resolveJsonModule": true,
-    "isolatedModules": true
+    "isolatedModules": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true
   },
   "include": ["**/*.ts", "**/*.tsx", "next-env.d.ts"],
   "exclude": ["node_modules"]

apps/factorioprints-scraper/src/app/populate-db.ts

@@ -10,7 +10,7 @@ const fsReadFile = promisify(fs.readFile);
 const CACHE_DIR = path.join(__dirname, "../../../.cache/factorioprints-data");
 
 export async function writeToDatastore() {
-  const filecontent = await fsReadFile(path.join(CACHE_DIR, `most-fav-json/page2.json`), "utf8");
+  const filecontent = await fsReadFile(path.join(CACHE_DIR, `most-fav-json/page1.json`), "utf8");
   const data = JSON.parse(filecontent);
 
   for (let i = 0; i < data.length; i++) {

apps/factorioprints-scraper/src/environments/environment.ts

@@ -1,3 +1,11 @@
 export const environment = {
   production: false,
 };
+
+process.env.GOOGLE_APPLICATION_CREDENTIALS =
+  "D:\\git\\factorio-sites\\credentials\\factorio-sites.json";
+process.env.POSTGRES_HOST = "127.0.0.1";
+process.env.POSTGRES_DB = "factorio-blueprints";
+process.env.POSTGRES_USER = "factorio-blueprints";
+process.env.POSTGRES_PASSWORD = "local";
+process.env.SYNCHRONIZE_DB = "true";

apps/factorioprints-scraper/src/main.ts

@@ -1,5 +1,12 @@
+import "./environments/environment";
+if (process.env.NODE_ENV === "development") {
+  console.log(process.env.GOOGLE_APPLICATION_CREDENTIALS);
+}
+
+// import { saveBlueprintFromFactorioprints } from "@factorio-sites/database";
+// import { parseBlueprintString } from "@factorio-sites/node-utils";
+// import { scanFactorioPrints } from "./app/scan";
 import { writeToDatastore } from "./app/populate-db";
-import { scanFactorioPrints } from "./app/scan";
 
 // async function writeTestBP() {
 //   const source =
@@ -11,12 +18,10 @@ import { scanFactorioPrints } from "./app/scan";
 //   // use createBlueprint
 //   saveBlueprintFromFactorioprints(
 //     {
-//       factorioprints_id: null,
+//       factorioprints_id: "-id",
 //       title: "my blueprint",
 //       description_markdown: "",
-//       tags: [],
-//       updated_at: Date.now() / 1000,
-//       created_at: Date.now() / 1000,
+//       tags: ["tag1", "tag2"],
 //     },
 //     string
 //   );
@@ -24,8 +29,8 @@ import { scanFactorioPrints } from "./app/scan";
 
 async function main() {
   // scanFactorioPrints(1, 3);
-  writeToDatastore();
   // writeTestBP();
+  writeToDatastore();
 }
 
 main().catch((reason) => {

docker/factorio-blueprints/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3.8"
+services:
+  postgres:
+    image: postgres:13
+    container_name: factorio-blueprints-database
+    restart: always
+    volumes:
+      - postgres-data:/var/lib/postgresql/data/
+    env_file:
+      - postgres.env
+    ports:
+      - 5432:5432
+  adminer:
+    image: dpage/pgadmin4:4
+    container_name: factorio-blueprints-database-admin
+    restart: always
+    volumes:
+      - pgadmin-data:/var/lib/pgadmin
+    env_file:
+      - pgadmin.env
+    ports:
+      - 8080:80
+
+volumes:
+  postgres-data:
+  pgadmin-data:

docker/factorio-blueprints/pgadmin.env

@@ -0,0 +1,3 @@
+PGADMIN_DEFAULT_EMAIL=me@barrykun.com
+PGADMIN_DEFAULT_PASSWORD=local
+PGADMIN_LISTEN_PORT=80

docker/factorio-blueprints/postgres.env

@@ -0,0 +1,3 @@
+POSTGRES_DB=factorio-blueprints
+POSTGRES_USER=factorio-blueprints
+POSTGRES_PASSWORD=local

libs/common-utils/src/lib/common-utils.ts

@@ -4,7 +4,7 @@ interface Entity {
   position: { x: number; y: number };
 }
 
-export interface Blueprint {
+export interface BlueprintData {
   entities: Entity[];
   tiles?: { name: string; position: { x: number; y: number } }[];
   icons: { signal: { type: "item" | "fluid"; name: string } }[];
@@ -14,33 +14,44 @@ export interface Blueprint {
   version: number;
 }
 
-export interface BlueprintBook {
+export interface BlueprintBookData {
   active_index: number;
-  blueprints: Array<{ index: number } & BlueprintData>;
+  blueprints: Array<{ index: number } & BlueprintStringData>;
   item: string;
   label: string;
   description?: string;
   version: number;
 }
 
-export interface BlueprintData {
-  blueprint_book?: BlueprintBook;
-  blueprint?: Blueprint;
+export interface BlueprintPageData {
+  id: string;
+  blueprint_id?: string;
+  blueprint_book_id?: string;
+  title: string;
+  description_markdown: string;
+  created_at: number;
+  updated_at: number;
+  factorioprints_id?: string;
 }
 
-export const getBlueprintContentForImageHash = (blueprint: Blueprint): string => {
+export interface BlueprintStringData {
+  blueprint_book?: BlueprintBookData;
+  blueprint?: BlueprintData;
+}
+
+export const getBlueprintContentForImageHash = (blueprint: BlueprintData): string => {
   return JSON.stringify({
     entities: blueprint.entities,
     tiles: blueprint.tiles,
   });
 };
 
-export const flattenBlueprintData = (data: BlueprintData) => {
-  const blueprints: Blueprint[] = [];
-  const books: BlueprintBook[] = [];
+export const flattenBlueprintData = (data: BlueprintStringData) => {
+  const blueprints: BlueprintData[] = [];
+  const books: BlueprintBookData[] = [];
 
   // Recursively go through the string to find all blueprints
-  const findAndPushBlueprints = (data: BlueprintData) => {
+  const findAndPushBlueprints = (data: BlueprintStringData) => {
     if (data.blueprint) {
       blueprints.push(data.blueprint);
     } else if (data.blueprint_book) {
@@ -59,25 +70,28 @@ export const flattenBlueprintData = (data: BlueprintData) => {
   };
 };
 
-export const findBlueprintByPath = (data: BlueprintData, path: number[]): Blueprint | null => {
+export const findBlueprintByPath = (
+  data: BlueprintStringData,
+  path: number[]
+): BlueprintStringData | null => {
   if (path.length === 0) {
-    return (data.blueprint || data.blueprint_book?.blueprints[0]) as Blueprint;
+    return (data.blueprint || data.blueprint_book?.blueprints[0]) as BlueprintStringData;
   } else if (data.blueprint_book && path.length === 1) {
-    return data.blueprint_book.blueprints[path[0]].blueprint as Blueprint;
+    return data.blueprint_book.blueprints[path[0]].blueprint as BlueprintStringData;
   }
   return null;
 };
 
 export const findActiveBlueprint = (
-  data: BlueprintData
-): { blueprint: Blueprint; path: number[] } => {
+  data: BlueprintStringData
+): { blueprint: BlueprintData; path: number[] } => {
   if (data.blueprint) {
     return { blueprint: data.blueprint, path: [0] };
   } else if (data.blueprint_book) {
     const findActive = (
-      book: BlueprintBook,
+      book: BlueprintBookData,
       _path: number[] = []
-    ): { blueprint: Blueprint; path: number[] } => {
+    ): { blueprint: BlueprintData; path: number[] } => {
       const active = book.blueprints.find((bp) => bp.index === book.active_index);
 
       if (active && active.blueprint) {

libs/database/jest.config.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-undef */
 module.exports = {
   displayName: "database",
   preset: "../../jest.preset.js",

libs/database/src/index.ts

@@ -1,4 +1,6 @@
-export * from "./lib/gcp-datastore";
+export * from "./lib/data";
 export * from "./lib/gcp-pubsub";
 export * from "./lib/gcp-storage";
 export * from "./lib/types";
+export * from "./lib/postgres/database";
+export * from "./lib/postgres/models";

libs/database/src/lib/data/blueprint.ts

@@ -0,0 +1,85 @@
+import { BlueprintData, getBlueprintContentForImageHash } from "@factorio-sites/common-utils";
+import { encodeBlueprint, hashString } from "@factorio-sites/node-utils";
+import { getBlueprintImageRequestTopic } from "../gcp-pubsub";
+import { saveBlueprintString } from "../gcp-storage";
+import { BlueprintInstance, BlueprintModel, BlueprintStringModel } from "../postgres/models";
+import { Blueprint } from "../types";
+
+const blueprintImageRequestTopic = getBlueprintImageRequestTopic();
+
+const mapBlueprintInstanceToEntry = (entity: BlueprintInstance): Blueprint => ({
+  id: entity.id,
+  blueprint_hash: entity.blueprint_hash,
+  image_hash: entity.image_hash,
+  label: entity.label || "",
+  description: entity.description || "",
+  tags: entity.tags,
+  created_at: entity.created_at && entity.created_at.getTime() / 1000,
+  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+  factorioprints_id: entity.factorioprints_id || undefined,
+  game_version: entity.game_version || null,
+});
+
+export async function getBlueprintById(id: string): Promise<Blueprint | null> {
+  const result = await BlueprintModel.findByPk(id).catch(() => null);
+  return result ? mapBlueprintInstanceToEntry(result) : null;
+}
+
+export async function getBlueprintByHash(hash: string): Promise<Blueprint | null> {
+  const result = await BlueprintModel.findOne({
+    where: { blueprint_hash: hash },
+  }).catch(() => null);
+  return result ? mapBlueprintInstanceToEntry(result) : null;
+}
+
+export async function createBlueprint(
+  blueprint: BlueprintData,
+  extraInfo: {
+    tags: string[];
+    created_at?: number;
+    updated_at?: number;
+    factorioprints_id?: string;
+  }
+) {
+  const string = await encodeBlueprint({ blueprint });
+  const blueprint_hash = hashString(string);
+  const image_hash = hashString(getBlueprintContentForImageHash(blueprint));
+
+  const exists = await getBlueprintByHash(blueprint_hash);
+  if (exists) {
+    return { insertedId: exists.id };
+  }
+
+  // Write string to google storage
+  await saveBlueprintString(blueprint_hash, string);
+
+  // Write blueprint details to datastore
+  const result = await BlueprintModel.create({
+    label: blueprint.label,
+    description: blueprint.description,
+    blueprint_hash: blueprint_hash,
+    image_hash: image_hash,
+    tags: extraInfo.tags,
+    game_version: `${blueprint.version}`,
+    image_version: 1,
+    factorioprints_id: extraInfo.factorioprints_id,
+    updated_at: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : undefined,
+    created_at: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : undefined,
+  });
+
+  console.log(`Created Blueprint ${result.id}`);
+
+  blueprintImageRequestTopic.publishJSON({
+    blueprintId: result.id,
+  });
+
+  await BlueprintStringModel.create({
+    blueprint_id: result.id,
+    blueprint_hash: blueprint_hash,
+    image_hash: image_hash,
+    version: 1,
+    changes_markdown: null,
+  });
+
+  return { insertedId: result.id };
+}

libs/database/src/lib/data/blueprint_book.ts

@@ -0,0 +1,97 @@
+import { BlueprintBookData } from "@factorio-sites/common-utils";
+import { encodeBlueprint, hashString } from "@factorio-sites/node-utils";
+import { saveBlueprintString } from "../gcp-storage";
+import { BlueprintBookModel, BlueprintBookInstance } from "../postgres/models";
+import { BlueprintBook, ChildTree } from "../types";
+import { createBlueprint } from "./blueprint";
+
+const mapBlueprintBookEntityToObject = (entity: BlueprintBookInstance): BlueprintBook => ({
+  id: entity.id,
+  // blueprint_ids: entity.blueprint_ids.map((key: any) => key.id),
+  // blueprint_book_ids: entity.blueprint_book_ids.map((key: any) => key.id),
+  child_tree: entity.child_tree ? entity.child_tree : [],
+  blueprint_hash: entity.blueprint_hash,
+  label: entity.label,
+  description: entity.description,
+  created_at: entity.created_at && entity.created_at.getTime() / 1000,
+  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+  is_modded: entity.is_modded || false,
+  factorioprints_id: entity.factorioprints_id,
+});
+
+export async function getBlueprintBookById(id: string): Promise<BlueprintBook | null> {
+  const result = await BlueprintBookModel.findByPk(id).catch(() => null);
+  return result ? mapBlueprintBookEntityToObject(result) : null;
+}
+
+export async function getBlueprintBookByHash(hash: string): Promise<BlueprintBook | null> {
+  const result = await BlueprintBookModel.findOne({
+    where: { blueprint_hash: hash },
+  }).catch(() => null);
+  return result ? mapBlueprintBookEntityToObject(result) : null;
+}
+
+export async function createBlueprintBook(
+  blueprintBook: BlueprintBookData,
+  extraInfo: {
+    tags: string[];
+    created_at?: number;
+    updated_at?: number;
+    factorioprints_id?: string;
+  }
+): Promise<{ insertedId: string; child_tree: ChildTree }> {
+  const string = await encodeBlueprint({ blueprint_book: blueprintBook });
+  const blueprint_hash = hashString(string);
+
+  const exists = await getBlueprintBookByHash(blueprint_hash);
+  if (exists) {
+    const book = await getBlueprintBookById(exists.id);
+    if (!book) throw Error("this is impossible, just pleasing typescript");
+    return { insertedId: exists.id, child_tree: book.child_tree };
+  }
+
+  // Write string to google storage
+  await saveBlueprintString(blueprint_hash, string);
+
+  const blueprint_ids = [];
+  const blueprint_book_ids = [];
+  const child_tree: ChildTree = [];
+
+  // Create the book's child objects
+  for (let i = 0; i < blueprintBook.blueprints.length; i++) {
+    const blueprint = blueprintBook.blueprints[i];
+    if (blueprint.blueprint) {
+      const result = await createBlueprint(blueprint.blueprint, extraInfo);
+      child_tree.push({
+        type: "blueprint",
+        id: result.insertedId,
+        name: blueprint.blueprint.label,
+      });
+      blueprint_ids.push(result.insertedId);
+    } else if (blueprint.blueprint_book) {
+      const result = await createBlueprintBook(blueprint.blueprint_book, extraInfo);
+      child_tree.push({
+        type: "blueprint_book",
+        id: result.insertedId,
+        name: blueprint.blueprint_book.label,
+        children: result.child_tree,
+      });
+      blueprint_book_ids.push(result.insertedId);
+    }
+  }
+
+  const result = await BlueprintBookModel.create({
+    label: blueprintBook.label,
+    description: blueprintBook.description,
+    blueprint_hash: blueprint_hash,
+    factorioprints_id: extraInfo.factorioprints_id,
+    is_modded: false,
+    child_tree,
+    updated_at: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : undefined,
+    created_at: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : undefined,
+  });
+
+  console.log(`Created Blueprint book ${result.id}`);
+
+  return { insertedId: result.id, child_tree };
+}

libs/database/src/lib/data/blueprint_page.ts

@@ -0,0 +1,63 @@
+import { BlueprintPageModel, BlueprintPageInstance } from "../postgres/models";
+import { BlueprintPage } from "../types";
+
+const mapBlueprintPageEntityToObject = (entity: BlueprintPageInstance): BlueprintPage => ({
+  id: entity.id,
+  blueprint_id: entity.blueprint_id ?? null,
+  blueprint_book_id: entity.blueprint_book_id ?? null,
+  title: entity.title,
+  description_markdown: entity.description_markdown,
+  created_at: entity.created_at && entity.created_at.getTime() / 1000,
+  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+  factorioprints_id: entity.factorioprints_id ?? null,
+});
+
+export async function getBlueprintPageById(id: string): Promise<BlueprintPage | null> {
+  const result = await BlueprintPageModel.findByPk(id).catch(() => null);
+  return result ? mapBlueprintPageEntityToObject(result) : null;
+}
+
+export async function getBlueprintPageByFactorioprintsId(
+  id: string
+): Promise<BlueprintPage | null> {
+  const result = await BlueprintPageModel.findOne({
+    where: { factorioprints_id: id },
+  }).catch(() => null);
+  return result ? mapBlueprintPageEntityToObject(result) : null;
+}
+
+export async function getMostRecentBlueprintPages(page = 1): Promise<BlueprintPage[]> {
+  const perPage = 10;
+  const result =
+    (await BlueprintPageModel.findAll({
+      order: [["updated_at", "DESC"]],
+      limit: perPage,
+      offset: (page - 1) * perPage,
+      raw: true,
+    }).catch(() => null)) || [];
+  return result.map(mapBlueprintPageEntityToObject);
+}
+
+export async function createBlueprintPage(
+  type: "blueprint" | "blueprint_book",
+  targetId: string,
+  extraInfo: {
+    title: string;
+    description_markdown: string;
+    created_at?: number;
+    updated_at?: number;
+    factorioprints_id?: string;
+  }
+) {
+  await BlueprintPageModel.create({
+    title: extraInfo.title,
+    description_markdown: extraInfo.description_markdown,
+    factorioprints_id: extraInfo.factorioprints_id,
+    blueprint_id: type === "blueprint" ? targetId : undefined,
+    blueprint_book_id: type === "blueprint_book" ? targetId : undefined,
+    updated_at: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : undefined,
+    created_at: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : undefined,
+  });
+
+  console.log(`Created Blueprint Page`);
+}

libs/database/src/lib/data/factorioprints.ts

@@ -0,0 +1,50 @@
+import { parseBlueprintString } from "@factorio-sites/node-utils";
+import { createBlueprint } from "./blueprint";
+import { createBlueprintBook } from "./blueprint_book";
+import { createBlueprintPage } from "./blueprint_page";
+
+interface BlueprintDataFromFactorioprints {
+  description_markdown: string;
+  title: string;
+  updated_at?: number;
+  created_at?: number;
+  tags: string[];
+  factorioprints_id: string;
+}
+export async function saveBlueprintFromFactorioprints(
+  factorioprintData: BlueprintDataFromFactorioprints,
+  blueprintString: string
+) {
+  const parsed = await parseBlueprintString(blueprintString);
+
+  // not needed for inserting, just printing
+  // const { blueprints, books } = flattenBlueprintData(parsed.data);
+  // console.log(`string has ${books.length} books with ${blueprints.length} blueprints`);
+
+  const extraInfo = {
+    created_at: factorioprintData.created_at,
+    updated_at: factorioprintData.updated_at,
+    tags: factorioprintData.tags,
+    factorioprints_id: factorioprintData.factorioprints_id,
+  };
+
+  const extraInfoPage = {
+    title: factorioprintData.title,
+    description_markdown: factorioprintData.description_markdown,
+    created_at: factorioprintData.created_at,
+    updated_at: factorioprintData.updated_at,
+    factorioprints_id: factorioprintData.factorioprints_id,
+  };
+
+  if (parsed.data.blueprint) {
+    console.log("string has one blueprint...");
+    const { insertedId } = await createBlueprint(parsed.data.blueprint, extraInfo);
+    await createBlueprintPage("blueprint", insertedId, extraInfoPage);
+  } else if (parsed.data.blueprint_book) {
+    console.log("string has a blueprint book...");
+    const { insertedId } = await createBlueprintBook(parsed.data.blueprint_book, extraInfo);
+    await createBlueprintPage("blueprint_book", insertedId, extraInfoPage);
+  }
+
+  return true;
+}

libs/database/src/lib/data/index.ts

@@ -0,0 +1,4 @@
+export * from "./blueprint";
+export * from "./blueprint_book";
+export * from "./blueprint_page";
+export * from "./factorioprints";

libs/database/src/lib/datastore/gcp-datastore.ts

@@ -0,0 +1,438 @@
+// import { Datastore } from "@google-cloud/datastore";
+// import { encodeBlueprint, hashString, parseBlueprintString } from "@factorio-sites/node-utils";
+// import {
+//   BlueprintStringData,
+//   BlueprintBookData,
+//   getBlueprintContentForImageHash,
+// } from "@factorio-sites/common-utils";
+// import { getBlueprintImageRequestTopic } from "./gcp-pubsub";
+// import { saveBlueprintString } from "./gcp-storage";
+// import { BlueprintBook, Blueprint, BlueprintPage } from "./types";
+
+// const datastore = new Datastore();
+// const BlueprintEntity = "Blueprint";
+// const BlueprintBookEntity = "BlueprintBook";
+// const BlueprintPageEntity = "BlueprintPage";
+// const BlueprintStringEntity = "BlueprintString";
+// const blueprintImageRequestTopic = getBlueprintImageRequestTopic();
+
+// class DatastoreExistsError extends Error {
+//   constructor(public existingId: string, message: string) {
+//     super(message);
+//     this.name = this.constructor.name;
+//     Error.captureStackTrace(this, this.constructor);
+//   }
+// }
+
+// /*
+//  * BlueprintBook
+//  */
+
+// const mapBlueprintBookEntityToObject = (entity: any): BlueprintBook => ({
+//   id: entity[datastore.KEY].id,
+//   // blueprint_ids: entity.blueprint_ids.map((key: any) => key.id),
+//   // blueprint_book_ids: entity.blueprint_book_ids.map((key: any) => key.id),
+//   child_tree: entity.child_tree ? entity.child_tree : [],
+//   blueprint_hash: entity.blueprint_hash,
+//   label: entity.label,
+//   description: entity.description,
+//   created_at: entity.created_at && entity.created_at.getTime() / 1000,
+//   updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+//   is_modded: entity.is_modded || false,
+//   factorioprints_id: entity.factorioprints_id || null,
+// });
+
+// export async function getBlueprintBookById(id: string): Promise<BlueprintBook | null> {
+//   const result = await datastore.get(datastore.key([BlueprintBookEntity, Number(id)]));
+//   return result[0] ? mapBlueprintBookEntityToObject(result[0]) : null;
+// }
+
+// export async function getBlueprintBookByHash(hash: string): Promise<BlueprintBook | null> {
+//   const query = datastore
+//     .createQuery(BlueprintBookEntity)
+//     .filter("blueprint_hash", "=", hash)
+//     .limit(1);
+//   const [result] = await datastore.runQuery(query);
+//   return result[0] ? mapBlueprintBookEntityToObject(result[0]) : null;
+// }
+
+// export async function createBlueprintBook(
+//   blueprintBook: BlueprintBookData,
+//   extraInfo: { tags: string[]; created_at: number; updated_at: number; factorioprints_id: string }
+// ): Promise<{ insertedId: string; child_tree: BlueprintBook["child_tree"] }> {
+//   const string = await encodeBlueprint({ blueprint_book: blueprintBook });
+//   const blueprint_hash = hashString(string);
+
+//   const exists = await getBlueprintBookByHash(blueprint_hash);
+//   if (exists) {
+//     const book = await getBlueprintBookById(exists.id);
+//     if (!book) throw Error("this is impossible, just pleasing typescript");
+//     return { insertedId: exists.id, child_tree: book.child_tree };
+//   }
+
+//   // Write string to google storage
+//   await saveBlueprintString(blueprint_hash, string);
+
+//   const blueprint_ids = [];
+//   const blueprint_book_ids = [];
+//   const child_tree: BlueprintBook["child_tree"] = [];
+
+//   // Create the book's child objects
+//   for (let i = 0; i < blueprintBook.blueprints.length; i++) {
+//     const blueprint = blueprintBook.blueprints[i];
+//     if (blueprint.blueprint) {
+//       const result = await createBlueprint(blueprint.blueprint, extraInfo);
+//       child_tree.push({
+//         type: "blueprint",
+//         id: result.insertedId,
+//         name: blueprint.blueprint.label,
+//       });
+//       blueprint_ids.push(result.insertedId);
+//     } else if (blueprint.blueprint_book) {
+//       const result = await createBlueprintBook(blueprint.blueprint_book, extraInfo);
+//       child_tree.push({
+//         type: "blueprint_book",
+//         id: result.insertedId,
+//         name: blueprint.blueprint_book.label,
+//         children: result.child_tree,
+//       });
+//       blueprint_book_ids.push(result.insertedId);
+//     }
+//   }
+
+//   // Write blueprint details to datastore
+//   const [result] = await datastore.insert({
+//     key: datastore.key([BlueprintBookEntity]),
+//     data: [
+//       {
+//         name: "blueprint_ids",
+//         value: blueprint_ids.map((id) => datastore.key([BlueprintEntity, datastore.int(id)])),
+//       },
+//       {
+//         name: "blueprint_book_ids",
+//         value: blueprint_book_ids.map((id) =>
+//           datastore.key([BlueprintBookEntity, datastore.int(id)])
+//         ),
+//       },
+//       { name: "child_tree", value: child_tree, excludeFromIndexes: true },
+//       { name: "label", value: blueprintBook.label },
+//       {
+//         name: "description",
+//         value: blueprintBook.description || null,
+//         excludeFromIndexes: true,
+//       },
+//       { name: "blueprint_hash", value: blueprint_hash },
+//       {
+//         name: "created_at",
+//         value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : null,
+//         excludeFromIndexes: true,
+//       },
+//       {
+//         name: "updated_at",
+//         value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : null,
+//         excludeFromIndexes: true,
+//       },
+//       { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
+//     ],
+//   });
+
+//   const insertedId = String(result.mutationResults?.[0]?.key?.path?.[0]?.id) as string;
+//   if (!insertedId) throw Error("Something went wrong inserting entity");
+
+//   console.log(`Created Blueprint book ${insertedId}`);
+
+//   return { insertedId, child_tree };
+// }
+
+// /*
+//  * Blueprint
+//  */
+
+// const mapBlueprintEntityToObject = (entity: any): Blueprint => ({
+//   id: entity[datastore.KEY].id,
+//   blueprint_hash: entity.blueprint_hash,
+//   image_hash: entity.image_hash,
+//   label: entity.label,
+//   description: entity.description,
+//   tags: entity.tags,
+//   created_at: entity.created_at && entity.created_at.getTime() / 1000,
+//   updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+//   factorioprints_id: entity.factorioprints_id || null,
+//   game_version: entity.game_version,
+// });
+
+// const mapBlueprintObjectToEntity = (object: Omit<Blueprint, "id"> & { id?: string }): any => ({
+//   key: datastore.key(object.id ? [BlueprintEntity, datastore.int(object.id)] : [BlueprintEntity]),
+//   data: [
+//     { name: "label", value: object.label },
+//     {
+//       name: "description",
+//       value: object.description || null,
+//       excludeFromIndexes: true,
+//     },
+//     { name: "blueprint_hash", value: object.blueprint_hash },
+//     { name: "image_hash", value: object.image_hash },
+//     { name: "tags", value: object.tags || [] },
+//     {
+//       name: "created_at",
+//       value: object.created_at ? new Date(object.created_at * 1000) : new Date(),
+//       excludeFromIndexes: true,
+//     },
+//     {
+//       name: "updated_at",
+//       value: object.updated_at ? new Date(object.updated_at * 1000) : new Date(),
+//       excludeFromIndexes: true,
+//     },
+//     { name: "game_version", value: object.game_version, excludeFromIndexes: true },
+//     { name: "factorioprints_id", value: object.factorioprints_id || null },
+//   ],
+// });
+
+// export async function getBlueprintById(id: string): Promise<Blueprint | null> {
+//   const result = await datastore.get(datastore.key([BlueprintEntity, datastore.int(id)]));
+//   return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
+// }
+
+// export async function getBlueprintByHash(hash: string): Promise<Blueprint | null> {
+//   const query = datastore.createQuery(BlueprintEntity).filter("blueprint_hash", "=", hash).limit(1);
+//   const [result] = await datastore.runQuery(query);
+//   return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
+// }
+
+// export async function getBlueprintByImageHash(hash: string): Promise<Blueprint | null> {
+//   const query = datastore.createQuery(BlueprintEntity).filter("image_hash", "=", hash).limit(1);
+//   const [result] = await datastore.runQuery(query);
+//   return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
+// }
+
+// export async function createBlueprint(
+//   blueprint: BlueprintStringData,
+//   extraInfo: { tags: string[]; created_at: number; updated_at: number; factorioprints_id: string }
+// ) {
+//   const string = await encodeBlueprint({ blueprint });
+//   const blueprint_hash = hashString(string);
+//   const image_hash = hashString(getBlueprintContentForImageHash(blueprint));
+
+//   const exists = await getBlueprintByHash(blueprint_hash);
+//   if (exists) {
+//     return { insertedId: exists.id };
+//   }
+
+//   // Write string to google storage
+//   await saveBlueprintString(blueprint_hash, string);
+
+//   // Write blueprint details to datastore
+//   const [result] = await datastore.insert({
+//     key: datastore.key([BlueprintEntity]),
+//     data: [
+//       { name: "label", value: blueprint.label },
+//       {
+//         name: "description",
+//         value: blueprint.description || null,
+//         excludeFromIndexes: true,
+//       },
+//       { name: "blueprint_hash", value: blueprint_hash },
+//       { name: "image_hash", value: image_hash },
+//       { name: "tags", value: extraInfo.tags ? extraInfo.tags : [] },
+//       {
+//         name: "created_at",
+//         value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
+//         excludeFromIndexes: true,
+//       },
+//       {
+//         name: "updated_at",
+//         value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
+//         excludeFromIndexes: true,
+//       },
+//       { name: "game_version", value: blueprint.version, excludeFromIndexes: true },
+//       { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
+//     ],
+//   });
+
+//   const insertedId = String(result.mutationResults?.[0]?.key?.path?.[0]?.id);
+//   if (!insertedId) throw Error("Something went wrong inserting entity");
+
+//   console.log(`Created Blueprint ${insertedId}`);
+
+//   blueprintImageRequestTopic.publishJSON({
+//     blueprintId: insertedId,
+//   });
+
+//   await datastore.insert({
+//     key: datastore.key([BlueprintEntity, datastore.int(insertedId), BlueprintStringEntity]),
+//     data: [
+//       { name: "blueprint_hash", value: blueprint_hash },
+//       { name: "image_hash", value: image_hash },
+//       { name: "version", value: 1 },
+//       { name: "changes_markdown", value: null },
+//       {
+//         name: "created_at",
+//         value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
+//         excludeFromIndexes: true,
+//       },
+//     ],
+//   });
+
+//   return { insertedId };
+// }
+
+// export async function updateBlueprint(blueprint: Blueprint) {
+//   datastore.save(mapBlueprintObjectToEntity(blueprint));
+// }
+
+// /*
+//  * BlueprintPage
+//  */
+// const mapBlueprintPageEntityToObject = (entity: any): BlueprintPage => ({
+//   id: entity[datastore.KEY].id,
+//   blueprint_id: entity.blueprint_id ? entity.blueprint_id.id : null,
+//   blueprint_book_id: entity.blueprint_book_id ? entity.blueprint_book_id.id : null,
+//   title: entity.title,
+//   description_markdown: entity.description_markdown,
+//   created_at: entity.created_at && entity.created_at.getTime() / 1000,
+//   updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+//   factorioprints_id: entity.factorioprints_id || null,
+// });
+
+// export async function getBlueprintPageById(id: string): Promise<BlueprintPage | null> {
+//   const result = await datastore.get(datastore.key([BlueprintPageEntity, datastore.int(id)]));
+//   return result[0] ? mapBlueprintPageEntityToObject(result[0]) : null;
+// }
+
+// export async function getBlueprintPageByFactorioprintsId(
+//   id: string
+// ): Promise<BlueprintPage | null> {
+//   const query = datastore
+//     .createQuery(BlueprintPageEntity)
+//     .filter("factorioprints_id", "=", id)
+//     .limit(1);
+//   const [result] = await datastore.runQuery(query);
+//   return result[0] ? mapBlueprintPageEntityToObject(result[0]) : null;
+// }
+
+// async function createBlueprintPage(
+//   type: "blueprint" | "blueprint_book",
+//   targetId: string,
+//   extraInfo: {
+//     title: string;
+//     description_markdown: string;
+//     created_at: number;
+//     updated_at: number;
+//     factorioprints_id?: string;
+//   }
+// ) {
+//   const insertData: any = [
+//     { name: "title", value: extraInfo.title },
+//     {
+//       name: "description_markdown",
+//       value: extraInfo.description_markdown,
+//       excludeFromIndexes: true,
+//     },
+//     {
+//       name: "created_at",
+//       value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
+//       excludeFromIndexes: true,
+//     },
+//     {
+//       name: "updated_at",
+//       value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
+//     },
+//     { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
+//   ];
+
+//   if (type === "blueprint") {
+//     insertData.push({
+//       name: "blueprint_id",
+//       value: datastore.key([BlueprintEntity, datastore.int(targetId)]),
+//     });
+//   } else if (type === "blueprint_book") {
+//     insertData.push({
+//       name: "blueprint_book_id",
+//       value: datastore.key([BlueprintBookEntity, datastore.int(targetId)]),
+//     });
+//   } else {
+//     throw Error("Invalid type given");
+//   }
+
+//   await datastore.insert({
+//     key: datastore.key([BlueprintPageEntity]),
+//     data: insertData,
+//   });
+
+//   console.log(`Created Blueprint Page`);
+// }
+
+// /*
+//  * Other
+//  */
+
+// interface BlueprintDataFromFactorioprints {
+//   description_markdown: string;
+//   title: string;
+//   updated_at: number;
+//   created_at: number;
+//   tags: string[];
+//   factorioprints_id: string;
+// }
+// export async function saveBlueprintFromFactorioprints(
+//   factorioprintData: BlueprintDataFromFactorioprints,
+//   blueprintString: string
+// ) {
+//   const parsed = await parseBlueprintString(blueprintString);
+
+//   // not needed for inserting, just printing
+//   // const { blueprints, books } = flattenBlueprintData(parsed.data);
+//   // console.log(`string has ${books.length} books with ${blueprints.length} blueprints`);
+
+//   const extraInfo = {
+//     created_at: factorioprintData.created_at,
+//     updated_at: factorioprintData.updated_at,
+//     tags: factorioprintData.tags,
+//     factorioprints_id: factorioprintData.factorioprints_id,
+//   };
+
+//   const extraInfoPage = {
+//     title: factorioprintData.title,
+//     description_markdown: factorioprintData.description_markdown,
+//     created_at: factorioprintData.created_at,
+//     updated_at: factorioprintData.updated_at,
+//     factorioprints_id: factorioprintData.factorioprints_id,
+//   };
+
+//   if (parsed.data.blueprint) {
+//     console.log("string has one blueprint...");
+//     const { insertedId } = await createBlueprint(parsed.data.blueprint, extraInfo).catch(
+//       (error) => {
+//         if (error instanceof DatastoreExistsError) {
+//           console.log(`Blueprint already exists with id ${error.existingId}`);
+//           return { insertedId: error.existingId };
+//         } else throw error;
+//       }
+//     );
+//     await createBlueprintPage("blueprint", insertedId, extraInfoPage);
+//   } else if (parsed.data.blueprint_book) {
+//     console.log("string has a blueprint book...");
+//     const { insertedId } = await createBlueprintBook(parsed.data.blueprint_book, extraInfo);
+//     await createBlueprintPage("blueprint_book", insertedId, extraInfoPage);
+//   }
+//   return true;
+// }
+
+// export async function getMostRecentBlueprintPages(page = 1): Promise<BlueprintPage[]> {
+//   const perPage = 10;
+//   const query = datastore
+//     .createQuery(BlueprintPageEntity)
+//     .limit(perPage)
+//     .offset((page - 1) * perPage);
+//   const [result] = await datastore.runQuery(query);
+//   return result.map(mapBlueprintPageEntityToObject);
+// }
+
+// export async function getPaginatedBlueprints(page = 1, perPage = 10): Promise<Blueprint[]> {
+//   const query = datastore
+//     .createQuery(BlueprintEntity)
+//     .limit(perPage)
+//     .offset((page - 1) * perPage);
+//   const [result] = await datastore.runQuery(query);
+//   return result.map(mapBlueprintEntityToObject);
+// }

libs/database/src/lib/gcp-datastore.ts

@@ -1,438 +0,0 @@
-import { Datastore } from "@google-cloud/datastore";
-import { encodeBlueprint, hashString, parseBlueprintString } from "@factorio-sites/node-utils";
-import {
-  Blueprint,
-  BlueprintBook,
-  getBlueprintContentForImageHash,
-} from "@factorio-sites/common-utils";
-import { getBlueprintImageRequestTopic } from "./gcp-pubsub";
-import { saveBlueprintString } from "./gcp-storage";
-import { BlueprintBookEntry, BlueprintEntry, BlueprintPageEntry } from "./types";
-
-const datastore = new Datastore();
-const BlueprintEntity = "Blueprint";
-const BlueprintBookEntity = "BlueprintBook";
-const BlueprintPageEntity = "BlueprintPage";
-const BlueprintStringEntity = "BlueprintString";
-const blueprintImageRequestTopic = getBlueprintImageRequestTopic();
-
-class DatastoreExistsError extends Error {
-  constructor(public existingId: string, message: string) {
-    super(message);
-    this.name = this.constructor.name;
-    Error.captureStackTrace(this, this.constructor);
-  }
-}
-
-/*
- * BlueprintBook
- */
-
-const mapBlueprintBookEntityToObject = (entity: any): BlueprintBookEntry => ({
-  id: entity[datastore.KEY].id,
-  blueprint_ids: entity.blueprint_ids.map((key: any) => key.id),
-  blueprint_book_ids: entity.blueprint_book_ids.map((key: any) => key.id),
-  child_tree: entity.child_tree ? entity.child_tree : [],
-  blueprint_hash: entity.blueprint_hash,
-  label: entity.label,
-  description: entity.description,
-  created_at: entity.created_at && entity.created_at.getTime() / 1000,
-  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
-  is_modded: entity.is_modded || false,
-  factorioprints_id: entity.factorioprints_id || null,
-});
-
-export async function getBlueprintBookById(id: string): Promise<BlueprintBookEntry | null> {
-  const result = await datastore.get(datastore.key([BlueprintBookEntity, Number(id)]));
-  return result[0] ? mapBlueprintBookEntityToObject(result[0]) : null;
-}
-
-export async function getBlueprintBookByHash(hash: string): Promise<BlueprintBookEntry | null> {
-  const query = datastore
-    .createQuery(BlueprintBookEntity)
-    .filter("blueprint_hash", "=", hash)
-    .limit(1);
-  const [result] = await datastore.runQuery(query);
-  return result[0] ? mapBlueprintBookEntityToObject(result[0]) : null;
-}
-
-export async function createBlueprintBook(
-  blueprintBook: BlueprintBook,
-  extraInfo: { tags: string[]; created_at: number; updated_at: number; factorioprints_id: string }
-): Promise<{ insertedId: string; child_tree: BlueprintBookEntry["child_tree"] }> {
-  const string = await encodeBlueprint({ blueprint_book: blueprintBook });
-  const blueprint_hash = hashString(string);
-
-  const exists = await getBlueprintBookByHash(blueprint_hash);
-  if (exists) {
-    const book = await getBlueprintBookById(exists.id);
-    if (!book) throw Error("this is impossible, just pleasing typescript");
-    return { insertedId: exists.id, child_tree: book.child_tree };
-  }
-
-  // Write string to google storage
-  await saveBlueprintString(blueprint_hash, string);
-
-  const blueprint_ids = [];
-  const blueprint_book_ids = [];
-  const child_tree: BlueprintBookEntry["child_tree"] = [];
-
-  // Create the book's child objects
-  for (let i = 0; i < blueprintBook.blueprints.length; i++) {
-    const blueprint = blueprintBook.blueprints[i];
-    if (blueprint.blueprint) {
-      const result = await createBlueprint(blueprint.blueprint, extraInfo);
-      child_tree.push({
-        type: "blueprint",
-        id: result.insertedId,
-        name: blueprint.blueprint.label,
-      });
-      blueprint_ids.push(result.insertedId);
-    } else if (blueprint.blueprint_book) {
-      const result = await createBlueprintBook(blueprint.blueprint_book, extraInfo);
-      child_tree.push({
-        type: "blueprint_book",
-        id: result.insertedId,
-        name: blueprint.blueprint_book.label,
-        children: result.child_tree,
-      });
-      blueprint_book_ids.push(result.insertedId);
-    }
-  }
-
-  // Write blueprint details to datastore
-  const [result] = await datastore.insert({
-    key: datastore.key([BlueprintBookEntity]),
-    data: [
-      {
-        name: "blueprint_ids",
-        value: blueprint_ids.map((id) => datastore.key([BlueprintEntity, datastore.int(id)])),
-      },
-      {
-        name: "blueprint_book_ids",
-        value: blueprint_book_ids.map((id) =>
-          datastore.key([BlueprintBookEntity, datastore.int(id)])
-        ),
-      },
-      { name: "child_tree", value: child_tree, excludeFromIndexes: true },
-      { name: "label", value: blueprintBook.label },
-      {
-        name: "description",
-        value: blueprintBook.description || null,
-        excludeFromIndexes: true,
-      },
-      { name: "blueprint_hash", value: blueprint_hash },
-      {
-        name: "created_at",
-        value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : null,
-        excludeFromIndexes: true,
-      },
-      {
-        name: "updated_at",
-        value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : null,
-        excludeFromIndexes: true,
-      },
-      { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
-    ],
-  });
-
-  const insertedId = String(result.mutationResults?.[0]?.key?.path?.[0]?.id) as string;
-  if (!insertedId) throw Error("Something went wrong inserting entity");
-
-  console.log(`Created Blueprint book ${insertedId}`);
-
-  return { insertedId, child_tree };
-}
-
-/*
- * Blueprint
- */
-
-const mapBlueprintEntityToObject = (entity: any): BlueprintEntry => ({
-  id: entity[datastore.KEY].id,
-  blueprint_hash: entity.blueprint_hash,
-  image_hash: entity.image_hash,
-  label: entity.label,
-  description: entity.description,
-  tags: entity.tags,
-  created_at: entity.created_at && entity.created_at.getTime() / 1000,
-  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
-  factorioprints_id: entity.factorioprints_id || null,
-  game_version: entity.game_version,
-});
-
-const mapBlueprintObjectToEntity = (object: Omit<BlueprintEntry, "id"> & { id?: string }): any => ({
-  key: datastore.key(object.id ? [BlueprintEntity, datastore.int(object.id)] : [BlueprintEntity]),
-  data: [
-    { name: "label", value: object.label },
-    {
-      name: "description",
-      value: object.description || null,
-      excludeFromIndexes: true,
-    },
-    { name: "blueprint_hash", value: object.blueprint_hash },
-    { name: "image_hash", value: object.image_hash },
-    { name: "tags", value: object.tags || [] },
-    {
-      name: "created_at",
-      value: object.created_at ? new Date(object.created_at * 1000) : new Date(),
-      excludeFromIndexes: true,
-    },
-    {
-      name: "updated_at",
-      value: object.updated_at ? new Date(object.updated_at * 1000) : new Date(),
-      excludeFromIndexes: true,
-    },
-    { name: "game_version", value: object.game_version, excludeFromIndexes: true },
-    { name: "factorioprints_id", value: object.factorioprints_id || null },
-  ],
-});
-
-export async function getBlueprintById(id: string): Promise<BlueprintEntry | null> {
-  const result = await datastore.get(datastore.key([BlueprintEntity, datastore.int(id)]));
-  return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
-}
-
-export async function getBlueprintByHash(hash: string): Promise<BlueprintEntry | null> {
-  const query = datastore.createQuery(BlueprintEntity).filter("blueprint_hash", "=", hash).limit(1);
-  const [result] = await datastore.runQuery(query);
-  return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
-}
-
-export async function getBlueprintByImageHash(hash: string): Promise<BlueprintEntry | null> {
-  const query = datastore.createQuery(BlueprintEntity).filter("image_hash", "=", hash).limit(1);
-  const [result] = await datastore.runQuery(query);
-  return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
-}
-
-export async function createBlueprint(
-  blueprint: Blueprint,
-  extraInfo: { tags: string[]; created_at: number; updated_at: number; factorioprints_id: string }
-) {
-  const string = await encodeBlueprint({ blueprint });
-  const blueprint_hash = hashString(string);
-  const image_hash = hashString(getBlueprintContentForImageHash(blueprint));
-
-  const exists = await getBlueprintByHash(blueprint_hash);
-  if (exists) {
-    return { insertedId: exists.id };
-  }
-
-  // Write string to google storage
-  await saveBlueprintString(blueprint_hash, string);
-
-  // Write blueprint details to datastore
-  const [result] = await datastore.insert({
-    key: datastore.key([BlueprintEntity]),
-    data: [
-      { name: "label", value: blueprint.label },
-      {
-        name: "description",
-        value: blueprint.description || null,
-        excludeFromIndexes: true,
-      },
-      { name: "blueprint_hash", value: blueprint_hash },
-      { name: "image_hash", value: image_hash },
-      { name: "tags", value: extraInfo.tags ? extraInfo.tags : [] },
-      {
-        name: "created_at",
-        value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
-        excludeFromIndexes: true,
-      },
-      {
-        name: "updated_at",
-        value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
-        excludeFromIndexes: true,
-      },
-      { name: "game_version", value: blueprint.version, excludeFromIndexes: true },
-      { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
-    ],
-  });
-
-  const insertedId = String(result.mutationResults?.[0]?.key?.path?.[0]?.id);
-  if (!insertedId) throw Error("Something went wrong inserting entity");
-
-  console.log(`Created Blueprint ${insertedId}`);
-
-  blueprintImageRequestTopic.publishJSON({
-    blueprintId: insertedId,
-  });
-
-  await datastore.insert({
-    key: datastore.key([BlueprintEntity, datastore.int(insertedId), BlueprintStringEntity]),
-    data: [
-      { name: "blueprint_hash", value: blueprint_hash },
-      { name: "image_hash", value: image_hash },
-      { name: "version", value: 1 },
-      { name: "changes_markdown", value: null },
-      {
-        name: "created_at",
-        value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
-        excludeFromIndexes: true,
-      },
-    ],
-  });
-
-  return { insertedId };
-}
-
-export async function updateBlueprint(blueprint: BlueprintEntry) {
-  datastore.save(mapBlueprintObjectToEntity(blueprint));
-}
-
-/*
- * BlueprintPage
- */
-const mapBlueprintPageEntityToObject = (entity: any): BlueprintPageEntry => ({
-  id: entity[datastore.KEY].id,
-  blueprint_id: entity.blueprint_id ? entity.blueprint_id.id : null,
-  blueprint_book_id: entity.blueprint_book_id ? entity.blueprint_book_id.id : null,
-  title: entity.title,
-  description_markdown: entity.description_markdown,
-  created_at: entity.created_at && entity.created_at.getTime() / 1000,
-  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
-  factorioprints_id: entity.factorioprints_id || null,
-});
-
-export async function getBlueprintPageById(id: string): Promise<BlueprintPageEntry | null> {
-  const result = await datastore.get(datastore.key([BlueprintPageEntity, datastore.int(id)]));
-  return result[0] ? mapBlueprintPageEntityToObject(result[0]) : null;
-}
-
-export async function getBlueprintPageByFactorioprintsId(
-  id: string
-): Promise<BlueprintPageEntry | null> {
-  const query = datastore
-    .createQuery(BlueprintPageEntity)
-    .filter("factorioprints_id", "=", id)
-    .limit(1);
-  const [result] = await datastore.runQuery(query);
-  return result[0] ? mapBlueprintPageEntityToObject(result[0]) : null;
-}
-
-async function createBlueprintPage(
-  type: "blueprint" | "blueprint_book",
-  targetId: string,
-  extraInfo: {
-    title: string;
-    description_markdown: string;
-    created_at: number;
-    updated_at: number;
-    factorioprints_id?: string;
-  }
-) {
-  const insertData: any = [
-    { name: "title", value: extraInfo.title },
-    {
-      name: "description_markdown",
-      value: extraInfo.description_markdown,
-      excludeFromIndexes: true,
-    },
-    {
-      name: "created_at",
-      value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
-      excludeFromIndexes: true,
-    },
-    {
-      name: "updated_at",
-      value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
-    },
-    { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
-  ];
-
-  if (type === "blueprint") {
-    insertData.push({
-      name: "blueprint_id",
-      value: datastore.key([BlueprintEntity, datastore.int(targetId)]),
-    });
-  } else if (type === "blueprint_book") {
-    insertData.push({
-      name: "blueprint_book_id",
-      value: datastore.key([BlueprintBookEntity, datastore.int(targetId)]),
-    });
-  } else {
-    throw Error("Invalid type given");
-  }
-
-  await datastore.insert({
-    key: datastore.key([BlueprintPageEntity]),
-    data: insertData,
-  });
-
-  console.log(`Created Blueprint Page`);
-}
-
-/*
- * Other
- */
-
-interface BlueprintDataFromFactorioprints {
-  description_markdown: string;
-  title: string;
-  updated_at: number;
-  created_at: number;
-  tags: string[];
-  factorioprints_id: string;
-}
-export async function saveBlueprintFromFactorioprints(
-  factorioprintData: BlueprintDataFromFactorioprints,
-  blueprintString: string
-) {
-  const parsed = await parseBlueprintString(blueprintString);
-
-  // not needed for inserting, just printing
-  // const { blueprints, books } = flattenBlueprintData(parsed.data);
-  // console.log(`string has ${books.length} books with ${blueprints.length} blueprints`);
-
-  const extraInfo = {
-    created_at: factorioprintData.created_at,
-    updated_at: factorioprintData.updated_at,
-    tags: factorioprintData.tags,
-    factorioprints_id: factorioprintData.factorioprints_id,
-  };
-
-  const extraInfoPage = {
-    title: factorioprintData.title,
-    description_markdown: factorioprintData.description_markdown,
-    created_at: factorioprintData.created_at,
-    updated_at: factorioprintData.updated_at,
-    factorioprints_id: factorioprintData.factorioprints_id,
-  };
-
-  if (parsed.data.blueprint) {
-    console.log("string has one blueprint...");
-    const { insertedId } = await createBlueprint(parsed.data.blueprint, extraInfo).catch(
-      (error) => {
-        if (error instanceof DatastoreExistsError) {
-          console.log(`Blueprint already exists with id ${error.existingId}`);
-          return { insertedId: error.existingId };
-        } else throw error;
-      }
-    );
-    await createBlueprintPage("blueprint", insertedId, extraInfoPage);
-  } else if (parsed.data.blueprint_book) {
-    console.log("string has a blueprint book...");
-    const { insertedId } = await createBlueprintBook(parsed.data.blueprint_book, extraInfo);
-    await createBlueprintPage("blueprint_book", insertedId, extraInfoPage);
-  }
-  return true;
-}
-
-export async function getMostRecentBlueprintPages(page = 1): Promise<BlueprintPageEntry[]> {
-  const perPage = 10;
-  const query = datastore
-    .createQuery(BlueprintPageEntity)
-    .limit(perPage)
-    .offset((page - 1) * perPage);
-  const [result] = await datastore.runQuery(query);
-  return result.map(mapBlueprintPageEntityToObject);
-}
-
-export async function getPaginatedBlueprints(page = 1, perPage = 10): Promise<BlueprintEntry[]> {
-  const query = datastore
-    .createQuery(BlueprintEntity)
-    .limit(perPage)
-    .offset((page - 1) * perPage);
-  const [result] = await datastore.runQuery(query);
-  return result.map(mapBlueprintEntityToObject);
-}

libs/database/src/lib/postgres/connection.ts

@@ -1,16 +0,0 @@
-import { createConnection } from "typeorm";
-import { Blueprint } from "./entities";
-
-export async function init() {
-  await createConnection({
-    type: "postgres",
-    host: "127.0.0.1",
-    port: 5432,
-    username: "postgres",
-    password: "local",
-    database: "factorio-blueprints",
-    entities: [Blueprint],
-  });
-}
-
-init();

libs/database/src/lib/postgres/database.ts

@@ -0,0 +1,61 @@
+import { sequelize } from "./sequelize";
+import { BlueprintModel } from "./models/Blueprint";
+import { BlueprintStringModel } from "./models/BlueprintString";
+import { BlueprintBookModel } from "./models/BlueprintBook";
+import { BlueprintPageModel } from "./models/BlueprintPage";
+
+export const init = async () => {
+  BlueprintModel.hasOne(BlueprintStringModel, { foreignKey: "blueprint_id" });
+  BlueprintModel.hasMany(BlueprintPageModel, { foreignKey: "blueprint_id" });
+
+  BlueprintStringModel.belongsTo(BlueprintModel, { foreignKey: "blueprint_id" });
+
+  BlueprintPageModel.belongsTo(BlueprintModel, { foreignKey: "blueprint_id" });
+  BlueprintPageModel.belongsTo(BlueprintBookModel, { foreignKey: "blueprint_book_id" });
+
+  BlueprintBookModel.hasMany(BlueprintPageModel, { foreignKey: "blueprint_book_id" });
+  BlueprintBookModel.belongsToMany(BlueprintBookModel, {
+    through: "blueprint_book_books",
+    as: "blueprint_books",
+    foreignKey: "blueprint_book_1_id",
+    otherKey: "blueprint_book_2_id",
+    timestamps: false,
+  });
+  BlueprintBookModel.belongsToMany(BlueprintModel, {
+    through: "blueprint_book_blueprints",
+    as: "blueprints",
+    foreignKey: "blueprint_book_id",
+    otherKey: "blueprint_id",
+    timestamps: false,
+  });
+
+  if (process.env.SYNCHRONIZE_DB) {
+    console.log(`[SYNCHRONIZE_DB] syncing database ${process.env.POSTGRES_DB}`);
+    await BlueprintModel.sync({ force: true });
+    await BlueprintStringModel.sync({ force: true });
+    await BlueprintBookModel.sync({ force: true });
+    await BlueprintPageModel.sync({ force: true });
+    await sequelize.models["blueprint_book_books"].sync({ force: true });
+    await sequelize.models["blueprint_book_blueprints"].sync({ force: true });
+
+    // SEED
+    {
+      const bp = await BlueprintModel.create({
+        blueprint_hash: "blueprint_hash",
+        label: "label",
+        image_hash: "image-hash",
+        image_version: 1,
+        tags: ["tag1"],
+      });
+      await BlueprintStringModel.create({
+        blueprint_hash: "blueprint_hash",
+        blueprint_id: bp.id,
+        changes_markdown: "markdown",
+        image_hash: "image-hash",
+        version: 1,
+      });
+    }
+  }
+
+  return sequelize;
+};

libs/database/src/lib/postgres/entities.ts

@@ -1,37 +0,0 @@
-import { Entity, PrimaryGeneratedColumn, Column } from "typeorm";
-
-@Entity()
-export class Blueprint {
-  @PrimaryGeneratedColumn("uuid")
-  id!: number;
-
-  @Column("text")
-  label?: string;
-
-  @Column("text", { nullable: true })
-  description?: string;
-
-  @Column("text")
-  game_version?: string;
-
-  @Column("text", { unique: true })
-  blueprint_hash?: string;
-
-  @Column("varchar", { length: 40 })
-  image_hash?: string;
-
-  @Column("text")
-  image_version?: string;
-
-  @Column("time without time zone")
-  created_at?: number;
-
-  @Column("time without time zone")
-  updated_at?: number;
-
-  @Column("text", { array: true })
-  tags?: string;
-
-  @Column("text", { nullable: true })
-  factorioprints_id?: number;
-}

libs/database/src/lib/postgres/models/Blueprint.ts

@@ -0,0 +1,67 @@
+import { DataTypes, UUIDV4, Optional, Model } from "sequelize";
+import { sequelize } from "../sequelize";
+
+interface BlueprintAttributes {
+  id: string;
+  label?: string;
+  description?: string;
+  game_version?: string;
+  blueprint_hash: string;
+  image_hash: string;
+  image_version: number;
+  tags: string[];
+  factorioprints_id?: string;
+  created_at: Date;
+  updated_at: Date;
+}
+
+export interface BlueprintInstance
+  extends Model<
+      Omit<BlueprintAttributes, "created_at" | "updated_at">,
+      Optional<BlueprintAttributes, "id" | "created_at" | "updated_at">
+    >,
+    BlueprintAttributes {}
+
+export const BlueprintModel = sequelize.define<BlueprintInstance>(
+  "blueprint",
+  {
+    id: {
+      primaryKey: true,
+      type: DataTypes.UUID,
+      defaultValue: UUIDV4,
+    },
+    label: {
+      type: DataTypes.STRING,
+    },
+    description: {
+      type: DataTypes.TEXT,
+    },
+    game_version: {
+      type: DataTypes.STRING,
+    },
+    blueprint_hash: {
+      type: DataTypes.STRING(40),
+      unique: true,
+      allowNull: false,
+    },
+    image_hash: {
+      type: DataTypes.STRING(40),
+      allowNull: false,
+    },
+    image_version: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+      defaultValue: 1,
+    },
+    tags: {
+      type: DataTypes.ARRAY(DataTypes.STRING),
+      set(value: string[]) {
+        this.setDataValue("tags", Array.isArray(value) ? value : []);
+      },
+    },
+    factorioprints_id: {
+      type: DataTypes.STRING,
+    },
+  },
+  {}
+);

libs/database/src/lib/postgres/models/BlueprintBook.ts

@@ -0,0 +1,55 @@
+import { DataTypes, UUIDV4, Optional, Model } from "sequelize";
+import { ChildTree } from "../../types";
+import { sequelize } from "../sequelize";
+
+interface BlueprintBookAttributes {
+  id: string;
+  label: string;
+  description?: string;
+  child_tree: ChildTree;
+  blueprint_hash: string;
+  is_modded: boolean;
+  factorioprints_id?: string;
+  created_at: Date;
+  updated_at: Date;
+}
+
+export interface BlueprintBookInstance
+  extends Model<
+      Omit<BlueprintBookAttributes, "created_at" | "updated_at">,
+      Optional<BlueprintBookAttributes, "id" | "created_at" | "updated_at">
+    >,
+    BlueprintBookAttributes {}
+
+export const BlueprintBookModel = sequelize.define<BlueprintBookInstance>(
+  "blueprint_book",
+  {
+    id: {
+      primaryKey: true,
+      type: DataTypes.UUID,
+      defaultValue: UUIDV4,
+    },
+    label: {
+      type: DataTypes.STRING,
+    },
+    description: {
+      type: DataTypes.TEXT,
+    },
+    child_tree: {
+      type: DataTypes.JSON,
+      allowNull: false,
+    },
+    blueprint_hash: {
+      type: DataTypes.STRING(40),
+      allowNull: false,
+    },
+    is_modded: {
+      type: DataTypes.BOOLEAN,
+      allowNull: false,
+    },
+    factorioprints_id: {
+      type: DataTypes.STRING,
+    },
+  },
+  {}
+);

libs/database/src/lib/postgres/models/BlueprintPage.ts

@@ -0,0 +1,48 @@
+import { DataTypes, UUIDV4, Optional, Model } from "sequelize";
+import { sequelize } from "../sequelize";
+
+interface BlueprintPageAttributes {
+  id: string;
+  blueprint_id?: string;
+  blueprint_book_id?: string;
+  title: string;
+  description_markdown: string;
+  factorioprints_id?: string;
+  created_at: Date;
+  updated_at: Date;
+}
+
+export interface BlueprintPageInstance
+  extends Model<
+      Omit<BlueprintPageAttributes, "created_at" | "updated_at">,
+      Optional<BlueprintPageAttributes, "id" | "created_at" | "updated_at">
+    >,
+    BlueprintPageAttributes {}
+
+export const BlueprintPageModel = sequelize.define<BlueprintPageInstance>(
+  "blueprint_page",
+  {
+    id: {
+      primaryKey: true,
+      type: DataTypes.UUID,
+      defaultValue: UUIDV4,
+    },
+    blueprint_id: {
+      type: DataTypes.UUID,
+    },
+    blueprint_book_id: {
+      type: DataTypes.UUID,
+    },
+    title: {
+      type: DataTypes.STRING,
+      allowNull: false,
+    },
+    description_markdown: {
+      type: DataTypes.TEXT,
+    },
+    factorioprints_id: {
+      type: DataTypes.STRING,
+    },
+  },
+  {}
+);

libs/database/src/lib/postgres/models/BlueprintString.ts

@@ -0,0 +1,52 @@
+import { DataTypes, UUIDV4, Optional, Model } from "sequelize";
+import { sequelize } from "../sequelize";
+
+interface BlueprintStringAttributes {
+  id: string;
+  blueprint_id: string;
+  blueprint_hash: string;
+  image_hash: string;
+  version: number;
+  changes_markdown: string | null;
+  created_at: Date;
+  updated_at: Date;
+}
+
+interface BlueprintStringInstance
+  extends Model<
+      Omit<BlueprintStringAttributes, "created_at" | "updated_at">,
+      Optional<BlueprintStringAttributes, "id" | "created_at" | "updated_at">
+    >,
+    BlueprintStringAttributes {}
+
+export const BlueprintStringModel = sequelize.define<BlueprintStringInstance>(
+  "blueprint_string",
+  {
+    id: {
+      primaryKey: true,
+      type: DataTypes.UUID,
+      defaultValue: UUIDV4,
+    },
+    blueprint_id: {
+      type: DataTypes.UUID,
+      allowNull: false,
+    },
+    blueprint_hash: {
+      type: DataTypes.STRING(40),
+      allowNull: false,
+      unique: true,
+    },
+    image_hash: {
+      type: DataTypes.STRING(40),
+      allowNull: false,
+    },
+    version: {
+      type: DataTypes.INTEGER,
+      allowNull: false,
+    },
+    changes_markdown: {
+      type: DataTypes.STRING(40),
+    },
+  },
+  {}
+);

libs/database/src/lib/postgres/models/index.ts

@@ -0,0 +1,4 @@
+export * from "./Blueprint";
+export * from "./BlueprintString";
+export * from "./BlueprintBook";
+export * from "./BlueprintPage";

libs/database/src/lib/postgres/sequelize.ts

@@ -0,0 +1,16 @@
+import { Sequelize } from "sequelize";
+
+export const sequelize = new Sequelize({
+  dialect: "postgres",
+  host: process.env.POSTGRES_HOST,
+  port: 5432,
+  database: process.env.POSTGRES_DB,
+  username: process.env.POSTGRES_USER,
+  password: process.env.POSTGRES_PASSWORD,
+  define: {
+    underscored: true,
+    freezeTableName: true,
+    updatedAt: "updated_at",
+    createdAt: "created_at",
+  },
+});

libs/database/src/lib/types.ts

@@ -3,19 +3,21 @@ interface BlueprintChild {
   id: string;
   name: string;
 }
+
 interface BlueprintBookChild {
   type: "blueprint_book";
   id: string;
   name: string;
   children: ChildTree;
 }
-type ChildTree = Array<BlueprintChild | BlueprintBookChild>;
 
-export interface BlueprintEntry {
+export type ChildTree = Array<BlueprintChild | BlueprintBookChild>;
+
+export interface Blueprint {
   id: string;
   label: string; // from source
   description: string | null; // from source
-  game_version: number; // from source
+  game_version: string | null; // from source
   blueprint_hash: string;
   image_hash: string;
   created_at: number;
@@ -26,14 +28,14 @@ export interface BlueprintEntry {
   // BlueprintEntry->BlueprintPageEntry n:m
 }
 
-export interface BlueprintBookEntry {
+export interface BlueprintBook {
   id: string;
   label: string;
   description?: string;
   /** strings as keys of BlueprintEntry */
-  blueprint_ids: string[];
+  // blueprint_ids: string[];
   /** strings as keys of BlueprintBookEntry (currently unsupported) */
-  blueprint_book_ids: string[];
+  // blueprint_book_ids: string[];
   child_tree: ChildTree;
   blueprint_hash: string;
   created_at: number;
@@ -44,20 +46,24 @@ export interface BlueprintBookEntry {
   // BlueprintBook:BlueprintEntry 1:m
 }
 
-export interface BlueprintPageEntry {
+/**
+ * Blueprint page data object for app use
+ * must be JSON serializable
+ */
+export interface BlueprintPage {
   id: string;
-  blueprint_id?: string;
-  blueprint_book_id?: string;
+  blueprint_id: string | null;
+  blueprint_book_id: string | null;
   title: string;
   description_markdown: string;
   created_at: number;
   updated_at: number;
-  factorioprints_id?: string;
+  factorioprints_id: string | null;
   // BlueprintPageEntry->BlueprintEntry 1:m
   // BlueprintPageEntry->BlueprintBook 1:m
 }
 
-export interface BlueprintStringEntry {
+export interface BlueprintString {
   blueprint_id: string;
   blueprint_hash: string;
   image_hash: string;

libs/node-utils/src/lib/node-utils.ts

@@ -1,11 +1,11 @@
 import * as crypto from "crypto";
 import * as pako from "pako";
 // import * as phin from "phin";
-import { BlueprintData } from "@factorio-sites/common-utils";
+import { BlueprintStringData } from "@factorio-sites/common-utils";
 
 export const parseBlueprintString = async (
   string: string
-): Promise<{ hash: string; data: BlueprintData; string: string }> => {
+): Promise<{ hash: string; data: BlueprintStringData; string: string }> => {
   // if (string.startsWith("http:") || string.startsWith("https:")) {
   //   const result = await phin(string);
   //   string = result.body.toString();
@@ -24,7 +24,7 @@ export const parseBlueprintString = async (
   };
 };
 
-export const encodeBlueprint = async (data: BlueprintData): Promise<string> => {
+export const encodeBlueprint = async (data: BlueprintStringData): Promise<string> => {
   const json = JSON.stringify(data);
   const encoded = new TextEncoder().encode(json);
   const compressed = pako.deflate(encoded);

libs/web-utils/src/lib/web-utils.ts

@@ -1,7 +1,7 @@
 import * as pako from "pako";
-import { BlueprintData } from "@factorio-sites/common-utils";
+import { BlueprintStringData } from "@factorio-sites/common-utils";
 
-export function parseBlueprintStringClient(source: string): { data: BlueprintData } {
+export function parseBlueprintStringClient(source: string): { data: BlueprintStringData } {
   const encoded = atob(source.substring(1));
   const decoded = pako.inflate(encoded);
   const string = new TextDecoder("utf-8").decode(decoded);
@@ -9,7 +9,7 @@ export function parseBlueprintStringClient(source: string): { data: BlueprintDat
   return { data: jsonObject };
 }
 
-export function encodeBlueprintStringClient(data: BlueprintData): string {
+export function encodeBlueprintStringClient(data: BlueprintStringData): string {
   const json = JSON.stringify(data);
   const encoded = new TextEncoder().encode(json);
   const compressed = pako.deflate(encoded, { to: "string" });

package.json

@@ -27,74 +27,75 @@
     "help": "nx help"
   },
   "dependencies": {
-    "@chakra-ui/react": "1.0.4",
-    "@emotion/react": "11.1.3",
+    "@chakra-ui/react": "1.1.2",
+    "@emotion/react": "11.1.4",
     "@emotion/server": "11.0.0",
     "@emotion/styled": "11.0.0",
-    "@google-cloud/datastore": "6.2.0",
-    "@google-cloud/pubsub": "2.6.0",
-    "@google-cloud/storage": "5.3.0",
+    "@fbe/editor": "file:.yalc/@fbe/editor",
+    "@google-cloud/datastore": "6.3.1",
+    "@google-cloud/pubsub": "2.7.0",
+    "@google-cloud/storage": "5.7.0",
     "bbcode-to-react": "0.2.9",
     "document-register-element": "1.14.10",
-    "framer-motion": "3.1.1",
-    "next": "10.0.3",
+    "framer-motion": "3.1.4",
+    "next": "10.0.5",
     "nprogress": "0.2.0",
     "pako": "1.0.11",
-    "pg": "8.4.1",
-    "phin": "3.5.0",
-    "puppeteer": "5.3.1",
-    "react": "16.13.1",
-    "react-dom": "16.13.1",
+    "pg": "8.5.1",
+    "phin": "3.5.1",
+    "puppeteer": "5.5.0",
+    "react": "17.0.1",
+    "react-dom": "17.0.1",
     "react-icons": "4.1.0",
     "react-map-interaction": "2.0.0",
-    "react-markdown": "5.0.1",
-    "sharp": "0.26.2",
-    "typeorm": "0.2.28",
-    "ws": "7.3.1"
+    "react-markdown": "5.0.3",
+    "sequelize": "6.3.5",
+    "sharp": "0.27.0",
+    "ws": "7.4.2"
   },
   "devDependencies": {
-    "@babel/core": "7.9.6",
-    "@babel/preset-env": "7.9.6",
-    "@babel/preset-react": "7.9.4",
-    "@babel/preset-typescript": "7.9.0",
-    "@emotion/babel-preset-css-prop": "11.0.0",
-    "@nrwl/cli": "10.3.1",
-    "@nrwl/cypress": "11.0.16",
-    "@nrwl/eslint-plugin-nx": "11.0.16",
-    "@nrwl/jest": "11.0.16",
-    "@nrwl/next": "11.0.16",
-    "@nrwl/node": "11.0.16",
-    "@nrwl/react": "11.0.16",
-    "@nrwl/web": "11.0.16",
-    "@nrwl/workspace": "11.0.16",
-    "@testing-library/react": "10.4.1",
+    "@babel/core": "7.12.10",
+    "@babel/preset-env": "7.12.11",
+    "@babel/preset-react": "7.12.10",
+    "@babel/preset-typescript": "7.12.7",
+    "@nrwl/cli": "11.0.20",
+    "@nrwl/cypress": "11.0.20",
+    "@nrwl/eslint-plugin-nx": "11.0.20",
+    "@nrwl/jest": "11.0.20",
+    "@nrwl/next": "11.0.20",
+    "@nrwl/node": "11.0.20",
+    "@nrwl/react": "11.0.20",
+    "@nrwl/web": "11.0.20",
+    "@nrwl/workspace": "11.0.20",
+    "@testing-library/react": "11.2.2",
     "@types/bbcode-to-react": "0.2.0",
-    "@types/jest": "26.0.8",
+    "@types/jest": "26.0.20",
     "@types/node": "14.14.14",
     "@types/nprogress": "0.2.0",
     "@types/pako": "1.0.1",
-    "@types/puppeteer": "3.0.2",
-    "@types/react": "16.9.53",
-    "@types/react-dom": "16.9.8",
-    "@types/sharp": "0.26.0",
-    "@types/ws": "7.2.7",
-    "@typescript-eslint/eslint-plugin": "4.3.0",
-    "@typescript-eslint/parser": "4.3.0",
-    "babel-jest": "26.2.2",
-    "cypress": "4.1.0",
-    "dotenv": "6.2.0",
-    "eslint": "7.10.0",
-    "eslint-config-prettier": "6.0.0",
-    "eslint-plugin-cypress": "2.10.3",
-    "eslint-plugin-import": "2.21.2",
-    "eslint-plugin-jsx-a11y": "6.3.1",
-    "eslint-plugin-react": "7.20.0",
-    "eslint-plugin-react-hooks": "4.0.4",
-    "jest": "26.2.2",
-    "prettier": "2.1.2",
-    "ts-jest": "26.4.0",
+    "@types/puppeteer": "5.4.2",
+    "@types/react": "17.0.0",
+    "@types/react-dom": "17.0.0",
+    "@types/sharp": "0.27.1",
+    "@types/ws": "7.4.0",
+    "@typescript-eslint/eslint-plugin": "4.12.0",
+    "@typescript-eslint/parser": "4.12.0",
+    "babel-jest": "26.6.3",
+    "cypress": "6.2.1",
+    "dotenv": "8.2.0",
+    "eslint": "7.17.0",
+    "eslint-config-prettier": "7.1.0",
+    "eslint-plugin-cypress": "2.11.2",
+    "eslint-plugin-import": "2.22.1",
+    "eslint-plugin-jsx-a11y": "6.4.1",
+    "eslint-plugin-react": "7.22.0",
+    "eslint-plugin-react-hooks": "4.2.0",
+    "jest": "26.6.3",
+    "prettier": "2.2.1",
+    "ts-jest": "26.4.4",
     "ts-node": "9.1.1",
     "tslint": "6.1.3",
-    "typescript": "4.1.3"
+    "typescript": "4.1.3",
+    "wasm-loader": "1.3.0"
   }
 }

yalc.lock

@@ -0,0 +1,9 @@
+{
+  "version": "v1",
+  "packages": {
+    "@fbe/editor": {
+      "signature": "fcd9eadabd31e2c2000a5ea3e4b000f8",
+      "file": true
+    }
+  }
+}