Added several ways of generating images, factorioprints scraper and a lot of updates to the main site

.dockerignore

@@ -1,2 +1,6 @@
 node_modules
-*.Dockerfile
+*.Dockerfile
+/credentials
+/dist
+/.vscode
+/.cache

.gitignore

@@ -37,3 +37,7 @@ testem.log
 # System Files
 .DS_Store
 Thumbs.db
+
+# custom
+/.cache
+/credentials

CREDITS.md

@@ -0,0 +1,6 @@
+### Factorio blueprint editor
+
+https://teoxoy.github.io/factorio-blueprint-editor/
+LISENCE: MIT
+
+Used for generating images based on blueprint string

README.md

@@ -12,6 +12,8 @@ Run `nx serve my-app` for a dev server. Navigate to http://localhost:4200/. The
 
 Run `nx build my-app` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `--prod` flag for a production build.
 
+### blueprints app
+
 ## Running unit tests
 
 Run `nx test my-app` to execute the unit tests via [Jest](https://jestjs.io).

apps/blueprint-image-function/src/localFileUpload.ts

@@ -0,0 +1,125 @@
+import * as fs from "fs";
+import * as path from "path";
+import { promisify } from "util";
+import * as sharp from "sharp";
+import {
+  hasBlueprintImage,
+  getBlueprintByImageHash,
+  saveBlueprintImage,
+} from "@factorio-sites/database";
+
+if (!process.env.DIR) throw Error("no 'DIR' environment variable")
+
+// const fsReaddir = promisify(fs.readdir);
+const fsReadFile = promisify(fs.readFile);
+const fsUnlink = promisify(fs.unlink);
+const FILE_DIR = path.normalize(process.env.DIR);
+
+const RESIZE_ENABLED = false;
+
+const uploadFile = async (image_path: string) => {
+  const image_hash = path.basename(image_path, ".png");
+
+  if (!(await getBlueprintByImageHash(image_hash))) {
+    console.log(`Image ${image_hash} has no database record, skipping...`);
+    return;
+  }
+  if (image_hash.includes("(")) {
+    console.log(`Image ${image_hash} is a duplicate, deleting...`);
+    await fsUnlink(image_path);
+    return;
+  }
+  if (await hasBlueprintImage(image_hash)) {
+    console.log(`Image ${image_hash} already exists, deleting...`);
+    await fsUnlink(image_path);
+    return;
+  }
+
+  console.log(`Processing ${image_hash}...`);
+  const image = await fsReadFile(image_path);
+
+  // const calculateImageSizeMod = (pixels: number) =>
+  // Math.min(Math.max((-pixels + 500) / 20500 + 1, 0.3), 1);
+
+  const calculateImageSizeMod = (pixels: number) =>
+    Math.min(Math.max((-pixels + 3000) / 33000 + 1, 0.3), 1);
+
+  let sharp_image = sharp(image);
+  if (RESIZE_ENABLED) {
+    const MAX_IMAGE_DIMENTION = 5000;
+    sharp_image = await sharp_image
+      .metadata()
+      .then((meta) => {
+        if (
+          meta.width &&
+          meta.height &&
+          (meta.width > MAX_IMAGE_DIMENTION || meta.height > MAX_IMAGE_DIMENTION)
+        ) {
+          const mod = calculateImageSizeMod(Math.max(meta.width, meta.height));
+          console.log({
+            file: image_path,
+            width: meta.width,
+            height: meta.height,
+            mod,
+            size_mb: image.byteLength / 1024_000,
+          });
+          return sharp_image.resize({
+            width: Math.round(meta.width * mod),
+            height: Math.round(meta.height * mod),
+          });
+        }
+        return sharp_image;
+      })
+      .then((image) => image.webp({ lossless: true }));
+  } else {
+    sharp_image = sharp_image.webp({ lossless: true });
+  }
+
+  const min_image = await sharp_image.toBuffer();
+
+  console.log({
+    input_size_mb: image.byteLength / 1024_000,
+    output_size_mb: min_image.byteLength / 1024_000,
+  });
+
+  // console.log(`Image ${image_hash} processed, writing...`);
+  // fs.writeFileSync(`${image_path}.webp`, min_image);
+  console.log(`Image ${image_hash} processed, uploading...`);
+  await saveBlueprintImage(image_hash, min_image);
+  await fsUnlink(image_path);
+};
+
+export async function uploadLocalFiles() {
+  // console.log(`Reading directory`, FILE_DIR);
+  // const files = await fsReaddir(FILE_DIR);
+  // for (let i = 0; i < files.length; i++) {
+  //   if (fs.statSync(path.join(FILE_DIR, files[i])).isDirectory()) continue;
+  //   await uploadFile(path.join(FILE_DIR, files[i]));
+  // }
+  console.log(`Watching directory`, FILE_DIR);
+  const work_done_buffeer: string[] = [];
+  const work_buffer: string[] = [];
+  fs.watch(FILE_DIR, (type, file) => {
+    if (type === "change" && file && file.endsWith(".png")) {
+      const file_path = path.join(FILE_DIR, file);
+      if (work_buffer.includes(file_path) || work_done_buffeer.includes(file_path)) {
+        return;
+      }
+      work_buffer.push(file_path);
+    }
+  });
+
+  let working = false;
+  const doWork = async () => {
+    if (working || !work_buffer.length) return setTimeout(doWork, 1000);
+    working = true;
+    const file_path = work_buffer.shift();
+    if (file_path) {
+      await uploadFile(file_path);
+      work_done_buffeer.push(file_path);
+    }
+    working = false;
+    doWork();
+  };
+  doWork();
+}

apps/blueprint-image-function/src/main.ts

@@ -1,15 +1,142 @@
 import { generateScreenshot } from "@factorio-sites/generate-bp-image";
+import {
+  saveBlueprintImage,
+  hasBlueprintImage,
+  getBlueprintById,
+  BlueprintEntry,
+  getBlueprintImageRequestTopic,
+  PubSubMessage,
+  getPaginatedBlueprints,
+} from "@factorio-sites/database";
+import { environment } from "./environments/environment";
+import { uploadLocalFiles } from "./localFileUpload";
+
+const generateImageForSource = async (blueprint: BlueprintEntry) => {
+  if (await hasBlueprintImage(blueprint.image_hash)) {
+    throw Error("Image already exists");
+  }
+
+  const imageBuffer = await generateScreenshot(
+    blueprint,
+    environment.production ? "/tmp" : undefined
+  );
+
+  if (!imageBuffer) return false;
+
+  await saveBlueprintImage(blueprint.image_hash, imageBuffer);
+
+  console.log(`[generateImageForSource] image hash ${blueprint.image_hash} successfully saved`);
+
+  return true;
+};
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type Handler = (req: any, res: any) => void; // Don't want to install express types just for this
 
+/**
+ * Handler method supports Google cloud functions
+ * @param req express request object
+ * @param res express response object
+ */
 export const handler: Handler = async (req, res) => {
   if (!req.query.source) {
     return res.status(400).end("No source string given");
   }
-  const string = (req.query.source as string).replace(/ /g, "+");
+
-  const stream = await generateScreenshot(string, "/tmp");
+  // generateImageForSource((req.query.source as string).replace(/ /g, "+"))
-  res.status(200);
+  //   .then(() => {
-  res.setHeader("content-type", "image/png");
+  //     res.status(200).end("done");
-  stream.pipe(res);
+  //   })
+  //   .catch((reason) => {
+  //     res.status(200).end(reason.message);
+  //   });
 };
+
+async function subscribeToPubSub() {
+  // Wait to make sure puppeteer browser started
+  await new Promise((resolve) => setTimeout(resolve, 2000));
+
+  const topic = getBlueprintImageRequestTopic();
+  const [subscription] = await topic
+    .subscription("blueprint-image-function-app", {
+      flowControl: { allowExcessMessages: false, maxMessages: 1, maxExtension: 3600 },
+    })
+    .get();
+
+  console.log(`[pubsub] Listening to subscription`);
+
+  let handlerBusy = false;
+  const messageHandler = async (message: PubSubMessage) => {
+    if (!handlerBusy) handlerBusy = true;
+    else {
+      console.log(`nack'd message because handler is busy ${message.data.toString()}`);
+      return message.nack();
+    }
+
+    try {
+      const data = JSON.parse(message.data.toString());
+      if (!data.blueprintId) return console.log("blueprintId not found in message body");
+      console.log("------------------------------------------------");
+      console.log("[pubsub] generating image for", data.blueprintId);
+      const blueprint = await getBlueprintById(data.blueprintId);
+      if (!blueprint) return console.log("Blueprint not found");
+      const start_time = Date.now();
+      await generateImageForSource(blueprint);
+      const duration = Date.now() - start_time;
+      console.log(`[pubsub] image generated in ${duration}ms`);
+      message.ack();
+
+      if (duration > 30000) {
+        console.log("Process too slow, closing...");
+        subscription.off("message", messageHandler);
+        return setTimeout(() => process.exit(1), 1000);
+      }
+    } catch (reason) {
+      if (reason.message === "Image already exists") {
+        console.log(`Image already exists`);
+        message.ack();
+      } else if (reason.message === "Failed to parse blueprint string") {
+        console.log(`Blueprint editor could not handle string`);
+        message.ack();
+      } else {
+        console.error("[pubsub:error]", reason);
+        message.nack();
+      }
+    }
+    handlerBusy = false;
+  };
+
+  subscription.on("message", messageHandler);
+  // image hash = a99525f97c26c7242ecdd96679043b1a5e65dd0c
+  // SELECT * FROM BlueprintBook WHERE blueprint_ids CONTAINS Key(Blueprint, 4532736400293888)
+  // bp = Key(Blueprint, 4532736400293888)
+  // book = Key(BlueprintBook, 5034207050989568)
+  // page = 6225886932107264
+
+  subscription.on("error", (error) => {
+    console.error("[pubsub] Received error:", error);
+  });
+}
+
+async function rePublishAllBlueprints() {
+  const topic = getBlueprintImageRequestTopic();
+  const fetchPage = async (page = 1) => {
+    const blueprints = await getPaginatedBlueprints(page);
+    if (blueprints.length === 0) {
+      return console.log("No more blueprints found");
+    }
+    console.log(`Publishing page ${page} with ${blueprints.length} blueprints`);
+
+    await Promise.all(
+      blueprints.map((blueprint) => {
+        return topic.publishJSON({ blueprintId: blueprint.id });
+      })
+    );
+    fetchPage(page + 1);
+  };
+  await fetchPage();
+}
+
+uploadLocalFiles().catch((reason) => console.error("Fatal error:", reason));
+// subscribeToPubSub().catch((reason) => console.error("Fatal error:", reason));
+// rePublishAllBlueprints().catch((reason) => console.error("Fatal error:", reason));

apps/blueprints/.eslintrc.json

@@ -194,10 +194,7 @@
     "jsx-a11y/accessible-emoji": "warn",
     "jsx-a11y/alt-text": "warn",
     "jsx-a11y/anchor-has-content": "warn",
-    "jsx-a11y/anchor-is-valid": [
+    "jsx-a11y/anchor-is-valid": ["warn", { "aspects": ["noHref", "invalidHref"] }],
-      "warn",
-      { "aspects": ["noHref", "invalidHref"] }
-    ],
     "jsx-a11y/aria-activedescendant-has-tabindex": "warn",
     "jsx-a11y/aria-props": "warn",
     "jsx-a11y/aria-proptypes": "warn",
@@ -231,10 +228,7 @@
       }
     ],
     "no-unused-vars": "off",
-    "@typescript-eslint/no-unused-vars": [
+    "@typescript-eslint/no-unused-vars": ["warn", { "args": "none", "ignoreRestSiblings": true }],
-      "warn",
-      { "args": "none", "ignoreRestSiblings": true }
-    ],
     "no-useless-constructor": "off",
     "@typescript-eslint/no-useless-constructor": "warn",
     "@typescript-eslint/no-unused-expressions": [

apps/blueprints/README.md

@@ -0,0 +1,16 @@
+## Before deploying
+
+Make sure `prod.package.json` is up to date on packages from the root `package.json`
+
+## Deploying
+
+`docker build -t eu.gcr.io/factorio-sites/blueprints --file blueprints.Dockerfile .`
+`docker push eu.gcr.io/factorio-sites/blueprints`
+
+### Testing deployment locally
+
+`docker run --rm -p 3000:3000 eu.gcr.io/factorio-sites/blueprints`
+
+### windows env
+
+`$env:GOOGLE_APPLICATION_CREDENTIALS="D:\git\factorio-sites\credentials\factorio-sites.json"`

apps/blueprints/index.d.ts

@@ -1,6 +1,10 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
-declare module '*.svg' {
+declare module "*.svg" {
   const content: any;
   export const ReactComponent: any;
   export default content;
 }
+
+declare module "react-map-interaction" {
+  export const MapInteractionCSS: any;
+}

apps/blueprints/pages/[blueprint]/[blueprintId].tsx

@@ -0,0 +1,331 @@
+/** @jsx jsx */
+import React, { ReactNode, useEffect, useState } from "react";
+import { jsx, css } from "@emotion/core";
+import { NextPage, NextPageContext } from "next";
+import BBCode from "bbcode-to-react";
+import { Button, Grid } from "@chakra-ui/core";
+import {
+  BlueprintBookEntry,
+  BlueprintEntry,
+  BlueprintPageEntry,
+  getBlueprintBookById,
+  getBlueprintById,
+  getBlueprintPageById,
+  hasBlueprintImage,
+} from "@factorio-sites/database";
+import { BlueprintData, timeLogger } from "@factorio-sites/common-utils";
+import { chakraResponsive, parseBlueprintStringClient } from "@factorio-sites/web-utils";
+import { Panel } from "../../src/Panel";
+import { Markdown } from "../../src/Markdown";
+import { FullscreenImage } from "../../src/FullscreenImage";
+import { BookChildTree } from "../../src/BookChildTree";
+import { CopyButton } from "../../src/CopyButton";
+
+const imageStyle = css`
+  display: flex;
+  justify-content: center;
+  &:hover {
+    cursor: pointer;
+  }
+`;
+
+type Selected =
+  | { type: "blueprint"; data: Pick<BlueprintEntry, "id" | "blueprint_hash" | "image_hash"> }
+  | { type: "blueprint_book"; data: Pick<BlueprintBookEntry, "id" | "blueprint_hash"> };
+
+interface IndexProps {
+  image_exists: boolean;
+  selected: Selected;
+  blueprint: BlueprintEntry | null;
+  blueprint_book: BlueprintBookEntry | null;
+  blueprint_page: BlueprintPageEntry;
+}
+
+export const Index: NextPage<IndexProps> = ({
+  image_exists,
+  selected,
+  blueprint,
+  blueprint_book,
+  blueprint_page,
+}) => {
+  const [imageZoom, setImageZoom] = useState(false);
+  const [blueprintString, setBlueprintString] = useState<string | null>(null);
+  const [data, setData] = useState<BlueprintData | null>(null);
+  const [showJson, setShowJson] = useState(false);
+
+  const selectedHash = selected.data.blueprint_hash;
+
+  useEffect(() => {
+    fetch(`/api/string/${selectedHash}`)
+      .then((res) => res.text())
+      .then((string) => {
+        setShowJson(false);
+        setBlueprintString(string);
+        if (selected.type === "blueprint") {
+          const { data } = parseBlueprintStringClient(string);
+          setData(data);
+        } else {
+          setData(null);
+        }
+      });
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [selectedHash]);
+
+  useEffect(() => {
+    console.log({
+      image_exists,
+      selected,
+      blueprint,
+      blueprint_book,
+      blueprint_page,
+    });
+  }, []);
+
+  const renderImage = () => {
+    let render: ReactNode;
+    if (selected.type === "blueprint_book") {
+      render = <div>Can't show image for a book, select a blueprint to the the image</div>;
+    } else if (!image_exists) {
+      render = <div>The image is not generated yet</div>;
+    } else if (imageZoom) {
+      render = (
+        <FullscreenImage
+          close={() => setImageZoom(false)}
+          alt="blueprint"
+          src={`https://storage.googleapis.com/blueprint-images/${selected.data.image_hash}.webp`}
+        />
+      );
+    } else {
+      render = (
+        <div onClick={() => setImageZoom(true)}>
+          <img
+            alt="blueprint"
+            src={`https://storage.googleapis.com/blueprint-images/${selected.data.image_hash}.webp`}
+          />
+        </div>
+      );
+    }
+    return <div css={imageStyle}>{render}</div>;
+  };
+
+  return (
+    <Grid
+      margin="0.7rem"
+      templateColumns={chakraResponsive({ mobile: "1fr", desktop: "1fr 1fr" })}
+      gap={6}
+    >
+      <Panel title={blueprint_page.title} gridColumn="1">
+        {blueprint_book ? (
+          <>
+            <div>This string contains a blueprint book </div>
+            <br />
+            <BookChildTree
+              child_tree={[
+                {
+                  id: blueprint_book.id,
+                  name: blueprint_book.label,
+                  type: "blueprint_book",
+                  children: blueprint_book.child_tree,
+                },
+              ]}
+              base_url={`/blueprint/${blueprint_page.id}`}
+              selected_id={selected.data.id}
+            />
+          </>
+        ) : blueprint ? (
+          <>
+            <div>This string contains one blueprint</div>
+            <div>tags: {blueprint.tags.join(", ")}</div>
+          </>
+        ) : null}
+      </Panel>
+      <Panel
+        title={image_exists ? undefined : "Image"}
+        gridColumn={chakraResponsive({ mobile: "1", desktop: "2" })}
+        gridRow={chakraResponsive({ mobile: "1", desktop: undefined })}
+      >
+        {renderImage()}
+      </Panel>
+
+      <Panel
+        title="Description"
+        gridColumn={chakraResponsive({ mobile: "1", desktop: "1 / span 2" })}
+      >
+        <Markdown>{blueprint_page.description_markdown}</Markdown>
+      </Panel>
+      {selected.type === "blueprint" && data && (
+        <Panel
+          title={(<span>Entities for {BBCode.toReact(data.blueprint.label)}</span>) as any}
+          gridColumn={chakraResponsive({ mobile: "1", desktop: "1 / span 2" })}
+        >
+          <table>
+            <tbody>
+              {Object.entries(
+                data.blueprint.entities.reduce<Record<string, number>>((entities, entity) => {
+                  if (entities[entity.name]) {
+                    entities[entity.name]++;
+                  } else {
+                    entities[entity.name] = 1;
+                  }
+                  return entities;
+                }, {})
+              )
+                .sort((a, b) => b[1] - a[1])
+                .map(([entry_name, entry]) => (
+                  <tr key={entry_name} css={{}}>
+                    <td css={{ border: "1px solid #909090" }}>
+                      <img
+                        alt={entry_name.replace(/-/g, " ")}
+                        src={`https://factorioprints.com/icons/${entry_name}.png`}
+                      />
+                    </td>
+                    <td css={{ padding: "5px 10px", border: "1px solid #909090" }}>{entry_name}</td>
+                    <td css={{ padding: "5px 10px", border: "1px solid #909090" }}>{entry}</td>
+                  </tr>
+                ))}
+            </tbody>
+          </table>
+        </Panel>
+      )}
+      <Panel title="string" gridColumn={chakraResponsive({ mobile: "1", desktop: "1 / span 2" })}>
+        <>
+          {blueprintString && <CopyButton content={blueprintString} marginBottom="0.5rem" />}
+          <textarea
+            value={blueprintString || "Loading..."}
+            readOnly
+            css={{
+              width: "100%",
+              height: "100px",
+              resize: "none",
+              color: "#fff",
+              backgroundColor: "#414040",
+            }}
+          />
+        </>
+      </Panel>
+      <Panel title="json" gridColumn={chakraResponsive({ mobile: "1", desktop: "1 / span 2" })}>
+        {showJson ? (
+          !data ? (
+            <div>Loading...</div>
+          ) : (
+            <>
+              <Button
+                variantColor="green"
+                css={{ position: "absolute", right: "65px" }}
+                onClick={() => {
+                  setShowJson(false);
+                  if (selected.type === "blueprint_book") {
+                    setData(null);
+                  }
+                }}
+              >
+                hide
+              </Button>
+              <pre css={{ maxHeight: "500px", overflowY: "scroll" }}>
+                {JSON.stringify(data, null, 2)}
+              </pre>
+            </>
+          )
+        ) : (
+          <Button
+            variantColor="green"
+            onClick={() => {
+              setShowJson(true);
+              if (selected.type === "blueprint_book") {
+                fetch(`/api/string/${selectedHash}`)
+                  .then((res) => res.text())
+                  .then((string) => {
+                    const { data } = parseBlueprintStringClient(string);
+                    setData(data);
+                  });
+              }
+            }}
+          >
+            show
+          </Button>
+        )}
+      </Panel>
+    </Grid>
+  );
+};
+
+export async function getServerSideProps(context: NextPageContext) {
+  const throwError = (message: string) => {
+    if (!blueprint_page && context.res) {
+      context.res.statusCode = 404;
+      context.res.end({ error: message });
+      return {};
+    }
+  };
+
+  const tl = timeLogger("getServerSideProps");
+  const selected_id = context.query.selected ? (context.query.selected as string) : null;
+  const type = context.query.type ? (context.query.type as string) : null;
+  const blueprintId = context.query.blueprintId ? (context.query.blueprintId as string) : null;
+
+  if (!blueprintId) return throwError("Blueprint ID not found");
+
+  const blueprint_page = await getBlueprintPageById(blueprintId);
+  tl("getBlueprintPageById");
+
+  if (!blueprint_page) return throwError("Blueprint page not found");
+
+  let blueprint: IndexProps["blueprint"] = null;
+  let blueprint_book: IndexProps["blueprint_book"] = null;
+  let selected!: IndexProps["selected"];
+  let selected_blueprint!: BlueprintEntry | null;
+  let selected_blueprint_book!: BlueprintBookEntry | null;
+
+  if (blueprint_page.blueprint_id) {
+    blueprint = await getBlueprintById(blueprint_page.blueprint_id);
+    selected_blueprint = blueprint;
+    tl("getBlueprintById");
+    // blueprint_string = await getBlueprintStringByHash(blueprint.blueprint_hash);
+  } else if (blueprint_page.blueprint_book_id) {
+    blueprint_book = await getBlueprintBookById(blueprint_page.blueprint_book_id);
+    if (selected_id && type === "book") {
+      selected_blueprint_book = await getBlueprintBookById(selected_id);
+      tl("getBlueprintBookById");
+    } else if (selected_id && type !== "book") {
+      selected_blueprint = await getBlueprintById(selected_id);
+      tl("getBlueprintById");
+    } else if (blueprint_book) {
+      selected_blueprint_book = blueprint_book;
+    }
+  }
+
+  if (selected_blueprint) {
+    selected = {
+      type: "blueprint",
+      data: {
+        id: selected_blueprint.id,
+        blueprint_hash: selected_blueprint.blueprint_hash,
+        image_hash: selected_blueprint.image_hash,
+      },
+    };
+  } else if (selected_blueprint_book) {
+    selected = {
+      type: "blueprint_book",
+      data: {
+        id: selected_blueprint_book.id,
+        blueprint_hash: selected_blueprint_book.blueprint_hash,
+      },
+    };
+  }
+
+  // selected = {type: 'blueprint', data: {id: blueprint.id}}
+  const image_exists =
+    selected.type === "blueprint" ? await hasBlueprintImage(selected.data.image_hash) : false;
+
+  return {
+    props: {
+      image_exists,
+      blueprint,
+      blueprint_book,
+      selected,
+      blueprint_page,
+    } as IndexProps,
+  };
+}
+
+export default Index;

apps/blueprints/pages/_app.tsx

@@ -1,52 +1,78 @@
 /** @jsx jsx */
-import React from "react";
+import { jsx, css, Global } from "@emotion/core";
 import { AppProps } from "next/app";
 import Head from "next/head";
-import { jsx, css } from "@emotion/core";
+import Router from "next/router";
-import { normalize } from "./normalize";
+import { CSSReset, ITheme, theme } from "@chakra-ui/core";
-import { Global } from "@emotion/core";
+import { ThemeProvider } from "emotion-theming";
+import NProgress from "nprogress";
+import { Header } from "../src/Header";
 
-const mainStyles = css`
+const globalStyles = css`
-  font-family: sans-serif;
+  html {
-
+    height: 100%;
-  header {
+  }
+  body {
+    background-color: #201810;
+    background: #201810;
+    background-image: url(https://cdn.factorio.com/assets/img/web/bg_v4-85.jpg),
+      url(data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMFBQX/wQARCABgAEADABEAAREAAhEA/8QAXgAAAgMBAQEAAAAAAAAAAAAAAQIAAwQFBgkQAAECBAQCBQcHCQkAAAAAAAECEQADEiEEIjFBMlETQlJhcQVicoGCkfAjM5KhorHiFFOjssHC0tPhFUOTw9Hj8fLz/9oADAMAAAEAAgAAPwD5rSkTJqqOJXped3PFcGOgPJWIbgT9Jf8ALhKoemB/Zs5KgCEAuw4yXa4AoS553A3h0HNCKTbWMM2WpDp23LqPuYGLFQiYgJX2mKk6+kIVUMmLjhlqICQS+42HIsvTT3Q1MHN2VRtT5LxKwnIunfi/igUQ30oyYmTNkZJlSSFfagwkWYBaZE5KrFIL5rpsz5QqUlVxZz1ntrCqQadYKSArbQc49qnyhKUAgGQxSpyrDzEguAFVEzgAVAEkXcJfMIy0cz+ii8F2ACfpzE8PtRnmYmTUM0qwCglEqkqAAAAKlqpSeHh4XzcrZYbrRWq/VfNHHmYdOLUSnIlKyXYswOjp207jfh4YvUOGKWzacP0Y52KwPQJqqQLpVmPD7S1fh0iUw1UDCJSqZeals/DMkKao8i7N5zCFVBEemlz5ACUpxEurpMQfnsCeLDoSn5upb1fm6pnmQkStMYfLkyRNWlclQHyaK5YS1JCUozKrdRFKe16So0LlTpXzsVS8ThcV8wurKj+7ojzLj64SHh0zpiVKQk1JAYpAAO7MQQw2Gni8LTSetDgu1otTiJpVd84YZhqna2+uj8MOmFqj2fknDdJhlHIbmorWoKRxqq0KSSlFWalKaYjqqRCVJ4M9Suzwxg8p4foZy0lIcPolwMoIrGV+xuRTYxnUtSl1Z4sl09Gnh6360cmVilSiaCQA3NP7sWcUJ8nFkzyjOPWFD6VDxqpKdEn19mAkZ4GVPBHNnTlTSXbN2fv/AKZYvqX14RCZUuvo4xAOQC4vrbmL+FuUL6UWZvNjqSsOSmoJLgEbBxZ2BsQyg5DUlg7QhUAWSWqNtGIN3ZrBVyx8LRclJzVC6dqjUDcHe5TbQ71MztoVhwEhRDsdLu4FJs1v+XiVpUpMV9H/ANY14XyirBpouynFNRZOVizmwAJSKQMxqMRRrqTCJSy0qpByqp+nFPlDyoZwKQ6glXN3YbqURYqvzhRLy5YJVVxcUcxExSklpZu4spD6X60PRCMmM6krUS6F6E2Va9wAAogWfu8IjdqqHT6vVCUhLpch7i5Oh1Bzd+kFMGGUQ4ax5Xv8eMOoQsepwKkfk630ILuCU8KCkLIzAFQZIQwU7K+caMigapTck6Fuqp8peo38NBGp3Uol7lZNgWdW6tg/u9cYcap5yiaKemVxpClEVDdKS4Z+FO1VSYaV1YRfW9uOXjFJEykOVakWtp2bs1MMBmir+H9+MxlzVaS5qwrlZX7PXFg4YEFKJiOrN/xP9yJBjdKkqUoNKmaMV1BSeYCvlDbvSmCKExOKM84BMy4T7uXc31wevAjGVZhYvvfv5czAy1QY6uFxikS6GqsRbVLpAdLEEOU3FyWOhTCUVFDdWGra5gTZipylgDjUotowUb1JfK4Ga94iUUxFGqIcKsrrWMrupZTr4sdOX6sMIWn0o9RJly1SkKRMw6UpB6RK5K1my0sUgTpfCOq6kq5ZYYy19rrwqFoTk+l8JiuZhDMLJ6BSnPDIWk+PzresXtAo9KDX6P2YtlyJsiSqpMopukqEtjp2ukV32pzdaFohq48jjCBMCQAWJYgBx3PblveLCqEA+1GADMCS4u4s45ftiuHiwkMCKtKtmfkcoJ3g5YkdHyZ0Znp6UqCUqA1472qqtSD+xLwe1A7PDHpMaiWiTdaSoolFgSCAy+IVa65kkeEImpVdMMKctXD+Psx5oTp6FEomBtnWAkX4db8s1XfBqqiU0q/DDIRiyqpKJijz6SYX9SV0J9lNPfEqgURoQvGUN0E1u0DM9pXFTA6VEHo1xzcRLnPXMCnq1Um/cD3+qIFwFJjHLzK/oPh4TLBjUZWVKmS5JLENoTqG1YXbW+0WAgD2oU/Bh5DpmS1FdLqQdcoIWka22vdx4Qz8MCNk9c+YEkKcBKQljZhWksbU3202eFSwqhqT6Pne1GFHSpV1vj2ffCtEj1GAnFQQlX1q1Sl1XFCmB6ztl7SYShMPHaEjDKSyxl615tYFWelKZEsP+a6y+1TeEql+3D0q9mPHeU0IlzVhPC6rpUo/r0j8XDFyaerFSvOjkyJVQPDwNxDiq9UVLgRtmJmSZCTSCAmxBCmZSteLKerypaIheWJGNOIIKWI5aM7EvbR7+HW2jQFcMCOph5hWEoCgRfLVLTaqp2VzV52sVHL/AOkXVK7X6KNkvydOnkqapi6i6FEIIssMGKedKreuJdXbgVU9j2paExknSl4OeSqQJiglgwSSHTqqoi2iu8MYCSUmGUpKxwZsvBCnHFvmU/RkfxfZi2K8scedNFavk6atbI1zXVSpVUCBAkzTLUD1R3P96ffFKoEegTPlYiWlOQKuhCQg+0ilKG6MvV5pitoLxx8RhJiFFctqTo52seTsCpns9ouDjeF6RKS2/oxkRNmoW3R9IKrhSQfVppyEGDUnzY9V5JVh1KlflNOepKkKQjopWdCUhlJd26/F5yYrzxK+PInLHfxuBkYxSVS5Uo1KBQqXhwyBnNQVSjhy1IyIppp7yuej0IEpWX+ZMjzeN8nJw8sqcmpJoYU3VYcUxb+GyYPS/HRRb5v+ZHm5msWQsFUsqKjZCRTvmZVqma/rVCQsXyq0FNHItqlrMSbMX8YWirqwI3SUEhNbhIQLFQJKlJJYJdiAqwCmIYGo6GSqgQ9gN4VS0XCg5vto0YZg+U63Wi+FdEUS0KrQGAD1KJpJNLEnhIt39nSFCc1MWBXFHRTicSgUy8TPloTZKECWEoS9gkGWSAN9iauZimYlJU9KSNnfmSw5C/vfnFqQCMwu3ZSdgL2J29wEU4nEYmeljjJiw9gsSyQNWcIB18A20RLJ0SB4W5D7gPcIYlwz25MGG+wHP745pCgFPcgBlB73SC4YC5uANL84s2+OYt4RWXeNxCEVKspV3pJuqpTcP1ebDQDFRWmolt1DR7U96b34fviZYVvGGMypsykvdho4sHYbcm0DwrZoIDCKd9Va7w8CNqFM7hII0sklmJcja31xN4baK1KBZSnSQWDDfckOxcOzWFiLBoA7ybvt3WOoZidG9xgnwFm+Dr7/AA2hWlrIGcF30QNfBNiw5sdABoD4l7cm076n/wBd4ngGvz/C0GZLISpQOg0O/q5QvWpibR//2Q==);
+    background-size: 2048px 3072px;
+    background-position: center top;
     display: flex;
-    align-items: center;
+    flex-direction: column;
-    justify-content: left;
+    min-height: 100%;
-    background-color: #143055;
+    font-family: titillium web, sans-serif;
-    color: white;
-    padding: 5px 20px;
-
-    h1 {
-      margin: 10px 0;
-    }
-  }
-
-  main {
-    padding: 0 20px;
-  }
-
-  .sidebar {
-    background: #ccc;
   }
 `;
 
+if (typeof window !== "undefined") {
+  NProgress.configure({
+    showSpinner: false,
+    speed: 800,
+    trickleSpeed: 150,
+    template:
+      '<div class="bar" role="bar" style="background: #00a1ff;position: fixed;z-index: 1031;top: 0;left: 0;width: 100%;height: 4px;" />',
+  });
+  Router.events.on("routeChangeStart", () => NProgress.start());
+  Router.events.on("routeChangeComplete", () => NProgress.done());
+  Router.events.on("routeChangeError", () => NProgress.done());
+}
+
+const config = (theme: ITheme) => ({
+  light: {
+    color: theme.colors.gray[800],
+    bg: theme.colors.gray[300],
+    borderColor: theme.colors.gray[200],
+    placeholderColor: theme.colors.gray[500],
+  },
+  dark: {
+    color: theme.colors.whiteAlpha[900],
+    bg: theme.colors.gray[800],
+    borderColor: theme.colors.whiteAlpha[300],
+    placeholderColor: theme.colors.whiteAlpha[400],
+  },
+});
+
 const CustomApp = ({ Component, pageProps }: AppProps) => {
   return (
-    <>
+    <ThemeProvider theme={theme}>
-      <Global styles={normalize} />
+      <Global styles={globalStyles} />
+      <CSSReset config={config} />
       <Head>
         <title>Welcome to blueprints!</title>
+        <link
+          href="https://cdn.factorio.com/assets/fonts/titillium-web.css"
+          rel="stylesheet"
+        ></link>
       </Head>
-      <div css={mainStyles}>
+      <div>
-        <header>
+        <Header />
-          <h1>Factorio Blueprints</h1>
-        </header>
         <main>
           <Component {...pageProps} />
         </main>
       </div>
-    </>
+    </ThemeProvider>
   );
 };
 

apps/blueprints/pages/api/image-to-gen.ts

@@ -0,0 +1,65 @@
+import { NextApiHandler } from "next";
+import {
+  getBlueprintImageRequestTopic,
+  PubSubMessage,
+  getBlueprintById,
+  getBlueprintStringByHash,
+  hasBlueprintImage,
+  BlueprintEntry,
+} from "@factorio-sites/database";
+
+const getOneMessage = async (): Promise<BlueprintEntry> => {
+  const topic = getBlueprintImageRequestTopic();
+  const [subscription] = await topic
+    .subscription("blueprint-image-function-app", {
+      flowControl: { allowExcessMessages: false, maxMessages: 1 },
+    })
+    .get();
+
+  return new Promise((resolve) => {
+    let acked_msg = false;
+    const messageHandler = async (message: PubSubMessage) => {
+      console.log("[pubsub]" + message.data.toString());
+      if (acked_msg) return;
+
+      const data = JSON.parse(message.data.toString());
+      const blueprint = await getBlueprintById(data.blueprintId);
+
+      if (await hasBlueprintImage(blueprint.image_hash)) {
+        console.log(`Blueprint ${data.blueprintId} image already exists ${blueprint.image_hash}`);
+        return message.ack();
+      }
+
+      if (acked_msg) return; // check again if it changes during async calls
+      acked_msg = true;
+      message.ack();
+      subscription.off("message", messageHandler);
+      resolve(blueprint);
+    };
+    subscription.on("message", messageHandler);
+  });
+};
+
+const handler: NextApiHandler = async (req, res) => {
+  // Allow the url to be used in the blueprint editor
+  if (
+    req.headers.origin === "https://teoxoy.github.io" ||
+    req.headers.origin.startsWith("http://localhost")
+  ) {
+    res.setHeader("Access-Control-Allow-Origin", req.headers.origin);
+  }
+
+  const blueprint = await getOneMessage();
+  const string = await getBlueprintStringByHash(blueprint.blueprint_hash);
+
+  res.setHeader("Content-Type", "application/json");
+  res.status(200).end(
+    JSON.stringify({
+      blueprintId: blueprint.id,
+      image_hash: blueprint.image_hash,
+      string: string,
+    })
+  );
+};
+
+export default handler;

apps/blueprints/pages/api/post-image.ts

@@ -0,0 +1,43 @@
+import { getBlueprintById, saveBlueprintImage } from "@factorio-sites/database";
+// import imagemin from "imagemin";
+// import imageminWebp from "imagemin-webp";
+import { NextApiHandler } from "next";
+
+const handler: NextApiHandler = async (req, res) => {
+  // Allow the url to be used in the blueprint editor
+  if (
+    (req.method === "OPTIONS" || req.method === "POST") &&
+    req.headers.origin === "https://teoxoy.github.io"
+  ) {
+    res.setHeader("Access-Control-Allow-Origin", "https://teoxoy.github.io");
+    res.setHeader("Access-Control-Allow-Headers", "content-type");
+  }
+  if (req.method === "OPTIONS") return res.status(200).end();
+  else if (req.method !== "POST") {
+    return res.status(400).end("Only accepts POST");
+  }
+
+  if (req.body.blueprintId && req.body.image) {
+    console.log(`store image for blueprint ${req.body.blueprintId}`);
+    const buffer = Buffer.from(req.body.image, "base64");
+    const buffermin = buffer;
+    // const buffermin = await imagemin.buffer(buffer, {
+    //   plugins: [imageminWebp({ quality: 50 })],
+    // });
+
+    console.log("minified image buffer length", buffermin.byteLength);
+
+    const blueprint = await getBlueprintById(req.body.blueprintId);
+    await saveBlueprintImage(blueprint.image_hash, buffermin);
+    console.log(`Successfuly saved image ${blueprint.image_hash}`);
+
+    res.setHeader("Content-Type", "text/plain");
+    res.status(200).end("ok");
+    return;
+  }
+
+  res.setHeader("Content-Type", "text/plain");
+  res.status(400).end("no post body");
+};
+
+export default handler;

apps/blueprints/pages/api/string/[hash].ts

@@ -0,0 +1,20 @@
+import { NextApiHandler } from "next";
+import { getBlueprintStringByHash } from "@factorio-sites/database";
+
+const handler: NextApiHandler = async (req, res) => {
+  if (!req.query.hash) {
+    return res.status(400).end("No string hash provided");
+  }
+
+  const blueprintString = await getBlueprintStringByHash(req.query.hash as string);
+
+  // Allow the url to be used in the blueprint editor
+  if (req.headers.origin === "https://teoxoy.github.io") {
+    res.setHeader("Access-Control-Allow-Origin", "https://teoxoy.github.io");
+  }
+  res.setHeader("Cache-Control", "public, max-age=86400");
+  res.setHeader("Content-Type", "text/plain");
+  res.status(200).end(blueprintString);
+};
+
+export default handler;

apps/blueprints/pages/index.tsx

@@ -1,12 +1,64 @@
+/** @jsx jsx */
+/* eslint-disable jsx-a11y/anchor-is-valid */
 import React from "react";
+import { NextPage, NextPageContext } from "next";
+import Link from "next/link";
+import { jsx, css } from "@emotion/core";
+import { BlueprintPageEntry, getMostRecentBlueprintPages } from "@factorio-sites/database";
+import { Panel } from "../src/Panel";
+import { SimpleGrid } from "@chakra-ui/core";
+import { Pagination } from "../src/Pagination";
 
-export const Index = () => {
+const linkStyles = css`
-  /*
+  width: 100%;
-   * Replace the elements below with your own.
+  margin: 5px 0;
-   *
+  a {
-   * Note: The corresponding styles are in the ./${fileName}.${style} file.
+    display: block;
-   */
+    padding: 5px;
-  return <div>app</div>;
+    color: #fff;
+  }
+  &:hover {
+    cursor: pointer;
+    background: #ccc;
+  }
+`;
+
+const BlueprintComponent: React.FC<{ blueprint: BlueprintPageEntry }> = ({ blueprint }) => (
+  <div css={linkStyles}>
+    <Link href={`/blueprint/${blueprint.id}`} passHref>
+      <a>{blueprint.title}</a>
+    </Link>
+  </div>
+);
+
+interface IndexProps {
+  page: number;
+  blueprints: BlueprintPageEntry[];
+}
+
+export const Index: NextPage<IndexProps> = ({ page, blueprints }) => {
+  return (
+    <SimpleGrid columns={1} margin="0.7rem">
+      <Panel title="Blueprints" w="100%">
+        {blueprints.map((bp) => (
+          <BlueprintComponent key={bp.id} blueprint={bp} />
+        ))}
+        <Pagination page={page} />
+      </Panel>
+    </SimpleGrid>
+  );
 };
 
+export async function getServerSideProps(context: NextPageContext) {
+  const page = Number(context.query.page || "1");
+  const blueprints = await getMostRecentBlueprintPages(page);
+
+  return {
+    props: {
+      page,
+      blueprints,
+    },
+  };
+}
+
 export default Index;

apps/blueprints/pages/normalize.ts

@@ -1,355 +0,0 @@
-import { css } from "@emotion/core";
-
-export const normalize = css`
-  /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */
-
-  /* Document
-   ========================================================================== */
-
-  /**
- * 1. Correct the line height in all browsers.
- * 2. Prevent adjustments of font size after orientation changes in iOS.
- */
-
-  html {
-    line-height: 1.15; /* 1 */
-    -webkit-text-size-adjust: 100%; /* 2 */
-  }
-
-  /* Sections
-   ========================================================================== */
-
-  /**
- * Remove the margin in all browsers.
- */
-
-  body {
-    margin: 0;
-  }
-
-  /**
- * Render the \`main\` element consistently in IE.
- */
-
-  main {
-    display: block;
-  }
-
-  /**
- * Correct the font size and margin on \`h1\` elements within \`section\` and
- * \`article\` contexts in Chrome, Firefox, and Safari.
- */
-
-  h1 {
-    font-size: 2em;
-    margin: 0.67em 0;
-  }
-
-  /* Grouping content
-   ========================================================================== */
-
-  /**
- * 1. Add the correct box sizing in Firefox.
- * 2. Show the overflow in Edge and IE.
- */
-
-  hr {
-    box-sizing: content-box; /* 1 */
-    height: 0; /* 1 */
-    overflow: visible; /* 2 */
-  }
-
-  /**
- * 1. Correct the inheritance and scaling of font size in all browsers.
- * 2. Correct the odd \`em\` font sizing in all browsers.
- */
-
-  pre {
-    font-family: monospace, monospace; /* 1 */
-    font-size: 1em; /* 2 */
-  }
-
-  /* Text-level semantics
-   ========================================================================== */
-
-  /**
- * Remove the gray background on active links in IE 10.
- */
-
-  a {
-    background-color: transparent;
-  }
-
-  /**
- * 1. Remove the bottom border in Chrome 57-
- * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari.
- */
-
-  abbr[title] {
-    border-bottom: none; /* 1 */
-    text-decoration: underline; /* 2 */
-    text-decoration: underline dotted; /* 2 */
-  }
-
-  /**
- * Add the correct font weight in Chrome, Edge, and Safari.
- */
-
-  b,
-  strong {
-    font-weight: bolder;
-  }
-
-  /**
- * 1. Correct the inheritance and scaling of font size in all browsers.
- * 2. Correct the odd \`em\` font sizing in all browsers.
- */
-
-  code,
-  kbd,
-  samp {
-    font-family: monospace, monospace; /* 1 */
-    font-size: 1em; /* 2 */
-  }
-
-  /**
- * Add the correct font size in all browsers.
- */
-
-  small {
-    font-size: 80%;
-  }
-
-  /**
- * Prevent \`sub\` and \`sup\` elements from affecting the line height in
- * all browsers.
- */
-
-  sub,
-  sup {
-    font-size: 75%;
-    line-height: 0;
-    position: relative;
-    vertical-align: baseline;
-  }
-
-  sub {
-    bottom: -0.25em;
-  }
-
-  sup {
-    top: -0.5em;
-  }
-
-  /* Embedded content
-   ========================================================================== */
-
-  /**
- * Remove the border on images inside links in IE 10.
- */
-
-  img {
-    border-style: none;
-  }
-
-  /* Forms
-   ========================================================================== */
-
-  /**
- * 1. Change the font styles in all browsers.
- * 2. Remove the margin in Firefox and Safari.
- */
-
-  button,
-  input,
-  optgroup,
-  select,
-  textarea {
-    font-family: inherit; /* 1 */
-    font-size: 100%; /* 1 */
-    line-height: 1.15; /* 1 */
-    margin: 0; /* 2 */
-  }
-
-  /**
- * Show the overflow in IE.
- * 1. Show the overflow in Edge.
- */
-
-  button,
-  input {
-    /* 1 */
-    overflow: visible;
-  }
-
-  /**
- * Remove the inheritance of text transform in Edge, Firefox, and IE.
- * 1. Remove the inheritance of text transform in Firefox.
- */
-
-  button,
-  select {
-    /* 1 */
-    text-transform: none;
-  }
-
-  /**
- * Correct the inability to style clickable types in iOS and Safari.
- */
-
-  button,
-  [type="button"],
-  [type="reset"],
-  [type="submit"] {
-    -webkit-appearance: button;
-  }
-
-  /**
- * Remove the inner border and padding in Firefox.
- */
-
-  button::-moz-focus-inner,
-  [type="button"]::-moz-focus-inner,
-  [type="reset"]::-moz-focus-inner,
-  [type="submit"]::-moz-focus-inner {
-    border-style: none;
-    padding: 0;
-  }
-
-  /**
- * Restore the focus styles unset by the previous rule.
- */
-
-  button:-moz-focusring,
-  [type="button"]:-moz-focusring,
-  [type="reset"]:-moz-focusring,
-  [type="submit"]:-moz-focusring {
-    outline: 1px dotted ButtonText;
-  }
-
-  /**
- * Correct the padding in Firefox.
- */
-
-  fieldset {
-    padding: 0.35em 0.75em 0.625em;
-  }
-
-  /**
- * 1. Correct the text wrapping in Edge and IE.
- * 2. Correct the color inheritance from \`fieldset\` elements in IE.
- * 3. Remove the padding so developers are not caught out when they zero out
- *    \`fieldset\` elements in all browsers.
- */
-
-  legend {
-    box-sizing: border-box; /* 1 */
-    color: inherit; /* 2 */
-    display: table; /* 1 */
-    max-width: 100%; /* 1 */
-    padding: 0; /* 3 */
-    white-space: normal; /* 1 */
-  }
-
-  /**
- * Add the correct vertical alignment in Chrome, Firefox, and Opera.
- */
-
-  progress {
-    vertical-align: baseline;
-  }
-
-  /**
- * Remove the default vertical scrollbar in IE 10+.
- */
-
-  textarea {
-    overflow: auto;
-  }
-
-  /**
- * 1. Add the correct box sizing in IE 10.
- * 2. Remove the padding in IE 10.
- */
-
-  [type="checkbox"],
-  [type="radio"] {
-    box-sizing: border-box; /* 1 */
-    padding: 0; /* 2 */
-  }
-
-  /**
- * Correct the cursor style of increment and decrement buttons in Chrome.
- */
-
-  [type="number"]::-webkit-inner-spin-button,
-  [type="number"]::-webkit-outer-spin-button {
-    height: auto;
-  }
-
-  /**
- * 1. Correct the odd appearance in Chrome and Safari.
- * 2. Correct the outline style in Safari.
- */
-
-  [type="search"] {
-    -webkit-appearance: textfield; /* 1 */
-    outline-offset: -2px; /* 2 */
-  }
-
-  /**
- * Remove the inner padding in Chrome and Safari on macOS.
- */
-
-  [type="search"]::-webkit-search-decoration {
-    -webkit-appearance: none;
-  }
-
-  /**
- * 1. Correct the inability to style clickable types in iOS and Safari.
- * 2. Change font properties to \`inherit\` in Safari.
- */
-
-  ::-webkit-file-upload-button {
-    -webkit-appearance: button; /* 1 */
-    font: inherit; /* 2 */
-  }
-
-  /* Interactive
-   ========================================================================== */
-
-  /*
- * Add the correct display in Edge, IE 10+, and Firefox.
- */
-
-  details {
-    display: block;
-  }
-
-  /*
- * Add the correct display in all browsers.
- */
-
-  summary {
-    display: list-item;
-  }
-
-  /* Misc
-   ========================================================================== */
-
-  /**
- * Add the correct display in IE 10+.
- */
-
-  template {
-    display: none;
-  }
-
-  /**
- * Add the correct display in IE 10.
- */
-
-  [hidden] {
-    display: none;
-  }
-`;

apps/blueprints/prod.package.json

@@ -1,11 +1,22 @@
 {
   "dependencies": {
-    "@emotion/core": "10.0.28",
+    "@chakra-ui/core": "0.8.0",
+    "@emotion/core": "10.0.35",
     "@emotion/styled": "10.0.27",
-    "document-register-element": "1.13.1",
+    "@google-cloud/datastore": "6.2.0",
+    "@google-cloud/pubsub": "2.6.0",
+    "@google-cloud/storage": "5.3.0",
+    "bbcode-to-react": "0.2.9",
+    "document-register-element": "1.14.10",
     "emotion-server": "10.0.27",
-    "next": "9.5.2",
+    "emotion-theming": "10.0.27",
+    "next": "9.5.5",
+    "nprogress": "0.2.0",
+    "pako": "1.0.11",
+    "phin": "3.5.0",
     "react": "16.13.1",
-    "react-dom": "16.13.1"
+    "react-dom": "16.13.1",
+    "react-map-interaction": "2.0.0",
+    "react-markdown": "5.0.1"
   }
 }

apps/blueprints/src/BookChildTree.tsx

@@ -0,0 +1,63 @@
+/** @jsx jsx */
+/* eslint-disable jsx-a11y/anchor-is-valid */
+import { jsx, css } from "@emotion/core";
+import Link from "next/link";
+import BBCode from "bbcode-to-react";
+import { BlueprintBookEntry } from "@factorio-sites/database";
+
+const componentStyles = css`
+  .blueprint,
+  .book {
+    display: block;
+    color: white;
+    padding: 4px 0;
+    &:hover {
+      background: #636363;
+    }
+  }
+  .active {
+    background: #41a73a;
+    color: #000;
+  }
+`;
+
+interface BookChildTreeProps {
+  child_tree: BlueprintBookEntry["child_tree"];
+  base_url: string;
+  selected_id: string;
+}
+
+export const BookChildTree: React.FC<BookChildTreeProps> = ({
+  child_tree,
+  base_url,
+  selected_id,
+}) => {
+  return (
+    <div css={componentStyles}>
+      {child_tree.map((bp, key) => {
+        return bp.type === "blueprint" ? (
+          <Link key={bp.id} href={`${base_url}?selected=${bp.id}`}>
+            <a className={"blueprint" + (selected_id === bp.id ? " active" : "")}>
+              {BBCode.toReact(bp.name)}
+            </a>
+          </Link>
+        ) : (
+          <div key={key}>
+            <Link key={bp.id} href={`${base_url}?selected=${bp.id}&type=book`}>
+              <a className={"book" + (selected_id === bp.id ? " active" : "")}>
+                [book] {BBCode.toReact(bp.name)}
+              </a>
+            </Link>
+            <div css={{ marginLeft: `20px` }}>
+              <BookChildTree
+                child_tree={bp.children}
+                base_url={base_url}
+                selected_id={selected_id}
+              />
+            </div>
+          </div>
+        );
+      })}
+    </div>
+  );
+};

apps/blueprints/src/CopyButton.tsx

@@ -0,0 +1,35 @@
+import React, { useState } from "react";
+import { Button, ButtonProps } from "@chakra-ui/core";
+
+export const CopyButton: React.FC<Omit<ButtonProps, "children"> & { content: string }> = ({
+  content,
+  ...props
+}) => {
+  const [loading, setLoading] = useState(false);
+  const [icon, setIcon] = useState<"check" | "small-close" | null>(null);
+
+  return (
+    <Button
+      {...props}
+      isLoading={loading}
+      leftIcon={icon}
+      variantColor={icon === "small-close" ? "red" : "green"}
+      onClick={() => {
+        setLoading(true);
+        navigator.clipboard
+          .writeText(content)
+          .then(() => {
+            setLoading(false);
+            setIcon("check");
+            setTimeout(() => setIcon(null), 2500);
+          })
+          .catch(() => {
+            setLoading(false);
+            setIcon("small-close");
+          });
+      }}
+    >
+      copy
+    </Button>
+  );
+};

apps/blueprints/src/FullscreenImage.tsx

@@ -0,0 +1,54 @@
+/** @jsx jsx */
+/* eslint-disable jsx-a11y/anchor-is-valid */
+import React, { useState } from "react";
+import { jsx, css } from "@emotion/core";
+import { MapInteractionCSS } from "react-map-interaction";
+
+const elementStyle = css`
+  display: flex;
+  position: fixed;
+  top: 0;
+  bottom: 0;
+  left: 0;
+  right: 0;
+  padding: 0px;
+  background: rgba(0, 0, 0, 0.5);
+  justify-content: center;
+  & > div > div {
+    display: flex;
+    justify-content: center;
+  }
+`;
+
+interface FullscreenImageProps {
+  alt: string;
+  src: string;
+  close: () => void;
+}
+
+export const FullscreenImage: React.FC<FullscreenImageProps> = ({ alt, src, close }) => {
+  const [state, setState] = useState({
+    scale: 0.9,
+    translation: { x: window.innerWidth * 0.05, y: 30 },
+  });
+
+  return (
+    <div
+      css={elementStyle}
+      onClick={(e) => {
+        if ((e as any).target.nodeName.toUpperCase() !== "IMG") {
+          close();
+        }
+      }}
+      onTouchEnd={(e) => {
+        if ((e as any).target.nodeName.toUpperCase() !== "IMG") {
+          close();
+        }
+      }}
+    >
+      <MapInteractionCSS value={state} onChange={setState}>
+        <img alt={alt} src={src} />
+      </MapInteractionCSS>
+    </div>
+  );
+};

apps/blueprints/src/Header.tsx

@@ -0,0 +1,61 @@
+import React from "react";
+import { Box, Heading, Flex } from "@chakra-ui/core";
+import Link from "next/link";
+
+// const MenuItems = ({ children }) => (
+//   <Text mt={{ base: 4, md: 0 }} mr={6} display="block">
+//     {children}
+//   </Text>
+// );
+
+export const Header: React.FC = (props) => {
+  const [show, setShow] = React.useState(false);
+  const handleToggle = () => setShow(!show);
+
+  return (
+    <Flex
+      as="header"
+      align="center"
+      justify="space-between"
+      wrap="wrap"
+      padding="1.5rem"
+      bg="rgba(0,0,0,.7)"
+      color="#ffe6c0"
+      {...props}
+    >
+      <Flex align="center" mr={5}>
+        <Heading as="h1" size="lg">
+          <Link href="/">
+            {/* eslint-disable-next-line jsx-a11y/anchor-is-valid*/}
+            <a>Factorio Blueprints</a>
+          </Link>
+        </Heading>
+      </Flex>
+
+      <Box display={{ base: "block", md: "none" }} onClick={handleToggle}>
+        <svg fill="white" width="12px" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg">
+          <title>Menu</title>
+          <path d="M0 3h20v2H0V3zm0 6h20v2H0V9zm0 6h20v2H0v-2z" />
+        </svg>
+      </Box>
+
+      {/* <Box
+        as="nav"
+        display={{ sm: show ? "block" : "none", md: "flex" }}
+        width={{ sm: "full", md: "auto" }}
+        alignItems="center"
+        flexGrow={1}
+      >
+        <MenuItems>Docs</MenuItems>
+        <MenuItems>Examples</MenuItems>
+        <MenuItems>Blog</MenuItems>
+      </Box> */}
+
+      {/* <Box display={{ sm: show ? "block" : "none", md: "block" }} mt={{ base: 4, md: 0 }}>
+        <Button bg="transparent" border="1px">
+          Create account
+        </Button>
+      </Box> */}
+    </Flex>
+  );
+};

apps/blueprints/src/Markdown.tsx

@@ -0,0 +1,25 @@
+/** @jsx jsx */
+/* eslint-disable jsx-a11y/anchor-is-valid */
+import { jsx, css } from "@emotion/core";
+import ReactMarkdown from "react-markdown";
+
+const markdownStyle = css`
+  overflow: hidden;
+  img {
+    max-width: 100%;
+  }
+  ul,
+  ol {
+    margin-block-start: 1em;
+    margin-block-end: 1em;
+    margin-inline-start: 0px;
+    margin-inline-end: 0px;
+    padding-inline-start: 40px;
+  }
+`;
+
+export const Markdown: React.FC<{ children: string }> = ({ children }) => (
+  <div css={markdownStyle}>
+    <ReactMarkdown>{children}</ReactMarkdown>
+  </div>
+);

apps/blueprints/src/Pagination.tsx

@@ -0,0 +1,35 @@
+/** @jsx jsx */
+import { jsx } from "@emotion/core";
+import React from "react";
+import Link from "next/link";
+import { useRouter } from "next/router";
+import { Box, BoxProps, Button } from "@chakra-ui/core";
+
+interface PaginationProps {
+  page: number;
+}
+
+const PaginationLink: React.FC<PaginationProps> = ({ page }) => {
+  const router = useRouter();
+  const query = { ...router.query, page };
+  const href =
+    "?" +
+    Object.keys(query)
+      .map((key) => `${key}=${query[key]}`)
+      .join("&");
+
+  return (
+    <Link href={href} passHref>
+      <Button as="a" size="sm" color="black" css={{ marginRight: "1rem" }}>
+        {page}
+      </Button>
+    </Link>
+  );
+};
+
+export const Pagination: React.FC<BoxProps & PaginationProps> = ({ page, ...props }) => (
+  <Box {...props}>
+    {page > 1 && <PaginationLink page={page - 1} />}
+    <PaginationLink page={page + 1} />
+  </Box>
+);

apps/blueprints/src/Panel.tsx

@@ -0,0 +1,39 @@
+/** @jsx jsx */
+import { jsx, css, SerializedStyles } from "@emotion/core";
+import { Box, BoxProps } from "@chakra-ui/core";
+import { ReactNode } from "react";
+
+const panelStyles = css`
+  display: flex;
+  flex-direction: column;
+  padding: 12px;
+  background-color: #313031;
+  overflow: hidden;
+  box-shadow: inset 3px 0 3px -3px #201815, inset 2px 0 2px -2px #201815,
+    inset 1px 0 1px -1px #201815, inset 0 3px 3px -3px #8f8c8b, inset 0 2px 2px -2px #8f8c8b,
+    inset 0 1px 1px -1px #8f8c8b, inset -3px 0 3px -3px #201815, inset -2px 0 2px -2px #201815,
+    inset -2px 0 1px -1px #201815, inset 0 -3px 3px -3px #000, inset 0 -2px 2px -2px #000,
+    inset 0 -1px 1px -1px #000, 0 0 2px 0 #201815, 0 0 4px 0 #201815;
+
+  h2 {
+    font-weight: 900;
+    color: #ffe6c0;
+    line-height: 1.25;
+    margin: 0 0 12px 0;
+    font-size: 120%;
+  }
+`;
+const boxShadow = `inset 0 0 3px 0 #000, 0 -2px 2px -1px #000, -2px 0 2px -2px #28221f,
+-2px 0 2px -2px #28221f, 2px 0 2px -2px #28221f, 2px 0 2px -2px #28221f,
+0 3px 3px -3px #8f8c8b, 0 2px 2px -2px #8f8c8b, 0 1px 1px -1px #8f8c8b`;
+
+export const Panel: React.FC<
+  BoxProps & { title?: string | ReactNode | null; css?: SerializedStyles }
+> = ({ children, title, css: prop_css, ...props }) => (
+  <Box css={prop_css ? [panelStyles, prop_css] : [panelStyles]} {...props}>
+    {title ? <h2>{title}</h2> : null}
+    <Box color="white" height="100%" padding="12px" bg="#414040" boxShadow={boxShadow}>
+      {children}
+    </Box>
+  </Box>
+);

apps/factorioprints-scraper/.eslintrc.json

@@ -0,0 +1 @@
+{ "extends": "../../.eslintrc.json", "ignorePatterns": ["!**/*"], "rules": {} }

apps/factorioprints-scraper/data/tags.ts

@@ -0,0 +1,97 @@
+export const tags = {
+  belt: {
+    "0": "balancer",
+    "1": "prioritizer",
+    "2": "tap",
+    "3": "transport belt (yellow)",
+    "4": "fast transport belt (red)",
+    "5": "express transport belt (blue)",
+  },
+  circuit: { "0": "indicator", "1": "counter" },
+  general: {
+    "0": "early game",
+    "1": "mid game",
+    "2": "late game (megabase)",
+    "3": "beaconized",
+    "4": "tileable",
+    "5": "compact",
+    "6": "marathon",
+    "7": "storage",
+    "8": "chunk aligned",
+  },
+  moderation: {
+    "0": "scheduled for deletion",
+    "1": "screenshot missing alt info",
+    "2": "uncropped screenshot",
+    "3": "add english translation",
+  },
+  mods: {
+    "0": "angels",
+    "1": "bobs",
+    "2": "creative",
+    "3": "factorissimo",
+    "4": "warehousing",
+    "5": "lighted-electric-poles",
+    "6": "other",
+    "7": "vanilla",
+    "8": "ltn",
+    "9": "aai",
+    "10": "5dim",
+    "11": "seablock",
+    "12": "industrial revolution",
+    "13": "krastorio",
+    "14": "transport drones",
+  },
+  power: {
+    "0": "nuclear",
+    "1": "kovarex enrichment",
+    "2": "solar",
+    "3": "steam",
+    "4": "accumulator",
+  },
+  production: {
+    "0": "oil processing",
+    "1": "coal liquification",
+    "2": "electronic circuit (green)",
+    "3": "advanced circuit (red)",
+    "4": "processing unit (blue)",
+    "5": "batteries",
+    "6": "rocket parts",
+    "7": "science",
+    "8": "research (labs)",
+    "9": "belts",
+    "10": "smelting",
+    "11": "mining",
+    "12": "uranium",
+    "13": "plastic",
+    "14": "modules",
+    "15": "mall (make everything)",
+    "16": "inserters",
+    "17": "guns and ammo",
+    "18": "robots",
+    "19": "other",
+    "20": "belt based",
+    "21": "logistic (bot) based",
+  },
+  train: {
+    "0": "loading station",
+    "1": "unloading station",
+    "2": "pax",
+    "3": "junction",
+    "4": "roundabout",
+    "5": "crossing",
+    "6": "stacker",
+    "7": "track",
+    "8": "left-hand-drive",
+    "9": "right-hand-drive",
+  },
+  version: {
+    "0": "0,14",
+    "1": "0,15",
+    "2": "0,16",
+    "3": "unknown",
+    "4": "0,17",
+    "5": "0,18",
+    "6": "1,0",
+  },
+};

apps/factorioprints-scraper/jest.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  displayName: "factorioprints-scraper",
+  preset: "../../jest.preset.js",
+  globals: {
+    "ts-jest": {
+      tsConfig: "<rootDir>/tsconfig.spec.json",
+    },
+  },
+  transform: {
+    "^.+\\.[tj]s$": "ts-jest",
+  },
+  moduleFileExtensions: ["ts", "js", "html"],
+  coverageDirectory: "../../coverage/apps/factorioprints-scraper",
+};

apps/factorioprints-scraper/src/app/populate-db.ts

@@ -0,0 +1,45 @@
+import * as fs from "fs";
+import * as path from "path";
+import { promisify } from "util";
+import {
+  getBlueprintPageByFactorioprintsId,
+  saveBlueprintFromFactorioprints,
+} from "@factorio-sites/database";
+
+const fsReadFile = promisify(fs.readFile);
+const CACHE_DIR = path.join(__dirname, "../../../.cache");
+
+export async function writeToDatastore() {
+  const filecontent = await fsReadFile(path.join(CACHE_DIR, `most-fav-json/page1.json`), "utf8");
+  const data = JSON.parse(filecontent);
+
+  for (let i = 0; i < data.length; i++) {
+    const bpfilecontent = await fsReadFile(
+      path.join(CACHE_DIR, `blueprint/${data[i].factorioprintsId}.json`),
+      "utf8"
+    );
+    const bp = JSON.parse(bpfilecontent);
+
+    const exists = await getBlueprintPageByFactorioprintsId(bp.factorioprintsId);
+    if (exists) {
+      console.log(`entity ${bp.factorioprintsId} already exists`);
+      continue;
+    }
+
+    await saveBlueprintFromFactorioprints(
+      {
+        title: bp.title,
+        factorioprints_id: bp.factorioprintsId,
+        description_markdown: bp.descriptionMarkdown,
+        tags: bp.tags ? Object.values(bp.tags) : [],
+        updated_at: bp.lastUpdatedDate / 1000,
+        created_at: bp.createdDate / 1000,
+      },
+      bp.blueprintString
+    ).catch((reason) => {
+      console.error(reason);
+    });
+
+    console.log(`saved entity ${bp.factorioprintsId} to database`);
+  }
+}

apps/factorioprints-scraper/src/app/scan.ts

@@ -0,0 +1,165 @@
+import * as crypto from "crypto";
+import * as fs from "fs";
+import * as path from "path";
+import { promisify } from "util";
+import * as WebSocket from "ws";
+
+const fsWriteFile = promisify(fs.writeFile);
+const fsExists = promisify(fs.exists);
+const CACHE_DIR = path.join(__dirname, "../../../.cache");
+
+const getMostFavoriteRequestObject = ({
+  lastId,
+  lastNumOfFavorites,
+}: {
+  lastId?: string;
+  lastNumOfFavorites?: number;
+}) => ({
+  t: "d",
+  d: {
+    r: 2,
+    a: "q",
+    b: {
+      p: "/blueprintSummaries",
+      q: { ep: lastNumOfFavorites, en: lastId, l: 61, vf: "r", i: "numberOfFavorites" },
+      t: 1,
+      h: "",
+    },
+  },
+});
+
+const getBlueprintRequestObject = ({ id }: { id: string }) => ({
+  t: "d",
+  d: { r: 7, a: "q", b: { p: `/blueprints/${id}`, h: "" } },
+});
+
+const openWebsocket = (): Promise<WebSocket> => {
+  const ws = new WebSocket("wss://s-usc1c-nss-239.firebaseio.com/.ws?v=5&ns=facorio-blueprints");
+  return new Promise((resolve) => {
+    ws.on("open", function open() {
+      resolve(ws);
+    });
+    ws.on("error", (error) => {
+      console.log("[ws:error]", error);
+    });
+    ws.on("close", () => {
+      console.log("[ws:close]");
+    });
+  });
+};
+
+export async function scanFactorioPrints() {
+  const ws = await openWebsocket();
+
+  const sendMessage = (data: any) => {
+    const string = JSON.stringify(data);
+    console.log("[ws:send]", string);
+    ws.send(string);
+  };
+
+  const sendMessageAndWaitForResponse = (
+    sent_data: any,
+    filter: (data: any) => boolean
+  ): Promise<any> => {
+    return new Promise((resolve) => {
+      const buffer = [] as string[];
+      const onmessage = (_message: WebSocket.Data) => {
+        const message = _message.toString();
+        let data;
+        try {
+          data = JSON.parse(message);
+        } catch (e) {
+          buffer.push(message);
+
+          if (buffer.length > 1) {
+            try {
+              data = JSON.parse(buffer.join(""));
+              console.log(`\nValid json found after reading ${buffer.length} messages`);
+            } catch (e) {
+              /**/
+            }
+          }
+          if (!data) return process.stdout.write(buffer.length === 1 ? "Buffering messages." : ".");
+        }
+
+        if (filter(data)) {
+          ws.off("message", onmessage);
+          resolve(data);
+        }
+      };
+      ws.on("message", onmessage);
+      sendMessage(sent_data);
+    });
+  };
+
+  const getMostFavorited = async (
+    page = 1,
+    reqObj: Parameters<typeof getMostFavoriteRequestObject>[0]
+  ) => {
+    const data = await sendMessageAndWaitForResponse(
+      getMostFavoriteRequestObject(reqObj),
+      (message) => message?.d?.b?.p === "blueprintSummaries" && message.d.b?.d
+    );
+
+    const _blueprints = data.d.b.d;
+    const ids = Object.keys(_blueprints);
+
+    const blueprints = ids
+      .map((id) => ({
+        factorioprintsId: id,
+        ..._blueprints[id],
+      }))
+      .sort((a, b) => b.numberOfFavorites - a.numberOfFavorites);
+
+    // First fetch details of each individual blueprint
+    for (let i = 0; i < blueprints.length; i++) {
+      const blueprint = blueprints[i];
+      const file_path = path.join(CACHE_DIR, `blueprint/${blueprint.factorioprintsId}.json`);
+
+      if (await fsExists(file_path)) {
+        console.log(`Blueprint ${blueprint.factorioprintsId} exists, skipping...`);
+        continue;
+      }
+
+      const result = await sendMessageAndWaitForResponse(
+        getBlueprintRequestObject({ id: blueprint.factorioprintsId }),
+        (data) => data?.d?.b?.p === "blueprints/" + blueprint.factorioprintsId
+      );
+
+      const bp = result.d.b.d;
+      const hash = crypto.createHash("sha1").update(bp.blueprintString).digest("hex");
+      bp.factorioprintsId = blueprint.factorioprintsId;
+      bp.hash = hash;
+      blueprint.hash = hash;
+
+      console.log(`Writing blueprint /blueprint/${bp.factorioprintsId}.json`);
+
+      fsWriteFile(
+        path.join(CACHE_DIR, `blueprint/${bp.factorioprintsId}.json`),
+        JSON.stringify(bp, null, 2)
+      ).catch((reason) => {
+        console.error(reason);
+      });
+    }
+
+    // Write the result of entire page including the hashes
+    console.log(`Writing blueprints /most-fav-json/page${page}.json`);
+
+    fsWriteFile(
+      path.join(CACHE_DIR, `most-fav-json/page${page}.json`),
+      JSON.stringify(blueprints, null, 2)
+    ).catch((reason) => {
+      console.error(reason);
+    });
+
+    if (page < 3) {
+      const lastBp = blueprints[blueprints.length - 1];
+      getMostFavorited(page + 1, {
+        lastId: lastBp.id,
+        lastNumOfFavorites: lastBp.numberOfFavorites,
+      });
+    }
+  };
+
+  getMostFavorited(1, {});
+}

apps/factorioprints-scraper/src/environments/environment.prod.ts

@@ -0,0 +1,3 @@
+export const environment = {
+  production: true,
+};

apps/factorioprints-scraper/src/environments/environment.ts

@@ -0,0 +1,3 @@
+export const environment = {
+  production: false,
+};

apps/factorioprints-scraper/src/main.ts

@@ -0,0 +1,32 @@
+import { writeToDatastore } from "./app/populate-db";
+
+// async function writeTestBP() {
+//   const source =
+//     "0eNrtkmFrgzAQhv/KuM+xNBq1zV8pQ9QdW5gmYtIykfz3mZRhh2VLpfsw6Me75J73eO8doWqO2PVCmqJS6h34OHc08MNF6d5EreS5rcWrLBvXM0OHwEEYbIGALFtXaYPYRPUbagOWgJAv+AGc2mcCKI0wAs8YXwyFPLYV9tOHqwACndLTjJJOb+JEyX6TEhiAs3yTWkf18vxiWwJNWeG0IXjIE506J+y1p8Q7ynK2z7OcbrM0mzfcWjL+V0923zwhC1D8d+bGD3PXmbsEJaGg7BcQCwXlN5+bBZ77CivyV5+BrnS4sjbihMXXaAifhvJ9PFcJPPJ6v7zeHrPkvjHzvBUxiIMF6EKA/iBgPwEPcWsm";
+//   const { hash, string } = await parseBlueprintString(source);
+
+//   console.log(`saving ${hash}`);
+
+//   // use createBlueprint
+//   saveBlueprintFromFactorioprints(
+//     {
+//       factorioprints_id: null,
+//       title: "my blueprint",
+//       description_markdown: "",
+//       tags: [],
+//       updated_at: Date.now() / 1000,
+//       created_at: Date.now() / 1000,
+//     },
+//     string
+//   );
+// }
+
+async function main() {
+  // scanFactorioPrints();
+  writeToDatastore();
+  // writeTestBP();
+}
+
+main().catch((reason) => {
+  console.error(reason);
+});

apps/factorioprints-scraper/tsconfig.app.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "types": ["node"]
+  },
+  "exclude": ["**/*.spec.ts"],
+  "include": ["**/*.ts"]
+}

apps/factorioprints-scraper/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.app.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

apps/factorioprints-scraper/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*.spec.ts", "**/*.d.ts"]
+}

blueprints.Dockerfile

@@ -14,6 +14,7 @@ RUN yarn nx build blueprints
 FROM node:14-slim
 
 WORKDIR /usr/src/app
+
 COPY apps/blueprints/prod.package.json ./package.json
 COPY yarn.lock .
 

jest.config.js

@@ -3,5 +3,10 @@ module.exports = {
     "<rootDir>/apps/blueprints",
     "<rootDir>/libs/generate-bp-image",
     "<rootDir>/apps/blueprint-image-function",
+    "<rootDir>/apps/factorioprints-scraper",
+    "<rootDir>/libs/database",
+    "<rootDir>/libs/utils",
+    "<rootDir>/libs/common-utils",
+    "<rootDir>/libs/web-utils",
   ],
 };

libs/common-utils/.eslintrc.json

@@ -0,0 +1 @@
+{ "extends": "../../.eslintrc.json", "ignorePatterns": ["!**/*"], "rules": {} }

libs/common-utils/README.md

@@ -0,0 +1,7 @@
+# common-utils
+
+This library was generated with [Nx](https://nx.dev).
+
+## Running unit tests
+
+Run `nx test common-utils` to execute the unit tests via [Jest](https://jestjs.io).

libs/common-utils/jest.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  displayName: "common-utils",
+  preset: "../../jest.preset.js",
+  globals: {
+    "ts-jest": {
+      tsConfig: "<rootDir>/tsconfig.spec.json",
+    },
+  },
+  transform: {
+    "^.+\\.[tj]sx?$": "ts-jest",
+  },
+  moduleFileExtensions: ["ts", "tsx", "js", "jsx"],
+  coverageDirectory: "../../coverage/libs/common-utils",
+};

libs/common-utils/src/index.ts

@@ -0,0 +1 @@
+export * from "./lib/common-utils";

libs/common-utils/src/lib/common-utils.spec.ts

@@ -0,0 +1,7 @@
+// import { commonUtils } from "./common-utils";
+
+describe("commonUtils", () => {
+  it("should work", () => {
+    // expect(commonUtils()).toEqual("common-utils");
+  });
+});

libs/common-utils/src/lib/common-utils.ts

@@ -0,0 +1,108 @@
+interface Entity {
+  entity_number: number;
+  name: string;
+  position: { x: number; y: number };
+}
+
+export interface Blueprint {
+  entities: Entity[];
+  tiles?: { name: string; position: { x: number; y: number } }[];
+  icons: { signal: { type: "item" | "fluid"; name: string } }[];
+  item: string;
+  label: string;
+  description?: string;
+  version: number;
+}
+
+export interface BlueprintBook {
+  active_index: number;
+  blueprints: Array<{ index: number } & BlueprintData>;
+  item: string;
+  label: string;
+  description?: string;
+  version: number;
+}
+
+export interface BlueprintData {
+  blueprint_book?: BlueprintBook;
+  blueprint?: Blueprint;
+}
+
+export const getBlueprintContentForImageHash = (blueprint: Blueprint): string => {
+  return JSON.stringify({
+    entities: blueprint.entities,
+    tiles: blueprint.tiles,
+  });
+};
+
+export const flattenBlueprintData = (data: BlueprintData) => {
+  const blueprints: Blueprint[] = [];
+  const books: BlueprintBook[] = [];
+
+  // Recursively go through the string to find all blueprints
+  const findAndPushBlueprints = (data: BlueprintData) => {
+    if (data.blueprint) {
+      blueprints.push(data.blueprint);
+    } else if (data.blueprint_book) {
+      books.push(data.blueprint_book);
+      data.blueprint_book.blueprints.forEach(({ index, ...bp }) => {
+        findAndPushBlueprints(bp);
+      });
+    }
+  };
+
+  findAndPushBlueprints(data);
+
+  return {
+    blueprints,
+    books,
+  };
+};
+
+export const findBlueprintByPath = (data: BlueprintData, path: number[]): Blueprint | null => {
+  if (path.length === 0) {
+    return (data.blueprint || data.blueprint_book?.blueprints[0]) as Blueprint;
+  } else if (data.blueprint_book && path.length === 1) {
+    return data.blueprint_book.blueprints[path[0]].blueprint as Blueprint;
+  }
+  return null;
+};
+
+export const findActiveBlueprint = (
+  data: BlueprintData
+): { blueprint: Blueprint; path: number[] } => {
+  if (data.blueprint) {
+    return { blueprint: data.blueprint, path: [0] };
+  } else if (data.blueprint_book) {
+    const findActive = (
+      book: BlueprintBook,
+      _path: number[] = []
+    ): { blueprint: Blueprint; path: number[] } => {
+      const active = book.blueprints.find((bp) => bp.index === book.active_index);
+
+      if (active && active.blueprint) {
+        return {
+          blueprint: active.blueprint,
+          path: _path.concat(book.active_index),
+        };
+      } else if (active && active.blueprint_book) {
+        return findActive(active.blueprint_book, _path.concat(book.active_index));
+      }
+
+      throw Error("Could not find active blueprint");
+    };
+
+    return findActive(data.blueprint_book);
+  }
+  throw Error("Could not find active blueprint");
+};
+
+export const timeLogger = (base_msg: string) => {
+  const start_time = Date.now();
+  let last_time = Date.now();
+  return (message: string) => {
+    const now = Date.now();
+    console.log(`[${base_msg}] ${message} in ${now - last_time} (${now - start_time} total)`);
+    last_time = now;
+  };
+};

libs/common-utils/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

libs/common-utils/tsconfig.lib.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "exclude": ["**/*.spec.ts"],
+  "include": ["**/*.ts"]
+}

libs/common-utils/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.spec.js", "**/*.spec.jsx", "**/*.d.ts"]
+}

libs/database/.eslintrc.json

@@ -0,0 +1 @@
+{ "extends": "../../.eslintrc.json", "ignorePatterns": ["!**/*"], "rules": {} }

libs/database/README.md

@@ -0,0 +1,7 @@
+# database
+
+This library was generated with [Nx](https://nx.dev).
+
+## Running unit tests
+
+Run `nx test database` to execute the unit tests via [Jest](https://jestjs.io).

libs/database/jest.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  displayName: "database",
+  preset: "../../jest.preset.js",
+  globals: {
+    "ts-jest": {
+      tsConfig: "<rootDir>/tsconfig.spec.json",
+    },
+  },
+  transform: {
+    "^.+\\.[tj]sx?$": "ts-jest",
+  },
+  moduleFileExtensions: ["ts", "tsx", "js", "jsx"],
+  coverageDirectory: "../../coverage/libs/database",
+};

libs/database/src/index.ts

@@ -0,0 +1,2 @@
+export * from "./lib/database";
+export * from "./lib/pubsub";

libs/database/src/lib/database.spec.ts

@@ -0,0 +1,7 @@
+import { database } from "./database";
+
+describe("database", () => {
+  it("should work", () => {
+    expect(database()).toEqual("database");
+  });
+});

libs/database/src/lib/database.ts

@@ -0,0 +1,547 @@
+import { Storage } from "@google-cloud/storage";
+import { Datastore } from "@google-cloud/datastore";
+import { encodeBlueprint, hashString, parseBlueprintString } from "@factorio-sites/node-utils";
+import {
+  Blueprint,
+  BlueprintBook,
+  getBlueprintContentForImageHash,
+} from "@factorio-sites/common-utils";
+import { getBlueprintImageRequestTopic } from "./pubsub";
+
+// to dev on windows run: $env:GOOGLE_APPLICATION_CREDENTIALS="FULL_PATH"
+const storage = new Storage();
+const datastore = new Datastore();
+
+const BLUEPRINT_BUCKET = storage.bucket("blueprint-strings");
+const IMAGE_BUCKET = storage.bucket("blueprint-images");
+const BlueprintEntity = "Blueprint";
+const BlueprintBookEntity = "BlueprintBook";
+const BlueprintPageEntity = "BlueprintPage";
+const BlueprintStringEntity = "BlueprintString";
+const blueprintImageRequestTopic = getBlueprintImageRequestTopic();
+
+interface BlueprintChild {
+  type: "blueprint";
+  id: string;
+  name: string;
+}
+interface BlueprintBookChild {
+  type: "blueprint_book";
+  id: string;
+  name: string;
+  children: ChildTree;
+}
+type ChildTree = Array<BlueprintChild | BlueprintBookChild>;
+
+export interface BlueprintEntry {
+  id: string;
+  label: string; // from source
+  description: string | null; // from source
+  game_version: number; // from source
+  blueprint_hash: string;
+  image_hash: string;
+  created_at: number;
+  updated_at: number;
+  tags: string[];
+  factorioprints_id?: string;
+  // BlueprintEntry->BlueprintString 1:m
+  // BlueprintEntry->BlueprintPageEntry n:m
+}
+
+export interface BlueprintBookEntry {
+  id: string;
+  label: string;
+  description?: string;
+  /** strings as keys of BlueprintEntry */
+  blueprint_ids: string[];
+  /** strings as keys of BlueprintBookEntry (currently unsupported) */
+  blueprint_book_ids: string[];
+  child_tree: ChildTree;
+  blueprint_hash: string;
+  created_at: number;
+  updated_at: number;
+  is_modded: boolean;
+  factorioprints_id?: string;
+  // BlueprintBook:BlueprintBook n:m
+  // BlueprintBook:BlueprintEntry 1:m
+}
+
+export interface BlueprintPageEntry {
+  id: string;
+  blueprint_id?: string;
+  blueprint_book_id?: string;
+  title: string;
+  description_markdown: string;
+  created_at: number;
+  updated_at: number;
+  factorioprints_id?: string;
+  // BlueprintPageEntry->BlueprintEntry 1:m
+  // BlueprintPageEntry->BlueprintBook 1:m
+}
+
+export interface BlueprintStringEntry {
+  blueprint_id: string;
+  blueprint_hash: string;
+  image_hash: string;
+  version: number;
+  changes_markdown: string;
+  created_at: Date;
+  // BlueprintString->BlueprintEntry m:1
+}
+
+class DatastoreExistsError extends Error {
+  constructor(public existingId: string, message: string) {
+    super(message);
+    this.name = this.constructor.name;
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+/*
+ * BlueprintBook
+ */
+
+const mapBlueprintBookEntityToObject = (entity: any): BlueprintBookEntry => ({
+  id: entity[datastore.KEY].id,
+  blueprint_ids: entity.blueprint_ids.map((key: any) => key.id),
+  blueprint_book_ids: entity.blueprint_book_ids.map((key: any) => key.id),
+  child_tree: entity.child_tree ? entity.child_tree : [],
+  blueprint_hash: entity.blueprint_hash,
+  label: entity.label,
+  description: entity.description,
+  created_at: entity.created_at && entity.created_at.getTime() / 1000,
+  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+  is_modded: entity.is_modded || false,
+  factorioprints_id: entity.factorioprints_id || null,
+});
+
+export async function getBlueprintBookById(id: string): Promise<BlueprintBookEntry | null> {
+  const result = await datastore.get(datastore.key([BlueprintBookEntity, Number(id)]));
+  return result[0] ? mapBlueprintBookEntityToObject(result[0]) : null;
+}
+
+export async function getBlueprintBookByHash(hash: string): Promise<BlueprintBookEntry | null> {
+  const query = datastore
+    .createQuery(BlueprintBookEntity)
+    .filter("blueprint_hash", "=", hash)
+    .limit(1);
+  const [result] = await datastore.runQuery(query);
+  return result[0] ? mapBlueprintBookEntityToObject(result[0]) : null;
+}
+
+async function createBlueprintBook(
+  blueprintBook: BlueprintBook,
+  extraInfo: { tags: string[]; created_at: number; updated_at: number; factorioprints_id: string }
+): Promise<{ insertedId: string; child_tree: ChildTree }> {
+  const string = await encodeBlueprint({ blueprint_book: blueprintBook });
+  const blueprint_hash = hashString(string);
+
+  const exists = await getBlueprintBookByHash(blueprint_hash);
+  if (exists) {
+    throw new DatastoreExistsError(exists.id, "Blueprint book already exists");
+  }
+
+  // Write string to google storage
+  await BLUEPRINT_BUCKET.file(blueprint_hash).save(string);
+
+  const blueprint_ids = [];
+  const blueprint_book_ids = [];
+  const child_tree: ChildTree = [];
+
+  // Create the book's child objects
+  for (let i = 0; i < blueprintBook.blueprints.length; i++) {
+    const blueprint = blueprintBook.blueprints[i];
+    if (blueprint.blueprint) {
+      const result = await createBlueprint(blueprint.blueprint, extraInfo).catch((error) => {
+        if (error instanceof DatastoreExistsError) {
+          console.log(`Blueprint already exists with id ${error.existingId}`);
+          return { insertedId: error.existingId };
+        } else throw error;
+      });
+      child_tree.push({
+        type: "blueprint",
+        id: result.insertedId,
+        name: blueprint.blueprint.label,
+      });
+      blueprint_ids.push(result.insertedId);
+    } else if (blueprint.blueprint_book) {
+      const result = await createBlueprintBook(blueprint.blueprint_book, extraInfo).catch(
+        (error) => {
+          if (error instanceof DatastoreExistsError) {
+            console.log(`Blueprint book already exists with id ${error.existingId}`);
+            // TODO: query blueprint book to get child_tree
+            return { insertedId: error.existingId, child_tree: [] as ChildTree };
+          } else throw error;
+        }
+      );
+      child_tree.push({
+        type: "blueprint_book",
+        id: result.insertedId,
+        name: blueprint.blueprint_book.label,
+        children: result.child_tree,
+      });
+      blueprint_book_ids.push(result.insertedId);
+    }
+  }
+
+  // Write blueprint details to datastore
+  const [result] = await datastore.insert({
+    key: datastore.key([BlueprintBookEntity]),
+    data: [
+      {
+        name: "blueprint_ids",
+        value: blueprint_ids.map((id) => datastore.key([BlueprintEntity, datastore.int(id)])),
+      },
+      {
+        name: "blueprint_book_ids",
+        value: blueprint_book_ids.map((id) =>
+          datastore.key([BlueprintBookEntity, datastore.int(id)])
+        ),
+      },
+      { name: "child_tree", value: child_tree, excludeFromIndexes: true },
+      { name: "label", value: blueprintBook.label },
+      {
+        name: "description",
+        value: blueprintBook.description || null,
+        excludeFromIndexes: true,
+      },
+      { name: "blueprint_hash", value: blueprint_hash },
+      {
+        name: "created_at",
+        value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : null,
+        excludeFromIndexes: true,
+      },
+      {
+        name: "updated_at",
+        value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : null,
+        excludeFromIndexes: true,
+      },
+      { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
+    ],
+  });
+
+  const insertedId = String(result.mutationResults?.[0]?.key?.path?.[0]?.id) as string;
+  if (!insertedId) throw Error("Something went wrong inserting entity");
+
+  console.log(`Created Blueprint book ${insertedId}`);
+
+  return { insertedId, child_tree };
+}
+
+/*
+ * Blueprint
+ */
+
+const mapBlueprintEntityToObject = (entity: any): BlueprintEntry => ({
+  id: entity[datastore.KEY].id,
+  blueprint_hash: entity.blueprint_hash,
+  image_hash: entity.image_hash,
+  label: entity.label,
+  description: entity.description,
+  tags: entity.tags,
+  created_at: entity.created_at && entity.created_at.getTime() / 1000,
+  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+  factorioprints_id: entity.factorioprints_id || null,
+  game_version: entity.game_version,
+});
+
+const mapBlueprintObjectToEntity = (object: Omit<BlueprintEntry, "id"> & { id?: string }): any => ({
+  key: datastore.key(object.id ? [BlueprintEntity, datastore.int(object.id)] : [BlueprintEntity]),
+  data: [
+    { name: "label", value: object.label },
+    {
+      name: "description",
+      value: object.description || null,
+      excludeFromIndexes: true,
+    },
+    { name: "blueprint_hash", value: object.blueprint_hash },
+    { name: "image_hash", value: object.image_hash },
+    { name: "tags", value: object.tags || [] },
+    {
+      name: "created_at",
+      value: object.created_at ? new Date(object.created_at * 1000) : new Date(),
+      excludeFromIndexes: true,
+    },
+    {
+      name: "updated_at",
+      value: object.updated_at ? new Date(object.updated_at * 1000) : new Date(),
+      excludeFromIndexes: true,
+    },
+    { name: "game_version", value: object.game_version, excludeFromIndexes: true },
+    { name: "factorioprints_id", value: object.factorioprints_id || null },
+  ],
+});
+
+export async function getBlueprintById(id: string): Promise<BlueprintEntry | null> {
+  const result = await datastore.get(datastore.key([BlueprintEntity, datastore.int(id)]));
+  return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
+}
+
+export async function getBlueprintByHash(hash: string): Promise<BlueprintEntry | null> {
+  const query = datastore.createQuery(BlueprintEntity).filter("blueprint_hash", "=", hash).limit(1);
+  const [result] = await datastore.runQuery(query);
+  return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
+}
+
+export async function getBlueprintByImageHash(hash: string): Promise<BlueprintEntry | null> {
+  const query = datastore.createQuery(BlueprintEntity).filter("image_hash", "=", hash).limit(1);
+  const [result] = await datastore.runQuery(query);
+  return result[0] ? mapBlueprintEntityToObject(result[0]) : null;
+}
+
+export async function createBlueprint(
+  blueprint: Blueprint,
+  extraInfo: { tags: string[]; created_at: number; updated_at: number; factorioprints_id: string }
+) {
+  const string = await encodeBlueprint({ blueprint });
+  const blueprint_hash = hashString(string);
+  const image_hash = hashString(getBlueprintContentForImageHash(blueprint));
+
+  const exists = await getBlueprintByHash(blueprint_hash);
+  if (exists) {
+    throw new DatastoreExistsError(exists.id, "Blueprint already exists");
+  }
+
+  // Write string to google storage
+  await BLUEPRINT_BUCKET.file(blueprint_hash).save(string);
+
+  // Write blueprint details to datastore
+  const [result] = await datastore.insert({
+    key: datastore.key([BlueprintEntity]),
+    data: [
+      { name: "label", value: blueprint.label },
+      {
+        name: "description",
+        value: blueprint.description || null,
+        excludeFromIndexes: true,
+      },
+      { name: "blueprint_hash", value: blueprint_hash },
+      { name: "image_hash", value: image_hash },
+      { name: "tags", value: extraInfo.tags ? extraInfo.tags : [] },
+      {
+        name: "created_at",
+        value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
+        excludeFromIndexes: true,
+      },
+      {
+        name: "updated_at",
+        value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
+        excludeFromIndexes: true,
+      },
+      { name: "game_version", value: blueprint.version, excludeFromIndexes: true },
+      { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
+    ],
+  });
+
+  const insertedId = String(result.mutationResults?.[0]?.key?.path?.[0]?.id);
+  if (!insertedId) throw Error("Something went wrong inserting entity");
+
+  console.log(`Created Blueprint ${insertedId}`);
+
+  blueprintImageRequestTopic.publishJSON({
+    blueprintId: insertedId,
+  });
+
+  await datastore.insert({
+    key: datastore.key([BlueprintEntity, datastore.int(insertedId), BlueprintStringEntity]),
+    data: [
+      { name: "blueprint_hash", value: blueprint_hash },
+      { name: "image_hash", value: image_hash },
+      { name: "version", value: 1 },
+      { name: "changes_markdown", value: null },
+      {
+        name: "created_at",
+        value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
+        excludeFromIndexes: true,
+      },
+    ],
+  });
+
+  return { insertedId };
+}
+
+export async function updateBlueprint(blueprint: BlueprintEntry) {
+  datastore.save(mapBlueprintObjectToEntity(blueprint));
+}
+
+/*
+ * BlueprintString
+ */
+
+export async function getBlueprintStringByHash(hash: string): Promise<string | null> {
+  const [buffer] = await BLUEPRINT_BUCKET.file(hash).download();
+  return buffer ? buffer.toString() : null;
+}
+
+/*
+ * BlueprintPage
+ */
+const mapBlueprintPageEntityToObject = (entity: any): BlueprintPageEntry => ({
+  id: entity[datastore.KEY].id,
+  blueprint_id: entity.blueprint_id ? entity.blueprint_id.id : null,
+  blueprint_book_id: entity.blueprint_book_id ? entity.blueprint_book_id.id : null,
+  title: entity.title,
+  description_markdown: entity.description_markdown,
+  created_at: entity.created_at && entity.created_at.getTime() / 1000,
+  updated_at: entity.updated_at && entity.updated_at.getTime() / 1000,
+  factorioprints_id: entity.factorioprints_id || null,
+});
+
+export async function getBlueprintPageById(id: string): Promise<BlueprintPageEntry | null> {
+  const result = await datastore.get(datastore.key([BlueprintPageEntity, datastore.int(id)]));
+  return result[0] ? mapBlueprintPageEntityToObject(result[0]) : null;
+}
+
+export async function getBlueprintPageByFactorioprintsId(
+  id: string
+): Promise<BlueprintPageEntry | null> {
+  const query = datastore
+    .createQuery(BlueprintPageEntity)
+    .filter("factorioprints_id", "=", id)
+    .limit(1);
+  const [result] = await datastore.runQuery(query);
+  return result[0] ? mapBlueprintPageEntityToObject(result[0]) : null;
+}
+
+async function createBlueprintPage(
+  type: "blueprint" | "blueprint_book",
+  targetId: string,
+  extraInfo: {
+    title: string;
+    description_markdown: string;
+    created_at: number;
+    updated_at: number;
+    factorioprints_id?: string;
+  }
+) {
+  const insertData: any = [
+    { name: "title", value: extraInfo.title },
+    {
+      name: "description_markdown",
+      value: extraInfo.description_markdown,
+      excludeFromIndexes: true,
+    },
+    {
+      name: "created_at",
+      value: extraInfo.created_at ? new Date(extraInfo.created_at * 1000) : new Date(),
+      excludeFromIndexes: true,
+    },
+    {
+      name: "updated_at",
+      value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
+      excludeFromIndexes: true,
+    },
+    { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
+  ];
+
+  if (type === "blueprint") {
+    insertData.push({
+      name: "blueprint_id",
+      value: datastore.key([BlueprintEntity, datastore.int(targetId)]),
+    });
+  } else if (type === "blueprint_book") {
+    insertData.push({
+      name: "blueprint_book_id",
+      value: datastore.key([BlueprintBookEntity, datastore.int(targetId)]),
+    });
+  } else {
+    throw Error("Invalid type given");
+  }
+
+  await datastore.insert({
+    key: datastore.key([BlueprintPageEntity]),
+    data: insertData,
+  });
+
+  console.log(`Created Blueprint Page`);
+}
+
+/*
+ * BlueprintImage
+ */
+
+export async function saveBlueprintImage(hash: string, image: Buffer): Promise<void> {
+  return IMAGE_BUCKET.file(`${hash}.webp`).save(image, {
+    contentType: "image/webp",
+  });
+}
+
+export async function hasBlueprintImage(hash: string): Promise<boolean> {
+  const [result] = await IMAGE_BUCKET.file(`${hash}.webp`).exists();
+  return result;
+}
+
+/*
+ * Other
+ */
+
+interface BlueprintDataFromFactorioprints {
+  description_markdown: string;
+  title: string;
+  updated_at: number;
+  created_at: number;
+  tags: string[];
+  factorioprints_id: string;
+}
+export async function saveBlueprintFromFactorioprints(
+  factorioprintData: BlueprintDataFromFactorioprints,
+  blueprintString: string
+) {
+  const parsed = await parseBlueprintString(blueprintString);
+
+  // not needed for inserting, just printing
+  // const { blueprints, books } = flattenBlueprintData(parsed.data);
+  // console.log(`string has ${books.length} books with ${blueprints.length} blueprints`);
+
+  const extraInfo = {
+    created_at: factorioprintData.created_at,
+    updated_at: factorioprintData.updated_at,
+    tags: factorioprintData.tags,
+    factorioprints_id: factorioprintData.factorioprints_id,
+  };
+
+  const extraInfoPage = {
+    title: factorioprintData.title,
+    description_markdown: factorioprintData.description_markdown,
+    created_at: factorioprintData.created_at,
+    updated_at: factorioprintData.updated_at,
+    factorioprints_id: factorioprintData.factorioprints_id,
+  };
+
+  if (parsed.data.blueprint) {
+    console.log(`string has one blueprint...`);
+    const { insertedId } = await createBlueprint(parsed.data.blueprint, extraInfo).catch(
+      (error) => {
+        if (error instanceof DatastoreExistsError) {
+          console.log(`Blueprint already exists with id ${error.existingId}`);
+          return { insertedId: error.existingId };
+        } else throw error;
+      }
+    );
+    await createBlueprintPage("blueprint", insertedId, extraInfoPage);
+  } else if (parsed.data.blueprint_book) {
+    console.log(`string has a blueprint book...`);
+    const { insertedId } = await createBlueprintBook(parsed.data.blueprint_book, extraInfo);
+    await createBlueprintPage("blueprint_book", insertedId, extraInfoPage);
+  }
+  return true;
+}
+
+export async function getMostRecentBlueprintPages(page = 1): Promise<BlueprintPageEntry[]> {
+  const perPage = 10;
+  const query = datastore
+    .createQuery(BlueprintPageEntity)
+    .limit(perPage)
+    .offset((page - 1) * perPage);
+  const [result] = await datastore.runQuery(query);
+  return result.map(mapBlueprintPageEntityToObject);
+}
+
+export async function getPaginatedBlueprints(page = 1, perPage = 10): Promise<BlueprintEntry[]> {
+  const query = datastore
+    .createQuery(BlueprintEntity)
+    .limit(perPage)
+    .offset((page - 1) * perPage);
+  const [result] = await datastore.runQuery(query);
+  return result.map(mapBlueprintEntityToObject);
+}

libs/database/src/lib/pubsub.ts

@@ -0,0 +1,10 @@
+import { PubSub, Message } from "@google-cloud/pubsub";
+// export { Message } from "@google-cloud/pubsub";
+
+const pubsub = new PubSub();
+
+export function getBlueprintImageRequestTopic() {
+  return pubsub.topic("projects/factorio-sites/topics/blueprint-image-request");
+}
+
+export type PubSubMessage = Message;

libs/database/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

libs/database/tsconfig.lib.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "exclude": ["**/*.spec.ts"],
+  "include": ["**/*.ts"]
+}

libs/database/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.spec.js", "**/*.spec.jsx", "**/*.d.ts"]
+}

libs/generate-bp-image/src/lib/generate-bp-image.ts

@@ -1,54 +1,80 @@
-import * as Puppeteer from "puppeteer";
-import * as crypto from "crypto";
 import * as path from "path";
 import * as fs from "fs";
 import { promisify } from "util";
+import * as Puppeteer from "puppeteer";
+import { BlueprintEntry } from "@factorio-sites/database";
+import { timeLogger } from "@factorio-sites/common-utils";
 
 const fsMkdir = promisify(fs.mkdir);
 const fsStat = promisify(fs.stat);
-const fsRmdir = promisify(fs.rmdir);
+const fsReadFile = promisify(fs.readFile);
 
-async function downloadScreenshot(blueprint: string, dir: string, on_complete: Promise<any>) {
+let browser: Puppeteer.Browser;
-  const browser = await Puppeteer.launch({
+Puppeteer.launch({
-    headless: true,
+  headless: true,
-    args: ["--no-sandbox"],
+  args: ["--no-sandbox"],
-  });
+}).then((_browser) => (browser = _browser));
 
-  const [page] = await browser.pages();
+async function downloadScreenshot(
+  blueprint: BlueprintEntry,
+  dir: string,
+  on_complete: Promise<any>
+) {
+  // const browser = await Puppeteer.launch({
+  //   headless: true,
+  //   args: ["--no-sandbox"],
+  // });
+  const tl = timeLogger("downloadScreenshot");
+
+  // const [page] = await browser.pages();
+  const page = await browser.newPage();
 
   await (page as any)._client.send("Page.setDownloadBehavior", {
     behavior: "allow",
     downloadPath: dir,
   });
 
-  const start = Date.now();
+  let pageErroredOnBlueprint = false;
 
-  await page.goto("https://teoxoy.github.io/factorio-blueprint-editor/?source=" + blueprint, {
+  page.on("console", (message) => {
+    if (message.type() === "error" && message.text() === "JSHandle@array")
+      pageErroredOnBlueprint = true;
+  });
+
+  const bp_url = `https://factorio-blueprints-nleb5djksq-ey.a.run.app/api/string/${blueprint.blueprint_hash}`;
+  tl(`Opening web page with blueprint_hash ${blueprint.blueprint_hash}`);
+  await page.goto("https://teoxoy.github.io/factorio-blueprint-editor/?source=" + bp_url, {
     waitUntil: "load",
   });
-  console.log(`[status] page load complete at ${Date.now() - start}`);
+
+  tl(`page load complete`);
 
   await page.waitForFunction(`!!document.querySelector('.toasts-text')`);
 
-  console.log(`[status] app initialized at ${Date.now() - start}`);
+  if (pageErroredOnBlueprint) throw Error("Failed to parse blueprint string");
 
-  await page.click("canvas");
+  tl(`app initialized`);
+
+  await page.focus("canvas");
   await page.keyboard.down("Control");
   await page.keyboard.press("S");
   await page.keyboard.up("Control");
 
-  console.log("[status] save image command entered");
+  tl("save image command entered");
 
   on_complete.finally(() => {
-    browser.close();
+    // browser.close();
+    page.close();
   });
 }
 
-export async function generateScreenshot(blueprint: string, _cache_dir?: string) {
+export async function generateScreenshot(blueprint: BlueprintEntry, _cache_dir?: string) {
-  const start_time = Date.now();
+  const cache_dir = _cache_dir || path.join(process.cwd(), ".cache/image-downloads");
-  const hash = crypto.createHash("sha256").update(blueprint).digest("hex");
+  const dir = path.join(cache_dir, String(blueprint.id));
-  const cache_dir = _cache_dir || path.join(process.cwd(), ".cache");
+
-  const dir = path.join(cache_dir, hash);
+  if (!blueprint.blueprint_hash) {
+    throw Error("Failed to generate screenshot, no blueprint hash found");
+  }
 
   await fsMkdir(cache_dir).catch((error) => {
     if (error.code !== "EEXIST") throw error;
@@ -57,6 +83,19 @@ export async function generateScreenshot(blueprint: string, _cache_dir?: string)
     if (error.code !== "EEXIST") throw error;
   });
 
+  const result = await _generateScreenshot(blueprint, dir);
+  // .finally(() => {
+  //     fsRmdir(dir, { recursive: true, maxRetries: 3, retryDelay: 100 }).catch((reason) => {
+  //       console.log(`clearing directory failed: ${reason.code}`);
+  //     });
+  // });
+
+  return result;
+}
+
+async function _generateScreenshot(blueprint: BlueprintEntry, dir: string) {
+  const tl = timeLogger("generateScreenshot");
+
   const promise = new Promise<string>((resolve, reject) => {
     const watcher = fs.watch(dir, async (type, file) => {
       if (type === "change" && !file.endsWith(".crdownload")) {
@@ -64,7 +103,10 @@ export async function generateScreenshot(blueprint: string, _cache_dir?: string)
         fsStat(file_path)
           .then(() => resolve(file_path))
           .catch(reject)
-          .finally(() => watcher.close());
+          .finally(() => {
+            console.log(`closing watcher ${dir}`);
+            watcher.close();
+          });
       }
     });
   });
@@ -73,11 +115,16 @@ export async function generateScreenshot(blueprint: string, _cache_dir?: string)
 
   const file_path = await promise;
 
-  console.log(`Downloaded image in ${Date.now() - start_time}ms`, file_path);
+  tl(`Downloaded image ${file_path}`);
-  const stream = fs.createReadStream(file_path);
-  stream.on("close", () => {
-    fsRmdir(dir, { recursive: true }).catch((reason) => console.error(reason));
-  });
 
-  return stream;
+  const buffer = await fsReadFile(file_path);
+
+  const buffermin = buffer;
+  // const buffermin = await imagemin.buffer(buffer, {
+  //   plugins: [imageminWebp({ quality: 50 })],
+  // });
+
+  tl("imageminWebp");
+
+  return buffermin;
 }

libs/node-utils/.eslintrc.json

@@ -0,0 +1 @@
+{ "extends": "../../.eslintrc.json", "ignorePatterns": ["!**/*"], "rules": {} }

libs/node-utils/README.md

@@ -0,0 +1,7 @@
+# node-utils
+
+This library was generated with [Nx](https://nx.dev).
+
+## Running unit tests
+
+Run `nx test node-utils` to execute the unit tests via [Jest](https://jestjs.io).

libs/node-utils/jest.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  displayName: "node-utils",
+  preset: "../../jest.preset.js",
+  globals: {
+    "ts-jest": {
+      tsConfig: "<rootDir>/tsconfig.spec.json",
+    },
+  },
+  transform: {
+    "^.+\\.[tj]sx?$": "ts-jest",
+  },
+  moduleFileExtensions: ["ts", "tsx", "js", "jsx"],
+  coverageDirectory: "../../coverage/libs/node-utils",
+};

libs/node-utils/src/index.ts

@@ -0,0 +1 @@
+export * from "./lib/node-utils";

libs/node-utils/src/lib/node-utils.spec.ts

@@ -0,0 +1,7 @@
+// import { utils } from "./node-utils";
+
+describe("utils", () => {
+  it("should work", () => {
+    // expect(true).toEqual("utils");
+  });
+});

libs/node-utils/src/lib/node-utils.ts

@@ -0,0 +1,37 @@
+import * as crypto from "crypto";
+import * as pako from "pako";
+// import * as phin from "phin";
+import { BlueprintData } from "@factorio-sites/common-utils";
+
+export const parseBlueprintString = async (
+  string: string
+): Promise<{ hash: string; data: BlueprintData; string: string }> => {
+  // if (string.startsWith("http:") || string.startsWith("https:")) {
+  //   const result = await phin(string);
+  //   string = result.body.toString();
+  // }
+
+  const hash = crypto.createHash("sha1").update(string).digest("hex");
+  const buffer = Buffer.from(string.substr(1), "base64");
+  const decoded = pako.inflate(buffer);
+  const json = new TextDecoder("utf-8").decode(decoded);
+  const data = JSON.parse(json);
+
+  return {
+    hash,
+    data,
+    string,
+  };
+};
+
+export const encodeBlueprint = async (data: BlueprintData): Promise<string> => {
+  const json = JSON.stringify(data);
+  const encoded = new TextEncoder().encode(json);
+  const compressed = pako.deflate(encoded);
+  const base64 = Buffer.from(compressed).toString("base64");
+  return "0" + base64;
+};
+
+export const hashString = (string: string) => {
+  return crypto.createHash("sha1").update(string).digest("hex");
+};

libs/node-utils/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

libs/node-utils/tsconfig.lib.json

@@ -0,0 +1,11 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "outDir": "../../dist/out-tsc",
+    "declaration": true,
+    "types": ["node"]
+  },
+  "exclude": ["**/*.spec.ts"],
+  "include": ["**/*.ts"]
+}

libs/node-utils/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.spec.js", "**/*.spec.jsx", "**/*.d.ts"]
+}

libs/web-utils/.eslintrc.json

@@ -0,0 +1 @@
+{ "extends": "../../.eslintrc.json", "ignorePatterns": ["!**/*"], "rules": {} }

libs/web-utils/README.md

@@ -0,0 +1,7 @@
+# web-utils
+
+This library was generated with [Nx](https://nx.dev).
+
+## Running unit tests
+
+Run `nx test web-utils` to execute the unit tests via [Jest](https://jestjs.io).

libs/web-utils/jest.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  displayName: "web-utils",
+  preset: "../../jest.preset.js",
+  globals: {
+    "ts-jest": {
+      tsConfig: "<rootDir>/tsconfig.spec.json",
+    },
+  },
+  transform: {
+    "^.+\\.[tj]sx?$": "ts-jest",
+  },
+  moduleFileExtensions: ["ts", "tsx", "js", "jsx"],
+  coverageDirectory: "../../coverage/libs/web-utils",
+};

libs/web-utils/src/index.ts

@@ -0,0 +1 @@
+export * from "./lib/web-utils";

libs/web-utils/src/lib/web-utils.ts

@@ -0,0 +1,28 @@
+import * as pako from "pako";
+import { BlueprintData } from "@factorio-sites/common-utils";
+
+export function parseBlueprintStringClient(source: string): { data: BlueprintData } {
+  const encoded = atob(source.substring(1));
+  const decoded = pako.inflate(encoded);
+  const string = new TextDecoder("utf-8").decode(decoded);
+  const jsonObject = JSON.parse(string);
+  return { data: jsonObject };
+}
+
+export function encodeBlueprintStringClient(data: BlueprintData): string {
+  const json = JSON.stringify(data);
+  const encoded = new TextEncoder().encode(json);
+  const compressed = pako.deflate(encoded, { to: "string" });
+  const base64 = btoa(compressed);
+  return "0" + base64;
+}
+
+export function chakraResponsive({
+  mobile,
+  desktop,
+}: {
+  mobile: string;
+  desktop: string;
+}): string[] {
+  return [mobile, mobile, desktop, desktop];
+}

libs/web-utils/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "files": [],
+  "include": [],
+  "references": [
+    {
+      "path": "./tsconfig.lib.json"
+    },
+    {
+      "path": "./tsconfig.spec.json"
+    }
+  ]
+}

libs/web-utils/tsconfig.lib.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "types": []
+  },
+  "exclude": ["**/*.spec.ts"],
+  "include": ["**/*.ts"]
+}

libs/web-utils/tsconfig.spec.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": ["**/*.spec.ts", "**/*.spec.tsx", "**/*.spec.js", "**/*.spec.jsx", "**/*.d.ts"]
+}

nx.json

@@ -35,6 +35,21 @@
     },
     "blueprint-image-function": {
       "tags": []
+    },
+    "factorioprints-scraper": {
+      "tags": []
+    },
+    "database": {
+      "tags": []
+    },
+    "node-utils": {
+      "tags": []
+    },
+    "common-utils": {
+      "tags": []
+    },
+    "web-utils": {
+      "tags": []
     }
   }
 }

package.json

@@ -27,14 +27,27 @@
   },
   "private": true,
   "dependencies": {
-    "@emotion/core": "10.0.28",
+    "@chakra-ui/core": "0.8.0",
+    "@emotion/core": "10.0.35",
     "@emotion/styled": "10.0.27",
-    "document-register-element": "1.13.1",
+    "@google-cloud/datastore": "6.2.0",
+    "@google-cloud/pubsub": "2.6.0",
+    "@google-cloud/storage": "5.3.0",
+    "bbcode-to-react": "0.2.9",
+    "document-register-element": "1.14.10",
     "emotion-server": "10.0.27",
-    "next": "9.5.2",
+    "emotion-theming": "10.0.27",
+    "next": "9.5.5",
+    "nprogress": "0.2.0",
+    "pako": "1.0.11",
+    "phin": "3.5.0",
     "puppeteer": "5.3.1",
     "react": "16.13.1",
-    "react-dom": "16.13.1"
+    "react-dom": "16.13.1",
+    "react-map-interaction": "2.0.0",
+    "react-markdown": "5.0.1",
+    "sharp": "0.26.2",
+    "ws": "7.3.1"
   },
   "devDependencies": {
     "@babel/core": "7.9.6",
@@ -52,25 +65,32 @@
     "@nrwl/web": "10.3.1",
     "@nrwl/workspace": "10.3.1",
     "@testing-library/react": "10.4.1",
+    "@types/bbcode-to-react": "0.2.0",
+    "@types/imagemin": "7.0.0",
+    "@types/imagemin-webp": "5.1.1",
     "@types/jest": "26.0.8",
-    "@types/node": "14.11.10",
+    "@types/node": "14.14.1",
+    "@types/nprogress": "0.2.0",
+    "@types/pako": "1.0.1",
     "@types/puppeteer": "3.0.2",
-    "@types/react": "16.9.38",
+    "@types/react": "16.9.53",
     "@types/react-dom": "16.9.8",
+    "@types/sharp": "0.26.0",
+    "@types/ws": "7.2.7",
     "@typescript-eslint/eslint-plugin": "4.3.0",
     "@typescript-eslint/parser": "4.3.0",
     "babel-jest": "26.2.2",
-    "cypress": "^4.1.0",
+    "cypress": "4.1.0",
     "dotenv": "6.2.0",
     "eslint": "7.10.0",
     "eslint-config-prettier": "6.0.0",
-    "eslint-plugin-cypress": "^2.10.3",
+    "eslint-plugin-cypress": "2.10.3",
     "eslint-plugin-import": "2.21.2",
     "eslint-plugin-jsx-a11y": "6.3.1",
     "eslint-plugin-react": "7.20.0",
     "eslint-plugin-react-hooks": "4.0.4",
     "jest": "26.2.2",
-    "prettier": "2.0.4",
+    "prettier": "2.1.2",
     "ts-jest": "26.4.0",
     "ts-node": "~7.0.0",
     "tslint": "~6.0.0",

tsconfig.base.json

@@ -17,7 +17,11 @@
     "strict": true,
     "baseUrl": ".",
     "paths": {
-      "@factorio-sites/generate-bp-image": ["libs/generate-bp-image/src/index.ts"]
+      "@factorio-sites/generate-bp-image": ["libs/generate-bp-image/src/index.ts"],
+      "@factorio-sites/database": ["libs/database/src/index.ts"],
+      "@factorio-sites/node-utils": ["libs/node-utils/src/index.ts"],
+      "@factorio-sites/common-utils": ["libs/common-utils/src/index.ts"],
+      "@factorio-sites/web-utils": ["libs/web-utils/src/index.ts"]
     }
   },
   "exclude": ["node_modules", "tmp"]

workspace.json

@@ -147,6 +147,140 @@
           }
         }
       }
+    },
+    "factorioprints-scraper": {
+      "root": "apps/factorioprints-scraper",
+      "sourceRoot": "apps/factorioprints-scraper/src",
+      "projectType": "application",
+      "prefix": "factorioprints-scraper",
+      "schematics": {},
+      "architect": {
+        "build": {
+          "builder": "@nrwl/node:build",
+          "options": {
+            "outputPath": "dist/apps/factorioprints-scraper",
+            "main": "apps/factorioprints-scraper/src/main.ts",
+            "tsConfig": "apps/factorioprints-scraper/tsconfig.app.json",
+            "assets": ["apps/factorioprints-scraper/src/assets"]
+          },
+          "configurations": {
+            "production": {
+              "optimization": true,
+              "extractLicenses": true,
+              "inspect": false,
+              "fileReplacements": [
+                {
+                  "replace": "apps/factorioprints-scraper/src/environments/environment.ts",
+                  "with": "apps/factorioprints-scraper/src/environments/environment.prod.ts"
+                }
+              ]
+            }
+          }
+        },
+        "serve": {
+          "builder": "@nrwl/node:execute",
+          "options": {
+            "buildTarget": "factorioprints-scraper:build"
+          }
+        },
+        "lint": {
+          "builder": "@nrwl/linter:eslint",
+          "options": {
+            "lintFilePatterns": ["apps/factorioprints-scraper/**/*.ts"]
+          }
+        },
+        "test": {
+          "builder": "@nrwl/jest:jest",
+          "options": {
+            "jestConfig": "apps/factorioprints-scraper/jest.config.js",
+            "passWithNoTests": true
+          }
+        }
+      }
+    },
+    "database": {
+      "root": "libs/database",
+      "sourceRoot": "libs/database/src",
+      "projectType": "library",
+      "schematics": {},
+      "architect": {
+        "lint": {
+          "builder": "@nrwl/linter:eslint",
+          "options": {
+            "lintFilePatterns": ["libs/database/**/*.ts"]
+          }
+        },
+        "test": {
+          "builder": "@nrwl/jest:jest",
+          "options": {
+            "jestConfig": "libs/database/jest.config.js",
+            "passWithNoTests": true
+          }
+        }
+      }
+    },
+    "node-utils": {
+      "root": "libs/node-utils",
+      "sourceRoot": "libs/node-utils/src",
+      "projectType": "library",
+      "schematics": {},
+      "architect": {
+        "lint": {
+          "builder": "@nrwl/linter:eslint",
+          "options": {
+            "lintFilePatterns": ["libs/node-utils/**/*.ts"]
+          }
+        },
+        "test": {
+          "builder": "@nrwl/jest:jest",
+          "options": {
+            "jestConfig": "libs/node-utils/jest.config.js",
+            "passWithNoTests": true
+          }
+        }
+      }
+    },
+    "common-utils": {
+      "root": "libs/common-utils",
+      "sourceRoot": "libs/common-utils/src",
+      "projectType": "library",
+      "schematics": {},
+      "architect": {
+        "lint": {
+          "builder": "@nrwl/linter:eslint",
+          "options": {
+            "lintFilePatterns": ["libs/common-utils/**/*.ts"]
+          }
+        },
+        "test": {
+          "builder": "@nrwl/jest:jest",
+          "options": {
+            "jestConfig": "libs/common-utils/jest.config.js",
+            "passWithNoTests": true
+          }
+        }
+      }
+    },
+    "web-utils": {
+      "root": "libs/web-utils",
+      "sourceRoot": "libs/web-utils/src",
+      "projectType": "library",
+      "schematics": {},
+      "architect": {
+        "lint": {
+          "builder": "@nrwl/linter:eslint",
+          "options": {
+            "lintFilePatterns": ["libs/web-utils/**/*.ts"]
+          }
+        },
+        "test": {
+          "builder": "@nrwl/jest:jest",
+          "options": {
+            "jestConfig": "libs/web-utils/jest.config.js",
+            "passWithNoTests": true
+          }
+        }
+      }
     }
   },
   "cli": {