1
0
mirror of https://github.com/barthuijgen/factorio-sites.git synced 2024-11-24 08:52:36 +02:00

Upgrade packages, refactoring db access

This commit is contained in:
Bart Huijgen 2021-01-07 14:09:56 +01:00
parent dc4fe41f38
commit 3f1bc9924a
25 changed files with 2129 additions and 1300 deletions

1
.gitignore vendored
View File

@ -41,3 +41,4 @@ Thumbs.db
# custom
/.cache
/credentials
.env.local

1
.yarnrc Normal file
View File

@ -0,0 +1 @@
save-prefix=

View File

@ -7,10 +7,14 @@ Make sure `prod.package.json` is up to date on packages from the root `package.j
`docker build -t eu.gcr.io/factorio-sites/blueprints --file blueprints.Dockerfile .`
`docker push eu.gcr.io/factorio-sites/blueprints`
### Testing deployment locally
## Testing deployment locally
`docker run --rm -p 3000:3000 eu.gcr.io/factorio-sites/blueprints`
### windows env
`$env:GOOGLE_APPLICATION_CREDENTIALS="D:\git\factorio-sites\credentials\factorio-sites.json"`
create a `.env.local` with
```
GOOGLE_APPLICATION_CREDENTIALS="D:\git\factorio-sites\credentials\factorio-sites.json"
```

View File

@ -1 +1,4 @@
module.exports = {};
module.exports = {
poweredByHeader: false,
reactStrictMode: true,
};

View File

@ -1,9 +1,9 @@
/** @jsx jsx */
import React, { ReactNode, useEffect, useState } from "react";
import { jsx, css } from "@emotion/core";
import { jsx, css } from "@emotion/react";
import { NextPage, NextPageContext } from "next";
import BBCode from "bbcode-to-react";
import { Button, Grid, Image } from "@chakra-ui/core";
import { Button, Grid, Image } from "@chakra-ui/react";
import {
BlueprintBookEntry,
BlueprintEntry,
@ -67,7 +67,8 @@ export const Index: NextPage<IndexProps> = ({
} else {
setData(null);
}
});
})
.catch((reason) => console.error(reason));
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedHash]);
@ -213,7 +214,7 @@ export const Index: NextPage<IndexProps> = ({
) : (
<>
<Button
variantColor="green"
colorScheme="green"
css={{ position: "absolute", right: "65px" }}
onClick={() => {
setShowJson(false);
@ -231,7 +232,7 @@ export const Index: NextPage<IndexProps> = ({
)
) : (
<Button
variantColor="green"
colorScheme="green"
onClick={() => {
setShowJson(true);
if (selected.type === "blueprint_book") {

View File

@ -1,10 +1,9 @@
/** @jsx jsx */
import { jsx, css, Global } from "@emotion/core";
import { AppProps } from "next/app";
import { AppProps, NextWebVitalsMetric } from "next/app";
import Head from "next/head";
import Router from "next/router";
import { CSSReset, ITheme, theme } from "@chakra-ui/core";
import { ThemeProvider } from "emotion-theming";
import { jsx, css, Global } from "@emotion/react";
import { ChakraProvider } from "@chakra-ui/react";
import NProgress from "nprogress";
import { Header } from "../src/Header";
@ -39,26 +38,10 @@ if (typeof window !== "undefined") {
Router.events.on("routeChangeError", () => NProgress.done());
}
const config = (theme: ITheme) => ({
light: {
color: theme.colors.gray[800],
bg: theme.colors.gray[300],
borderColor: theme.colors.gray[200],
placeholderColor: theme.colors.gray[500],
},
dark: {
color: theme.colors.whiteAlpha[900],
bg: theme.colors.gray[800],
borderColor: theme.colors.whiteAlpha[300],
placeholderColor: theme.colors.whiteAlpha[400],
},
});
const CustomApp = ({ Component, pageProps }: AppProps) => {
const BlueprintsApp = ({ Component, pageProps }: AppProps) => {
return (
<ThemeProvider theme={theme}>
<ChakraProvider>
<Global styles={globalStyles} />
<CSSReset config={config} />
<Head>
<title>Welcome to blueprints!</title>
<link
@ -72,8 +55,12 @@ const CustomApp = ({ Component, pageProps }: AppProps) => {
<Component {...pageProps} />
</main>
</div>
</ThemeProvider>
</ChakraProvider>
);
};
export default CustomApp;
export function reportWebVitals(metric: NextWebVitalsMetric) {
console.log(metric);
}
export default BlueprintsApp;

View File

@ -3,10 +3,10 @@
import React from "react";
import { NextPage, NextPageContext } from "next";
import Link from "next/link";
import { jsx, css } from "@emotion/core";
import { jsx, css } from "@emotion/react";
import { BlueprintPageEntry, getMostRecentBlueprintPages } from "@factorio-sites/database";
import { Panel } from "../src/Panel";
import { SimpleGrid } from "@chakra-ui/core";
import { SimpleGrid } from "@chakra-ui/react";
import { Pagination } from "../src/Pagination";
const linkStyles = css`

View File

@ -1,15 +1,13 @@
{
"dependencies": {
"@chakra-ui/core": "0.8.0",
"@emotion/core": "10.0.35",
"@chakra-ui/react": "1.0.0",
"@emotion/react": "11.1.3",
"@emotion/styled": "10.0.27",
"@google-cloud/datastore": "6.2.0",
"@google-cloud/pubsub": "2.6.0",
"@google-cloud/storage": "5.3.0",
"bbcode-to-react": "0.2.9",
"document-register-element": "1.14.10",
"emotion-server": "10.0.27",
"emotion-theming": "10.0.27",
"next": "9.5.5",
"nprogress": "0.2.0",
"pako": "1.0.11",

View File

@ -1,6 +1,6 @@
/** @jsx jsx */
/* eslint-disable jsx-a11y/anchor-is-valid */
import { jsx, css } from "@emotion/core";
import { jsx, css } from "@emotion/react";
import Link from "next/link";
import BBCode from "bbcode-to-react";
import { BlueprintBookEntry } from "@factorio-sites/database";

View File

@ -1,31 +1,49 @@
import React, { useState } from "react";
import { Button, ButtonProps } from "@chakra-ui/core";
import React, { useMemo, useState } from "react";
import { Button, ButtonProps } from "@chakra-ui/react";
import { MdCheck, MdClose } from "react-icons/md";
const SUCCESS_ICON_DURATION = 2000;
export const CopyButton: React.FC<Omit<ButtonProps, "children"> & { content: string }> = ({
content,
...props
}) => {
const [loading, setLoading] = useState(false);
const [icon, setIcon] = useState<"check" | "small-close" | null>(null);
const [icon, setIcon] = useState<"red" | "green" | null>(null);
const iconProps = useMemo(() => {
if (icon === "green") {
return {
colorScheme: "green",
leftIcon: <MdCheck />,
};
} else if (icon === "red") {
return {
colorScheme: "red",
leftIcon: <MdClose />,
};
} else {
return { colorScheme: "green" };
}
}, [icon]);
return (
<Button
{...props}
isLoading={loading}
leftIcon={icon ?? undefined}
variantColor={icon === "small-close" ? "red" : "green"}
{...iconProps}
onClick={() => {
setLoading(true);
navigator.clipboard
.writeText(content)
.then(() => {
setLoading(false);
setIcon("check");
setTimeout(() => setIcon(null), 2500);
setIcon("green");
setTimeout(() => setIcon(null), SUCCESS_ICON_DURATION);
})
.catch(() => {
setLoading(false);
setIcon("small-close");
setIcon("red");
});
}}
>

View File

@ -1,7 +1,7 @@
/** @jsx jsx */
/* eslint-disable jsx-a11y/anchor-is-valid */
import React, { useState } from "react";
import { jsx, css } from "@emotion/core";
import { jsx, css } from "@emotion/react";
import { MapInteractionCSS } from "react-map-interaction";
const elementStyle = css`

View File

@ -1,5 +1,5 @@
import React from "react";
import { Box, Heading, Flex } from "@chakra-ui/core";
import { Box, Heading, Flex } from "@chakra-ui/react";
import Link from "next/link";
// const MenuItems = ({ children }) => (

View File

@ -1,6 +1,6 @@
/** @jsx jsx */
/* eslint-disable jsx-a11y/anchor-is-valid */
import { jsx, css } from "@emotion/core";
import { jsx, css } from "@emotion/react";
import ReactMarkdown from "react-markdown";
const markdownStyle = css`

View File

@ -1,9 +1,9 @@
/** @jsx jsx */
import { jsx } from "@emotion/core";
import { jsx } from "@emotion/react";
import React from "react";
import Link from "next/link";
import { useRouter } from "next/router";
import { Box, BoxProps, Button } from "@chakra-ui/core";
import { Box, BoxProps, Button } from "@chakra-ui/react";
interface PaginationProps {
page: number;
@ -13,7 +13,7 @@ const PaginationLink: React.FC<PaginationProps> = ({ page }) => {
const router = useRouter();
const query: Record<string, string> = { ...router.query, page: page.toString() };
const href =
"?" +
"/?" +
Object.keys(query)
.map((key) => `${key}=${query[key]}`)
.join("&");

View File

@ -1,6 +1,6 @@
/** @jsx jsx */
import { jsx, css, SerializedStyles } from "@emotion/core";
import { Box, BoxProps } from "@chakra-ui/core";
import { jsx, css, SerializedStyles } from "@emotion/react";
import { Box, BoxProps } from "@chakra-ui/react";
import { ReactNode } from "react";
const panelStyles = css`

View File

@ -1,2 +1,4 @@
export * from "./lib/database";
export * from "./lib/pubsub";
export * from "./lib/gcp-datastore";
export * from "./lib/gcp-pubsub";
export * from "./lib/gcp-storage";
export * from "./lib/types";

View File

@ -1,7 +0,0 @@
import { database } from "./database";
describe("database", () => {
it("should work", () => {
expect(database()).toEqual("database");
});
});

View File

@ -1,4 +1,3 @@
import { Storage } from "@google-cloud/storage";
import { Datastore } from "@google-cloud/datastore";
import { encodeBlueprint, hashString, parseBlueprintString } from "@factorio-sites/node-utils";
import {
@ -6,89 +5,17 @@ import {
BlueprintBook,
getBlueprintContentForImageHash,
} from "@factorio-sites/common-utils";
import { getBlueprintImageRequestTopic } from "./pubsub";
import { getBlueprintImageRequestTopic } from "./gcp-pubsub";
import { saveBlueprintString } from "./gcp-storage";
import { BlueprintBookEntry, BlueprintEntry, BlueprintPageEntry } from "./types";
// to dev on windows run: $env:GOOGLE_APPLICATION_CREDENTIALS="FULL_PATH"
const storage = new Storage();
const datastore = new Datastore();
const BLUEPRINT_BUCKET = storage.bucket("blueprint-strings");
const IMAGE_BUCKET = storage.bucket("blueprint-images");
const BlueprintEntity = "Blueprint";
const BlueprintBookEntity = "BlueprintBook";
const BlueprintPageEntity = "BlueprintPage";
const BlueprintStringEntity = "BlueprintString";
const blueprintImageRequestTopic = getBlueprintImageRequestTopic();
interface BlueprintChild {
type: "blueprint";
id: string;
name: string;
}
interface BlueprintBookChild {
type: "blueprint_book";
id: string;
name: string;
children: ChildTree;
}
type ChildTree = Array<BlueprintChild | BlueprintBookChild>;
export interface BlueprintEntry {
id: string;
label: string; // from source
description: string | null; // from source
game_version: number; // from source
blueprint_hash: string;
image_hash: string;
created_at: number;
updated_at: number;
tags: string[];
factorioprints_id?: string;
// BlueprintEntry->BlueprintString 1:m
// BlueprintEntry->BlueprintPageEntry n:m
}
export interface BlueprintBookEntry {
id: string;
label: string;
description?: string;
/** strings as keys of BlueprintEntry */
blueprint_ids: string[];
/** strings as keys of BlueprintBookEntry (currently unsupported) */
blueprint_book_ids: string[];
child_tree: ChildTree;
blueprint_hash: string;
created_at: number;
updated_at: number;
is_modded: boolean;
factorioprints_id?: string;
// BlueprintBook:BlueprintBook n:m
// BlueprintBook:BlueprintEntry 1:m
}
export interface BlueprintPageEntry {
id: string;
blueprint_id?: string;
blueprint_book_id?: string;
title: string;
description_markdown: string;
created_at: number;
updated_at: number;
factorioprints_id?: string;
// BlueprintPageEntry->BlueprintEntry 1:m
// BlueprintPageEntry->BlueprintBook 1:m
}
export interface BlueprintStringEntry {
blueprint_id: string;
blueprint_hash: string;
image_hash: string;
version: number;
changes_markdown: string;
created_at: Date;
// BlueprintString->BlueprintEntry m:1
}
class DatastoreExistsError extends Error {
constructor(public existingId: string, message: string) {
super(message);
@ -129,35 +56,32 @@ export async function getBlueprintBookByHash(hash: string): Promise<BlueprintBoo
return result[0] ? mapBlueprintBookEntityToObject(result[0]) : null;
}
async function createBlueprintBook(
export async function createBlueprintBook(
blueprintBook: BlueprintBook,
extraInfo: { tags: string[]; created_at: number; updated_at: number; factorioprints_id: string }
): Promise<{ insertedId: string; child_tree: ChildTree }> {
): Promise<{ insertedId: string; child_tree: BlueprintBookEntry["child_tree"] }> {
const string = await encodeBlueprint({ blueprint_book: blueprintBook });
const blueprint_hash = hashString(string);
const exists = await getBlueprintBookByHash(blueprint_hash);
if (exists) {
throw new DatastoreExistsError(exists.id, "Blueprint book already exists");
const book = await getBlueprintBookById(exists.id);
if (!book) throw Error("this is impossible, just pleasing typescript");
return { insertedId: exists.id, child_tree: book.child_tree };
}
// Write string to google storage
await BLUEPRINT_BUCKET.file(blueprint_hash).save(string);
await saveBlueprintString(blueprint_hash, string);
const blueprint_ids = [];
const blueprint_book_ids = [];
const child_tree: ChildTree = [];
const child_tree: BlueprintBookEntry["child_tree"] = [];
// Create the book's child objects
for (let i = 0; i < blueprintBook.blueprints.length; i++) {
const blueprint = blueprintBook.blueprints[i];
if (blueprint.blueprint) {
const result = await createBlueprint(blueprint.blueprint, extraInfo).catch((error) => {
if (error instanceof DatastoreExistsError) {
console.log(`Blueprint already exists with id ${error.existingId}`);
return { insertedId: error.existingId };
} else throw error;
});
const result = await createBlueprint(blueprint.blueprint, extraInfo);
child_tree.push({
type: "blueprint",
id: result.insertedId,
@ -165,15 +89,7 @@ async function createBlueprintBook(
});
blueprint_ids.push(result.insertedId);
} else if (blueprint.blueprint_book) {
const result = await createBlueprintBook(blueprint.blueprint_book, extraInfo).catch(
(error) => {
if (error instanceof DatastoreExistsError) {
console.log(`Blueprint book already exists with id ${error.existingId}`);
// TODO: query blueprint book to get child_tree
return { insertedId: error.existingId, child_tree: [] as ChildTree };
} else throw error;
}
);
const result = await createBlueprintBook(blueprint.blueprint_book, extraInfo);
child_tree.push({
type: "blueprint_book",
id: result.insertedId,
@ -299,11 +215,11 @@ export async function createBlueprint(
const exists = await getBlueprintByHash(blueprint_hash);
if (exists) {
throw new DatastoreExistsError(exists.id, "Blueprint already exists");
return { insertedId: exists.id };
}
// Write string to google storage
await BLUEPRINT_BUCKET.file(blueprint_hash).save(string);
await saveBlueprintString(blueprint_hash, string);
// Write blueprint details to datastore
const [result] = await datastore.insert({
@ -364,15 +280,6 @@ export async function updateBlueprint(blueprint: BlueprintEntry) {
datastore.save(mapBlueprintObjectToEntity(blueprint));
}
/*
* BlueprintString
*/
export async function getBlueprintStringByHash(hash: string): Promise<string | null> {
const [buffer] = await BLUEPRINT_BUCKET.file(hash).download();
return buffer ? buffer.toString() : null;
}
/*
* BlueprintPage
*/
@ -455,21 +362,6 @@ async function createBlueprintPage(
console.log(`Created Blueprint Page`);
}
/*
* BlueprintImage
*/
export async function saveBlueprintImage(hash: string, image: Buffer): Promise<void> {
return IMAGE_BUCKET.file(`${hash}.webp`).save(image, {
contentType: "image/webp",
});
}
export async function hasBlueprintImage(hash: string): Promise<boolean> {
const [result] = await IMAGE_BUCKET.file(`${hash}.webp`).exists();
return result;
}
/*
* Other
*/

View File

@ -0,0 +1,34 @@
import { Storage } from "@google-cloud/storage";
const storage = new Storage();
const BLUEPRINT_BUCKET = storage.bucket("blueprint-strings");
const IMAGE_BUCKET = storage.bucket("blueprint-images");
/*
* BlueprintString
*/
export async function getBlueprintStringByHash(hash: string): Promise<string | null> {
const [buffer] = await BLUEPRINT_BUCKET.file(hash).download();
return buffer ? buffer.toString() : null;
}
export async function saveBlueprintString(hash: string, content: string) {
await BLUEPRINT_BUCKET.file(hash).save(content);
}
/*
* BlueprintImage
*/
export async function saveBlueprintImage(hash: string, image: Buffer): Promise<void> {
return IMAGE_BUCKET.file(`${hash}.webp`).save(image, {
contentType: "image/webp",
});
}
export async function hasBlueprintImage(hash: string): Promise<boolean> {
const [result] = await IMAGE_BUCKET.file(`${hash}.webp`).exists();
return result;
}

View File

@ -0,0 +1,16 @@
import { createConnection } from "typeorm";
import { Blueprint } from "./entities";
export async function init() {
await createConnection({
type: "postgres",
host: "127.0.0.1",
port: 5432,
username: "postgres",
password: "local",
database: "factorio-blueprints",
entities: [Blueprint],
});
}
init();

View File

@ -0,0 +1,37 @@
import { Entity, PrimaryGeneratedColumn, Column } from "typeorm";
@Entity()
export class Blueprint {
@PrimaryGeneratedColumn("uuid")
id!: number;
@Column("text")
label?: string;
@Column("text", { nullable: true })
description?: string;
@Column("text")
game_version?: string;
@Column("text", { unique: true })
blueprint_hash?: string;
@Column("varchar", { length: 40 })
image_hash?: string;
@Column("text")
image_version?: string;
@Column("time without time zone")
created_at?: number;
@Column("time without time zone")
updated_at?: number;
@Column("text", { array: true })
tags?: string;
@Column("text", { nullable: true })
factorioprints_id?: number;
}

View File

@ -0,0 +1,68 @@
interface BlueprintChild {
type: "blueprint";
id: string;
name: string;
}
interface BlueprintBookChild {
type: "blueprint_book";
id: string;
name: string;
children: ChildTree;
}
type ChildTree = Array<BlueprintChild | BlueprintBookChild>;
export interface BlueprintEntry {
id: string;
label: string; // from source
description: string | null; // from source
game_version: number; // from source
blueprint_hash: string;
image_hash: string;
created_at: number;
updated_at: number;
tags: string[];
factorioprints_id?: string;
// BlueprintEntry->BlueprintString 1:m
// BlueprintEntry->BlueprintPageEntry n:m
}
export interface BlueprintBookEntry {
id: string;
label: string;
description?: string;
/** strings as keys of BlueprintEntry */
blueprint_ids: string[];
/** strings as keys of BlueprintBookEntry (currently unsupported) */
blueprint_book_ids: string[];
child_tree: ChildTree;
blueprint_hash: string;
created_at: number;
updated_at: number;
is_modded: boolean;
factorioprints_id?: string;
// BlueprintBook:BlueprintBook n:m
// BlueprintBook:BlueprintEntry 1:m
}
export interface BlueprintPageEntry {
id: string;
blueprint_id?: string;
blueprint_book_id?: string;
title: string;
description_markdown: string;
created_at: number;
updated_at: number;
factorioprints_id?: string;
// BlueprintPageEntry->BlueprintEntry 1:m
// BlueprintPageEntry->BlueprintBook 1:m
}
export interface BlueprintStringEntry {
blueprint_id: string;
blueprint_hash: string;
image_hash: string;
version: number;
changes_markdown: string;
created_at: Date;
// BlueprintString->BlueprintEntry m:1
}

View File

@ -2,6 +2,7 @@
"name": "factorio-sites",
"version": "0.0.0",
"license": "MIT",
"private": true,
"scripts": {
"nx": "nx",
"start": "nx serve",
@ -25,28 +26,30 @@
"dep-graph": "nx dep-graph",
"help": "nx help"
},
"private": true,
"dependencies": {
"@chakra-ui/core": "0.8.0",
"@emotion/core": "10.0.35",
"@emotion/styled": "10.0.27",
"@chakra-ui/react": "1.0.4",
"@emotion/react": "11.1.3",
"@emotion/server": "11.0.0",
"@emotion/styled": "11.0.0",
"@google-cloud/datastore": "6.2.0",
"@google-cloud/pubsub": "2.6.0",
"@google-cloud/storage": "5.3.0",
"bbcode-to-react": "0.2.9",
"document-register-element": "1.14.10",
"emotion-server": "10.0.27",
"emotion-theming": "10.0.27",
"next": "9.5.5",
"framer-motion": "3.1.1",
"next": "10.0.3",
"nprogress": "0.2.0",
"pako": "1.0.11",
"pg": "8.4.1",
"phin": "3.5.0",
"puppeteer": "5.3.1",
"react": "16.13.1",
"react-dom": "16.13.1",
"react-icons": "4.1.0",
"react-map-interaction": "2.0.0",
"react-markdown": "5.0.1",
"sharp": "0.26.2",
"typeorm": "0.2.28",
"ws": "7.3.1"
},
"devDependencies": {
@ -54,22 +57,20 @@
"@babel/preset-env": "7.9.6",
"@babel/preset-react": "7.9.4",
"@babel/preset-typescript": "7.9.0",
"@emotion/babel-preset-css-prop": "10.0.27",
"@emotion/babel-preset-css-prop": "11.0.0",
"@nrwl/cli": "10.3.1",
"@nrwl/cypress": "10.3.1",
"@nrwl/eslint-plugin-nx": "10.3.1",
"@nrwl/jest": "10.3.1",
"@nrwl/next": "10.3.1",
"@nrwl/node": "10.3.1",
"@nrwl/react": "10.3.1",
"@nrwl/web": "10.3.1",
"@nrwl/workspace": "10.3.1",
"@nrwl/cypress": "11.0.16",
"@nrwl/eslint-plugin-nx": "11.0.16",
"@nrwl/jest": "11.0.16",
"@nrwl/next": "11.0.16",
"@nrwl/node": "11.0.16",
"@nrwl/react": "11.0.16",
"@nrwl/web": "11.0.16",
"@nrwl/workspace": "11.0.16",
"@testing-library/react": "10.4.1",
"@types/bbcode-to-react": "0.2.0",
"@types/imagemin": "7.0.0",
"@types/imagemin-webp": "5.1.1",
"@types/jest": "26.0.8",
"@types/node": "14.14.1",
"@types/node": "14.14.14",
"@types/nprogress": "0.2.0",
"@types/pako": "1.0.1",
"@types/puppeteer": "3.0.2",
@ -92,8 +93,8 @@
"jest": "26.2.2",
"prettier": "2.1.2",
"ts-jest": "26.4.0",
"ts-node": "~7.0.0",
"tslint": "~6.0.0",
"typescript": "~4.0.3"
"ts-node": "9.1.1",
"tslint": "6.1.3",
"typescript": "4.1.3"
}
}

2955
yarn.lock

File diff suppressed because it is too large Load Diff