games/factorio-sites
1
0
Fork 0
mirror of https://github.com/barthuijgen/factorio-sites.git synced 2025-02-09 14:33:12 +02:00
Code Issues Releases Activity

Small fixes to scan logic

Browse Source
This commit is contained in:
Bart Huijgen 2020-10-25 23:53:21 +01:00
parent d79b13de8a
commit dc4fe41f38
4 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/factorioprints-scraper/src/app/populate-db.ts
View File

@ -7,10 +7,10 @@ import {
} from "@factorio-sites/database";
const fsReadFile = promisify(fs.readFile);
const CACHE_DIR = path.join(__dirname, "../../../.cache");
const CACHE_DIR = path.join(__dirname, "../../../.cache/factorioprints-data");
export async function writeToDatastore() {
const filecontent = await fsReadFile(path.join(CACHE_DIR, `most-fav-json/page1.json`), "utf8");
const filecontent = await fsReadFile(path.join(CACHE_DIR, `most-fav-json/page2.json`), "utf8");
const data = JSON.parse(filecontent);
for (let i = 0; i < data.length; i++) {

10
apps/factorioprints-scraper/src/app/scan.ts
View File

@ -6,7 +6,7 @@ import * as WebSocket from "ws";
const fsWriteFile = promisify(fs.writeFile);
const fsExists = promisify(fs.exists);
const CACHE_DIR = path.join(__dirname, "../../../.cache");
const CACHE_DIR = path.join(__dirname, "../../../.cache/factorioprints-data");
const getMostFavoriteRequestObject = ({
lastId,
@ -34,7 +34,7 @@ const getBlueprintRequestObject = ({ id }: { id: string }) => ({
});
const openWebsocket = (): Promise<WebSocket> => {
const ws = new WebSocket("wss://s-usc1c-nss-239.firebaseio.com/.ws?v=5&ns=facorio-blueprints");
const ws = new WebSocket("wss://s-usc1c-nss-238.firebaseio.com/.ws?v=5&ns=facorio-blueprints");
return new Promise((resolve) => {
ws.on("open", function open() {
resolve(ws);
@ -48,7 +48,7 @@ const openWebsocket = (): Promise<WebSocket> => {
});
};
export async function scanFactorioPrints() {
export async function scanFactorioPrints(page_from: number, page_to: number) {
const ws = await openWebsocket();
const sendMessage = (data: any) => {
@ -93,7 +93,7 @@ export async function scanFactorioPrints() {
};
const getMostFavorited = async (
page = 1,
page = page_from,
reqObj: Parameters<typeof getMostFavoriteRequestObject>[0]
) => {
const data = await sendMessageAndWaitForResponse(
@ -152,7 +152,7 @@ export async function scanFactorioPrints() {
console.error(reason);
});
if (page < 3) {
if (page < page_to) {
const lastBp = blueprints[blueprints.length - 1];
getMostFavorited(page + 1, {
lastId: lastBp.id,

3
apps/factorioprints-scraper/src/main.ts
View File

@ -1,4 +1,5 @@
import { writeToDatastore } from "./app/populate-db";
import { scanFactorioPrints } from "./app/scan";
// async function writeTestBP() {
// const source =
@ -22,7 +23,7 @@ import { writeToDatastore } from "./app/populate-db";
// }
async function main() {
// scanFactorioPrints();
// scanFactorioPrints(1, 3);
writeToDatastore();
// writeTestBP();
}

5
libs/database/src/lib/database.ts
View File

@ -429,7 +429,6 @@ async function createBlueprintPage(
{
name: "updated_at",
value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
excludeFromIndexes: true,
},
{ name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
];
@ -509,7 +508,7 @@ export async function saveBlueprintFromFactorioprints(
};
if (parsed.data.blueprint) {
console.log(`string has one blueprint...`);
console.log("string has one blueprint...");
const { insertedId } = await createBlueprint(parsed.data.blueprint, extraInfo).catch(
(error) => {
if (error instanceof DatastoreExistsError) {
@ -520,7 +519,7 @@ export async function saveBlueprintFromFactorioprints(
);
await createBlueprintPage("blueprint", insertedId, extraInfoPage);
} else if (parsed.data.blueprint_book) {
console.log(`string has a blueprint book...`);
console.log("string has a blueprint book...");
const { insertedId } = await createBlueprintBook(parsed.data.blueprint_book, extraInfo);
await createBlueprintPage("blueprint_book", insertedId, extraInfoPage);
}