From dc4fe41f384206248c29da460db90b505868b960 Mon Sep 17 00:00:00 2001
From: Bart Huijgen <bart_huijgen@hotmail.com>
Date: Sun, 25 Oct 2020 23:53:21 +0100
Subject: [PATCH] Small fixes to scan logic

---
 apps/factorioprints-scraper/src/app/populate-db.ts |  4 ++--
 apps/factorioprints-scraper/src/app/scan.ts        | 10 +++++-----
 apps/factorioprints-scraper/src/main.ts            |  3 ++-
 libs/database/src/lib/database.ts                  |  5 ++---
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/apps/factorioprints-scraper/src/app/populate-db.ts b/apps/factorioprints-scraper/src/app/populate-db.ts
index 1c7b132..7a632d5 100644
--- a/apps/factorioprints-scraper/src/app/populate-db.ts
+++ b/apps/factorioprints-scraper/src/app/populate-db.ts
@@ -7,10 +7,10 @@ import {
 } from "@factorio-sites/database";
 
 const fsReadFile = promisify(fs.readFile);
-const CACHE_DIR = path.join(__dirname, "../../../.cache");
+const CACHE_DIR = path.join(__dirname, "../../../.cache/factorioprints-data");
 
 export async function writeToDatastore() {
-  const filecontent = await fsReadFile(path.join(CACHE_DIR, `most-fav-json/page1.json`), "utf8");
+  const filecontent = await fsReadFile(path.join(CACHE_DIR, `most-fav-json/page2.json`), "utf8");
   const data = JSON.parse(filecontent);
 
   for (let i = 0; i < data.length; i++) {
diff --git a/apps/factorioprints-scraper/src/app/scan.ts b/apps/factorioprints-scraper/src/app/scan.ts
index e09e56b..643abd2 100644
--- a/apps/factorioprints-scraper/src/app/scan.ts
+++ b/apps/factorioprints-scraper/src/app/scan.ts
@@ -6,7 +6,7 @@ import * as WebSocket from "ws";
 
 const fsWriteFile = promisify(fs.writeFile);
 const fsExists = promisify(fs.exists);
-const CACHE_DIR = path.join(__dirname, "../../../.cache");
+const CACHE_DIR = path.join(__dirname, "../../../.cache/factorioprints-data");
 
 const getMostFavoriteRequestObject = ({
   lastId,
@@ -34,7 +34,7 @@ const getBlueprintRequestObject = ({ id }: { id: string }) => ({
 });
 
 const openWebsocket = (): Promise<WebSocket> => {
-  const ws = new WebSocket("wss://s-usc1c-nss-239.firebaseio.com/.ws?v=5&ns=facorio-blueprints");
+  const ws = new WebSocket("wss://s-usc1c-nss-238.firebaseio.com/.ws?v=5&ns=facorio-blueprints");
   return new Promise((resolve) => {
     ws.on("open", function open() {
       resolve(ws);
@@ -48,7 +48,7 @@ const openWebsocket = (): Promise<WebSocket> => {
   });
 };
 
-export async function scanFactorioPrints() {
+export async function scanFactorioPrints(page_from: number, page_to: number) {
   const ws = await openWebsocket();
 
   const sendMessage = (data: any) => {
@@ -93,7 +93,7 @@ export async function scanFactorioPrints() {
   };
 
   const getMostFavorited = async (
-    page = 1,
+    page = page_from,
     reqObj: Parameters<typeof getMostFavoriteRequestObject>[0]
   ) => {
     const data = await sendMessageAndWaitForResponse(
@@ -152,7 +152,7 @@ export async function scanFactorioPrints() {
       console.error(reason);
     });
 
-    if (page < 3) {
+    if (page < page_to) {
       const lastBp = blueprints[blueprints.length - 1];
       getMostFavorited(page + 1, {
         lastId: lastBp.id,
diff --git a/apps/factorioprints-scraper/src/main.ts b/apps/factorioprints-scraper/src/main.ts
index ac4f388..575f58a 100644
--- a/apps/factorioprints-scraper/src/main.ts
+++ b/apps/factorioprints-scraper/src/main.ts
@@ -1,4 +1,5 @@
 import { writeToDatastore } from "./app/populate-db";
+import { scanFactorioPrints } from "./app/scan";
 
 // async function writeTestBP() {
 //   const source =
@@ -22,7 +23,7 @@ import { writeToDatastore } from "./app/populate-db";
 // }
 
 async function main() {
-  // scanFactorioPrints();
+  // scanFactorioPrints(1, 3);
   writeToDatastore();
   // writeTestBP();
 }
diff --git a/libs/database/src/lib/database.ts b/libs/database/src/lib/database.ts
index 9508c1f..6c30746 100644
--- a/libs/database/src/lib/database.ts
+++ b/libs/database/src/lib/database.ts
@@ -429,7 +429,6 @@ async function createBlueprintPage(
     {
       name: "updated_at",
       value: extraInfo.updated_at ? new Date(extraInfo.updated_at * 1000) : new Date(),
-      excludeFromIndexes: true,
     },
     { name: "factorioprints_id", value: extraInfo.factorioprints_id || null },
   ];
@@ -509,7 +508,7 @@ export async function saveBlueprintFromFactorioprints(
   };
 
   if (parsed.data.blueprint) {
-    console.log(`string has one blueprint...`);
+    console.log("string has one blueprint...");
     const { insertedId } = await createBlueprint(parsed.data.blueprint, extraInfo).catch(
       (error) => {
         if (error instanceof DatastoreExistsError) {
@@ -520,7 +519,7 @@ export async function saveBlueprintFromFactorioprints(
     );
     await createBlueprintPage("blueprint", insertedId, extraInfoPage);
   } else if (parsed.data.blueprint_book) {
-    console.log(`string has a blueprint book...`);
+    console.log("string has a blueprint book...");
     const { insertedId } = await createBlueprintBook(parsed.data.blueprint_book, extraInfo);
     await createBlueprintPage("blueprint_book", insertedId, extraInfoPage);
   }