New utilities.

2025-08-10 22:11:45 +02:00 · 2024-02-06 11:03:51 +00:00
parent c024ed13d3
commit 27d71ca2fb
8 changed files with 491 additions and 0 deletions
--- a/utils/.gitignore
+++ b/utils/.gitignore
@@ -0,0 +1 @@
+fly.toml
--- a/utils/couchdb/couchdb-init.sh
+++ b/utils/couchdb/couchdb-init.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+if [[ -z "$hostname" ]]; then
+    echo "ERROR: Hostname missing"
+    exit 1
+fi
+if [[ -z "$username" ]]; then
+    echo "ERROR: Username missing"
+    exit 1
+fi
+
+if [[ -z "$password" ]]; then
+    echo "ERROR: Password missing"
+    exit 1
+fi
+
+echo "-- Configuring CouchDB by REST APIs... -->"
+
+until (curl -X POST "${hostname}/_cluster_setup" -H "Content-Type: application/json" -d "{\"action\":\"enable_single_node\",\"username\":\"${username}\",\"password\":\"${password}\",\"bind_address\":\"0.0.0.0\",\"port\":5984,\"singlenode\":true}" --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd/require_valid_user" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd_auth/require_valid_user" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/httpd/WWW-Authenticate" -H "Content-Type: application/json" -d '"Basic realm=\"couchdb\""' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/httpd/enable_cors" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd/enable_cors" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/chttpd/max_http_request_size" -H "Content-Type: application/json" -d '"4294967296"' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/couchdb/max_document_size" -H "Content-Type: application/json" -d '"50000000"' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/cors/credentials" -H "Content-Type: application/json" -d '"true"' --user "${username}:${password}"); do sleep 5; done
+until (curl -X PUT "${hostname}/_node/nonode@nohost/_config/cors/origins" -H "Content-Type: application/json" -d '"app://obsidian.md,capacitor://localhost,http://localhost"' --user "${username}:${password}"); do sleep 5; done
+
+echo "<-- Configuring CouchDB by REST APIs Done!"
--- a/utils/flyio/delete-server.sh
+++ b/utils/flyio/delete-server.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+fly scale count 0 -y
+fly apps destroy $(fly status -j | jq -r .Name) -y
--- a/utils/flyio/deploy-server.sh
+++ b/utils/flyio/deploy-server.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+## Script for deploy and automatic setup CouchDB onto fly.io.
+## We need Deno for generating the Setup-URI.
+
+source setenv.sh $@
+
+export hostname="https://$appname.fly.dev"
+
+echo "-- YOUR CONFIGURATION --"
+echo "URL     : $hostname"
+echo "username: $username"
+echo "password: $password"
+echo "region  : $region"
+echo ""
+echo "-- START DEPLOYING --> "
+
+set -e
+fly launch --name=$appname --env="COUCHDB_USER=$username" --copy-config=true --detach --no-deploy --region ${region} --yes
+fly secrets set COUCHDB_PASSWORD=$password
+fly deploy
+
+set +e
+../couchdb/couchdb-init.sh
+# flyctl deploy
+echo "OK!"
+
+if command -v deno >/dev/null 2>&1; then
+    echo "Setup finished! Also, we can set up Self-hosted LiveSync instantly, by the following setup uri."
+    echo "Passphrase of setup-uri is \`welcome\`".
+    echo "--- configured ---"
+    echo "database       : ${database}"
+    echo "E2EE passphrase: ${passphrase}"
+    echo "--- setup uri  ---"
+    deno run -A generate_setupuri.ts
+else
+    echo "Setup finished! Here is the configured values (reprise)!"
+    echo "-- YOUR CONFIGURATION --"
+    echo "URL     : $hostname"
+    echo "username: $username"
+    echo "password: $password"
+    echo "-- YOUR CONFIGURATION --"
+    echo "If we had Deno, we would got the setup uri directly!"
+fi
--- a/utils/flyio/fly.template.toml
+++ b/utils/flyio/fly.template.toml
@@ -0,0 +1,40 @@
+## CouchDB for fly.io image
+
+app = ''
+primary_region = 'nrt'
+swap_size_mb = 512
+
+[build]
+  image = "couchdb:latest"
+
+[mounts]
+  source = "couchdata"
+  destination = "/opt/couchdb/data"
+  initial_size = "1GB"
+  auto_extend_size_threshold = 90
+  auto_extend_size_increment = "1GB"
+  auto_extend_size_limit = "2GB"
+
+[env]
+  COUCHDB_USER = ""
+  ERL_FLAGS = "-couch_ini /opt/couchdb/etc/default.ini /opt/couchdb/etc/default.d/ /opt/couchdb/etc/local.d /opt/couchdb/etc/local.ini /opt/couchdb/data/persistence.ini"
+
+[http_service]
+  internal_port = 5984
+  force_https = true
+  auto_stop_machines = true
+  auto_start_machines = true
+  min_machines_running = 0
+  processes = ['app']
+
+[[vm]]
+  cpu_kind = 'shared'
+  cpus = 1
+  memory_mb = 256
+
+[[files]]
+  guest_path = "/docker-entrypoint2.sh"
+  raw_value = "#!/bin/bash\ntouch /opt/couchdb/data/persistence.ini\nchmod +w /opt/couchdb/data/persistence.ini\n/docker-entrypoint.sh $@"
+
+[experimental]
+  entrypoint = ["tini", "--", "/docker-entrypoint2.sh"]
--- a/utils/flyio/generate_setupuri.ts
+++ b/utils/flyio/generate_setupuri.ts
@@ -0,0 +1,180 @@
+import { webcrypto } from "node:crypto";
+
+const KEY_RECYCLE_COUNT = 100;
+type KeyBuffer = {
+    key: CryptoKey;
+    salt: Uint8Array;
+    count: number;
+};
+
+let semiStaticFieldBuffer: Uint8Array;
+const nonceBuffer: Uint32Array = new Uint32Array(1);
+const writeString = (string: string) => {
+    // Prepare enough buffer.
+    const buffer = new Uint8Array(string.length * 4);
+    const length = string.length;
+    let index = 0;
+    let chr = 0;
+    let idx = 0;
+    while (idx < length) {
+        chr = string.charCodeAt(idx++);
+        if (chr < 128) {
+            buffer[index++] = chr;
+        } else if (chr < 0x800) {
+            // 2 bytes
+            buffer[index++] = 0xC0 | (chr >>> 6);
+            buffer[index++] = 0x80 | (chr & 0x3F);
+        } else if (chr < 0xD800 || chr > 0xDFFF) {
+            // 3 bytes
+            buffer[index++] = 0xE0 | (chr >>> 12);
+            buffer[index++] = 0x80 | ((chr >>> 6) & 0x3F);
+            buffer[index++] = 0x80 | (chr & 0x3F);
+        } else {
+            // 4 bytes - surrogate pair
+            chr = (((chr - 0xD800) << 10) | (string.charCodeAt(idx++) - 0xDC00)) + 0x10000;
+            buffer[index++] = 0xF0 | (chr >>> 18);
+            buffer[index++] = 0x80 | ((chr >>> 12) & 0x3F);
+            buffer[index++] = 0x80 | ((chr >>> 6) & 0x3F);
+            buffer[index++] = 0x80 | (chr & 0x3F);
+        }
+    }
+    return buffer.slice(0, index);
+};
+const KeyBuffs = new Map<string, KeyBuffer>();
+async function getKeyForEncrypt(passphrase: string, autoCalculateIterations: boolean): Promise<[CryptoKey, Uint8Array]> {
+    // For performance, the plugin reuses the key KEY_RECYCLE_COUNT times.
+    const buffKey = `${passphrase}-${autoCalculateIterations}`;
+    const f = KeyBuffs.get(buffKey);
+    if (f) {
+        f.count--;
+        if (f.count > 0) {
+            return [f.key, f.salt];
+        }
+        f.count--;
+    }
+    const passphraseLen = 15 - passphrase.length;
+    const iteration = autoCalculateIterations ? ((passphraseLen > 0 ? passphraseLen : 0) * 1000) + 121 - passphraseLen : 100000;
+    const passphraseBin = new TextEncoder().encode(passphrase);
+    const digest = await webcrypto.subtle.digest({ name: "SHA-256" }, passphraseBin);
+    const keyMaterial = await webcrypto.subtle.importKey("raw", digest, { name: "PBKDF2" }, false, ["deriveKey"]);
+    const salt = webcrypto.getRandomValues(new Uint8Array(16));
+    const key = await webcrypto.subtle.deriveKey(
+        {
+            name: "PBKDF2",
+            salt,
+            iterations: iteration,
+            hash: "SHA-256",
+        },
+        keyMaterial,
+        { name: "AES-GCM", length: 256 },
+        false,
+        ["encrypt"]
+    );
+    KeyBuffs.set(buffKey, {
+        key,
+        salt,
+        count: KEY_RECYCLE_COUNT,
+    });
+    return [key, salt];
+}
+
+function getSemiStaticField(reset?: boolean) {
+    // return fixed field of iv.
+    if (semiStaticFieldBuffer != null && !reset) {
+        return semiStaticFieldBuffer;
+    }
+    semiStaticFieldBuffer = webcrypto.getRandomValues(new Uint8Array(12));
+    return semiStaticFieldBuffer;
+}
+
+function getNonce() {
+    // This is nonce, so do not send same thing.
+    nonceBuffer[0]++;
+    if (nonceBuffer[0] > 10000) {
+        // reset semi-static field.
+        getSemiStaticField(true);
+    }
+    return nonceBuffer;
+}
+function arrayBufferToBase64internalBrowser(buffer: DataView | Uint8Array): Promise<string> {
+    return new Promise((res, rej) => {
+        const blob = new Blob([buffer], { type: "application/octet-binary" });
+        const reader = new FileReader();
+        reader.onload = function (evt) {
+            const dataURI = evt.target?.result?.toString() || "";
+            if (buffer.byteLength != 0 && (dataURI == "" || dataURI == "data:")) return rej(new TypeError("Could not parse the encoded string"));
+            const result = dataURI.substring(dataURI.indexOf(",") + 1);
+            res(result);
+        };
+        reader.readAsDataURL(blob);
+    });
+}
+
+// Map for converting hexString
+const revMap: { [key: string]: number } = {};
+const numMap: { [key: number]: string } = {};
+for (let i = 0; i < 256; i++) {
+    revMap[(`00${i.toString(16)}`.slice(-2))] = i;
+    numMap[i] = (`00${i.toString(16)}`.slice(-2));
+}
+
+
+function uint8ArrayToHexString(src: Uint8Array): string {
+    return [...src].map(e => numMap[e]).join("");
+}
+
+const QUANTUM = 32768;
+async function arrayBufferToBase64Single(buffer: ArrayBuffer): Promise<string> {
+    const buf = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
+    if (buf.byteLength < QUANTUM) return btoa(String.fromCharCode.apply(null, [...buf]));
+    return await arrayBufferToBase64internalBrowser(buf);
+}
+
+
+export async function encrypt(input: string, passphrase: string, autoCalculateIterations: boolean) {
+    const [key, salt] = await getKeyForEncrypt(passphrase, autoCalculateIterations);
+    // Create initial vector with semi-fixed part and incremental part
+    // I think it's not good against related-key attacks.
+    const fixedPart = getSemiStaticField();
+    const invocationPart = getNonce();
+    const iv = new Uint8Array([...fixedPart, ...new Uint8Array(invocationPart.buffer)]);
+    const plainStringified = JSON.stringify(input);
+
+    // const plainStringBuffer: Uint8Array = tex.encode(plainStringified)
+    const plainStringBuffer: Uint8Array = writeString(plainStringified);
+    const encryptedDataArrayBuffer = await webcrypto.subtle.encrypt({ name: "AES-GCM", iv }, key, plainStringBuffer);
+    const encryptedData2 = (await arrayBufferToBase64Single(encryptedDataArrayBuffer));
+    //return data with iv and salt.
+    const ret = `["${encryptedData2}","${uint8ArrayToHexString(iv)}","${uint8ArrayToHexString(salt)}"]`;
+    return ret;
+}
+
+const URIBASE = "obsidian://setuplivesync?settings=";
+async function main() {
+    const conf = {
+        "couchDB_URI": `${Deno.env.get("hostname")}`,
+        "couchDB_USER": `${Deno.env.get("username")}`,
+        "couchDB_PASSWORD": `${Deno.env.get("password")}`,
+        "couchDB_DBNAME": `${Deno.env.get("database")}`,
+        "syncOnStart": true,
+        "gcDelay": 0,
+        "periodicReplication": true,
+        "syncOnFileOpen": true,
+        "encrypt": true,
+        "passphrase": `${Deno.env.get("passphrase")}`,
+        "usePathObfuscation": true,
+        "batchSave": true,
+        "batch_size": 50,
+        "batches_limit": 50,
+        "useHistory": true,
+        "disableRequestURI": true,
+        "customChunkSize": 50,
+        "syncAfterMerge": false,
+        "concurrencyOfReadChunksOnline": 100,
+        "minimumIntervalOfReadChunksOnline": 100,
+    }
+    const encryptedConf = encodeURIComponent(await encrypt(JSON.stringify(conf), "welcome", false));
+    const theURI = `${URIBASE}${encryptedConf}`;
+    console.log(theURI);
+}
+await main();
--- a/utils/flyio/setenv.sh
+++ b/utils/flyio/setenv.sh
@@ -0,0 +1,30 @@
+random_num() {
+    echo $RANDOM
+}
+random_noun() {
+    nouns=("waterfall" "river" "breeze" "moon" "rain" "wind" "sea" "morning" "snow" "lake" "sunset" "pine" "shadow" "leaf" "dawn" "glitter" "forest" "hill" "cloud" "meadow" "sun" "glade" "bird" "brook" "butterfly" "bush" "dew" "dust" "field" "fire" "flower" "firefly" "feather" "grass" "haze" "mountain" "night" "pond" "darkness" "snowflake" "silence" "sound" "sky" "shape" "surf" "thunder" "violet" "water" "wildflower" "wave" "water" "resonance" "sun" "log" "dream" "cherry" "tree" "fog" "frost" "voice" "paper" "frog" "smoke" "star")
+    echo ${nouns[$(($RANDOM % ${#nouns[*]}))]}
+}
+
+random_adjective() {
+    adjectives=("autumn" "hidden" "bitter" "misty" "silent" "empty" "dry" "dark" "summer" "icy" "delicate" "quiet" "white" "cool" "spring" "winter" "patient" "twilight" "dawn" "crimson" "wispy" "weathered" "blue" "billowing" "broken" "cold" "damp" "falling" "frosty" "green" "long" "late" "lingering" "bold" "little" "morning" "muddy" "old" "red" "rough" "still" "small" "sparkling" "thrumming" "shy" "wandering" "withered" "wild" "black" "young" "holy" "solitary" "fragrant" "aged" "snowy" "proud" "floral" "restless" "divine" "polished" "ancient" "purple" "lively" "nameless")
+    echo ${adjectives[$(($RANDOM % ${#adjectives[*]}))]}
+}
+
+cp ./fly.template.toml ./fly.toml
+
+if [ "$1" = "renew" ]; then
+    unset appname
+    unset username
+    unset password
+    unset database
+    unset passphrase
+    unset region
+fi
+
+[ -z $appname ] && export appname=$(random_adjective)-$(random_noun)-$(random_num)
+[ -z $username ] && export username=$(random_adjective)-$(random_noun)-$(random_num)
+[ -z $password ] && export password=$(random_adjective)-$(random_noun)-$(random_num)
+[ -z $database ] && export database="obsidiannotes"
+[ -z $passphrase ] && export passphrase=$(random_adjective)-$(random_noun)-$(random_num)
+[ -z $region ] && export region="nrt"
--- a/utils/readme.md
+++ b/utils/readme.md
@@ -0,0 +1,164 @@
+<!-- For translation: 20240206r0 -->
+# Utilities
+Here are some useful things.
+
+## couchdb
+
+### couchdb-init.sh
+This script can configure CouchDB with the necessary settings by REST APIs.
+
+#### Materials
+- Mandatory: curl
+
+#### Usage
+
+```sh
+export hostname=http://localhost:5984/
+export username=couchdb-admin-username
+export password=couchdb-admin-password
+./couchdb-init.sh
+```
+
+curl result will be shown, however, all of them can be ignored if the script has been run completely.
+
+## fly.io
+
+### deploy-server.sh
+
+A fully automated CouchDB deployment script. We can deploy CouchDB onto fly.io. The only we need is an account of it.
+
+All omitted configurations will be determined at random. (And, it is preferred). The region is configured to `nrt`.
+If Japan is not close to you, please choose a region closer to you. However, the deployed database will work if you leave it at all.
+
+#### Materials
+- Mandatory: curl, flyctl
+- Recommended: deno
+
+#### Usage
+```sh
+#export appname=
+#export username=
+#export password=
+#export database=
+#export passphrase=
+export region=nrt #pick your nearest location
+./deploy-server.sh
+```
+
+The result of this command is as follows.
+
+```
+-- YOUR CONFIGURATION --
+URL     : https://young-darkness-25342.fly.dev
+username: billowing-cherry-22580
+password: misty-dew-13571
+region  : nrt
+
+-- START DEPLOYING --> 
+An existing fly.toml file was found
+Using build strategies '[the "couchdb:latest" docker image]'. Remove [build] from fly.toml to force a rescan
+Creating app in /home/vorotamoroz/dev/obsidian-livesync/utils/flyio
+We're about to launch your app on Fly.io. Here's what you're getting:
+
+Organization: vorotamoroz              (fly launch defaults to the personal org)
+Name:         young-darkness-25342     (specified on the command line)
+Region:       Tokyo, Japan             (specified on the command line)
+App Machines: shared-cpu-1x, 256MB RAM (specified on the command line)
+Postgres:     <none>                   (not requested)
+Redis:        <none>                   (not requested)
+
+Created app 'young-darkness-25342' in organization 'personal'
+Admin URL: https://fly.io/apps/young-darkness-25342
+Hostname: young-darkness-25342.fly.dev
+Wrote config file fly.toml
+Validating /home/vorotamoroz/dev/obsidian-livesync/utils/flyio/fly.toml
+Platform: machines
+✓ Configuration is valid
+Your app is ready! Deploy with `flyctl deploy`
+Secrets are staged for the first deployment
+==> Verifying app config
+Validating /home/vorotamoroz/dev/obsidian-livesync/utils/flyio/fly.toml
+Platform: machines
+✓ Configuration is valid
+--> Verified app config
+==> Building image
+Searching for image 'couchdb:latest' remotely...
+image found: img_ox20prk63084j1zq
+
+Watch your deployment at https://fly.io/apps/young-darkness-25342/monitoring
+
+Provisioning ips for young-darkness-25342
+  Dedicated ipv6: 2a09:8280:1::37:fde9
+  Shared ipv4: 66.241.124.163
+  Add a dedicated ipv4 with: fly ips allocate-v4
+
+Creating a 1 GB volume named 'couchdata' for process group 'app'. Use 'fly vol extend' to increase its size
+This deployment will:
+ * create 1 "app" machine
+
+No machines in group app, launching a new machine
+
+WARNING The app is not listening on the expected address and will not be reachable by fly-proxy.
+You can fix this by configuring your app to listen on the following addresses:
+  - 0.0.0.0:5984
+Found these processes inside the machine with open listening sockets:
+  PROCESS        | ADDRESSES                             
+-----------------*---------------------------------------
+  /.fly/hallpass | [fdaa:0:73b9:a7b:22e:3851:7f28:2]:22  
+
+Finished launching new machines
+
+NOTE: The machines for [app] have services with 'auto_stop_machines = true' that will be stopped when idling
+
+-------
+Checking DNS configuration for young-darkness-25342.fly.dev
+
+Visit your newly deployed app at https://young-darkness-25342.fly.dev/
+-- Configuring CouchDB by REST APIs... -->
+curl: (35) OpenSSL SSL_connect: Connection reset by peer in connection to young-darkness-25342.fly.dev:443 
+{"ok":true}
+""
+""
+""
+""
+""
+""
+""
+""
+""
+<-- Configuring CouchDB by REST APIs Done!
+OK!
+Setup finished! Also, we can set up Self-hosted LiveSync instantly, by the following setup uri.
+Passphrase of setup-uri is `welcome`.
+--- configured ---
+database       : obsidiannotes
+E2EE passphrase: dark-wildflower-26467
+--- setup uri  ---
+obsidian://setuplivesync?settings=%5B%22gZkBwjFbLqxbdSIbJymU%2FmTPBPAKUiHVGDRKYiNnKhW0auQeBgJOfvnxexZtMCn8sNiIUTAlxNaMGF2t%2BCEhpJoeCP%2FO%2BrwfN5LaNDQyky1Uf7E%2B64A5UWyjOYvZDOgq4iCKSdBAXp9oO%2BwKh4MQjUZ78vIVvJp8Mo6NWHfm5fkiWoAoddki1xBMvi%2BmmN%2FhZatQGcslVb9oyYWpZocduTl0a5Dv%2FQviGwlYQ%2F4NY0dVDIoOdvaYS%2FX4GhNAnLzyJKMXhPEJHo9FvR%2FEOBuwyfMdftV1SQUZ8YDCuiR3T7fh7Kn1c6OFgaFMpFm%2BWgIJ%2FZpmAyhZFpEcjpd7ty%2BN9kfd9gQsZM4%2BYyU9OwDd2DahVMBWkqoV12QIJ8OlJScHHdcUfMW5ex%2F4UZTWKNEHJsigITXBrtq11qGk3rBfHys8O0vY6sz%2FaYNM3iAOsR1aoZGyvwZm4O6VwtzK8edg0T15TL4O%2B7UajQgtCGxgKNYxb8EMOGeskv7NifYhjCWcveeTYOJzBhnIDyRbYaWbkAXQgHPBxzJRkkG%2FpBPfBBoJarj7wgjMvhLJ9xtL4FbP6sBNlr8jtAUCoq4L7LJcRNF4hlgvjJpL2BpFZMzkRNtUBcsRYR5J%2BM1X2buWi2BHncbSiRRDKEwNOQkc%2FmhMJjbAn%2F8eNKRuIICOLD5OvxD7FZNCJ0R%2BWzgrzcNV%22%2C%22ec7edc900516b4fcedb4c7cc01000000%22%2C%22fceb5fe54f6619ee266ed9a887634e07%22%5D
+```
+
+All we have to do is copy the setup-URI (`obsidian`://...`) and open it from Self-hosted LiveSync on Obsidian.
+
+If you did not install Deno, configurations will be printed again, instead of the setup-URI. In this case, we should configure it manually.
+
+### delete-server.sh
+
+The pair script of `deploy-server.sh`. We can delete the deployed server by this with fly.toml.
+
+#### Materials
+
+- Mandatory: flyctl, jq
+- Recommended: none
+
+#### Usage
+```sh
+./delete-server.sh 
+```
+
+```
+App 'young-darkness-25342 is going to be scaled according to this plan:
+  -1 machines for group 'app' on region 'nrt' of size 'shared-cpu-1x'
+Executing scale plan
+  Destroyed e28667eec57158 group:app region:nrt size:shared-cpu-1x
+Destroyed app young-darkness-25342
+```