feat: postgres reverse geocoding (#5301)

* feat: add system metadata repository for storing key values for internal usage

* feat: add database entities for geodata

* feat: move reverse geocoding from local-reverse-geocoder to postgresql

* infra: disable synchronization for geodata_places table until typeorm supports earth column

* feat: remove cities override config as we will default all instances to cities500 now

* test: e2e tests don't clear geodata tables on reset

cli/src/api/open-api/api.ts

@@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto {
      */
     'existingIds': Array<string>;
 }
-/**
- * 
- * @export
- * @enum {string}
- */
-
-export const CitiesFile = {
-    Cities15000: 'cities15000',
-    Cities5000: 'cities5000',
-    Cities1000: 'cities1000',
-    Cities500: 'cities500'
-} as const;
-
-export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile];
-
-
 /**
  * 
  * @export
@@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto {
  * @interface SystemConfigReverseGeocodingDto
  */
 export interface SystemConfigReverseGeocodingDto {
-    /**
-     * 
-     * @type {CitiesFile}
-     * @memberof SystemConfigReverseGeocodingDto
-     */
-    'citiesFileOverride': CitiesFile;
     /**
      * 
      * @type {boolean}
@@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto {
      */
     'enabled': boolean;
 }
-
-
 /**
  * 
  * @export

mobile/openapi/.openapi-generator/FILES

@@ -46,7 +46,6 @@ doc/CQMode.md
 doc/ChangePasswordDto.md
 doc/CheckExistingAssetsDto.md
 doc/CheckExistingAssetsResponseDto.md
-doc/CitiesFile.md
 doc/ClassificationConfig.md
 doc/Colorspace.md
 doc/CreateAlbumDto.md
@@ -231,7 +230,6 @@ lib/model/bulk_ids_dto.dart
 lib/model/change_password_dto.dart
 lib/model/check_existing_assets_dto.dart
 lib/model/check_existing_assets_response_dto.dart
-lib/model/cities_file.dart
 lib/model/classification_config.dart
 lib/model/clip_config.dart
 lib/model/clip_mode.dart
@@ -388,7 +386,6 @@ test/bulk_ids_dto_test.dart
 test/change_password_dto_test.dart
 test/check_existing_assets_dto_test.dart
 test/check_existing_assets_response_dto_test.dart
-test/cities_file_test.dart
 test/classification_config_test.dart
 test/clip_config_test.dart
 test/clip_mode_test.dart

server/Dockerfile

@@ -31,7 +31,7 @@ COPY --from=prod /usr/src/app/node_modules ./node_modules
 COPY --from=prod /usr/src/app/dist ./dist
 COPY --from=prod /usr/src/app/bin ./bin
 COPY --from=web /usr/src/app/build ./www
-COPY server/assets assets
+COPY server/resources resources
 COPY server/package.json server/package-lock.json ./
 COPY server/start*.sh ./
 RUN npm link && npm cache clean --force

server/immich-openapi-specs.json

@@ -6989,15 +6989,6 @@
         ],
         "type": "object"
       },
-      "CitiesFile": {
-        "enum": [
-          "cities15000",
-          "cities5000",
-          "cities1000",
-          "cities500"
-        ],
-        "type": "string"
-      },
       "ClassificationConfig": {
         "properties": {
           "enabled": {
@@ -9112,15 +9103,11 @@
       },
       "SystemConfigReverseGeocodingDto": {
         "properties": {
-          "citiesFileOverride": {
-            "$ref": "#/components/schemas/CitiesFile"
-          },
           "enabled": {
             "type": "boolean"
           }
         },
         "required": [
-          "citiesFileOverride",
           "enabled"
         ],
         "type": "object"

server/package-lock.json

@@ -38,7 +38,6 @@
         "i18n-iso-countries": "^7.6.0",
         "ioredis": "^5.3.2",
         "joi": "^17.10.0",
-        "local-reverse-geocoder": "0.16.5",
         "lodash": "^4.17.21",
         "luxon": "^3.4.2",
         "mv": "^2.1.1",
@@ -4132,18 +4131,6 @@
         "node": ">=0.6"
       }
     },
-    "node_modules/binary": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
-      "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
-      "dependencies": {
-        "buffers": "~0.1.1",
-        "chainsaw": "~0.1.0"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -4329,14 +4316,6 @@
         "node": ">=4"
       }
     },
-    "node_modules/buffers": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
-      "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==",
-      "engines": {
-        "node": ">=0.2.0"
-      }
-    },
     "node_modules/buildcheck": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
@@ -4500,17 +4479,6 @@
         }
       ]
     },
-    "node_modules/chainsaw": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
-      "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
-      "dependencies": {
-        "traverse": ">=0.3.0 <0.4"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -5161,19 +5129,6 @@
         "node": ">= 8"
       }
     },
-    "node_modules/csv-parse": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz",
-      "integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw=="
-    },
-    "node_modules/data-uri-to-buffer": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
-      "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
-      "engines": {
-        "node": ">= 12"
-      }
-    },
     "node_modules/date-fns": {
       "version": "2.30.0",
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
@@ -6300,28 +6255,6 @@
         "bser": "2.1.1"
       }
     },
-    "node_modules/fetch-blob": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
-      "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/jimmywarting"
-        },
-        {
-          "type": "paypal",
-          "url": "https://paypal.me/jimmywarting"
-        }
-      ],
-      "dependencies": {
-        "node-domexception": "^1.0.0",
-        "web-streams-polyfill": "^3.0.3"
-      },
-      "engines": {
-        "node": "^12.20 || >= 14.13"
-      }
-    },
     "node_modules/figures": {
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
@@ -6575,17 +6508,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/formdata-polyfill": {
-      "version": "4.0.10",
-      "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
-      "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
-      "dependencies": {
-        "fetch-blob": "^3.1.2"
-      },
-      "engines": {
-        "node": ">=12.20.0"
-      }
-    },
     "node_modules/formidable": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
@@ -8323,11 +8245,6 @@
         "graceful-fs": "^4.1.6"
       }
     },
-    "node_modules/kdt": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz",
-      "integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg=="
-    },
     "node_modules/keyv": {
       "version": "4.5.3",
       "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz",
@@ -8425,41 +8342,6 @@
         "node": ">=6.11.5"
       }
     },
-    "node_modules/local-reverse-geocoder": {
-      "version": "0.16.5",
-      "resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz",
-      "integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==",
-      "hasInstallScript": true,
-      "dependencies": {
-        "async": "^3.2.4",
-        "csv-parse": "^5.5.0",
-        "debug": "^4.3.4",
-        "kdt": "^0.1.0",
-        "node-fetch": "^3.3.2",
-        "unzip-stream": "^0.3.1"
-      },
-      "engines": {
-        "node": ">=11.0.0",
-        "npm": ">=6.4.1"
-      }
-    },
-    "node_modules/local-reverse-geocoder/node_modules/node-fetch": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
-      "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
-      "dependencies": {
-        "data-uri-to-buffer": "^4.0.0",
-        "fetch-blob": "^3.1.4",
-        "formdata-polyfill": "^4.0.10"
-      },
-      "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/node-fetch"
-      }
-    },
     "node_modules/locate-path": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@@ -9077,24 +8959,6 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
       "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
     },
-    "node_modules/node-domexception": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
-      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/jimmywarting"
-        },
-        {
-          "type": "github",
-          "url": "https://paypal.me/jimmywarting"
-        }
-      ],
-      "engines": {
-        "node": ">=10.5.0"
-      }
-    },
     "node_modules/node-emoji": {
       "version": "1.11.0",
       "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
@@ -11717,14 +11581,6 @@
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
       "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
-    "node_modules/traverse": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
-      "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==",
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/tree-kill": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
@@ -12314,15 +12170,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/unzip-stream": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
-      "integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
-      "dependencies": {
-        "binary": "^0.3.0",
-        "mkdirp": "^0.5.1"
-      }
-    },
     "node_modules/update-browserslist-db": {
       "version": "1.0.13",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
@@ -12480,14 +12327,6 @@
         "defaults": "^1.0.3"
       }
     },
-    "node_modules/web-streams-polyfill": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
-      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
@@ -15937,15 +15776,6 @@
       "integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==",
       "dev": true
     },
-    "binary": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
-      "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==",
-      "requires": {
-        "buffers": "~0.1.1",
-        "chainsaw": "~0.1.0"
-      }
-    },
     "binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -16077,11 +15907,6 @@
       "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz",
       "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw=="
     },
-    "buffers": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
-      "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="
-    },
     "buildcheck": {
       "version": "0.0.6",
       "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz",
@@ -16194,14 +16019,6 @@
       "integrity": "sha512-UrtAXVcj1mvPBFQ4sKd38daP8dEcXXr5sQe6QNNinaPd0iA/cxg9/l3VrSdL73jgw5sKyuQ6jNgiKO12W3SsVA==",
       "dev": true
     },
-    "chainsaw": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
-      "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==",
-      "requires": {
-        "traverse": ">=0.3.0 <0.4"
-      }
-    },
     "chalk": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
@@ -16681,16 +16498,6 @@
         "which": "^2.0.1"
       }
     },
-    "csv-parse": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz",
-      "integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw=="
-    },
-    "data-uri-to-buffer": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
-      "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A=="
-    },
     "date-fns": {
       "version": "2.30.0",
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
@@ -17523,15 +17330,6 @@
         "bser": "2.1.1"
       }
     },
-    "fetch-blob": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
-      "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
-      "requires": {
-        "node-domexception": "^1.0.0",
-        "web-streams-polyfill": "^3.0.3"
-      }
-    },
     "figures": {
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz",
@@ -17717,14 +17515,6 @@
         "mime-types": "^2.1.12"
       }
     },
-    "formdata-polyfill": {
-      "version": "4.0.10",
-      "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
-      "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
-      "requires": {
-        "fetch-blob": "^3.1.2"
-      }
-    },
     "formidable": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz",
@@ -19005,11 +18795,6 @@
         "universalify": "^2.0.0"
       }
     },
-    "kdt": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/kdt/-/kdt-0.1.0.tgz",
-      "integrity": "sha512-ueX0gyv7tw4zBq9cQjaCr9qIhGTo5XYHUf/8aUUMHwoyb81KeCZHkSOoUwHGg/mgabvhTKCYjDUuYEmdak6Xjg=="
-    },
     "keyv": {
       "version": "4.5.3",
       "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.3.tgz",
@@ -19094,31 +18879,6 @@
       "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==",
       "dev": true
     },
-    "local-reverse-geocoder": {
-      "version": "0.16.5",
-      "resolved": "https://registry.npmjs.org/local-reverse-geocoder/-/local-reverse-geocoder-0.16.5.tgz",
-      "integrity": "sha512-MgJsyR3s8eeMfRfMvikwIdOG/jh9s78zPPX9kfx1qk5fwQLJnry5Qx5jreclqDPEpjOpNKIqz4aG5BbWGAGLbw==",
-      "requires": {
-        "async": "^3.2.4",
-        "csv-parse": "^5.5.0",
-        "debug": "^4.3.4",
-        "kdt": "^0.1.0",
-        "node-fetch": "^3.3.2",
-        "unzip-stream": "^0.3.1"
-      },
-      "dependencies": {
-        "node-fetch": {
-          "version": "3.3.2",
-          "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
-          "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
-          "requires": {
-            "data-uri-to-buffer": "^4.0.0",
-            "fetch-blob": "^3.1.4",
-            "formdata-polyfill": "^4.0.10"
-          }
-        }
-      }
-    },
     "locate-path": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@@ -19599,11 +19359,6 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
       "integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
     },
-    "node-domexception": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
-      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="
-    },
     "node-emoji": {
       "version": "1.11.0",
       "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.11.0.tgz",
@@ -21569,11 +21324,6 @@
       "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
       "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
     },
-    "traverse": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
-      "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ=="
-    },
     "tree-kill": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz",
@@ -21900,15 +21650,6 @@
       "integrity": "sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw==",
       "dev": true
     },
-    "unzip-stream": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/unzip-stream/-/unzip-stream-0.3.1.tgz",
-      "integrity": "sha512-RzaGXLNt+CW+T41h1zl6pGz3EaeVhYlK+rdAap+7DxW5kqsqePO8kRtWPaCiVqdhZc86EctSPVYNix30YOMzmw==",
-      "requires": {
-        "binary": "^0.3.0",
-        "mkdirp": "^0.5.1"
-      }
-    },
     "update-browserslist-db": {
       "version": "1.0.13",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz",
@@ -22028,11 +21769,6 @@
         "defaults": "^1.0.3"
       }
     },
-    "web-streams-polyfill": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
-      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q=="
-    },
     "webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",

server/package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "@babel/runtime": "^7.22.11",
+    "@immich/cli": "^2.0.3",
     "@nestjs/bullmq": "^10.0.1",
     "@nestjs/common": "^10.2.2",
     "@nestjs/config": "^3.0.0",
@@ -65,10 +66,8 @@
     "glob": "^10.3.3",
     "handlebars": "^4.7.8",
     "i18n-iso-countries": "^7.6.0",
-    "@immich/cli": "^2.0.3",
     "ioredis": "^5.3.2",
     "joi": "^17.10.0",
-    "local-reverse-geocoder": "0.16.5",
     "lodash": "^4.17.21",
     "luxon": "^3.4.2",
     "mv": "^2.1.1",

server/src/domain/metadata/metadata.service.spec.ts

@@ -1,4 +1,4 @@
-import { AssetType, CitiesFile, ExifEntity, SystemConfigKey } from '@app/infra/entities';
+import { AssetType, ExifEntity, SystemConfigKey } from '@app/infra/entities';
 import {
   assetStub,
   newAlbumRepositoryMock,
@@ -15,7 +15,7 @@ import { randomBytes } from 'crypto';
 import { Stats } from 'fs';
 import { constants } from 'fs/promises';
 import { when } from 'jest-when';
-import { JobName, QueueName } from '../job';
+import { JobName } from '../job';
 import {
   IAlbumRepository,
   IAssetRepository,
@@ -78,10 +78,7 @@ describe(MetadataService.name, () => {
 
   describe('init', () => {
     beforeEach(async () => {
-      configMock.load.mockResolvedValue([
+      configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true }]);
-        { key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: true },
-        { key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_500 },
-      ]);
 
       await sut.init();
     });
@@ -90,42 +87,10 @@ describe(MetadataService.name, () => {
       configMock.load.mockResolvedValue([{ key: SystemConfigKey.REVERSE_GEOCODING_ENABLED, value: false }]);
 
       await sut.init();
-      expect(metadataMock.deleteCache).not.toHaveBeenCalled();
       expect(jobMock.pause).toHaveBeenCalledTimes(1);
       expect(metadataMock.init).toHaveBeenCalledTimes(1);
       expect(jobMock.resume).toHaveBeenCalledTimes(1);
     });
-
-    it('should return if deleteCache is false and the cities precision has not changed', async () => {
-      await sut.init();
-
-      expect(metadataMock.deleteCache).not.toHaveBeenCalled();
-      expect(jobMock.pause).toHaveBeenCalledTimes(1);
-      expect(metadataMock.init).toHaveBeenCalledTimes(1);
-      expect(jobMock.resume).toHaveBeenCalledTimes(1);
-    });
-
-    it('should re-init if deleteCache is false but the cities precision has changed', async () => {
-      configMock.load.mockResolvedValue([
-        { key: SystemConfigKey.REVERSE_GEOCODING_CITIES_FILE_OVERRIDE, value: CitiesFile.CITIES_1000 },
-      ]);
-
-      await sut.init();
-
-      expect(metadataMock.deleteCache).not.toHaveBeenCalled();
-      expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
-      expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_1000 });
-      expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
-    });
-
-    it('should re-init and delete cache if deleteCache is true', async () => {
-      await sut.init(true);
-
-      expect(metadataMock.deleteCache).toHaveBeenCalled();
-      expect(jobMock.pause).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
-      expect(metadataMock.init).toHaveBeenCalledWith({ citiesFileOverride: CitiesFile.CITIES_500 });
-      expect(jobMock.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
-    });
   });
 
   describe('handleLivePhotoLinking', () => {

server/src/domain/metadata/metadata.service.ts

@@ -97,31 +97,24 @@ export class MetadataService {
     this.storageCore = StorageCore.create(assetRepository, moveRepository, personRepository, storageRepository);
   }
 
-  async init(deleteCache = false) {
+  async init() {
     if (!this.subscription) {
       this.subscription = this.configCore.config$.subscribe(() => this.init());
     }
 
     const { reverseGeocoding } = await this.configCore.getConfig();
-    const { citiesFileOverride } = reverseGeocoding;
+    const { enabled } = reverseGeocoding;
 
-    if (!reverseGeocoding.enabled) {
+    if (!enabled) {
       return;
     }
 
     try {
-      if (deleteCache) {
-        await this.repository.deleteCache();
-      } else if (this.oldCities && this.oldCities === citiesFileOverride) {
-        return;
-      }
-
       await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
-      await this.repository.init({ citiesFileOverride });
+      await this.repository.init();
       await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
 
-      this.logger.log(`Initialized local reverse geocoder with ${citiesFileOverride}`);
+      this.logger.log(`Initialized local reverse geocoder`);
-      this.oldCities = citiesFileOverride;
     } catch (error: Error | any) {
       this.logger.error(`Unable to initialize reverse geocoding: ${error}`, error?.stack);
     }
@@ -258,8 +251,9 @@ export class MetadataService {
     }
 
     try {
-      const { city, state, country } = await this.repository.reverseGeocode({ latitude, longitude });
+      const reverseGeocode = await this.repository.reverseGeocode({ latitude, longitude });
-      Object.assign(exifData, { city, state, country });
+      if (!reverseGeocode) return;
+      Object.assign(exifData, reverseGeocode);
     } catch (error: Error | any) {
       this.logger.warn(
         `Unable to run reverse geocoding due to ${error} for asset ${asset.id} at ${asset.originalPath}`,

server/src/domain/repositories/index.ts

@@ -20,6 +20,7 @@ export * from './shared-link.repository';
 export * from './smart-info.repository';
 export * from './storage.repository';
 export * from './system-config.repository';
+export * from './system-metadata.repository';
 export * from './tag.repository';
 export * from './user-token.repository';
 export * from './user.repository';

server/src/domain/repositories/metadata.repository.ts

@@ -1,5 +1,4 @@
 import { Tags } from 'exiftool-vendored';
-import { InitOptions } from 'local-reverse-geocoder';
 
 export const IMetadataRepository = 'IMetadataRepository';
 
@@ -31,9 +30,8 @@ export interface ImmichTags extends Omit<Tags, 'FocalLength' | 'Duration'> {
 }
 
 export interface IMetadataRepository {
-  init(options: Partial<InitOptions>): Promise<void>;
+  init(): Promise<void>;
   teardown(): Promise<void>;
-  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult>;
+  reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null>;
-  deleteCache(): Promise<void>;
   getExifTags(path: string): Promise<ImmichTags | null>;
 }

server/src/domain/repositories/system-metadata.repository.ts

@@ -0,0 +1,8 @@
+import { SystemMetadata } from '@app/infra/entities';
+
+export const ISystemMetadataRepository = 'ISystemMetadataRepository';
+
+export interface ISystemMetadataRepository {
+  get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null>;
+  set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void>;
+}

server/src/domain/system-config/dto/system-config-reverse-geocoding.dto.ts

@@ -1,12 +1,6 @@
-import { CitiesFile } from '@app/infra/entities';
+import { IsBoolean } from 'class-validator';
-import { ApiProperty } from '@nestjs/swagger';
-import { IsBoolean, IsEnum } from 'class-validator';
 
 export class SystemConfigReverseGeocodingDto {
   @IsBoolean()
   enabled!: boolean;
-
-  @IsEnum(CitiesFile)
-  @ApiProperty({ enum: CitiesFile, enumName: 'CitiesFile' })
-  citiesFileOverride!: CitiesFile;
 }

server/src/domain/system-config/system-config.core.ts

@@ -1,6 +1,5 @@
 import {
   AudioCodec,
-  CitiesFile,
   Colorspace,
   CQMode,
   SystemConfig,
@@ -85,7 +84,6 @@ export const defaults = Object.freeze<SystemConfig>({
   },
   reverseGeocoding: {
     enabled: true,
-    citiesFileOverride: CitiesFile.CITIES_500,
   },
   oauth: {
     enabled: false,

server/src/domain/system-config/system-config.service.spec.ts

@@ -1,6 +1,5 @@
 import {
   AudioCodec,
-  CitiesFile,
   Colorspace,
   CQMode,
   SystemConfig,
@@ -85,7 +84,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
   },
   reverseGeocoding: {
     enabled: true,
-    citiesFileOverride: CitiesFile.CITIES_500,
   },
   oauth: {
     autoLaunch: true,

server/src/domain/system-config/system-config.service.ts

@@ -79,7 +79,7 @@ export class SystemConfigService {
       return this.repository.fetchStyle(styleUrl);
     }
 
-    return JSON.parse(await this.repository.readFile(`./assets/style-${theme}.json`));
+    return JSON.parse(await this.repository.readFile(`./resources/style-${theme}.json`));
   }
 
   async getCustomCss(): Promise<string> {

server/src/infra/entities/geodata-admin1.entity.ts

@@ -0,0 +1,10 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity('geodata_admin1')
+export class GeodataAdmin1Entity {
+  @PrimaryColumn({ type: 'varchar' })
+  key!: string;
+
+  @Column({ type: 'varchar' })
+  name!: string;
+}

server/src/infra/entities/geodata-admin2.entity.ts

@@ -0,0 +1,10 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity('geodata_admin2')
+export class GeodataAdmin2Entity {
+  @PrimaryColumn({ type: 'varchar' })
+  key!: string;
+
+  @Column({ type: 'varchar' })
+  name!: string;
+}

server/src/infra/entities/geodata-places.entity.ts

@@ -0,0 +1,59 @@
+import { GeodataAdmin1Entity } from '@app/infra/entities/geodata-admin1.entity';
+import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity';
+import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm';
+
+@Entity('geodata_places', { synchronize: false })
+export class GeodataPlacesEntity {
+  @PrimaryColumn({ type: 'integer' })
+  id!: number;
+
+  @Column({ type: 'varchar', length: 200 })
+  name!: string;
+
+  @Column({ type: 'float' })
+  longitude!: number;
+
+  @Column({ type: 'float' })
+  latitude!: number;
+
+  // @Column({
+  //   generatedType: 'STORED',
+  //   asExpression: 'll_to_earth((latitude)::double precision, (longitude)::double precision)',
+  //   type: 'earth',
+  // })
+  earthCoord!: unknown;
+
+  @Column({ type: 'char', length: 2 })
+  countryCode!: string;
+
+  @Column({ type: 'varchar', length: 20, nullable: true })
+  admin1Code!: string;
+
+  @Column({ type: 'varchar', length: 80, nullable: true })
+  admin2Code!: string;
+
+  @Column({
+    type: 'varchar',
+    generatedType: 'STORED',
+    asExpression: `"countryCode" || '.' || "admin1Code"`,
+    nullable: true,
+  })
+  admin1Key!: string;
+
+  @ManyToOne(() => GeodataAdmin1Entity, { eager: true, nullable: true, createForeignKeyConstraints: false })
+  admin1!: GeodataAdmin1Entity;
+
+  @Column({
+    type: 'varchar',
+    generatedType: 'STORED',
+    asExpression: `"countryCode" || '.' || "admin1Code" || '.' || "admin2Code"`,
+    nullable: true,
+  })
+  admin2Key!: string;
+
+  @ManyToOne(() => GeodataAdmin2Entity, { eager: true, nullable: true, createForeignKeyConstraints: false })
+  admin2!: GeodataAdmin2Entity;
+
+  @Column({ type: 'date' })
+  modificationDate!: Date;
+}

server/src/infra/entities/index.ts

@@ -1,3 +1,4 @@
+import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity';
 import { ActivityEntity } from './activity.entity';
 import { AlbumEntity } from './album.entity';
 import { APIKeyEntity } from './api-key.entity';
@@ -6,6 +7,8 @@ import { AssetJobStatusEntity } from './asset-job-status.entity';
 import { AssetEntity } from './asset.entity';
 import { AuditEntity } from './audit.entity';
 import { ExifEntity } from './exif.entity';
+import { GeodataAdmin1Entity } from './geodata-admin1.entity';
+import { GeodataPlacesEntity } from './geodata-places.entity';
 import { LibraryEntity } from './library.entity';
 import { MoveEntity } from './move.entity';
 import { PartnerEntity } from './partner.entity';
@@ -13,6 +16,7 @@ import { PersonEntity } from './person.entity';
 import { SharedLinkEntity } from './shared-link.entity';
 import { SmartInfoEntity } from './smart-info.entity';
 import { SystemConfigEntity } from './system-config.entity';
+import { SystemMetadataEntity } from './system-metadata.entity';
 import { TagEntity } from './tag.entity';
 import { UserTokenEntity } from './user-token.entity';
 import { UserEntity } from './user.entity';
@@ -25,6 +29,9 @@ export * from './asset-job-status.entity';
 export * from './asset.entity';
 export * from './audit.entity';
 export * from './exif.entity';
+export * from './geodata-admin1.entity';
+export * from './geodata-admin2.entity';
+export * from './geodata-places.entity';
 export * from './library.entity';
 export * from './move.entity';
 export * from './partner.entity';
@@ -32,6 +39,7 @@ export * from './person.entity';
 export * from './shared-link.entity';
 export * from './smart-info.entity';
 export * from './system-config.entity';
+export * from './system-metadata.entity';
 export * from './tag.entity';
 export * from './user-token.entity';
 export * from './user.entity';
@@ -45,12 +53,16 @@ export const databaseEntities = [
   AssetJobStatusEntity,
   AuditEntity,
   ExifEntity,
+  GeodataPlacesEntity,
+  GeodataAdmin1Entity,
+  GeodataAdmin2Entity,
   MoveEntity,
   PartnerEntity,
   PersonEntity,
   SharedLinkEntity,
   SmartInfoEntity,
   SystemConfigEntity,
+  SystemMetadataEntity,
   TagEntity,
   UserEntity,
   UserTokenEntity,

server/src/infra/entities/system-config.entity.ts

@@ -66,7 +66,6 @@ export enum SystemConfigKey {
   MAP_DARK_STYLE = 'map.darkStyle',
 
   REVERSE_GEOCODING_ENABLED = 'reverseGeocoding.enabled',
-  REVERSE_GEOCODING_CITIES_FILE_OVERRIDE = 'reverseGeocoding.citiesFileOverride',
 
   NEW_VERSION_CHECK_ENABLED = 'newVersionCheck.enabled',
 
@@ -145,13 +144,6 @@ export enum Colorspace {
   P3 = 'p3',
 }
 
-export enum CitiesFile {
-  CITIES_15000 = 'cities15000',
-  CITIES_5000 = 'cities5000',
-  CITIES_1000 = 'cities1000',
-  CITIES_500 = 'cities500',
-}
-
 export interface SystemConfig {
   ffmpeg: {
     crf: number;
@@ -200,7 +192,6 @@ export interface SystemConfig {
   };
   reverseGeocoding: {
     enabled: boolean;
-    citiesFileOverride: CitiesFile;
   };
   oauth: {
     enabled: boolean;

server/src/infra/entities/system-metadata.entity.ts

@@ -0,0 +1,18 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity('system_metadata')
+export class SystemMetadataEntity {
+  @PrimaryColumn()
+  key!: string;
+
+  @Column({ type: 'jsonb', default: '{}', transformer: { to: JSON.stringify, from: JSON.parse } })
+  value!: { [key: string]: unknown };
+}
+
+export enum SystemMetadataKey {
+  REVERSE_GEOCODING_STATE = 'reverse-geocoding-state',
+}
+
+export interface SystemMetadata extends Record<SystemMetadataKey, { [key: string]: unknown }> {
+  [SystemMetadataKey.REVERSE_GEOCODING_STATE]: { lastUpdate?: string; lastImportFileName?: string };
+}

server/src/infra/infra.config.ts

@@ -74,6 +74,3 @@ function parseTypeSenseConfig(): ConfigurationOptions {
 }
 
 export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig();
-
-export const REVERSE_GEOCODING_DUMP_DIRECTORY =
-  process.env.REVERSE_GEOCODING_DUMP_DIRECTORY || process.cwd() + '/.reverse-geocoding-dump/';

server/src/infra/infra.module.ts

@@ -21,6 +21,7 @@ import {
   ISmartInfoRepository,
   IStorageRepository,
   ISystemConfigRepository,
+  ISystemMetadataRepository,
   ITagRepository,
   IUserRepository,
   IUserTokenRepository,
@@ -56,6 +57,7 @@ import {
   SharedLinkRepository,
   SmartInfoRepository,
   SystemConfigRepository,
+  SystemMetadataRepository,
   TagRepository,
   TypesenseRepository,
   UserRepository,
@@ -84,6 +86,7 @@ const providers: Provider[] = [
   { provide: ISmartInfoRepository, useClass: SmartInfoRepository },
   { provide: IStorageRepository, useClass: FilesystemProvider },
   { provide: ISystemConfigRepository, useClass: SystemConfigRepository },
+  { provide: ISystemMetadataRepository, useClass: SystemMetadataRepository },
   { provide: ITagRepository, useClass: TagRepository },
   { provide: IMediaRepository, useClass: MediaRepository },
   { provide: IUserRepository, useClass: UserRepository },

server/src/infra/migrations/1700345818045-SystemMetadata.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class SystemMetadata1700345818045 implements MigrationInterface {
+    name = 'SystemMetadata1700345818045'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TABLE "system_metadata" ("key" character varying NOT NULL, "value" jsonb NOT NULL DEFAULT '{}', CONSTRAINT "PK_fa94f6857470fb5b81ec6084465" PRIMARY KEY ("key"))`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE "system_metadata"`);
+    }
+
+}

server/src/infra/migrations/1700362016675-Geodata.ts

@@ -0,0 +1,29 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class Geodata1700362016675 implements MigrationInterface {
+    name = 'Geodata1700362016675'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS cube`)
+        await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS earthdistance`)
+        await queryRunner.query(`CREATE TABLE "geodata_admin2" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_1e3886455dbb684d6f6b4756726" PRIMARY KEY ("key"))`);
+        await queryRunner.query(`CREATE TABLE "geodata_admin1" ("key" character varying NOT NULL, "name" character varying NOT NULL, CONSTRAINT "PK_3fe3a89c5aac789d365871cb172" PRIMARY KEY ("key"))`);
+        await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin1Key","\"countryCode\" || '.' || \"admin1Code\""]);
+        await queryRunner.query(`INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") VALUES ($1, $2, $3, $4, $5, $6)`, ["immich","public","geodata_places","GENERATED_COLUMN","admin2Key","\"countryCode\" || '.' || \"admin1Code\" || '.' || \"admin2Code\""]);
+        await queryRunner.query(`CREATE TABLE "geodata_places" ("id" integer NOT NULL, "name" character varying(200) NOT NULL, "longitude" double precision NOT NULL, "latitude" double precision NOT NULL, "countryCode" character(2) NOT NULL, "admin1Code" character varying(20), "admin2Code" character varying(80), "admin1Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code") STORED, "admin2Key" character varying GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code" || '.' || "admin2Code") STORED, "modificationDate" date NOT NULL, CONSTRAINT "PK_c29918988912ef4036f3d7fbff4" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`ALTER TABLE "geodata_places" ADD "earthCoord" earth GENERATED ALWAYS AS (ll_to_earth(latitude, longitude)) STORED`)
+        await queryRunner.query(`CREATE INDEX "IDX_geodata_gist_earthcoord" ON "geodata_places" USING gist ("earthCoord");`)
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "IDX_geodata_gist_earthcoord"`);
+        await queryRunner.query(`DROP TABLE "geodata_places"`);
+        await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin2Key","immich","public","geodata_places"]);
+        await queryRunner.query(`DELETE FROM "typeorm_metadata" WHERE "type" = $1 AND "name" = $2 AND "database" = $3 AND "schema" = $4 AND "table" = $5`, ["GENERATED_COLUMN","admin1Key","immich","public","geodata_places"]);
+        await queryRunner.query(`DROP TABLE "geodata_admin1"`);
+        await queryRunner.query(`DROP TABLE "geodata_admin2"`);
+        await queryRunner.query(`DROP EXTENSION cube`);
+        await queryRunner.query(`DROP EXTENSION earthdistance`);
+    }
+
+}

server/src/infra/repositories/index.ts

@@ -19,6 +19,7 @@ export * from './server-info.repository';
 export * from './shared-link.repository';
 export * from './smart-info.repository';
 export * from './system-config.repository';
+export * from './system-metadata.repository';
 export * from './tag.repository';
 export * from './typesense.repository';
 export * from './user-token.repository';

server/src/infra/repositories/metadata.repository.ts

@@ -1,77 +1,182 @@
-import { GeoPoint, IMetadataRepository, ImmichTags, ReverseGeocodeResult } from '@app/domain';
+import {
-import { REVERSE_GEOCODING_DUMP_DIRECTORY } from '@app/infra';
+  GeoPoint,
-import { Injectable, Logger } from '@nestjs/common';
+  IMetadataRepository,
+  ImmichTags,
+  ISystemMetadataRepository,
+  ReverseGeocodeResult,
+} from '@app/domain';
+import { GeodataAdmin1Entity, GeodataAdmin2Entity, GeodataPlacesEntity, SystemMetadataKey } from '@app/infra/entities';
+import { DatabaseLock } from '@app/infra/utils/database-locks';
+import { Inject, Logger } from '@nestjs/common';
+import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
 import { DefaultReadTaskOptions, exiftool } from 'exiftool-vendored';
-import { readdir, rm } from 'fs/promises';
+import { createReadStream, existsSync } from 'fs';
+import { readFile } from 'fs/promises';
 import * as geotz from 'geo-tz';
 import { getName } from 'i18n-iso-countries';
-import geocoder, { AddressObject, InitOptions } from 'local-reverse-geocoder';
+import * as readLine from 'readline';
-import path from 'path';
+import { DataSource, DeepPartial, QueryRunner, Repository } from 'typeorm';
-import { promisify } from 'util';
 
-export interface AdminCode {
+type GeoEntity = GeodataPlacesEntity | GeodataAdmin1Entity | GeodataAdmin2Entity;
-  name: string;
+type GeoEntityClass = typeof GeodataPlacesEntity | typeof GeodataAdmin1Entity | typeof GeodataAdmin2Entity;
-  asciiName: string;
-  geoNameId: string;
-}
 
-export type GeoData = AddressObject & {
+const CITIES_FILE = 'cities500.txt';
-  admin1Code?: AdminCode | string;
-  admin2Code?: AdminCode | string;
-};
 
-const lookup = promisify<GeoPoint[], number, AddressObject[][]>(geocoder.lookUp).bind(geocoder);
-
-@Injectable()
 export class MetadataRepository implements IMetadataRepository {
+  constructor(
+    @InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository<GeodataPlacesEntity>,
+    @InjectRepository(GeodataAdmin1Entity) private readonly geodataAdmin1Repository: Repository<GeodataAdmin1Entity>,
+    @InjectRepository(GeodataAdmin2Entity) private readonly geodataAdmin2Repository: Repository<GeodataAdmin2Entity>,
+    @Inject(ISystemMetadataRepository) private readonly systemMetadataRepository: ISystemMetadataRepository,
+    @InjectDataSource() private dataSource: DataSource,
+  ) {}
+
   private logger = new Logger(MetadataRepository.name);
 
-  async init(options: Partial<InitOptions>): Promise<void> {
+  async init(): Promise<void> {
-    return new Promise<void>((resolve) => {
+    this.logger.log('Initializing metadata repository');
-      geocoder.init(
+    const geodataDate = await readFile('/usr/src/resources/geodata-date.txt', 'utf8');
-        {
+
-          load: {
+    await this.geodataPlacesRepository.query('SELECT pg_advisory_lock($1)', [DatabaseLock.GeodataImport]);
-            admin1: true,
+
-            admin2: true,
+    const geocodingMetadata = await this.systemMetadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
-            admin3And4: false,
+
-            alternateNames: false,
+    if (geocodingMetadata?.lastUpdate === geodataDate) {
-          },
+      await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
-          countries: [],
+      return;
-          dumpDirectory: REVERSE_GEOCODING_DUMP_DIRECTORY,
+    }
-          ...options,
+
-        },
+    this.logger.log('Importing geodata to database from file');
-        resolve,
+
-      );
+    const queryRunner = this.dataSource.createQueryRunner();
+    await queryRunner.connect();
+
+    try {
+      await queryRunner.startTransaction();
+
+      await this.loadCities500(queryRunner);
+      await this.loadAdmin1(queryRunner);
+      await this.loadAdmin2(queryRunner);
+
+      await queryRunner.commitTransaction();
+    } catch (e) {
+      this.logger.fatal('Error importing geodata', e);
+      await queryRunner.rollbackTransaction();
+      throw e;
+    } finally {
+      await queryRunner.release();
+    }
+
+    await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, {
+      lastUpdate: geodataDate,
+      lastImportFileName: CITIES_FILE,
     });
+
+    await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
+    this.logger.log('Geodata import completed');
+  }
+
+  private async loadGeodataToTableFromFile<T extends GeoEntity>(
+    queryRunner: QueryRunner,
+    lineToEntityMapper: (lineSplit: string[]) => T,
+    filePath: string,
+    entity: GeoEntityClass,
+  ) {
+    if (!existsSync(filePath)) {
+      this.logger.error(`Geodata file ${filePath} not found`);
+      throw new Error(`Geodata file ${filePath} not found`);
+    }
+    await queryRunner.manager.clear(entity);
+
+    const input = createReadStream(filePath);
+    let buffer: DeepPartial<T>[] = [];
+    const lineReader = readLine.createInterface({ input: input });
+
+    for await (const line of lineReader) {
+      const lineSplit = line.split('\t');
+      buffer.push(lineToEntityMapper(lineSplit));
+      if (buffer.length > 1000) {
+        await queryRunner.manager.save(buffer);
+        buffer = [];
+      }
+    }
+    await queryRunner.manager.save(buffer);
+  }
+
+  private async loadCities500(queryRunner: QueryRunner) {
+    await this.loadGeodataToTableFromFile<GeodataPlacesEntity>(
+      queryRunner,
+      (lineSplit: string[]) =>
+        this.geodataPlacesRepository.create({
+          id: parseInt(lineSplit[0]),
+          name: lineSplit[1],
+          latitude: parseFloat(lineSplit[4]),
+          longitude: parseFloat(lineSplit[5]),
+          countryCode: lineSplit[8],
+          admin1Code: lineSplit[10],
+          admin2Code: lineSplit[11],
+          modificationDate: lineSplit[18],
+        }),
+      `/usr/src/resources/${CITIES_FILE}`,
+      GeodataPlacesEntity,
+    );
+  }
+
+  private async loadAdmin1(queryRunner: QueryRunner) {
+    await this.loadGeodataToTableFromFile<GeodataAdmin1Entity>(
+      queryRunner,
+      (lineSplit: string[]) =>
+        this.geodataAdmin1Repository.create({
+          key: lineSplit[0],
+          name: lineSplit[1],
+        }),
+      '/usr/src/resources/admin1CodesASCII.txt',
+      GeodataAdmin1Entity,
+    );
+  }
+
+  private async loadAdmin2(queryRunner: QueryRunner) {
+    await this.loadGeodataToTableFromFile<GeodataAdmin2Entity>(
+      queryRunner,
+      (lineSplit: string[]) =>
+        this.geodataAdmin2Repository.create({
+          key: lineSplit[0],
+          name: lineSplit[1],
+        }),
+      '/usr/src/resources/admin2Codes.txt',
+      GeodataAdmin2Entity,
+    );
   }
 
   async teardown() {
     await exiftool.end();
   }
 
-  async deleteCache() {
+  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult | null> {
-    const dumpDirectory = REVERSE_GEOCODING_DUMP_DIRECTORY;
-    if (dumpDirectory) {
-      // delete contents
-      const items = await readdir(dumpDirectory, { withFileTypes: true });
-      const folders = items.filter((item) => item.isDirectory());
-      for (const { name } of folders) {
-        await rm(path.join(dumpDirectory, name), { recursive: true, force: true });
-      }
-    }
-  }
-
-  async reverseGeocode(point: GeoPoint): Promise<ReverseGeocodeResult> {
     this.logger.debug(`Request: ${point.latitude},${point.longitude}`);
 
-    const [address] = await lookup([point], 1);
+    const response = await this.geodataPlacesRepository
-    this.logger.verbose(`Raw: ${JSON.stringify(address, null, 2)}`);
+      .createQueryBuilder('geoplaces')
+      .leftJoinAndSelect('geoplaces.admin1', 'admin1')
+      .leftJoinAndSelect('geoplaces.admin2', 'admin2')
+      .where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point)
+      .orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")')
+      .limit(1)
+      .getOne();
 
-    const { countryCode, name: city, admin1Code, admin2Code } = address[0] as GeoData;
+    if (!response) {
+      this.logger.warn(
+        `Response from database for reverse geocoding latitude: ${point.latitude}, longitude: ${point.longitude} was null`,
+      );
+      return null;
+    }
+
+    this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`);
+
+    const { countryCode, name: city, admin1, admin2 } = response;
     const country = getName(countryCode, 'en') ?? null;
-    const stateParts = [(admin2Code as AdminCode)?.name, (admin1Code as AdminCode)?.name].filter((name) => !!name);
+    const stateParts = [admin2?.name, admin1?.name].filter((name) => !!name);
     const state = stateParts.length > 0 ? stateParts.join(', ') : null;
-    this.logger.debug(`Normalized: ${JSON.stringify({ country, state, city })}`);
 
     return { country, state, city };
   }

server/src/infra/repositories/system-metadata.repository.ts

@@ -0,0 +1,20 @@
+import { ISystemMetadataRepository } from '@app/domain/repositories/system-metadata.repository';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository } from 'typeorm';
+import { SystemMetadata, SystemMetadataEntity } from '../entities';
+
+export class SystemMetadataRepository implements ISystemMetadataRepository {
+  constructor(
+    @InjectRepository(SystemMetadataEntity)
+    private repository: Repository<SystemMetadataEntity>,
+  ) {}
+  async get<T extends keyof SystemMetadata>(key: T): Promise<SystemMetadata[T] | null> {
+    const metadata = await this.repository.findOne({ where: { key } });
+    if (!metadata) return null;
+    return metadata.value as SystemMetadata[T];
+  }
+
+  async set<T extends keyof SystemMetadata>(key: T, value: SystemMetadata[T]): Promise<void> {
+    await this.repository.upsert({ key, value }, { conflictPaths: { key: true } });
+  }
+}

server/src/infra/utils/database-locks.ts

@@ -0,0 +1,3 @@
+export enum DatabaseLock {
+  GeodataImport = 100,
+}

server/src/microservices/app.service.ts

@@ -92,16 +92,6 @@ export class AppService {
       [JobName.LIBRARY_QUEUE_CLEANUP]: () => this.libraryService.handleQueueCleanup(),
     });
 
-    process.on('uncaughtException', async (error: Error | any) => {
-      const isCsvError = error.code === 'CSV_RECORD_INCONSISTENT_FIELDS_LENGTH';
-      if (!isCsvError) {
-        throw error;
-      }
-
-      this.logger.warn('Geocoding csv parse error, trying again without cache...');
-      await this.metadataService.init(true);
-    });
-
     await this.metadataService.init();
     await this.searchService.init();
   }

server/test/repositories/metadata.repository.mock.ts

@@ -2,7 +2,6 @@ import { IMetadataRepository } from '@app/domain';
 
 export const newMetadataRepositoryMock = (): jest.Mocked<IMetadataRepository> => {
   return {
-    deleteCache: jest.fn(),
     getExifTags: jest.fn(),
     init: jest.fn(),
     teardown: jest.fn(),

server/test/test-utils.ts

@@ -25,7 +25,9 @@ export const db = {
       const tableNames =
         entities.length > 0
           ? entities.map((entity) => em.getRepository(entity).metadata.tableName)
-          : dataSource.entityMetadatas.map((entity) => entity.tableName);
+          : dataSource.entityMetadatas
+              .map((entity) => entity.tableName)
+              .filter((tableName) => !tableName.startsWith('geodata'));
 
       let deleteUsers = false;
       for (const tableName of tableNames) {

web/src/api/open-api/api.ts

@@ -1164,22 +1164,6 @@ export interface CheckExistingAssetsResponseDto {
      */
     'existingIds': Array<string>;
 }
-/**
- * 
- * @export
- * @enum {string}
- */
-
-export const CitiesFile = {
-    Cities15000: 'cities15000',
-    Cities5000: 'cities5000',
-    Cities1000: 'cities1000',
-    Cities500: 'cities500'
-} as const;
-
-export type CitiesFile = typeof CitiesFile[keyof typeof CitiesFile];
-
-
 /**
  * 
  * @export
@@ -3832,12 +3816,6 @@ export interface SystemConfigPasswordLoginDto {
  * @interface SystemConfigReverseGeocodingDto
  */
 export interface SystemConfigReverseGeocodingDto {
-    /**
-     * 
-     * @type {CitiesFile}
-     * @memberof SystemConfigReverseGeocodingDto
-     */
-    'citiesFileOverride': CitiesFile;
     /**
      * 
      * @type {boolean}
@@ -3845,8 +3823,6 @@ export interface SystemConfigReverseGeocodingDto {
      */
     'enabled': boolean;
 }
-
-
 /**
  * 
  * @export

web/src/lib/components/admin-page/settings/map-settings/map-settings.svelte

@@ -4,13 +4,12 @@
     NotificationType,
   } from '$lib/components/shared-components/notification/notification';
   import { handleError } from '$lib/utils/handle-error';
-  import { api, CitiesFile, SystemConfigDto } from '@api';
+  import { api, SystemConfigDto } from '@api';
   import { cloneDeep, isEqual } from 'lodash-es';
   import { fade } from 'svelte/transition';
   import SettingAccordion from '../setting-accordion.svelte';
   import SettingButtonsRow from '../setting-buttons-row.svelte';
   import SettingSwitch from '../setting-switch.svelte';
-  import SettingSelect from '../setting-select.svelte';
   import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
 
   export let config: SystemConfigDto; // this is the config that is being edited
@@ -39,7 +38,6 @@
           },
           reverseGeocoding: {
             enabled: config.reverseGeocoding.enabled,
-            citiesFileOverride: config.reverseGeocoding.citiesFileOverride,
           },
         },
       });
@@ -131,24 +129,6 @@
                 subtitle="Enable reverse geocoding"
                 bind:checked={config.reverseGeocoding.enabled}
               />
-
-              <hr />
-
-              <SettingSelect
-                label="Precision"
-                desc="Set reverse geocoding precision"
-                name="reverse-geocoding-precision"
-                bind:value={config.reverseGeocoding.citiesFileOverride}
-                options={[
-                  { value: CitiesFile.Cities500, text: 'Cities with more than 500 people' },
-                  { value: CitiesFile.Cities1000, text: 'Cities with more than 1000 people' },
-                  { value: CitiesFile.Cities5000, text: 'Cities with more than 5000 people' },
-                  { value: CitiesFile.Cities15000, text: 'Cities with more than 15000 people' },
-                ]}
-                disabled={disabled || !config.reverseGeocoding.enabled}
-                isEdited={config.reverseGeocoding.citiesFileOverride !==
-                  savedConfig.reverseGeocoding.citiesFileOverride}
-              />
             </div></SettingAccordion
           >