diff --git a/cmd/migrate.go b/cmd/migrate.go index 1d095266..c7e4f259 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -27,7 +27,7 @@ Supported arguments are: - up - runs all available migrations. - down [number] - reverts the last [number] applied migrations. - create name [folder] - creates new migration template file. -- collections [folder] - creates new migration file with the current collections configuration. +- collections [folder] - (Experimental) creates new migration file with the most recent local collections configuration. ` var databaseFlag string diff --git a/daos/collection.go b/daos/collection.go index d50625bd..8ca72840 100644 --- a/daos/collection.go +++ b/daos/collection.go @@ -190,7 +190,7 @@ func (dao *Dao) ImportCollections( mappedImported := make(map[string]*models.Collection, len(importedCollections)) for _, imported := range importedCollections { - // normalize + // normalize ids if !imported.HasId() { // generate id if not set imported.MarkAsNew() @@ -199,6 +199,15 @@ func (dao *Dao) ImportCollections( imported.MarkAsNew() } + // extend existing schema + if existing, ok := mappedExisting[imported.GetId()]; ok && !deleteMissing { + schema, _ := existing.Schema.Clone() + for _, f := range imported.Schema.Fields() { + schema.AddField(f) // add or replace + } + imported.Schema = *schema + } + mappedImported[imported.GetId()] = imported } diff --git a/forms/collection_upsert.go b/forms/collection_upsert.go index 54098f71..cf9f8df1 100644 --- a/forms/collection_upsert.go +++ b/forms/collection_upsert.go @@ -1,6 +1,7 @@ package forms import ( + "fmt" "regexp" "strings" @@ -115,6 +116,7 @@ func (form *CollectionUpsert) Validate() error { &form.Schema, validation.By(form.ensureNoSystemFieldsChange), validation.By(form.ensureNoFieldsTypeChange), + validation.By(form.ensureExistingRelationCollectionId), ), validation.Field(&form.ListRule, validation.By(form.checkRule)), validation.Field(&form.ViewRule, validation.By(form.checkRule)), @@ -161,11 +163,38 @@ func (form *CollectionUpsert) ensureNoSystemFlagChange(value any) error { func (form *CollectionUpsert) ensureNoFieldsTypeChange(value any) error { v, _ := value.(schema.Schema) - for _, field := range v.Fields() { + for i, field := range v.Fields() { oldField := form.collection.Schema.GetFieldById(field.Id) if oldField != nil && oldField.Type != field.Type { - return validation.NewError("validation_field_type_change", "Field type cannot be changed.") + return validation.Errors{fmt.Sprint(i): validation.NewError( + "validation_field_type_change", + "Field type cannot be changed.", + )} + } + } + + return nil +} + +func (form *CollectionUpsert) ensureExistingRelationCollectionId(value any) error { + v, _ := value.(schema.Schema) + + for i, field := range v.Fields() { + if field.Type != schema.FieldTypeRelation { + continue + } + + options, _ := field.Options.(*schema.RelationOptions) + if options == nil { + continue + } + + if _, err := form.config.TxDao.FindCollectionByNameOrId(options.CollectionId); err != nil { + return validation.Errors{fmt.Sprint(i): validation.NewError( + "validation_field_invalid_relation", + "The relation collection doesn't exist.", + )} } } diff --git a/forms/collections_import.go b/forms/collections_import.go index 3c31b8c5..70314c73 100644 --- a/forms/collections_import.go +++ b/forms/collections_import.go @@ -139,11 +139,11 @@ func (form *CollectionsImport) beforeRecordsSync(txDao *daos.Dao, mappedNew, map if err := upsertForm.Validate(); err != nil { // serialize the validation error(s) - serializedErr, _ := json.Marshal(err) + serializedErr, _ := json.MarshalIndent(err, "", " ") return validation.Errors{"collections": validation.NewError( "collections_import_validate_failure", - fmt.Sprintf("Data validations failed for collection %q (%s): %s", collection.Name, collection.Id, serializedErr), + fmt.Sprintf("Data validations failed for collection %q (%s):\n%s", collection.Name, collection.Id, serializedErr), )} } } diff --git a/ui/src/components/base/Field.svelte b/ui/src/components/base/Field.svelte index 5bd06e12..22a701d0 100644 --- a/ui/src/components/base/Field.svelte +++ b/ui/src/components/base/Field.svelte @@ -39,7 +39,7 @@ {#each fieldErrors as error}
{#if typeof error === "object"} - {error?.message || error?.code || defaultError} + 
{error?.message || error?.code || defaultError}
{:else} {error || defaultError} {/if} diff --git a/ui/src/components/base/Toasts.svelte b/ui/src/components/base/Toasts.svelte index baf4f3c3..d6be1db6 100644 --- a/ui/src/components/base/Toasts.svelte +++ b/ui/src/components/base/Toasts.svelte @@ -20,6 +20,8 @@ {:else if toast.type === "success"} + {:else if toast.type === "warning"} + {:else} {/if} diff --git a/ui/src/components/collections/CollectionsDiffTable.svelte b/ui/src/components/collections/CollectionsDiffTable.svelte new file mode 100644 index 00000000..43cec496 --- /dev/null +++ b/ui/src/components/collections/CollectionsDiffTable.svelte @@ -0,0 +1,264 @@ + + +
+ {#if !collectionA?.id} + {collectionB?.name} + Added + {:else if !collectionB?.id} + {collectionA?.name} + Removed + {:else} +
+ {#if collectionA.name !== collectionB.name} + {collectionA.name} + + {/if} + {collectionB.name} + {#if hasAnyChange} + Changed + {/if} +
+ {/if} +
+ + + + + + + + + + + + {#each mainModelProps as prop} + + + + + + {/each} + + {#if deleteMissing || isDeleteDiff} + {#each removedFields as field} + + + + + {#each Object.entries(field) as [key, value]} + + + + + {/each} + {/each} + {/if} + + {#each sharedFields as field} + + + + + {#each Object.entries(field) as [key, newValue]} + + + + + + {/each} + {/each} + + {#each addedFields as field} + + + + + {#each Object.entries(field) as [key, value]} + + + + + {/each} + {/each} + +
PropsOldNew
+ {prop} + + 
{displayValue(collectionA?.[prop])}
+ 		+ 
{displayValue(collectionB?.[prop])}
+
+ schema.{field.name} + + Removed - + All stored data related to {field.name} will be deleted! + + +
{key} + 
{displayValue(value)}
+ 		+
+ schema.{field.name} + {#if hasChanges(getFieldById(schemaA, field.id), getFieldById(schemaB, field.id))} + Changed + {/if} +
{key} + 
{displayValue(getFieldById(schemaA, field.id)?.[key])}
+ 		+ 
{displayValue(newValue)}
+
+ schema.{field.name} + Added +
{key} + + 
{displayValue(value)}
+
+ + diff --git a/ui/src/components/settings/ImportPopup.svelte b/ui/src/components/settings/ImportPopup.svelte index 74fff8e4..0c5e0c1f 100644 --- a/ui/src/components/settings/ImportPopup.svelte +++ b/ui/src/components/settings/ImportPopup.svelte @@ -2,29 +2,28 @@ import { createEventDispatcher } from "svelte"; import ApiClient from "@/utils/ApiClient"; import CommonHelper from "@/utils/CommonHelper"; - import OverlayPanel from "@/components/base/OverlayPanel.svelte"; import { addSuccessToast } from "@/stores/toasts"; import { confirm } from "@/stores/confirmation"; + import OverlayPanel from "@/components/base/OverlayPanel.svelte"; + import CollectionsDiffTable from "@/components/collections/CollectionsDiffTable.svelte"; const dispatch = createEventDispatcher(); let panel; let oldCollections = []; let newCollections = []; - let changes = []; + let pairs = []; + let deleteMissing = false; let isImporting = false; $: if (Array.isArray(oldCollections) && Array.isArray(newCollections)) { - loadChanges(); + loadPairs(); } - $: deletedCollections = oldCollections.filter((old) => { - return !CommonHelper.findByKey(newCollections, "id", old.id)?.id; - }); - - export function show(a, b) { - oldCollections = a; - newCollections = b; + export function show(oldCollectionsArg, newCollectionsArg, deleteMissingArg = false) { + oldCollections = oldCollectionsArg; + newCollections = newCollectionsArg; + deleteMissing = deleteMissingArg; panel?.show(); } @@ -33,25 +32,23 @@ return panel?.hide(); } - function loadChanges() { - changes = []; + function loadPairs() { + pairs = []; // add deleted and modified collections for (const oldCollection of oldCollections) { const newCollection = CommonHelper.findByKey(newCollections, "id", oldCollection.id) || null; - if (!newCollection?.id || JSON.stringify(oldCollection) != JSON.stringify(newCollection)) { - changes.push({ - old: oldCollection, - new: newCollection, - }); - } + pairs.push({ + old: oldCollection, + new: newCollection, + }); } // add only new collections for (const newCollection of newCollections) { const oldCollection = CommonHelper.findByKey(oldCollections, "id", newCollection.id) || null; if (!oldCollection?.id) { - changes.push({ + pairs.push({ old: oldCollection, new: newCollection, }); @@ -59,63 +56,32 @@ } } - function diffsToHtml(diffs, ops = [window.DIFF_INSERT, window.DIFF_DELETE, window.DIFF_EQUAL]) { - const html = []; - const pattern_amp = /&/g; - const pattern_lt = //g; - const pattern_para = /\n/g; - - for (let i = 0; i < diffs.length; i++) { - const op = diffs[i][0]; // operation (insert, delete, equal) - - if (!ops.includes(op)) { - continue; - } - - const text = diffs[i][1] - .replace(pattern_amp, "&") - .replace(pattern_lt, "<") - .replace(pattern_gt, ">") - .replace(pattern_para, "
"); - - switch (op) { - case DIFF_INSERT: - html[i] = '' + text + ""; - break; - case DIFF_DELETE: - html[i] = '' + text + ""; - break; - case DIFF_EQUAL: - html[i] = text; - break; + function submitWithConfirm() { + // find deleted fields + const deletedFieldNames = []; + if (deleteMissing) { + for (const old of oldCollections) { + const imported = !CommonHelper.findByKey(newCollections, "id", old.id); + if (!imported) { + // add all fields + deletedFieldNames.push(old.name + ".*"); + } else { + // add only deleted fields + const schema = Array.isArray(old.schema) ? old.schema : []; + for (const field of schema) { + if (!CommonHelper.findByKey(imported.schema, "id", field.id)) { + deletedFieldNames.push(old.name + "." + field.name); + } + } + } } } - return html.join(""); - } - - function diff(obj1, obj2, ops = [window.DIFF_INSERT, window.DIFF_DELETE, window.DIFF_EQUAL]) { - const dmp = new diff_match_patch(); - const lines = dmp.diff_linesToChars_( - obj1 ? JSON.stringify(obj1, null, 4) : "", - obj2 ? JSON.stringify(obj2, null, 4) : "" - ); - const diffs = dmp.diff_main(lines.chars1, lines.chars2, false); - - dmp.diff_charsToLines_(diffs, lines.lineArray); - - return diffsToHtml(diffs, ops); - } - - function submitWithConfirm() { - if (deletedCollections.length) { - const deletedNames = deletedCollections.map((c) => c.name); - + if (deletedFieldNames.length) { confirm( - `Do you really want to delete the following collections and their related records data:\n- ${deletedNames.join( + `Do you really want to delete the following collection fields and their related records data:\n- ${deletedFieldNames.join( "\n- " - )}?`, + )}`, () => { submit(); } @@ -133,8 +99,8 @@ isImporting = true; try { - await ApiClient.collections.import(newCollections, true); - addSuccessToast("Successfully imported the collections configuration."); + await ApiClient.collections.import(newCollections, deleteMissing); + addSuccessToast("Successfully imported collections configuration."); dispatch("submit"); } catch (err) { ApiClient.errorResponseHandler(err); @@ -148,7 +114,7 @@ !isImporting} @@ -160,40 +126,9 @@

Side-by-side diff

-
- {#each changes as pair (pair.old?.id + pair.new?.id)} -
-
- {#if !pair.old?.id} - New - {pair.new?.name} - {:else if !pair.new?.id} - Deleted - {pair.old?.name} - {:else} - Modified -
- {#if pair.old.name !== pair.new.name} - {pair.old.name} - - {/if} - {pair.new.name} -
- {/if} -
-
-
- - {@html diff(pair.old, pair.new, [window.DIFF_DELETE, window.DIFF_EQUAL]) || "N/A"} - -
-
- - {@html diff(pair.old, pair.new, [window.DIFF_INSERT, window.DIFF_EQUAL]) || "N/A"} - -
- {/each} -
+ {#each pairs as pair} + + {/each} @@ -208,10 +143,3 @@ - - diff --git a/ui/src/components/settings/PageExportCollections.svelte b/ui/src/components/settings/PageExportCollections.svelte index 9accfcca..fd599566 100644 --- a/ui/src/components/settings/PageExportCollections.svelte +++ b/ui/src/components/settings/PageExportCollections.svelte @@ -15,7 +15,7 @@ let collections = []; let isLoadingCollections = false; - $: schema = JSON.stringify(collections, null, 2); + $: schema = JSON.stringify(collections, null, 4); loadCollections(); diff --git a/ui/src/components/settings/PageImportCollections.svelte b/ui/src/components/settings/PageImportCollections.svelte index dc7bfafa..9557fa78 100644 --- a/ui/src/components/settings/PageImportCollections.svelte +++ b/ui/src/components/settings/PageImportCollections.svelte @@ -19,7 +19,8 @@ let isLoadingFile = false; let newCollections = []; let oldCollections = []; - let collectionsToModify = []; + let deleteMissing = false; + let collectionsToChange = []; let isLoadingOldCollections = false; $: if (typeof schemas !== "undefined") { @@ -44,10 +45,32 @@ } $: hasChanges = - !!schemas && (collectionsToDelete.length || collectionsToAdd.length || collectionsToModify.length); + !!schemas && (collectionsToDelete.length || collectionsToAdd.length || collectionsToChange.length); $: canImport = !isLoadingOldCollections && isValid && hasChanges; + $: idReplacableCollections = newCollections.filter((collection) => { + const old = CommonHelper.findByKey(oldCollections, "name", collection.name); + if (!old?.id) { + return false; + } + + if (old.id != collection.id) { + return true; + } + + const oldSchema = Array.isArray(old.schema) ? old.schema : []; + const newSchema = Array.isArray(collection.schema) ? collection.schema : []; + for (const field of newSchema) { + const oldField = CommonHelper.findByKey(oldSchema, "name", field.name); + if (oldField && field.id != oldField.id) { + return true; + } + } + + return false; + }); + loadOldCollections(); async function loadOldCollections() { @@ -68,7 +91,7 @@ } function loadCollectionsToModify() { - collectionsToModify = []; + collectionsToChange = []; if (!isValid) { return; @@ -85,7 +108,7 @@ continue; } - collectionsToModify.push({ + collectionsToChange.push({ new: newCollection, old: oldCollection, }); @@ -105,13 +128,52 @@ newCollections = CommonHelper.filterDuplicatesByKey(newCollections); } - // delete timestamps + // normalizations for (let collection of newCollections) { + // delete timestamps delete collection.created; delete collection.updated; + + // merge fields with duplicated ids + collection.schema = CommonHelper.filterDuplicatesByKey(collection.schema); } } + function replaceIds() { + for (let collection of newCollections) { + const old = CommonHelper.findByKey(oldCollections, "name", collection.name); + if (!old?.id) { + continue; + } + + const originalId = collection.id; + const replacedId = old.id; + collection.id = replacedId; + + // replace field ids + const oldSchema = Array.isArray(old.schema) ? old.schema : []; + const newSchema = Array.isArray(collection.schema) ? collection.schema : []; + for (const field of newSchema) { + const oldField = CommonHelper.findByKey(oldSchema, "name", field.name); + field.id = oldField.id; + } + + // update references + for (let ref of newCollections) { + if (!Array.isArray(ref.schema)) { + continue; + } + for (let field of ref.schema) { + if (field.options?.collectionId === originalId) { + field.options.collectionId = replacedId; + } + } + } + } + + schemas = JSON.stringify(newCollections, null, 4); + } + function loadFile(file) { isLoadingFile = true; @@ -207,6 +269,14 @@ {/if} + + + + + {#if isValid && newCollections.length && !hasChanges}
@@ -234,10 +304,10 @@ {/each} {/if} - {#if collectionsToModify.length} - {#each collectionsToModify as pair (pair.old.id + pair.new.id)} + {#if collectionsToChange.length} + {#each collectionsToChange as pair (pair.old.id + pair.new.id)}
- Modified + Changed {#if pair.old.name !== pair.new.name} {pair.old.name} - @@ -254,7 +324,7 @@ {#if collectionsToAdd.length} {#each collectionsToAdd as collection (collection.id)}
- New + Added {collection.name} {#if collection.id} ({collection.id}) @@ -265,6 +335,26 @@
{/if} + {#if idReplacableCollections.length} +
+
+ +
+
+ + Some of the imported collections shares the same name but has different IDs. + +
+ +
+ {/if} +
{#if !!schemas} diff --git a/ui/src/components/settings/SettingsSidebar.svelte b/ui/src/components/settings/SettingsSidebar.svelte index bfe49fca..998fc33f 100644 --- a/ui/src/components/settings/SettingsSidebar.svelte +++ b/ui/src/components/settings/SettingsSidebar.svelte @@ -29,7 +29,10 @@ Files storage -
Sync
+
+ Sync + Experimental +
= 0; i--) { if (arr[i] == value) { return true; @@ -126,6 +128,8 @@ export default class CommonHelper { * @param {Mixed} value */ static removeByValue(arr, value) { + arr = Array.isArray(arr) ? arr : []; + for (let i = arr.length - 1; i >= 0; i--) { if (arr[i] == value) { arr.splice(i, 1); @@ -155,6 +159,8 @@ export default class CommonHelper { * @return {Object} */ static findByKey(objectsArr, key, value) { + objectsArr = Array.isArray(objectsArr) ? objectsArr : []; + for (let i in objectsArr) { if (objectsArr[i][key] == value) { return objectsArr[i]; @@ -172,7 +178,9 @@ export default class CommonHelper { * @return {Object} */ static groupByKey(objectsArr, key) { - let result = {}; + objectsArr = Array.isArray(objectsArr) ? objectsArr : []; + + const result = {}; for (let i in objectsArr) { result[objectsArr[i][key]] = result[objectsArr[i][key]] || []; @@ -226,7 +234,10 @@ export default class CommonHelper { * @return {Array} */ static filterDuplicatesByKey(objectsArr, key = "id") { + objectsArr = Array.isArray(objectsArr) ? objectsArr : []; + const uniqueMap = {}; + for (const item of objectsArr) { uniqueMap[item[key]] = item; }