diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 89d79628..add44dc6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,7 +66,7 @@ jobs: build-args: | COMMIT_SHA=${{ steps.info.outputs.sha }} labels: | - org.opencontainers.image.title=${{ github.event.repository.name }} + org.opencontainers.image.title=SFTPGo org.opencontainers.image.description=Fully featured and highly configurable SFTP server with optional FTP/S and WebDAV support org.opencontainers.image.url=${{ github.event.repository.html_url }} org.opencontainers.image.source=${{ github.event.repository.clone_url }} diff --git a/README.md b/README.md index f21f79f3..5104ad9f 100644 --- a/README.md +++ b/README.md @@ -90,17 +90,17 @@ sftpgo serve Check out [this documentation](./docs/service.md) if you want to run SFTPGo as a service. -### Data provider initialization +### Data provider initialization and update -Before starting the SFTPGo server, please ensure that the configured data provider is properly initialized. +Before starting the SFTPGo server, please ensure that the configured data provider is properly initialized/updated. -SQL based data providers (SQLite, MySQL, PostgreSQL) require the creation of a database containing the required tables. Memory and bolt data providers do not require an initialization. +SQL based data providers (SQLite, MySQL, PostgreSQL) require the creation of a database containing the required tables. Memory and bolt data providers do not require an initialization but they could require an update to the existing data after upgrading SFTPGo. For PostgreSQL and MySQL providers, you need to create the configured database. -SFTPGo will attempt to automatically detect if the data privider has been initialized and if not, initialize it on startup. +SFTPGo will attempt to automatically detect if the data provider is initialized/updated and if not, will attempt to initialize/ update it on startup as needed. -Alternately, you can create the required data provider structure yourself using the `initprovider` command. +Alternately, you can create/update the required data provider structures yourself using the `initprovider` command. For example, you can simply execute the following command from the configuration directory: @@ -114,7 +114,7 @@ Take a look at the CLI usage to learn how to specify a different configuration f sftpgo initprovider --help ``` -After the first initialization (manual or automatic), the database structure will be automatically checked and updated, if required, at startup. +You can also disable automatic data provider checks at startup setting the `update_mode` configuration key to `1`. ## Tutorials diff --git a/cmd/initprovider.go b/cmd/initprovider.go index 14c313c4..9e15d06b 100644 --- a/cmd/initprovider.go +++ b/cmd/initprovider.go @@ -16,18 +16,20 @@ import ( var ( initProviderCmd = &cobra.Command{ Use: "initprovider", - Short: "Initializes the configured data provider", + Short: "Initializes and/or updates the configured data provider", Long: `This command reads the data provider connection details from the specified -configuration file and creates the initial structure. +configuration file and creates the initial structure or update the existing one, +as needed. -Some data providers such as bolt and memory does not require an initialization. +Some data providers such as bolt and memory does not require an initialization +but they could require an update to the existing data after upgrading SFTPGo. -For SQLite provider the database file will be auto created if missing. +For SQLite/bolt providers the database file will be auto-created if missing. For PostgreSQL and MySQL providers you need to create the configured database, -this command will create the required tables. +this command will create/update the required tables as needed. -To initialize the data provider from the configuration directory simply use: +To initialize/update the data provider from the configuration directory simply use: $ sftpgo initprovider @@ -42,14 +44,14 @@ Please take a look at the usage below to customize the options.`, return } providerConf := config.GetProviderConf() - logger.DebugToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed()) + logger.InfoToConsole("Initializing provider: %#v config file: %#v", providerConf.Driver, viper.ConfigFileUsed()) err = dataprovider.InitializeDatabase(providerConf, configDir) if err == nil { - logger.DebugToConsole("Data provider successfully initialized") + logger.InfoToConsole("Data provider successfully initialized/updated") } else if err == dataprovider.ErrNoInitRequired { - logger.DebugToConsole("%v", err.Error()) + logger.InfoToConsole("%v", err.Error()) } else { - logger.WarnToConsole("Unable to initialize data provider: %v", err) + logger.WarnToConsole("Unable to initialize/update the data provider: %v", err) os.Exit(1) } }, diff --git a/config/config.go b/config/config.go index f45d8570..505ddc7d 100644 --- a/config/config.go +++ b/config/config.go @@ -141,6 +141,7 @@ func init() { Parallelism: 2, }, }, + UpdateMode: 0, }, HTTPDConfig: httpd.Conf{ BindPort: 8080, diff --git a/dataprovider/bolt.go b/dataprovider/bolt.go index 3a3796ff..55bcd47e 100644 --- a/dataprovider/bolt.go +++ b/dataprovider/bolt.go @@ -712,8 +712,8 @@ func (p BoltProvider) migrateDatabase() error { return err } if dbVersion.Version == boltDatabaseVersion { - providerLog(logger.LevelDebug, "bolt database is updated, current version: %v", dbVersion.Version) - return nil + providerLog(logger.LevelDebug, "bolt database is up to date, current version: %v", dbVersion.Version) + return ErrNoInitRequired } switch dbVersion.Version { case 1: @@ -866,6 +866,7 @@ func getFolderBucket(tx *bolt.Tx) (*bolt.Bucket, error) { } func updateDatabaseFrom1To2(dbHandle *bolt.DB) error { + logger.InfoToConsole("updating bolt database version: 1 -> 2") providerLog(logger.LevelInfo, "updating bolt database version: 1 -> 2") usernames, err := getBoltAvailableUsernames(dbHandle) if err != nil { @@ -887,6 +888,7 @@ func updateDatabaseFrom1To2(dbHandle *bolt.DB) error { } func updateDatabaseFrom2To3(dbHandle *bolt.DB) error { + logger.InfoToConsole("updating bolt database version: 2 -> 3") providerLog(logger.LevelInfo, "updating bolt database version: 2 -> 3") users := []User{} err := dbHandle.View(func(tx *bolt.Tx) error { @@ -941,6 +943,7 @@ func updateDatabaseFrom2To3(dbHandle *bolt.DB) error { } func updateDatabaseFrom3To4(dbHandle *bolt.DB) error { + logger.InfoToConsole("updating bolt database version: 3 -> 4") providerLog(logger.LevelInfo, "updating bolt database version: 3 -> 4") foldersToScan := []string{} users := []userCompactVFolders{} diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 89a97a58..5dd80433 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -99,8 +99,8 @@ var ( ErrNoAuthTryed = errors.New("no auth tryed") // ValidProtocols defines all the valid protcols ValidProtocols = []string{"SSH", "FTP", "DAV"} - // ErrNoInitRequired defines the error returned by InitProvider if no inizialization is required - ErrNoInitRequired = errors.New("Data provider initialization is not required") + // ErrNoInitRequired defines the error returned by InitProvider if no inizialization/update is required + ErrNoInitRequired = errors.New("The data provider is already up to date") // ErrInvalidCredentials defines the error to return if the supplied credentials are invalid ErrInvalidCredentials = errors.New("Invalid credentials") webDAVUsersCache sync.Map @@ -251,6 +251,10 @@ type Config struct { CheckPasswordScope int `json:"check_password_scope" mapstructure:"check_password_scope"` // PasswordHashing defines the configuration for password hashing PasswordHashing PasswordHashing `json:"password_hashing" mapstructure:"password_hashing"` + // Defines how the database will be initialized/updated: + // - 0 means automatically + // - 1 means manually using the initprovider sub-command + UpdateMode int `json:"update_mode" mapstructure:"update_mode"` } // BackupData defines the structure for the backup/restore files @@ -383,19 +387,23 @@ func Initialize(cnf Config, basePath string) error { if err != nil { return err } - err = provider.initializeDatabase() - if err != nil && err != ErrNoInitRequired { - logger.WarnToConsole("Unable to initialize data provider: %v", err) - providerLog(logger.LevelWarn, "Unable to initialize data provider: %v", err) - return err - } - if err == nil { - logger.DebugToConsole("Data provider successfully initialized") - } - err = provider.migrateDatabase() - if err != nil { - providerLog(logger.LevelWarn, "database migration error: %v", err) - return err + if cnf.UpdateMode == 0 { + err = provider.initializeDatabase() + if err != nil && err != ErrNoInitRequired { + logger.WarnToConsole("Unable to initialize data provider: %v", err) + providerLog(logger.LevelWarn, "Unable to initialize data provider: %v", err) + return err + } + if err == nil { + logger.DebugToConsole("Data provider successfully initialized") + } + err = provider.migrateDatabase() + if err != nil && err != ErrNoInitRequired { + providerLog(logger.LevelWarn, "database migration error: %v", err) + return err + } + } else { + providerLog(logger.LevelInfo, "database initialization/migration skipped, manual mode is configured") } argon2Params = &argon2id.Params{ Memory: cnf.PasswordHashing.Argon2Options.Memory, @@ -458,14 +466,15 @@ func validateSQLTablesPrefix() error { func InitializeDatabase(cnf Config, basePath string) error { config = cnf - if config.Driver == BoltDataProviderName || config.Driver == MemoryDataProviderName { - return ErrNoInitRequired - } err := createProvider(basePath) if err != nil { return err } - return provider.initializeDatabase() + err = provider.initializeDatabase() + if err != nil && err != ErrNoInitRequired { + return err + } + return provider.migrateDatabase() } // CheckUserAndPass retrieves the SFTP user with the given username and password if a match is found or an error diff --git a/dataprovider/memory.go b/dataprovider/memory.go index ba396b41..1ad36a91 100644 --- a/dataprovider/memory.go +++ b/dataprovider/memory.go @@ -671,5 +671,5 @@ func (p MemoryProvider) initializeDatabase() error { } func (p MemoryProvider) migrateDatabase() error { - return nil + return ErrNoInitRequired } diff --git a/dataprovider/mysql.go b/dataprovider/mysql.go index bbd652a0..c5f11d3c 100644 --- a/dataprovider/mysql.go +++ b/dataprovider/mysql.go @@ -205,8 +205,8 @@ func (p MySQLProvider) migrateDatabase() error { return err } if dbVersion.Version == sqlDatabaseVersion { - providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version) - return nil + providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version) + return ErrNoInitRequired } switch dbVersion.Version { case 1: @@ -233,12 +233,14 @@ func (p MySQLProvider) migrateDatabase() error { } func updateMySQLDatabaseFrom1To2(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") sql := strings.Replace(mysqlV2SQL, "{{users}}", sqlTableUsers, 1) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2) } func updateMySQLDatabaseFrom2To3(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 2 -> 3") providerLog(logger.LevelInfo, "updating database version: 2 -> 3") sql := strings.Replace(mysqlV3SQL, "{{users}}", sqlTableUsers, 1) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3) diff --git a/dataprovider/pgsql.go b/dataprovider/pgsql.go index 7088783e..dd9b055d 100644 --- a/dataprovider/pgsql.go +++ b/dataprovider/pgsql.go @@ -204,8 +204,8 @@ func (p PGSQLProvider) migrateDatabase() error { return err } if dbVersion.Version == sqlDatabaseVersion { - providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version) - return nil + providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version) + return ErrNoInitRequired } switch dbVersion.Version { case 1: @@ -232,12 +232,14 @@ func (p PGSQLProvider) migrateDatabase() error { } func updatePGSQLDatabaseFrom1To2(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") sql := strings.Replace(pgsqlV2SQL, "{{users}}", sqlTableUsers, 1) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2) } func updatePGSQLDatabaseFrom2To3(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 2 -> 3") providerLog(logger.LevelInfo, "updating database version: 2 -> 3") sql := strings.Replace(pgsqlV3SQL, "{{users}}", sqlTableUsers, 1) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3) diff --git a/dataprovider/sqlcommon.go b/dataprovider/sqlcommon.go index 4dd64085..cca11bed 100644 --- a/dataprovider/sqlcommon.go +++ b/dataprovider/sqlcommon.go @@ -899,6 +899,7 @@ func sqlCommonRestoreCompatVirtualFolders(ctx context.Context, users []userCompa } func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 3 -> 4") providerLog(logger.LevelInfo, "updating database version: 3 -> 4") users, err := sqlCommonGetCompatVirtualFolders(dbHandle) if err != nil { diff --git a/dataprovider/sqlite.go b/dataprovider/sqlite.go index 7f3f485f..d96f5aca 100644 --- a/dataprovider/sqlite.go +++ b/dataprovider/sqlite.go @@ -227,8 +227,8 @@ func (p SQLiteProvider) migrateDatabase() error { return err } if dbVersion.Version == sqlDatabaseVersion { - providerLog(logger.LevelDebug, "sql database is updated, current version: %v", dbVersion.Version) - return nil + providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", dbVersion.Version) + return ErrNoInitRequired } switch dbVersion.Version { case 1: @@ -255,12 +255,14 @@ func (p SQLiteProvider) migrateDatabase() error { } func updateSQLiteDatabaseFrom1To2(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") sql := strings.Replace(sqliteV2SQL, "{{users}}", sqlTableUsers, 1) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 2) } func updateSQLiteDatabaseFrom2To3(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 2 -> 3") providerLog(logger.LevelInfo, "updating database version: 2 -> 3") sql := strings.ReplaceAll(sqliteV3SQL, "{{users}}", sqlTableUsers) return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 3) diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 4dfe1ade..d6c05ac7 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -11,7 +11,7 @@ Usage: Available Commands: gen A collection of useful generators help Help about any command - initprovider Initializes the configured data provider + initprovider Initializes and/or updates the configured data provider portable Serve a single directory serve Start the SFTP Server @@ -143,6 +143,7 @@ The configuration file contains the following sections: - `memory`, unsigned integer. The amount of memory used by the algorithm (in kibibytes). Default: 65536. - `iterations`, unsigned integer. The number of iterations over the memory. Default: 1. - `parallelism`. unsigned 8 bit integer. The number of threads (or lanes) used by the algorithm. Default: 2. + - `update_mode`, integer. Defines how the database will be initialized/updated. 0 means automatically. 1 means manually using the initprovider sub-command. - **"httpd"**, the configuration for the HTTP server used to serve REST API and to expose the built-in web interface - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080 - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1" diff --git a/sftpgo.json b/sftpgo.json index c9906a8e..9c8e0317 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -99,7 +99,8 @@ "iterations": 1, "parallelism": 2 } - } + }, + "update_mode": 0 }, "httpd": { "bind_port": 8080,