2018-02-20 00:24:10 +02:00
// Copyright 2018 Drone.IO Inc.
2018-03-10 21:09:14 +02:00
//
2018-02-20 00:24:10 +02:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
2018-03-10 21:09:14 +02:00
//
2018-02-20 00:24:10 +02:00
// http://www.apache.org/licenses/LICENSE-2.0
2018-03-10 21:09:14 +02:00
//
2018-02-20 00:24:10 +02:00
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2017-06-30 00:51:22 +02:00
package main
2017-05-03 23:25:33 +02:00
import (
2021-11-25 18:15:36 +02:00
"context"
2017-06-28 19:21:22 +02:00
"fmt"
2021-12-11 14:15:04 +02:00
"net/url"
2021-10-28 21:02:43 +02:00
"os"
2021-12-11 14:15:04 +02:00
"strings"
2017-09-20 21:29:57 +02:00
"time"
2017-06-28 19:21:22 +02:00
2021-10-12 09:25:13 +02:00
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/rs/zerolog/log"
2021-10-27 21:03:14 +02:00
"github.com/urfave/cli/v2"
2021-10-12 09:25:13 +02:00
"golang.org/x/sync/errgroup"
2021-09-22 20:48:01 +02:00
"github.com/woodpecker-ci/woodpecker/server"
2021-09-27 19:51:55 +02:00
"github.com/woodpecker-ci/woodpecker/server/model"
2021-09-23 16:12:46 +02:00
"github.com/woodpecker-ci/woodpecker/server/plugins/environments"
"github.com/woodpecker-ci/woodpecker/server/plugins/registry"
"github.com/woodpecker-ci/woodpecker/server/plugins/secrets"
2021-09-23 22:29:09 +02:00
"github.com/woodpecker-ci/woodpecker/server/queue"
2021-09-23 18:25:51 +02:00
"github.com/woodpecker-ci/woodpecker/server/remote"
"github.com/woodpecker-ci/woodpecker/server/remote/bitbucket"
"github.com/woodpecker-ci/woodpecker/server/remote/bitbucketserver"
"github.com/woodpecker-ci/woodpecker/server/remote/coding"
"github.com/woodpecker-ci/woodpecker/server/remote/gitea"
"github.com/woodpecker-ci/woodpecker/server/remote/github"
"github.com/woodpecker-ci/woodpecker/server/remote/gitlab"
"github.com/woodpecker-ci/woodpecker/server/remote/gogs"
2021-09-23 13:33:59 +02:00
"github.com/woodpecker-ci/woodpecker/server/store"
"github.com/woodpecker-ci/woodpecker/server/store/datastore"
2017-05-03 23:25:33 +02:00
)
2021-10-19 11:44:49 +02:00
func setupStore ( c * cli . Context ) ( store . Store , error ) {
2021-10-30 14:53:24 +02:00
datasource := c . String ( "datasource" )
driver := c . String ( "driver" )
2021-11-13 21:18:06 +02:00
if driver == "sqlite3" {
if datastore . SupportedDriver ( "sqlite3" ) {
log . Debug ( ) . Msgf ( "server has sqlite3 support" )
} else {
log . Debug ( ) . Msgf ( "server was build with no sqlite3 support!" )
}
}
if ! datastore . SupportedDriver ( driver ) {
log . Fatal ( ) . Msgf ( "database driver '%s' not supported" , driver )
}
if driver == "sqlite3" {
2021-10-30 14:53:24 +02:00
if new , err := fallbackSqlite3File ( datasource ) ; err != nil {
log . Fatal ( ) . Err ( err ) . Msg ( "fallback to old sqlite3 file failed" )
} else {
datasource = new
}
2021-10-28 21:02:43 +02:00
}
2021-11-13 21:18:06 +02:00
opts := & store . Opts {
2021-10-30 14:53:24 +02:00
Driver : driver ,
Config : datasource ,
2021-10-19 11:44:49 +02:00
}
2021-11-13 21:18:06 +02:00
log . Trace ( ) . Msgf ( "setup datastore: %#v" , * opts )
store , err := datastore . NewEngine ( opts )
if err != nil {
log . Fatal ( ) . Err ( err ) . Msg ( "could not open datastore" )
}
if err := store . Migrate ( ) ; err != nil {
log . Fatal ( ) . Err ( err ) . Msg ( "could not migrate datastore" )
}
return store , nil
2017-05-03 23:25:33 +02:00
}
2021-10-30 14:53:24 +02:00
// TODO: convert it to a check and fail hard only function in v0.16.0 to be able to remove it in v0.17.0
// TODO: add it to the "how to migrate from drone docs"
func fallbackSqlite3File ( path string ) ( string , error ) {
const dockerDefaultPath = "/var/lib/woodpecker/woodpecker.sqlite"
const dockerDefaultDir = "/var/lib/woodpecker/drone.sqlite"
const dockerOldPath = "/var/lib/drone/drone.sqlite"
const standaloneDefault = "woodpecker.sqlite"
const standaloneOld = "drone.sqlite"
// custom location was set, use that one
if path != dockerDefaultPath && path != standaloneDefault {
return path , nil
2021-10-28 21:02:43 +02:00
}
2021-10-30 14:53:24 +02:00
// file is at new default("/var/lib/woodpecker/woodpecker.sqlite")
_ , err := os . Stat ( dockerDefaultPath )
if err != nil && ! os . IsNotExist ( err ) {
return "" , err
}
if err == nil {
return dockerDefaultPath , nil
}
// file is at new default("woodpecker.sqlite")
_ , err = os . Stat ( standaloneDefault )
if err != nil && ! os . IsNotExist ( err ) {
return "" , err
}
if err == nil {
return standaloneDefault , nil
}
// woodpecker run in standalone mode, file is in same folder but not renamed
_ , err = os . Stat ( standaloneOld )
if err != nil && ! os . IsNotExist ( err ) {
return "" , err
}
if err == nil {
// rename in same folder should be fine as it should be same docker volume
log . Warn ( ) . Msgf ( "found sqlite3 file at '%s' and moved to '%s'" , standaloneOld , standaloneDefault )
return standaloneDefault , os . Rename ( standaloneOld , standaloneDefault )
}
// file is in new folder but not renamed
_ , err = os . Stat ( dockerDefaultDir )
if err != nil && ! os . IsNotExist ( err ) {
return "" , err
}
if err == nil {
// rename in same folder should be fine as it should be same docker volume
log . Warn ( ) . Msgf ( "found sqlite3 file at '%s' and moved to '%s'" , dockerDefaultDir , dockerDefaultPath )
return dockerDefaultPath , os . Rename ( dockerDefaultDir , dockerDefaultPath )
}
// file is still at old location
_ , err = os . Stat ( dockerOldPath )
if err == nil {
// TODO: use log.Fatal()... in next version
log . Error ( ) . Msgf ( "found sqlite3 file at deprecated path '%s', please move it to '%s' and update your volume path if necessary" , dockerOldPath , dockerDefaultPath )
return dockerOldPath , nil
2021-10-28 21:02:43 +02:00
}
2021-10-30 14:53:24 +02:00
// file does not exist at all
log . Warn ( ) . Msgf ( "no sqlite3 file found, will create one at '%s'" , path )
return path , nil
2021-10-28 21:02:43 +02:00
}
2017-05-04 02:02:08 +02:00
func setupQueue ( c * cli . Context , s store . Store ) queue . Queue {
2021-12-11 14:15:04 +02:00
return queue . WithTaskStore ( queue . New ( c . Context ) , s )
2017-05-04 02:02:08 +02:00
}
2017-05-07 18:47:06 +02:00
func setupSecretService ( c * cli . Context , s store . Store ) model . SecretService {
2021-12-11 14:15:04 +02:00
return secrets . New ( c . Context , s )
2017-05-07 18:47:06 +02:00
}
func setupRegistryService ( c * cli . Context , s store . Store ) model . RegistryService {
2020-05-19 14:44:16 +02:00
if c . String ( "docker-config" ) != "" {
return registry . Combined (
registry . New ( s ) ,
registry . Filesystem ( c . String ( "docker-config" ) ) ,
)
}
2021-12-01 15:22:06 +02:00
return registry . New ( s )
2017-05-07 18:47:06 +02:00
}
2017-06-30 00:51:22 +02:00
func setupEnvironService ( c * cli . Context , s store . Store ) model . EnvironService {
2020-05-18 17:48:31 +02:00
return environments . Filesystem ( c . StringSlice ( "environment" ) )
2017-06-30 00:51:22 +02:00
}
2021-11-26 14:01:54 +02:00
// setupRemote helper function to setup the remote from the CLI arguments.
func setupRemote ( c * cli . Context ) ( remote . Remote , error ) {
2017-06-28 19:21:22 +02:00
switch {
case c . Bool ( "github" ) :
return setupGithub ( c )
case c . Bool ( "gitlab" ) :
return setupGitlab ( c )
case c . Bool ( "bitbucket" ) :
return setupBitbucket ( c )
case c . Bool ( "stash" ) :
return setupStash ( c )
case c . Bool ( "gogs" ) :
return setupGogs ( c )
case c . Bool ( "gitea" ) :
return setupGitea ( c )
2017-07-22 11:12:09 +02:00
case c . Bool ( "coding" ) :
return setupCoding ( c )
2017-06-28 19:21:22 +02:00
default :
return nil , fmt . Errorf ( "version control system not configured" )
}
}
// helper function to setup the Bitbucket remote from the CLI arguments.
func setupBitbucket ( c * cli . Context ) ( remote . Remote , error ) {
2021-10-19 11:44:49 +02:00
opts := & bitbucket . Opts {
Client : c . String ( "bitbucket-client" ) ,
Secret : c . String ( "bitbucket-secret" ) ,
}
log . Trace ( ) . Msgf ( "Remote (bitbucket) opts: %#v" , opts )
return bitbucket . New ( opts )
2017-06-28 19:21:22 +02:00
}
// helper function to setup the Gogs remote from the CLI arguments.
func setupGogs ( c * cli . Context ) ( remote . Remote , error ) {
2021-10-19 11:44:49 +02:00
opts := gogs . Opts {
2017-06-28 19:21:22 +02:00
URL : c . String ( "gogs-server" ) ,
Username : c . String ( "gogs-git-username" ) ,
Password : c . String ( "gogs-git-password" ) ,
PrivateMode : c . Bool ( "gogs-private-mode" ) ,
SkipVerify : c . Bool ( "gogs-skip-verify" ) ,
2021-10-19 11:44:49 +02:00
}
log . Trace ( ) . Msgf ( "Remote (gogs) opts: %#v" , opts )
return gogs . New ( opts )
2017-06-28 19:21:22 +02:00
}
// helper function to setup the Gitea remote from the CLI arguments.
func setupGitea ( c * cli . Context ) ( remote . Remote , error ) {
2021-12-11 14:15:04 +02:00
server , err := url . Parse ( c . String ( "gitea-server" ) )
if err != nil {
return nil , err
}
Clean up config environment variables for server and agent (#218)
The goal here is to make consistent use of configuration environment variables prefixed `WOODPECKER_`. Where several variants existed, this PR aims to remove all but one option, leaving the most explicit.
This PR only changes server and agent code, but not documentation, in order to keep the PR digestible. Once we have consensus that this is correct, I'll change docs accordingly.
User (rather: admin) facing changes in this PR:
- In general, support for all server and agent config environment variables (env vars) starting with `DRONE_` is removed. The according `WOODPECKER_*` variables must be used instead.
- The env var `WOODPECKER_HOST` replaces `DRONE_HOST`, and `DRONE_SERVER_HOST`.
- The env var `WOODPECKER_AGENT_SECRET` is used to configure the shared secret which agents use to authenticate against the server. It replaces `WOODPECKER_SECRET`, `DRONE_SECRET`, `WOODPECKER_PASSWORD`, `DRONE_PASSWORD`, and `DRONE_AGENT_SECRET`.
- The env var `WOODPECKER_DATABASE_DRIVER` replaces `DRONE_DATABASE_DRIVER` and `DATABASE_DRIVER`.
- The env var `WOODPECKER_DATABASE_DATASOURCE` replaces `DRONE_DATABASE_DATASOURCE` and `DATABASE_CONFIG`.
2021-09-28 15:43:44 +02:00
opts := gitea . Opts {
2022-01-31 16:38:00 +02:00
URL : strings . TrimRight ( server . String ( ) , "/" ) ,
Client : c . String ( "gitea-client" ) ,
Secret : c . String ( "gitea-secret" ) ,
SkipVerify : c . Bool ( "gitea-skip-verify" ) ,
Clean up config environment variables for server and agent (#218)
The goal here is to make consistent use of configuration environment variables prefixed `WOODPECKER_`. Where several variants existed, this PR aims to remove all but one option, leaving the most explicit.
This PR only changes server and agent code, but not documentation, in order to keep the PR digestible. Once we have consensus that this is correct, I'll change docs accordingly.
User (rather: admin) facing changes in this PR:
- In general, support for all server and agent config environment variables (env vars) starting with `DRONE_` is removed. The according `WOODPECKER_*` variables must be used instead.
- The env var `WOODPECKER_HOST` replaces `DRONE_HOST`, and `DRONE_SERVER_HOST`.
- The env var `WOODPECKER_AGENT_SECRET` is used to configure the shared secret which agents use to authenticate against the server. It replaces `WOODPECKER_SECRET`, `DRONE_SECRET`, `WOODPECKER_PASSWORD`, `DRONE_PASSWORD`, and `DRONE_AGENT_SECRET`.
- The env var `WOODPECKER_DATABASE_DRIVER` replaces `DRONE_DATABASE_DRIVER` and `DATABASE_DRIVER`.
- The env var `WOODPECKER_DATABASE_DATASOURCE` replaces `DRONE_DATABASE_DATASOURCE` and `DATABASE_CONFIG`.
2021-09-28 15:43:44 +02:00
}
if len ( opts . URL ) == 0 {
2021-10-12 09:25:13 +02:00
log . Fatal ( ) . Msg ( "WOODPECKER_GITEA_URL must be set" )
Clean up config environment variables for server and agent (#218)
The goal here is to make consistent use of configuration environment variables prefixed `WOODPECKER_`. Where several variants existed, this PR aims to remove all but one option, leaving the most explicit.
This PR only changes server and agent code, but not documentation, in order to keep the PR digestible. Once we have consensus that this is correct, I'll change docs accordingly.
User (rather: admin) facing changes in this PR:
- In general, support for all server and agent config environment variables (env vars) starting with `DRONE_` is removed. The according `WOODPECKER_*` variables must be used instead.
- The env var `WOODPECKER_HOST` replaces `DRONE_HOST`, and `DRONE_SERVER_HOST`.
- The env var `WOODPECKER_AGENT_SECRET` is used to configure the shared secret which agents use to authenticate against the server. It replaces `WOODPECKER_SECRET`, `DRONE_SECRET`, `WOODPECKER_PASSWORD`, `DRONE_PASSWORD`, and `DRONE_AGENT_SECRET`.
- The env var `WOODPECKER_DATABASE_DRIVER` replaces `DRONE_DATABASE_DRIVER` and `DATABASE_DRIVER`.
- The env var `WOODPECKER_DATABASE_DATASOURCE` replaces `DRONE_DATABASE_DATASOURCE` and `DATABASE_CONFIG`.
2021-09-28 15:43:44 +02:00
}
2021-10-19 11:44:49 +02:00
log . Trace ( ) . Msgf ( "Remote (gitea) opts: %#v" , opts )
Clean up config environment variables for server and agent (#218)
The goal here is to make consistent use of configuration environment variables prefixed `WOODPECKER_`. Where several variants existed, this PR aims to remove all but one option, leaving the most explicit.
This PR only changes server and agent code, but not documentation, in order to keep the PR digestible. Once we have consensus that this is correct, I'll change docs accordingly.
User (rather: admin) facing changes in this PR:
- In general, support for all server and agent config environment variables (env vars) starting with `DRONE_` is removed. The according `WOODPECKER_*` variables must be used instead.
- The env var `WOODPECKER_HOST` replaces `DRONE_HOST`, and `DRONE_SERVER_HOST`.
- The env var `WOODPECKER_AGENT_SECRET` is used to configure the shared secret which agents use to authenticate against the server. It replaces `WOODPECKER_SECRET`, `DRONE_SECRET`, `WOODPECKER_PASSWORD`, `DRONE_PASSWORD`, and `DRONE_AGENT_SECRET`.
- The env var `WOODPECKER_DATABASE_DRIVER` replaces `DRONE_DATABASE_DRIVER` and `DATABASE_DRIVER`.
- The env var `WOODPECKER_DATABASE_DATASOURCE` replaces `DRONE_DATABASE_DATASOURCE` and `DATABASE_CONFIG`.
2021-09-28 15:43:44 +02:00
return gitea . New ( opts )
2017-06-28 19:21:22 +02:00
}
// helper function to setup the Stash remote from the CLI arguments.
func setupStash ( c * cli . Context ) ( remote . Remote , error ) {
2021-10-19 11:44:49 +02:00
opts := bitbucketserver . Opts {
2017-06-28 19:21:22 +02:00
URL : c . String ( "stash-server" ) ,
Username : c . String ( "stash-git-username" ) ,
Password : c . String ( "stash-git-password" ) ,
ConsumerKey : c . String ( "stash-consumer-key" ) ,
ConsumerRSA : c . String ( "stash-consumer-rsa" ) ,
ConsumerRSAString : c . String ( "stash-consumer-rsa-string" ) ,
SkipVerify : c . Bool ( "stash-skip-verify" ) ,
2021-10-19 11:44:49 +02:00
}
log . Trace ( ) . Msgf ( "Remote (bitbucketserver) opts: %#v" , opts )
return bitbucketserver . New ( opts )
2017-06-28 19:21:22 +02:00
}
// helper function to setup the Gitlab remote from the CLI arguments.
func setupGitlab ( c * cli . Context ) ( remote . Remote , error ) {
return gitlab . New ( gitlab . Opts {
2021-10-03 14:42:47 +02:00
URL : c . String ( "gitlab-server" ) ,
ClientID : c . String ( "gitlab-client" ) ,
ClientSecret : c . String ( "gitlab-secret" ) ,
SkipVerify : c . Bool ( "gitlab-skip-verify" ) ,
2017-06-28 19:21:22 +02:00
} )
}
// helper function to setup the GitHub remote from the CLI arguments.
func setupGithub ( c * cli . Context ) ( remote . Remote , error ) {
2021-10-19 11:44:49 +02:00
opts := github . Opts {
2022-01-31 16:38:00 +02:00
URL : c . String ( "github-server" ) ,
Client : c . String ( "github-client" ) ,
Secret : c . String ( "github-secret" ) ,
SkipVerify : c . Bool ( "github-skip-verify" ) ,
MergeRef : c . Bool ( "github-merge-ref" ) ,
2021-10-19 11:44:49 +02:00
}
log . Trace ( ) . Msgf ( "Remote (github) opts: %#v" , opts )
return github . New ( opts )
2017-06-28 19:21:22 +02:00
}
2017-06-30 00:51:22 +02:00
2017-07-22 11:12:09 +02:00
// helper function to setup the Coding remote from the CLI arguments.
func setupCoding ( c * cli . Context ) ( remote . Remote , error ) {
2021-10-19 11:44:49 +02:00
opts := coding . Opts {
2017-07-22 11:12:09 +02:00
URL : c . String ( "coding-server" ) ,
Client : c . String ( "coding-client" ) ,
Secret : c . String ( "coding-secret" ) ,
Scopes : c . StringSlice ( "coding-scope" ) ,
Machine : c . String ( "coding-git-machine" ) ,
Username : c . String ( "coding-git-username" ) ,
Password : c . String ( "coding-git-password" ) ,
SkipVerify : c . Bool ( "coding-skip-verify" ) ,
2021-10-19 11:44:49 +02:00
}
log . Trace ( ) . Msgf ( "Remote (coding) opts: %#v" , opts )
return coding . New ( opts )
2017-07-22 11:12:09 +02:00
}
2021-12-01 15:22:06 +02:00
func setupMetrics ( g * errgroup . Group , _store store . Store ) {
2019-05-30 11:11:14 +02:00
pendingJobs := promauto . NewGauge ( prometheus . GaugeOpts {
2021-10-13 08:34:57 +02:00
Namespace : "woodpecker" ,
2019-05-30 11:11:14 +02:00
Name : "pending_jobs" ,
Help : "Total number of pending build processes." ,
} )
2019-07-10 10:00:38 +02:00
waitingJobs := promauto . NewGauge ( prometheus . GaugeOpts {
2021-10-13 08:34:57 +02:00
Namespace : "woodpecker" ,
2019-07-10 10:00:38 +02:00
Name : "waiting_jobs" ,
Help : "Total number of builds waiting on deps." ,
} )
2019-05-30 11:11:14 +02:00
runningJobs := promauto . NewGauge ( prometheus . GaugeOpts {
2021-10-13 08:34:57 +02:00
Namespace : "woodpecker" ,
2019-05-30 11:11:14 +02:00
Name : "running_jobs" ,
Help : "Total number of running build processes." ,
} )
workers := promauto . NewGauge ( prometheus . GaugeOpts {
2021-10-13 08:34:57 +02:00
Namespace : "woodpecker" ,
2019-05-30 11:11:14 +02:00
Name : "worker_count" ,
Help : "Total number of workers." ,
} )
builds := promauto . NewGauge ( prometheus . GaugeOpts {
2021-10-13 08:34:57 +02:00
Namespace : "woodpecker" ,
2019-06-28 14:23:52 +02:00
Name : "build_total_count" ,
2019-05-30 11:11:14 +02:00
Help : "Total number of builds." ,
} )
users := promauto . NewGauge ( prometheus . GaugeOpts {
2021-10-13 08:34:57 +02:00
Namespace : "woodpecker" ,
2019-05-30 11:11:14 +02:00
Name : "user_count" ,
Help : "Total number of users." ,
} )
repos := promauto . NewGauge ( prometheus . GaugeOpts {
2021-10-13 08:34:57 +02:00
Namespace : "woodpecker" ,
2019-05-30 11:11:14 +02:00
Name : "repo_count" ,
Help : "Total number of repos." ,
} )
g . Go ( func ( ) error {
for {
2021-11-25 18:15:36 +02:00
stats := server . Config . Services . Queue . Info ( context . TODO ( ) )
2019-05-30 11:11:14 +02:00
pendingJobs . Set ( float64 ( stats . Stats . Pending ) )
2019-07-10 10:00:38 +02:00
waitingJobs . Set ( float64 ( stats . Stats . WaitingOnDeps ) )
2019-05-30 11:11:14 +02:00
runningJobs . Set ( float64 ( stats . Stats . Running ) )
workers . Set ( float64 ( stats . Stats . Workers ) )
time . Sleep ( 500 * time . Millisecond )
}
} )
g . Go ( func ( ) error {
for {
2021-12-01 15:22:06 +02:00
repoCount , _ := _store . GetRepoCount ( )
userCount , _ := _store . GetUserCount ( )
buildCount , _ := _store . GetBuildCount ( )
2019-05-30 13:57:52 +02:00
builds . Set ( float64 ( buildCount ) )
2019-05-30 11:11:14 +02:00
users . Set ( float64 ( userCount ) )
repos . Set ( float64 ( repoCount ) )
time . Sleep ( 10 * time . Second )
}
} )
}