awesome/dockerfiles
1
0
Fork 0
mirror of https://github.com/vimagick/dockerfiles.git synced 2025-02-03 13:21:49 +02:00
Code Issues Releases Activity

Merge branch 'master' into update-mantis

Browse Source
This commit is contained in:
Kevin He 2019-10-27 07:42:03 +08:00 committed by GitHub
commit 23bb7b9e7d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
381 changed files with 19425 additions and 2034 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
README.md
View File

@ -21,7 +21,6 @@ A collection of delicious docker recipes.
- [ ] caddy
- [ ] dsniff
- [ ] ettercap
- [ ] freegeoip
- [ ] freelan
- [ ] gitbook
- [ ] gitolite
@ -32,8 +31,6 @@ A collection of delicious docker recipes.
- [ ] libreswan
- [ ] mitmproxy
- [ ] nagios
- [ ] nfs
- [ ] openldap
- [ ] openswan
- [ ] postfix
- [ ] pritunl
@ -46,8 +43,12 @@ A collection of delicious docker recipes.
## Big Data
- [x] airflow
- [x] ambari
- [x] kafka-arm
- [x] kafka-manager
- [x] prestodb (official)
- [x] prestosql (community)
- [x] superset-arm
- [x] zookeeper-arm
@ -70,11 +71,13 @@ A collection of delicious docker recipes.
## Daemon
- [x] alpine-arm :+1:
- [x] apacheds
- [x] aria2 :+1:
- [x] audiowaveform
- [x] cadvisor
- [x] casperjs :+1:
- [x] collectd
- [x] freegeoip
- [x] freeradius
- [x] frp :cn:
- [x] graphite
@ -91,10 +94,12 @@ A collection of delicious docker recipes.
- [x] motion-arm :+1:
- [x] nginx
- [x] nifi
- [x] ntopng
- [x] nullmailer
- [x] nullmailer-arm
- [x] openhab
- [x] openssh
- [x] ot-frontend-arm
- [x] ot-recorder
- [x] ot-recorder-arm
- [x] piknik
@ -161,7 +166,6 @@ A collection of delicious docker recipes.
- [x] json-server
- [x] mantisbt
- [x] mediagoblin
- [x] netdata
- [x] nginad
- [x] nodebb :+1:
- [x] openrefine
@ -180,6 +184,7 @@ A collection of delicious docker recipes.
## Security
- [x] aircrack-ng-arm
- [x] bro
- [x] clamav
- [x] dsniff
@ -187,6 +192,7 @@ A collection of delicious docker recipes.
- [x] grr
- [x] hydra
- [x] iptables
- [x] kismet
- [x] routersploit
- [x] snort :beetle:
- [x] sslsplit
@ -200,7 +206,9 @@ A collection of delicious docker recipes.
- [x] fteproxy-arm :+1:
- [x] hans
- [x] haproxy-arm
- [x] i2pd :ru:
- [x] kcptun :cn:
- [x] mtproxy
- [x] mysql-proxy
- [x] ngrok :+1:
- [x] obfsproxy
@ -223,6 +231,7 @@ A collection of delicious docker recipes.
## VPN
- [x] n2n :+1:
- [x] ocserv :+1:
- [x] openconnect
- [x] openconnect-arm
@ -234,6 +243,7 @@ A collection of delicious docker recipes.
- [x] strongswan :+1:
- [x] tinc :+1:
- [x] tinc-arm :+1:
- [x] wiregurad :beetle:
- [x] xl2tpd
## DNS
@ -248,50 +258,75 @@ A collection of delicious docker recipes.
## 3rd-party
- [x] docker.bintray.io/jfrog/artifactory-oss
- [x] tutum/builder
- [x] browserless/chrome
- [x] certbot
- [x] codercom/code-server
- [x] confluentinc/cp-kafka-mqtt
- [x] confluentinc/cp-kafka-rest
- [x] streamsets/datacollector
- [x] cachethq/docker
- [x] puckel/docker-airflow
- [x] drone/drone
- [x] drupal
- [x] elastalert
- [x] elk
- [x] flogo/flogo-docker
- [x] mher/flower
- [x] ghost
- [x] gitlab/gitlab-ce
- [x] atlassianlabs/gostatsd
- [x] gitea/gitea :cn:
- [x] gliderlabs/logspout
- [x] gliderlabs/registrator
- [ ] glot
- [ ] bash
- [ ] python
- [x] gogs :cn:
- [x] gogs/gogs :cn:
- [x] haproxy
- [x] jmxtrans/jmxtrans
- [x] wurstmeister/kafka
- [x] netdata/netdata
- [x] nextcloud
- [x] sonatype/nexus3
- [x] jazzdd/phpvirtualbox
- [x] jenkins
- [x] sonatype/nexus3
- [x] jupyter/notebook
- [x] kylemanna/openvpn
- [x] metabase/metabase
- [x] metasploitframework/metasploit-framework :skull:
- [x] minio/minio
- [x] mongo
- [x] neo4j
- [x] erichough/nfs-server
- [x] odoo
- [x] osixia/openldap
- [x] campbellsoftwaresolutions/osticket
- [x] owncloud
- [x] phpmyadmin
- [x] pihole/pihole
- [x] portainer/portainer :+1:
- [x] postgres
- [x] postgrest/postgrest
- [x] registry
- [x] rocket.chat
- [x] scrapinghub/splash
- [x] rundeck/rundeck
- [x] wonderfall/searx
- [ ] selenium
- [ ] hub
- [ ] node-firefox
- [x] standalone-firefox
- [x] sentry
- [x] scrapinghub/splash
- [x] amancevice/superset
- [x] v2ray/official :cn:
- [x] centurylink/watchtower
- [x] anapsix/webdis
- [x] wekanteam/wekan
- [x] yourls
- [x] zookeeper
- [x] elkozmon/zoonavigator
## auto-completion

9
aircrack-ng-arm/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
#
# Dockerfile for aircrack-ng-arm
#
FROM easypi/alpine-arm
RUN apk add --no-cache aircrack-ng bash coreutils tmux
ENTRYPOINT ["sleep", "inf"]

13
aircrack-ng-arm/README.md Normal file
View File

@ -0,0 +1,13 @@
aircrack-ng
===========
```bash
$ docker-compose up -d
$ docker-compose exec aircrack bash
>>> airmon-ng
>>> airmon-ng start wlan1
>>> ifconfig
>>> airodump-ng wlan1mon
>>> airmon-ng stop wlan1mon
>>> exit
```

7
aircrack-ng-arm/docker-compose.yml Normal file
View File

@ -0,0 +1,7 @@
aircrack:
image: easypi/aircrack-ng-arm
cap_add:
- NET_ADMIN
net: host
tty: true
restart: unless-stopped

43
airflow/Dockerfile Normal file
View File

@ -0,0 +1,43 @@
#
# Dockerfile for airflow
#
FROM python:3.7-alpine
ENV AIRFLOW_VERSION=1.10.5
ENV AIRFLOW_EXTRAS=async,all_dbs,celery,crypto,devel_hadoop,jdbc,ldap,password,redis,s3,samba,slack,ssh,statsd
ENV AIRFLOW_HOME=/opt/airflow
ENV AIRFLOW_CONFIG=/opt/airflow/airflow.cfg
RUN set -xe \
&& apk add --no-cache \
build-base \
cyrus-sasl-dev \
freetds \
freetds-dev \
krb5-dev \
libffi-dev \
mariadb-dev \
postgresql-dev \
python3-dev \
&& pip install cython numpy psycopg2-binary \
&& pip install apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION} \
&& pip install "websocket-client>=0.35,<0.55.0" \
&& apk del \
build-base \
cyrus-sasl-dev \
freetds-dev \
krb5-dev \
libffi-dev \
mariadb-dev \
postgresql-dev \
python3-dev \
&& rm -rf /root/.cache/pip
WORKDIR ${AIRFLOW_HOME}
VOLUME ${AIRFLOW_HOME}
EXPOSE 8080
ENTRYPOINT ["airflow"]
CMD ["--help"]

36
airflow/README.md
View File

@ -1,6 +1,8 @@
airflow
=======
![](https://airflow.apache.org/_images/pin_large.png)
## How It Works
```
@ -20,13 +22,31 @@ airflow
## Quick Start
```bash
# On Master
$ docker-compose up -d
$ chmod 777 data/airflow/dags
$ docker-compose exec webserver cp -r /usr/local/lib/python3.6/site-packages/airflow/example_dags dags
$ docker stack deploy -c docker-stack.yaml airflow
$ docker service update --replicas-max-per-node=1 airflow_worker
$ docker service update --replicas 3 airflow_worker
# On Workers
$ docker-compose up -d
$ chmod 777 data/airflow/dags
$ docker-compose exec worker cp -r /usr/local/lib/python3.6/site-packages/airflow/example_dags dags
$ docker stack services airflow
$ docker service ps airflow_webserver
$ docker exec -it airflow_webserver.1.xxxxxx sh
>>> airflow create_user -r Admin -u admin -e admin@borderxlab.com -f Super -l Admin -p secret
>>> airflow list_users
╒══════╤════════════╤══════════════════════╤══════════════╤═════════════╤═════════╕
│ Id │ Username │ Email │ First name │ Last name │ Roles │
╞══════╪════════════╪══════════════════════╪══════════════╪═════════════╪═════════╡
│ 1 │ admin │ admin@borderxlab.com │ Super │ Admin │ [Admin] │
╘══════╧════════════╧══════════════════════╧══════════════╧═════════════╧═════════╛
>>> exit
$ curl http://localhost:8080/
$ curl http://localhost:5555/
```
> :warning: You need to prepare nfs server with `airflow.cfg`.
```
$ python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'
CD2wL7G0zt1SLuO4JQpLJuHtBaBEcXWKbQyvkvf2cZ8=
```
> :warning: You should set another value to `fernet_key` in `airflow.cfg` to improve security.

829
airflow/data/airflow.cfg Normal file
View File

@ -0,0 +1,829 @@
[core]
# The folder where your airflow pipelines live, most likely a
# subfolder in a code repository
# This path must be absolute
dags_folder = /opt/airflow/dags
# The folder where airflow should store its log files
# This path must be absolute
base_log_folder = /opt/airflow/logs
# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search.
# Users must supply an Airflow connection id that provides access to the storage
# location. If remote_logging is set to true, see UPDATING.md for additional
# configuration requirements.
remote_logging = False
remote_log_conn_id =
remote_base_log_folder =
encrypt_s3_logs = False
# Logging level
logging_level = INFO
fab_logging_level = WARN
# Logging class
# Specify the class that will specify the logging configuration
# This class has to be on the python classpath
# logging_config_class = my.path.default_local_settings.LOGGING_CONFIG
logging_config_class =
# Log format
# Colour the logs when the controlling terminal is a TTY.
colored_console_log = True
colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s
colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter
log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s
simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
# Log filename format
log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log
log_processor_filename_template = {{ filename }}.log
dag_processor_manager_log_location = /opt/airflow/logs/dag_processor_manager/dag_processor_manager.log
# Hostname by providing a path to a callable, which will resolve the hostname
# The format is "package:function". For example,
# default value "socket:getfqdn" means that result from getfqdn() of "socket" package will be used as hostname
# No argument should be required in the function specified.
# If using IP address as hostname is preferred, use value "airflow.utils.net:get_host_ip_address"
hostname_callable = socket:getfqdn
# Default timezone in case supplied date times are naive
# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam)
default_timezone = utc
# The executor class that airflow should use. Choices include
# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor
executor = CeleryExecutor
# The SqlAlchemy connection string to the metadata database.
# SqlAlchemy supports many different database engine, more information
# their website
sql_alchemy_conn = postgresql+psycopg2://airflow:airflow@postges:5432/airflow
# The encoding for the databases
sql_engine_encoding = utf-8
# If SqlAlchemy should pool database connections.
sql_alchemy_pool_enabled = True
# The SqlAlchemy pool size is the maximum number of database connections
# in the pool. 0 indicates no limit.
sql_alchemy_pool_size = 5
# The maximum overflow size of the pool.
# When the number of checked-out connections reaches the size set in pool_size,
# additional connections will be returned up to this limit.
# When those additional connections are returned to the pool, they are disconnected and discarded.
# It follows then that the total number of simultaneous connections the pool will allow is pool_size + max_overflow,
# and the total number of "sleeping" connections the pool will allow is pool_size.
# max_overflow can be set to -1 to indicate no overflow limit;
# no limit will be placed on the total number of concurrent connections. Defaults to 10.
sql_alchemy_max_overflow = 10
# The SqlAlchemy pool recycle is the number of seconds a connection
# can be idle in the pool before it is invalidated. This config does
# not apply to sqlite. If the number of DB connections is ever exceeded,
# a lower config value will allow the system to recover faster.
sql_alchemy_pool_recycle = 1800
# How many seconds to retry re-establishing a DB connection after
# disconnects. Setting this to 0 disables retries.
sql_alchemy_reconnect_timeout = 300
# The schema to use for the metadata database
# SqlAlchemy supports databases with the concept of multiple schemas.
sql_alchemy_schema =
# The amount of parallelism as a setting to the executor. This defines
# the max number of task instances that should run simultaneously
# on this airflow installation
parallelism = 32
# The number of task instances allowed to run concurrently by the scheduler
dag_concurrency = 16
# Are DAGs paused by default at creation
dags_are_paused_at_creation = True
# The maximum number of active DAG runs per DAG
max_active_runs_per_dag = 16
# Whether to load the examples that ship with Airflow. It's good to
# get started, but you probably want to set this to False in a production
# environment
load_examples = False
# Where your Airflow plugins are stored
plugins_folder = /opt/airflow/plugins
# Secret key to save connection passwords in the db
fernet_key = CD2wL7G0zt1SLuO4JQpLJuHtBaBEcXWKbQyvkvf2cZ8=
# Whether to disable pickling dags
donot_pickle = False
# How long before timing out a python file import while filling the DagBag
dagbag_import_timeout = 30
# The class to use for running task instances in a subprocess
task_runner = StandardTaskRunner
# If set, tasks without a `run_as_user` argument will be run with this user
# Can be used to de-elevate a sudo user running Airflow when executing tasks
default_impersonation =
# What security module to use (for example kerberos):
security =
# If set to False enables some unsecure features like Charts and Ad Hoc Queries.
# In 2.0 will default to True.
secure_mode = False
# Turn unit test mode on (overwrites many configuration options with test
# values at runtime)
unit_test_mode = False
# Name of handler to read task instance logs.
# Default to use task handler.
task_log_reader = task
# Whether to enable pickling for xcom (note that this is insecure and allows for
# RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False).
enable_xcom_pickling = True
# When a task is killed forcefully, this is the amount of time in seconds that
# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED
killed_task_cleanup_time = 60
# Whether to override params with dag_run.conf. If you pass some key-value pairs through `airflow backfill -c` or
# `airflow trigger_dag -c`, the key-value pairs will override the existing ones in params.
dag_run_conf_overrides_params = False
# Worker initialisation check to validate Metadata Database connection
worker_precheck = False
# When discovering DAGs, ignore any files that don't contain the strings `DAG` and `airflow`.
dag_discovery_safe_mode = True
[cli]
# In what way should the cli access the API. The LocalClient will use the
# database directly, while the json_client will use the api running on the
# webserver
api_client = airflow.api.client.local_client
# If you set web_server_url_prefix, do NOT forget to append it here, ex:
# endpoint_url = http://localhost:8080/myroot
# So api will look like: http://localhost:8080/myroot/api/experimental/...
endpoint_url = http://localhost:8080
[api]
# How to authenticate users of the API
auth_backend = airflow.api.auth.backend.default
[lineage]
# what lineage backend to use
backend =
[atlas]
sasl_enabled = False
host =
port = 21000
username =
password =
[operators]
# The default owner assigned to each new operator, unless
# provided explicitly or passed via `default_args`
default_owner = airflow
default_cpus = 1
default_ram = 512
default_disk = 512
default_gpus = 0
[hive]
# Default mapreduce queue for HiveOperator tasks
default_hive_mapred_queue =
[webserver]
# The base url of your website as airflow cannot guess what domain or
# cname you are using. This is used in automated emails that
# airflow sends to point links to the right web server
base_url = http://localhost:8080
# The ip specified when starting the web server
web_server_host = 0.0.0.0
# The port on which to run the web server
web_server_port = 8080
# Paths to the SSL certificate and key for the web server. When both are
# provided SSL will be enabled. This does not change the web server port.
web_server_ssl_cert =
web_server_ssl_key =
# Number of seconds the webserver waits before killing gunicorn master that doesn't respond
web_server_master_timeout = 120
# Number of seconds the gunicorn webserver waits before timing out on a worker
web_server_worker_timeout = 120
# Number of workers to refresh at a time. When set to 0, worker refresh is
# disabled. When nonzero, airflow periodically refreshes webserver workers by
# bringing up new ones and killing old ones.
worker_refresh_batch_size = 1
# Number of seconds to wait before refreshing a batch of workers.
worker_refresh_interval = 30
# Secret key used to run your flask app
secret_key = temporary_key
# Number of workers to run the Gunicorn web server
workers = 4
# The worker class gunicorn should use. Choices include
# sync (default), eventlet, gevent
worker_class = sync
# Log files for the gunicorn webserver. '-' means log to stderr.
access_logfile = -
error_logfile = -
# Expose the configuration file in the web server
# This is only applicable for the flask-admin based web UI (non FAB-based).
# In the FAB-based web UI with RBAC feature,
# access to configuration is controlled by role permissions.
expose_config = False
# Set to true to turn on authentication:
# https://airflow.apache.org/security.html#web-authentication
authenticate = False
# Filter the list of dags by owner name (requires authentication to be enabled)
filter_by_owner = False
# Filtering mode. Choices include user (default) and ldapgroup.
# Ldap group filtering requires using the ldap backend
#
# Note that the ldap server needs the "memberOf" overlay to be set up
# in order to user the ldapgroup mode.
owner_mode = user
# Default DAG view. Valid values are:
# tree, graph, duration, gantt, landing_times
dag_default_view = tree
# Default DAG orientation. Valid values are:
# LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top)
dag_orientation = LR
# Puts the webserver in demonstration mode; blurs the names of Operators for
# privacy.
demo_mode = False
# The amount of time (in secs) webserver will wait for initial handshake
# while fetching logs from other worker machine
log_fetch_timeout_sec = 5
# By default, the webserver shows paused DAGs. Flip this to hide paused
# DAGs by default
hide_paused_dags_by_default = False
# Consistent page size across all listing views in the UI
page_size = 100
# Use FAB-based webserver with RBAC feature
rbac = True
# Define the color of navigation bar
navbar_color = #007A87
# Default dagrun to show in UI
default_dag_run_display_number = 25
# Enable werkzeug `ProxyFix` middleware
enable_proxy_fix = False
# Set secure flag on session cookie
cookie_secure = False
# Set samesite policy on session cookie
cookie_samesite =
# Default setting for wrap toggle on DAG code and TI log views.
default_wrap = False
# Send anonymous user activity to your analytics tool
# analytics_tool = # choose from google_analytics, segment, or metarouter
# analytics_id = XXXXXXXXXXX
[email]
email_backend = airflow.utils.email.send_email_smtp
[smtp]
# If you want airflow to send emails on retries, failure, and you want to use
# the airflow.utils.email.send_email_smtp function, you have to configure an
# smtp server here
smtp_host = localhost
smtp_starttls = True
smtp_ssl = False
# Uncomment and set the user/pass settings if you want to use SMTP AUTH
# smtp_user = airflow
# smtp_password = airflow
smtp_port = 25
smtp_mail_from = airflow@example.com
[celery]
# This section only applies if you are using the CeleryExecutor in
# [core] section above
# The app name that will be used by celery
celery_app_name = airflow.executors.celery_executor
# The concurrency that will be used when starting workers with the
# "airflow worker" command. This defines the number of task instances that
# a worker will take, so size up your workers based on the resources on
# your worker box and the nature of your tasks
worker_concurrency = 16
# The maximum and minimum concurrency that will be used when starting workers with the
# "airflow worker" command (always keep minimum processes, but grow to maximum if necessary).
# Note the value should be "max_concurrency,min_concurrency"
# Pick these numbers based on resources on worker box and the nature of the task.
# If autoscale option is available, worker_concurrency will be ignored.
# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale
# worker_autoscale = 16,12
# When you start an airflow worker, airflow starts a tiny web server
# subprocess to serve the workers local log files to the airflow main
# web server, who then builds pages and sends them to users. This defines
# the port on which the logs are served. It needs to be unused, and open
# visible from the main web server to connect into the workers.
worker_log_server_port = 8793
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
# a sqlalchemy database. Refer to the Celery documentation for more
# information.
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings
broker_url = redis://redis:6379/1
# The Celery result_backend. When a job finishes, it needs to update the
# metadata of the job. Therefore it will post a message on a message bus,
# or insert it into a database (depending of the backend)
# This status is used by the scheduler to update the state of the task
# The use of a database is highly recommended
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings
result_backend = db+postgresql://airflow:airflow@postges/airflow
# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
# it `airflow flower`. This defines the IP that Celery Flower runs on
flower_host = 0.0.0.0
# The root URL for Flower
# Ex: flower_url_prefix = /flower
flower_url_prefix =
# This defines the port that Celery Flower runs on
flower_port = 5555
# Securing Flower with Basic Authentication
# Accepts user:password pairs separated by a comma
# Example: flower_basic_auth = user1:password1,user2:password2
flower_basic_auth =
# Default queue that tasks get assigned to and that worker listen on.
default_queue = default
# How many processes CeleryExecutor uses to sync task state.
# 0 means to use max(1, number of cores - 1) processes.
sync_parallelism = 0
# Import path for celery configuration options
celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
# In case of using SSL
ssl_active = False
ssl_key =
ssl_cert =
ssl_cacert =
# Celery Pool implementation.
# Choices include: prefork (default), eventlet, gevent or solo.
# See:
# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency
# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html
pool = prefork
[celery_broker_transport_options]
# This section is for specifying options which can be passed to the
# underlying celery broker transport. See:
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options
# The visibility timeout defines the number of seconds to wait for the worker
# to acknowledge the task before the message is redelivered to another worker.
# Make sure to increase the visibility timeout to match the time of the longest
# ETA you're planning to use.
#
# visibility_timeout is only supported for Redis and SQS celery brokers.
# See:
# http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options
#
#visibility_timeout = 21600
[dask]
# This section only applies if you are using the DaskExecutor in
# [core] section above
# The IP address and port of the Dask cluster's scheduler.
cluster_address = 127.0.0.1:8786
# TLS/ SSL settings to access a secured Dask scheduler.
tls_ca =
tls_cert =
tls_key =
[scheduler]
# Task instances listen for external kill signal (when you clear tasks
# from the CLI or the UI), this defines the frequency at which they should
# listen (in seconds).
job_heartbeat_sec = 5
# The scheduler constantly tries to trigger new tasks (look at the
# scheduler section in the docs for more information). This defines
# how often the scheduler should run (in seconds).
scheduler_heartbeat_sec = 5
# after how much time should the scheduler terminate in seconds
# -1 indicates to run continuously (see also num_runs)
run_duration = -1
# after how much time (seconds) a new DAGs should be picked up from the filesystem
min_file_process_interval = 0
# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes.
dag_dir_list_interval = 300
# How often should stats be printed to the logs
print_stats_interval = 30
# If the last scheduler heartbeat happened more than scheduler_health_check_threshold ago (in seconds),
# scheduler is considered unhealthy.
# This is used by the health check in the "/health" endpoint
scheduler_health_check_threshold = 30
child_process_log_directory = /opt/airflow/logs/scheduler
# Local task jobs periodically heartbeat to the DB. If the job has
# not heartbeat in this many seconds, the scheduler will mark the
# associated task instance as failed and will re-schedule the task.
scheduler_zombie_task_threshold = 300
# Turn off scheduler catchup by setting this to False.
# Default behavior is unchanged and
# Command Line Backfills still work, but the scheduler
# will not do scheduler catchup if this is False,
# however it can be set on a per DAG basis in the
# DAG definition (catchup)
catchup_by_default = True
# This changes the batch size of queries in the scheduling main loop.
# If this is too high, SQL query performance may be impacted by one
# or more of the following:
# - reversion to full table scan
# - complexity of query predicate
# - excessive locking
#
# Additionally, you may hit the maximum allowable query length for your db.
#
# Set this to 0 for no limit (not advised)
max_tis_per_query = 512
# Statsd (https://github.com/etsy/statsd) integration settings
statsd_on = False
statsd_host = localhost
statsd_port = 8125
statsd_prefix = airflow
# The scheduler can run multiple threads in parallel to schedule dags.
# This defines how many threads will run.
max_threads = 2
authenticate = False
# Turn off scheduler use of cron intervals by setting this to False.
# DAGs submitted manually in the web UI or with trigger_dag will still run.
use_job_schedule = True
[ldap]
# set this to ldaps://<your.ldap.server>:<port>
uri =
user_filter = objectClass=*
user_name_attr = uid
group_member_attr = memberOf
superuser_filter =
data_profiler_filter =
bind_user = cn=Manager,dc=example,dc=com
bind_password = insecure
basedn = dc=example,dc=com
cacert = /etc/ca/ldap_ca.crt
search_scope = LEVEL
# This setting allows the use of LDAP servers that either return a
# broken schema, or do not return a schema.
ignore_malformed_schema = False
[mesos]
# Mesos master address which MesosExecutor will connect to.
master = localhost:5050
# The framework name which Airflow scheduler will register itself as on mesos
framework_name = Airflow
# Number of cpu cores required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_cpu = 1
# Memory in MB required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_memory = 256
# Enable framework checkpointing for mesos
# See http://mesos.apache.org/documentation/latest/slave-recovery/
checkpoint = False
# Failover timeout in milliseconds.
# When checkpointing is enabled and this option is set, Mesos waits
# until the configured timeout for
# the MesosExecutor framework to re-register after a failover. Mesos
# shuts down running tasks if the
# MesosExecutor framework fails to re-register within this timeframe.
# failover_timeout = 604800
# Enable framework authentication for mesos
# See http://mesos.apache.org/documentation/latest/configuration/
authenticate = False
# Mesos credentials, if authentication is enabled
# default_principal = admin
# default_secret = admin
# Optional Docker Image to run on slave before running the command
# This image should be accessible from mesos slave i.e mesos slave
# should be able to pull this docker image before executing the command.
# docker_image_slave = puckel/docker-airflow
[kerberos]
ccache = /tmp/airflow_krb5_ccache
# gets augmented with fqdn
principal = airflow
reinit_frequency = 3600
kinit_path = kinit
keytab = airflow.keytab
[github_enterprise]
api_rev = v3
[admin]
# UI to hide sensitive variable fields when set to True
hide_sensitive_variable_fields = True
[elasticsearch]
# Elasticsearch host
host =
# Format of the log_id, which is used to query for a given tasks logs
log_id_template = {dag_id}-{task_id}-{execution_date}-{try_number}
# Used to mark the end of a log stream for a task
end_of_log_mark = end_of_log
# Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id
# Code will construct log_id using the log_id template from the argument above.
# NOTE: The code will prefix the https:// automatically, don't include that here.
frontend =
# Write the task logs to the stdout of the worker, rather than the default files
write_stdout = False
# Instead of the default log formatter, write the log lines as JSON
json_format = False
# Log fields to also attach to the json output, if enabled
json_fields = asctime, filename, lineno, levelname, message
[elasticsearch_configs]
use_ssl = False
verify_certs = True
[kubernetes]
# The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run
worker_container_repository =
worker_container_tag =
worker_container_image_pull_policy = IfNotPresent
# If True (default), worker pods will be deleted upon termination
delete_worker_pods = True
# Number of Kubernetes Worker Pod creation calls per scheduler loop
worker_pods_creation_batch_size = 1
# The Kubernetes namespace where airflow workers should be created. Defaults to `default`
namespace = default
# The name of the Kubernetes ConfigMap Containing the Airflow Configuration (this file)
airflow_configmap =
# For docker image already contains DAGs, this is set to `True`, and the worker will search for dags in dags_folder,
# otherwise use git sync or dags volume claim to mount DAGs
dags_in_image = False
# For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs
dags_volume_subpath =
# For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path)
dags_volume_claim =
# For volume mounted logs, the worker will look in this subpath for logs
logs_volume_subpath =
# A shared volume claim for the logs
logs_volume_claim =
# For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync)
# Useful in local environment, discouraged in production
dags_volume_host =
# A hostPath volume for the logs
# Useful in local environment, discouraged in production
logs_volume_host =
# A list of configMapsRefs to envFrom. If more than one configMap is
# specified, provide a comma separated list: configmap_a,configmap_b
env_from_configmap_ref =
# A list of secretRefs to envFrom. If more than one secret is
# specified, provide a comma separated list: secret_a,secret_b
env_from_secret_ref =
# Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim)
git_repo =
git_branch =
git_subpath =
# Use git_user and git_password for user authentication or git_ssh_key_secret_name and git_ssh_key_secret_key
# for SSH authentication
git_user =
git_password =
git_sync_root = /git
git_sync_dest = repo
# Mount point of the volume if git-sync is being used.
# i.e. /opt/airflow/dags
git_dags_folder_mount_point =
# To get Git-sync SSH authentication set up follow this format
#
# airflow-secrets.yaml:
# ---
# apiVersion: v1
# kind: Secret
# metadata:
# name: airflow-secrets
# data:
# # key needs to be gitSshKey
# gitSshKey: <base64_encoded_data>
# ---
# airflow-configmap.yaml:
# apiVersion: v1
# kind: ConfigMap
# metadata:
# name: airflow-configmap
# data:
# known_hosts: |
# github.com ssh-rsa <...>
# airflow.cfg: |
# ...
#
# git_ssh_key_secret_name = airflow-secrets
# git_ssh_known_hosts_configmap_name = airflow-configmap
git_ssh_key_secret_name =
git_ssh_known_hosts_configmap_name =
# To give the git_sync init container credentials via a secret, create a secret
# with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and
# add `git_sync_credentials_secret = <secret_name>` to your airflow config under the kubernetes section
#
# Secret Example:
# apiVersion: v1
# kind: Secret
# metadata:
# name: git-credentials
# data:
# GIT_SYNC_USERNAME: <base64_encoded_git_username>
# GIT_SYNC_PASSWORD: <base64_encoded_git_password>
git_sync_credentials_secret =
# For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync
git_sync_container_repository = k8s.gcr.io/git-sync
git_sync_container_tag = v3.1.1
git_sync_init_container_name = git-sync-clone
git_sync_run_as_user = 65533
# The name of the Kubernetes service account to be associated with airflow workers, if any.
# Service accounts are required for workers that require access to secrets or cluster resources.
# See the Kubernetes RBAC documentation for more:
# https://kubernetes.io/docs/admin/authorization/rbac/
worker_service_account_name =
# Any image pull secrets to be given to worker pods, If more than one secret is
# required, provide a comma separated list: secret_a,secret_b
image_pull_secrets =
# GCP Service Account Keys to be provided to tasks run on Kubernetes Executors
# Should be supplied in the format: key-name-1:key-path-1,key-name-2:key-path-2
gcp_service_account_keys =
# Use the service account kubernetes gives to pods to connect to kubernetes cluster.
# It's intended for clients that expect to be running inside a pod running on kubernetes.
# It will raise an exception if called from a process not running in a kubernetes environment.
in_cluster = True
# When running with in_cluster=False change the default cluster_context or config_file
# options to Kubernetes client. Leave blank these to use default behaviour like `kubectl` has.
# cluster_context =
# config_file =
# Affinity configuration as a single line formatted JSON object.
# See the affinity model for top-level key names (e.g. `nodeAffinity`, etc.):
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core
affinity =
# A list of toleration objects as a single line formatted JSON array
# See:
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core
tolerations =
# **kwargs parameters to pass while calling a kubernetes client core_v1_api methods from Kubernetes Executor
# provided as a single line formatted JSON dictionary string.
# List of supported params in **kwargs are similar for all core_v1_apis, hence a single config variable for all apis
# See:
# https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py
# Note that if no _request_timeout is specified, the kubernetes client will wait indefinitely for kubernetes
# api responses, which will cause the scheduler to hang. The timeout is specified as [connect timeout, read timeout]
kube_client_request_args = {"_request_timeout" : [60,60] }
# Worker pods security context options
# See:
# https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
# Specifies the uid to run the first process of the worker pods containers as
run_as_user =
# Specifies a gid to associate with all containers in the worker pods
# if using a git_ssh_key_secret_name use an fs_group
# that allows for the key to be read, e.g. 65533
fs_group =
[kubernetes_node_selectors]
# The Key-value pairs to be given to worker pods.
# The worker pods will be scheduled to the nodes of the specified key-value pairs.
# Should be supplied in the format: key = value
[kubernetes_annotations]
# The Key-value annotations pairs to be given to worker pods.
# Should be supplied in the format: key = value
[kubernetes_environment_variables]
# The scheduler sets the following environment variables into your workers. You may define as
# many environment variables as needed and the kubernetes launcher will set them in the launched workers.
# Environment variables in this section are defined as follows
# <environment_variable_key> = <environment_variable_value>
#
# For example if you wanted to set an environment variable with value `prod` and key
# `ENVIRONMENT` you would follow the following format:
# ENVIRONMENT = prod
#
# Additionally you may override worker airflow settings with the AIRFLOW__<SECTION>__<KEY>
# formatting as supported by airflow normally.
[kubernetes_secrets]
# The scheduler mounts the following secrets into your workers as they are launched by the
# scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the
# defined secrets and mount them as secret environment variables in the launched workers.
# Secrets in this section are defined as follows
# <environment_variable_mount> = <kubernetes_secret_object>=<kubernetes_secret_key>
#
# For example if you wanted to mount a kubernetes secret key named `postgres_password` from the
# kubernetes secret object `airflow-secret` as the environment variable `POSTGRES_PASSWORD` into
# your workers you would follow the following format:
# POSTGRES_PASSWORD = airflow-secret=postgres_credentials
#
# Additionally you may override worker airflow settings with the AIRFLOW__<SECTION>__<KEY>
# formatting as supported by airflow normally.
[kubernetes_labels]
# The Key-value pairs to be given to worker pods.
# The worker pods will be given these static labels, as well as some additional dynamic labels
# to identify the task.
# Should be supplied in the format: key = value

0
airflow/data/dags/.gitkeep Normal file
View File

0
airflow/data/dags/__init__.py Normal file
View File

104
airflow/data/dags/tutorial.py Normal file
View File

@ -0,0 +1,104 @@
# -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""
### Tutorial Documentation
Documentation that goes along with the Airflow tutorial located
[here](https://airflow.apache.org/tutorial.html)
"""
from datetime import timedelta
import airflow
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
# These args will get passed on to each operator
# You can override them on a per-task basis during operator initialization
default_args = {
'owner': 'Airflow',
'depends_on_past': False,
'start_date': airflow.utils.dates.days_ago(2),
'email': ['airflow@example.com'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=5),
# 'queue': 'bash_queue',
# 'pool': 'backfill',
# 'priority_weight': 10,
# 'end_date': datetime(2016, 1, 1),
# 'wait_for_downstream': False,
# 'dag': dag,
# 'sla': timedelta(hours=2),
# 'execution_timeout': timedelta(seconds=300),
# 'on_failure_callback': some_function,
# 'on_success_callback': some_other_function,
# 'on_retry_callback': another_function,
# 'sla_miss_callback': yet_another_function,
# 'trigger_rule': 'all_success'
}
dag = DAG(
'tutorial',
default_args=default_args,
description='A simple tutorial DAG',
schedule_interval=timedelta(days=1),
)
# t1, t2 and t3 are examples of tasks created by instantiating operators
t1 = BashOperator(
task_id='print_date',
bash_command='date',
dag=dag,
)
t1.doc_md = """\
#### Task Documentation
You can document your task using the attributes `doc_md` (markdown),
`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
rendered in the UI's Task Instance Details page.
![img](http://montcs.bloomu.edu/~bobmon/Semesters/2012-01/491/import%20soul.png)
"""
dag.doc_md = __doc__
t2 = BashOperator(
task_id='sleep',
depends_on_past=False,
bash_command='sleep 5',
dag=dag,
)
templated_command = """
{% for i in range(5) %}
echo "{{ ds }}"
echo "{{ macros.ds_add(ds, 7)}}"
echo "{{ params.my_param }}"
{% endfor %}
"""
t3 = BashOperator(
task_id='templated',
depends_on_past=False,
bash_command=templated_command,
params={'my_param': 'Parameter I passed in'},
dag=dag,
)
t1 >> [t2, t3]

860
airflow/data/default_airflow.cfg Normal file
View File

@ -0,0 +1,860 @@
# -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# This is the template for Airflow's default configuration. When Airflow is
# imported, it looks for a configuration file at $AIRFLOW_HOME/airflow.cfg. If
# it doesn't exist, Airflow uses this template to generate it by replacing
# variables in curly braces with their global values from configuration.py.
# Users should not modify this file; they should customize the generated
# airflow.cfg instead.
# ----------------------- TEMPLATE BEGINS HERE -----------------------
[core]
# The folder where your airflow pipelines live, most likely a
# subfolder in a code repository
# This path must be absolute
dags_folder = {AIRFLOW_HOME}/dags
# The folder where airflow should store its log files
# This path must be absolute
base_log_folder = {AIRFLOW_HOME}/logs
# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search.
# Users must supply an Airflow connection id that provides access to the storage
# location. If remote_logging is set to true, see UPDATING.md for additional
# configuration requirements.
remote_logging = False
remote_log_conn_id =
remote_base_log_folder =
encrypt_s3_logs = False
# Logging level
logging_level = INFO
fab_logging_level = WARN
# Logging class
# Specify the class that will specify the logging configuration
# This class has to be on the python classpath
# logging_config_class = my.path.default_local_settings.LOGGING_CONFIG
logging_config_class =
# Log format
# Colour the logs when the controlling terminal is a TTY.
colored_console_log = True
colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s
colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter
log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s
simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
# Log filename format
log_filename_template = {{{{ ti.dag_id }}}}/{{{{ ti.task_id }}}}/{{{{ ts }}}}/{{{{ try_number }}}}.log
log_processor_filename_template = {{{{ filename }}}}.log
dag_processor_manager_log_location = {AIRFLOW_HOME}/logs/dag_processor_manager/dag_processor_manager.log
# Hostname by providing a path to a callable, which will resolve the hostname
# The format is "package:function". For example,
# default value "socket:getfqdn" means that result from getfqdn() of "socket" package will be used as hostname
# No argument should be required in the function specified.
# If using IP address as hostname is preferred, use value "airflow.utils.net:get_host_ip_address"
hostname_callable = socket:getfqdn
# Default timezone in case supplied date times are naive
# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam)
default_timezone = utc
# The executor class that airflow should use. Choices include
# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor
executor = SequentialExecutor
# The SqlAlchemy connection string to the metadata database.
# SqlAlchemy supports many different database engine, more information
# their website
sql_alchemy_conn = sqlite:///{AIRFLOW_HOME}/airflow.db
# The encoding for the databases
sql_engine_encoding = utf-8
# If SqlAlchemy should pool database connections.
sql_alchemy_pool_enabled = True
# The SqlAlchemy pool size is the maximum number of database connections
# in the pool. 0 indicates no limit.
sql_alchemy_pool_size = 5
# The maximum overflow size of the pool.
# When the number of checked-out connections reaches the size set in pool_size,
# additional connections will be returned up to this limit.
# When those additional connections are returned to the pool, they are disconnected and discarded.
# It follows then that the total number of simultaneous connections the pool will allow is pool_size + max_overflow,
# and the total number of "sleeping" connections the pool will allow is pool_size.
# max_overflow can be set to -1 to indicate no overflow limit;
# no limit will be placed on the total number of concurrent connections. Defaults to 10.
sql_alchemy_max_overflow = 10
# The SqlAlchemy pool recycle is the number of seconds a connection
# can be idle in the pool before it is invalidated. This config does
# not apply to sqlite. If the number of DB connections is ever exceeded,
# a lower config value will allow the system to recover faster.
sql_alchemy_pool_recycle = 1800
# How many seconds to retry re-establishing a DB connection after
# disconnects. Setting this to 0 disables retries.
sql_alchemy_reconnect_timeout = 300
# The schema to use for the metadata database
# SqlAlchemy supports databases with the concept of multiple schemas.
sql_alchemy_schema =
# The amount of parallelism as a setting to the executor. This defines
# the max number of task instances that should run simultaneously
# on this airflow installation
parallelism = 32
# The number of task instances allowed to run concurrently by the scheduler
dag_concurrency = 16
# Are DAGs paused by default at creation
dags_are_paused_at_creation = True
# The maximum number of active DAG runs per DAG
max_active_runs_per_dag = 16
# Whether to load the examples that ship with Airflow. It's good to
# get started, but you probably want to set this to False in a production
# environment
load_examples = True
# Where your Airflow plugins are stored
plugins_folder = {AIRFLOW_HOME}/plugins
# Secret key to save connection passwords in the db
fernet_key = {FERNET_KEY}
# Whether to disable pickling dags
donot_pickle = False
# How long before timing out a python file import while filling the DagBag
dagbag_import_timeout = 30
# The class to use for running task instances in a subprocess
task_runner = StandardTaskRunner
# If set, tasks without a `run_as_user` argument will be run with this user
# Can be used to de-elevate a sudo user running Airflow when executing tasks
default_impersonation =
# What security module to use (for example kerberos):
security =
# If set to False enables some unsecure features like Charts and Ad Hoc Queries.
# In 2.0 will default to True.
secure_mode = False
# Turn unit test mode on (overwrites many configuration options with test
# values at runtime)
unit_test_mode = False
# Name of handler to read task instance logs.
# Default to use task handler.
task_log_reader = task
# Whether to enable pickling for xcom (note that this is insecure and allows for
# RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False).
enable_xcom_pickling = True
# When a task is killed forcefully, this is the amount of time in seconds that
# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED
killed_task_cleanup_time = 60
# Whether to override params with dag_run.conf. If you pass some key-value pairs through `airflow backfill -c` or
# `airflow trigger_dag -c`, the key-value pairs will override the existing ones in params.
dag_run_conf_overrides_params = False
# Worker initialisation check to validate Metadata Database connection
worker_precheck = False
# When discovering DAGs, ignore any files that don't contain the strings `DAG` and `airflow`.
dag_discovery_safe_mode = True
[cli]
# In what way should the cli access the API. The LocalClient will use the
# database directly, while the json_client will use the api running on the
# webserver
api_client = airflow.api.client.local_client
# If you set web_server_url_prefix, do NOT forget to append it here, ex:
# endpoint_url = http://localhost:8080/myroot
# So api will look like: http://localhost:8080/myroot/api/experimental/...
endpoint_url = http://localhost:8080
[api]
# How to authenticate users of the API
auth_backend = airflow.api.auth.backend.default
[lineage]
# what lineage backend to use
backend =
[atlas]
sasl_enabled = False
host =
port = 21000
username =
password =
[operators]
# The default owner assigned to each new operator, unless
# provided explicitly or passed via `default_args`
default_owner = airflow
default_cpus = 1
default_ram = 512
default_disk = 512
default_gpus = 0
[hive]
# Default mapreduce queue for HiveOperator tasks
default_hive_mapred_queue =
[webserver]
# The base url of your website as airflow cannot guess what domain or
# cname you are using. This is used in automated emails that
# airflow sends to point links to the right web server
base_url = http://localhost:8080
# The ip specified when starting the web server
web_server_host = 0.0.0.0
# The port on which to run the web server
web_server_port = 8080
# Paths to the SSL certificate and key for the web server. When both are
# provided SSL will be enabled. This does not change the web server port.
web_server_ssl_cert =
web_server_ssl_key =
# Number of seconds the webserver waits before killing gunicorn master that doesn't respond
web_server_master_timeout = 120
# Number of seconds the gunicorn webserver waits before timing out on a worker
web_server_worker_timeout = 120
# Number of workers to refresh at a time. When set to 0, worker refresh is
# disabled. When nonzero, airflow periodically refreshes webserver workers by
# bringing up new ones and killing old ones.
worker_refresh_batch_size = 1
# Number of seconds to wait before refreshing a batch of workers.
worker_refresh_interval = 30
# Secret key used to run your flask app
secret_key = temporary_key
# Number of workers to run the Gunicorn web server
workers = 4
# The worker class gunicorn should use. Choices include
# sync (default), eventlet, gevent
worker_class = sync
# Log files for the gunicorn webserver. '-' means log to stderr.
access_logfile = -
error_logfile = -
# Expose the configuration file in the web server
# This is only applicable for the flask-admin based web UI (non FAB-based).
# In the FAB-based web UI with RBAC feature,
# access to configuration is controlled by role permissions.
expose_config = False
# Set to true to turn on authentication:
# https://airflow.apache.org/security.html#web-authentication
authenticate = False
# Filter the list of dags by owner name (requires authentication to be enabled)
filter_by_owner = False
# Filtering mode. Choices include user (default) and ldapgroup.
# Ldap group filtering requires using the ldap backend
#
# Note that the ldap server needs the "memberOf" overlay to be set up
# in order to user the ldapgroup mode.
owner_mode = user
# Default DAG view. Valid values are:
# tree, graph, duration, gantt, landing_times
dag_default_view = tree
# Default DAG orientation. Valid values are:
# LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top)
dag_orientation = LR
# Puts the webserver in demonstration mode; blurs the names of Operators for
# privacy.
demo_mode = False
# The amount of time (in secs) webserver will wait for initial handshake
# while fetching logs from other worker machine
log_fetch_timeout_sec = 5
# By default, the webserver shows paused DAGs. Flip this to hide paused
# DAGs by default
hide_paused_dags_by_default = False
# Consistent page size across all listing views in the UI
page_size = 100
# Use FAB-based webserver with RBAC feature
rbac = False
# Define the color of navigation bar
navbar_color = #007A87
# Default dagrun to show in UI
default_dag_run_display_number = 25
# Enable werkzeug `ProxyFix` middleware
enable_proxy_fix = False
# Set secure flag on session cookie
cookie_secure = False
# Set samesite policy on session cookie
cookie_samesite =
# Default setting for wrap toggle on DAG code and TI log views.
default_wrap = False
# Send anonymous user activity to your analytics tool
# analytics_tool = # choose from google_analytics, segment, or metarouter
# analytics_id = XXXXXXXXXXX
[email]
email_backend = airflow.utils.email.send_email_smtp
[smtp]
# If you want airflow to send emails on retries, failure, and you want to use
# the airflow.utils.email.send_email_smtp function, you have to configure an
# smtp server here
smtp_host = localhost
smtp_starttls = True
smtp_ssl = False
# Uncomment and set the user/pass settings if you want to use SMTP AUTH
# smtp_user = airflow
# smtp_password = airflow
smtp_port = 25
smtp_mail_from = airflow@example.com
[celery]
# This section only applies if you are using the CeleryExecutor in
# [core] section above
# The app name that will be used by celery
celery_app_name = airflow.executors.celery_executor
# The concurrency that will be used when starting workers with the
# "airflow worker" command. This defines the number of task instances that
# a worker will take, so size up your workers based on the resources on
# your worker box and the nature of your tasks
worker_concurrency = 16
# The maximum and minimum concurrency that will be used when starting workers with the
# "airflow worker" command (always keep minimum processes, but grow to maximum if necessary).
# Note the value should be "max_concurrency,min_concurrency"
# Pick these numbers based on resources on worker box and the nature of the task.
# If autoscale option is available, worker_concurrency will be ignored.
# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale
# worker_autoscale = 16,12
# When you start an airflow worker, airflow starts a tiny web server
# subprocess to serve the workers local log files to the airflow main
# web server, who then builds pages and sends them to users. This defines
# the port on which the logs are served. It needs to be unused, and open
# visible from the main web server to connect into the workers.
worker_log_server_port = 8793
# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally
# a sqlalchemy database. Refer to the Celery documentation for more
# information.
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings
broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow
# The Celery result_backend. When a job finishes, it needs to update the
# metadata of the job. Therefore it will post a message on a message bus,
# or insert it into a database (depending of the backend)
# This status is used by the scheduler to update the state of the task
# The use of a database is highly recommended
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings
result_backend = db+mysql://airflow:airflow@localhost:3306/airflow
# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start
# it `airflow flower`. This defines the IP that Celery Flower runs on
flower_host = 0.0.0.0
# The root URL for Flower
# Ex: flower_url_prefix = /flower
flower_url_prefix =
# This defines the port that Celery Flower runs on
flower_port = 5555
# Securing Flower with Basic Authentication
# Accepts user:password pairs separated by a comma
# Example: flower_basic_auth = user1:password1,user2:password2
flower_basic_auth =
# Default queue that tasks get assigned to and that worker listen on.
default_queue = default
# How many processes CeleryExecutor uses to sync task state.
# 0 means to use max(1, number of cores - 1) processes.
sync_parallelism = 0
# Import path for celery configuration options
celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG
# In case of using SSL
ssl_active = False
ssl_key =
ssl_cert =
ssl_cacert =
# Celery Pool implementation.
# Choices include: prefork (default), eventlet, gevent or solo.
# See:
# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency
# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html
pool = prefork
[celery_broker_transport_options]
# This section is for specifying options which can be passed to the
# underlying celery broker transport. See:
# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options
# The visibility timeout defines the number of seconds to wait for the worker
# to acknowledge the task before the message is redelivered to another worker.
# Make sure to increase the visibility timeout to match the time of the longest
# ETA you're planning to use.
#
# visibility_timeout is only supported for Redis and SQS celery brokers.
# See:
# http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options
#
#visibility_timeout = 21600
[dask]
# This section only applies if you are using the DaskExecutor in
# [core] section above
# The IP address and port of the Dask cluster's scheduler.
cluster_address = 127.0.0.1:8786
# TLS/ SSL settings to access a secured Dask scheduler.
tls_ca =
tls_cert =
tls_key =
[scheduler]
# Task instances listen for external kill signal (when you clear tasks
# from the CLI or the UI), this defines the frequency at which they should
# listen (in seconds).
job_heartbeat_sec = 5
# The scheduler constantly tries to trigger new tasks (look at the
# scheduler section in the docs for more information). This defines
# how often the scheduler should run (in seconds).
scheduler_heartbeat_sec = 5
# after how much time should the scheduler terminate in seconds
# -1 indicates to run continuously (see also num_runs)
run_duration = -1
# after how much time (seconds) a new DAGs should be picked up from the filesystem
min_file_process_interval = 0
# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes.
dag_dir_list_interval = 300
# How often should stats be printed to the logs
print_stats_interval = 30
# If the last scheduler heartbeat happened more than scheduler_health_check_threshold ago (in seconds),
# scheduler is considered unhealthy.
# This is used by the health check in the "/health" endpoint
scheduler_health_check_threshold = 30
child_process_log_directory = {AIRFLOW_HOME}/logs/scheduler
# Local task jobs periodically heartbeat to the DB. If the job has
# not heartbeat in this many seconds, the scheduler will mark the
# associated task instance as failed and will re-schedule the task.
scheduler_zombie_task_threshold = 300
# Turn off scheduler catchup by setting this to False.
# Default behavior is unchanged and
# Command Line Backfills still work, but the scheduler
# will not do scheduler catchup if this is False,
# however it can be set on a per DAG basis in the
# DAG definition (catchup)
catchup_by_default = True
# This changes the batch size of queries in the scheduling main loop.
# If this is too high, SQL query performance may be impacted by one
# or more of the following:
# - reversion to full table scan
# - complexity of query predicate
# - excessive locking
#
# Additionally, you may hit the maximum allowable query length for your db.
#
# Set this to 0 for no limit (not advised)
max_tis_per_query = 512
# Statsd (https://github.com/etsy/statsd) integration settings
statsd_on = False
statsd_host = localhost
statsd_port = 8125
statsd_prefix = airflow
# The scheduler can run multiple threads in parallel to schedule dags.
# This defines how many threads will run.
max_threads = 2
authenticate = False
# Turn off scheduler use of cron intervals by setting this to False.
# DAGs submitted manually in the web UI or with trigger_dag will still run.
use_job_schedule = True
[ldap]
# set this to ldaps://<your.ldap.server>:<port>
uri =
user_filter = objectClass=*
user_name_attr = uid
group_member_attr = memberOf
superuser_filter =
data_profiler_filter =
bind_user = cn=Manager,dc=example,dc=com
bind_password = insecure
basedn = dc=example,dc=com
cacert = /etc/ca/ldap_ca.crt
search_scope = LEVEL
# This setting allows the use of LDAP servers that either return a
# broken schema, or do not return a schema.
ignore_malformed_schema = False
[mesos]
# Mesos master address which MesosExecutor will connect to.
master = localhost:5050
# The framework name which Airflow scheduler will register itself as on mesos
framework_name = Airflow
# Number of cpu cores required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_cpu = 1
# Memory in MB required for running one task instance using
# 'airflow run <dag_id> <task_id> <execution_date> --local -p <pickle_id>'
# command on a mesos slave
task_memory = 256
# Enable framework checkpointing for mesos
# See http://mesos.apache.org/documentation/latest/slave-recovery/
checkpoint = False
# Failover timeout in milliseconds.
# When checkpointing is enabled and this option is set, Mesos waits
# until the configured timeout for
# the MesosExecutor framework to re-register after a failover. Mesos
# shuts down running tasks if the
# MesosExecutor framework fails to re-register within this timeframe.
# failover_timeout = 604800
# Enable framework authentication for mesos
# See http://mesos.apache.org/documentation/latest/configuration/
authenticate = False
# Mesos credentials, if authentication is enabled
# default_principal = admin
# default_secret = admin
# Optional Docker Image to run on slave before running the command
# This image should be accessible from mesos slave i.e mesos slave
# should be able to pull this docker image before executing the command.
# docker_image_slave = puckel/docker-airflow
[kerberos]
ccache = /tmp/airflow_krb5_ccache
# gets augmented with fqdn
principal = airflow
reinit_frequency = 3600
kinit_path = kinit
keytab = airflow.keytab
[github_enterprise]
api_rev = v3
[admin]
# UI to hide sensitive variable fields when set to True
hide_sensitive_variable_fields = True
[elasticsearch]
# Elasticsearch host
host =
# Format of the log_id, which is used to query for a given tasks logs
log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}}
# Used to mark the end of a log stream for a task
end_of_log_mark = end_of_log
# Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id
# Code will construct log_id using the log_id template from the argument above.
# NOTE: The code will prefix the https:// automatically, don't include that here.
frontend =
# Write the task logs to the stdout of the worker, rather than the default files
write_stdout = False
# Instead of the default log formatter, write the log lines as JSON
json_format = False
# Log fields to also attach to the json output, if enabled
json_fields = asctime, filename, lineno, levelname, message
[elasticsearch_configs]
use_ssl = False
verify_certs = True
[kubernetes]
# The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run
worker_container_repository =
worker_container_tag =
worker_container_image_pull_policy = IfNotPresent
# If True (default), worker pods will be deleted upon termination
delete_worker_pods = True
# Number of Kubernetes Worker Pod creation calls per scheduler loop
worker_pods_creation_batch_size = 1
# The Kubernetes namespace where airflow workers should be created. Defaults to `default`
namespace = default
# The name of the Kubernetes ConfigMap Containing the Airflow Configuration (this file)
airflow_configmap =
# For docker image already contains DAGs, this is set to `True`, and the worker will search for dags in dags_folder,
# otherwise use git sync or dags volume claim to mount DAGs
dags_in_image = False
# For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs
dags_volume_subpath =
# For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path)
dags_volume_claim =
# For volume mounted logs, the worker will look in this subpath for logs
logs_volume_subpath =
# A shared volume claim for the logs
logs_volume_claim =
# For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync)
# Useful in local environment, discouraged in production
dags_volume_host =
# A hostPath volume for the logs
# Useful in local environment, discouraged in production
logs_volume_host =
# A list of configMapsRefs to envFrom. If more than one configMap is
# specified, provide a comma separated list: configmap_a,configmap_b
env_from_configmap_ref =
# A list of secretRefs to envFrom. If more than one secret is
# specified, provide a comma separated list: secret_a,secret_b
env_from_secret_ref =
# Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim)
git_repo =
git_branch =
git_subpath =
# Use git_user and git_password for user authentication or git_ssh_key_secret_name and git_ssh_key_secret_key
# for SSH authentication
git_user =
git_password =
git_sync_root = /git
git_sync_dest = repo
# Mount point of the volume if git-sync is being used.
# i.e. {AIRFLOW_HOME}/dags
git_dags_folder_mount_point =
# To get Git-sync SSH authentication set up follow this format
#
# airflow-secrets.yaml:
# ---
# apiVersion: v1
# kind: Secret
# metadata:
# name: airflow-secrets
# data:
# # key needs to be gitSshKey
# gitSshKey: <base64_encoded_data>
# ---
# airflow-configmap.yaml:
# apiVersion: v1
# kind: ConfigMap
# metadata:
# name: airflow-configmap
# data:
# known_hosts: |
# github.com ssh-rsa <...>
# airflow.cfg: |
# ...
#
# git_ssh_key_secret_name = airflow-secrets
# git_ssh_known_hosts_configmap_name = airflow-configmap
git_ssh_key_secret_name =
git_ssh_known_hosts_configmap_name =
# To give the git_sync init container credentials via a secret, create a secret
# with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and
# add `git_sync_credentials_secret = <secret_name>` to your airflow config under the kubernetes section
#
# Secret Example:
# apiVersion: v1
# kind: Secret
# metadata:
# name: git-credentials
# data:
# GIT_SYNC_USERNAME: <base64_encoded_git_username>
# GIT_SYNC_PASSWORD: <base64_encoded_git_password>
git_sync_credentials_secret =
# For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync
git_sync_container_repository = k8s.gcr.io/git-sync
git_sync_container_tag = v3.1.1
git_sync_init_container_name = git-sync-clone
git_sync_run_as_user = 65533
# The name of the Kubernetes service account to be associated with airflow workers, if any.
# Service accounts are required for workers that require access to secrets or cluster resources.
# See the Kubernetes RBAC documentation for more:
# https://kubernetes.io/docs/admin/authorization/rbac/
worker_service_account_name =
# Any image pull secrets to be given to worker pods, If more than one secret is
# required, provide a comma separated list: secret_a,secret_b
image_pull_secrets =
# GCP Service Account Keys to be provided to tasks run on Kubernetes Executors
# Should be supplied in the format: key-name-1:key-path-1,key-name-2:key-path-2
gcp_service_account_keys =
# Use the service account kubernetes gives to pods to connect to kubernetes cluster.
# It's intended for clients that expect to be running inside a pod running on kubernetes.
# It will raise an exception if called from a process not running in a kubernetes environment.
in_cluster = True
# When running with in_cluster=False change the default cluster_context or config_file
# options to Kubernetes client. Leave blank these to use default behaviour like `kubectl` has.
# cluster_context =
# config_file =
# Affinity configuration as a single line formatted JSON object.
# See the affinity model for top-level key names (e.g. `nodeAffinity`, etc.):
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core
affinity =
# A list of toleration objects as a single line formatted JSON array
# See:
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core
tolerations =
# **kwargs parameters to pass while calling a kubernetes client core_v1_api methods from Kubernetes Executor
# provided as a single line formatted JSON dictionary string.
# List of supported params in **kwargs are similar for all core_v1_apis, hence a single config variable for all apis
# See:
# https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py
# Note that if no _request_timeout is specified, the kubernetes client will wait indefinitely for kubernetes
# api responses, which will cause the scheduler to hang. The timeout is specified as [connect timeout, read timeout]
kube_client_request_args = {{"_request_timeout" : [60,60] }}
# Worker pods security context options
# See:
# https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
# Specifies the uid to run the first process of the worker pods containers as
run_as_user =
# Specifies a gid to associate with all containers in the worker pods
# if using a git_ssh_key_secret_name use an fs_group
# that allows for the key to be read, e.g. 65533
fs_group =
[kubernetes_node_selectors]
# The Key-value pairs to be given to worker pods.
# The worker pods will be scheduled to the nodes of the specified key-value pairs.
# Should be supplied in the format: key = value
[kubernetes_annotations]
# The Key-value annotations pairs to be given to worker pods.
# Should be supplied in the format: key = value
[kubernetes_environment_variables]
# The scheduler sets the following environment variables into your workers. You may define as
# many environment variables as needed and the kubernetes launcher will set them in the launched workers.
# Environment variables in this section are defined as follows
# <environment_variable_key> = <environment_variable_value>
#
# For example if you wanted to set an environment variable with value `prod` and key
# `ENVIRONMENT` you would follow the following format:
# ENVIRONMENT = prod
#
# Additionally you may override worker airflow settings with the AIRFLOW__<SECTION>__<KEY>
# formatting as supported by airflow normally.
[kubernetes_secrets]
# The scheduler mounts the following secrets into your workers as they are launched by the
# scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the
# defined secrets and mount them as secret environment variables in the launched workers.
# Secrets in this section are defined as follows
# <environment_variable_mount> = <kubernetes_secret_object>=<kubernetes_secret_key>
#
# For example if you wanted to mount a kubernetes secret key named `postgres_password` from the
# kubernetes secret object `airflow-secret` as the environment variable `POSTGRES_PASSWORD` into
# your workers you would follow the following format:
# POSTGRES_PASSWORD = airflow-secret=postgres_credentials
#
# Additionally you may override worker airflow settings with the AIRFLOW__<SECTION>__<KEY>
# formatting as supported by airflow normally.
[kubernetes_labels]
# The Key-value pairs to be given to worker pods.
# The worker pods will be given these static labels, as well as some additional dynamic labels
# to identify the task.
# Should be supplied in the format: key = value

0
airflow/data/logs/.gitkeep Normal file
View File

0
airflow/data/plugins/.gitkeep Normal file
View File

6
airflow/docker-compose-master.yml → airflow/deprecated/docker-compose-master.yml
View File

@ -24,7 +24,7 @@ services:
restart: always
webserver:
image: puckel/docker-airflow:1.9.0-4
image: puckel/docker-airflow:1.10.4
command: webserver
hostname: master
ports:
@ -52,7 +52,7 @@ services:
restart: always
scheduler:
image: puckel/docker-airflow:1.9.0-4
image: puckel/docker-airflow:1.10.4
command: scheduler
volumes:
- ./data/airflow/dags:/usr/local/airflow/dags
@ -66,7 +66,7 @@ services:
restart: always
flower:
image: puckel/docker-airflow:1.9.0-4
image: puckel/docker-airflow:1.10.4
command: flower
ports:
- "5555:5555"

2
airflow/docker-compose-worker.yml → airflow/deprecated/docker-compose-worker.yml
View File

@ -3,7 +3,7 @@ version: '2.1'
services:
worker:
image: puckel/docker-airflow:1.9.0-4
image: puckel/docker-airflow:1.10.4
command: worker
hostname: worker1
ports:

110
airflow/docker-stack.yaml Normal file
View File

@ -0,0 +1,110 @@
version: "3.7"
services:
redis:
image: redis:alpine
command: --save 900 1
ports:
- "6379:6379"
volumes:
- /data/redis:/data
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure
postgres:
image: postgres:alpine
ports:
- "5432:5432"
volumes:
- /data/postgres:/var/lib/postgresql/data
environment:
- POSTGRES_USER=airflow
- POSTGRES_PASSWORD=airflow
- POSTGRES_DB=airflow
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure
webserver:
image: vimagick/airflow
command: webserver
ports:
- "8080:8080"
volumes:
- airflow_data:/opt/airflow
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure
depends_on:
- postgres
- redis
scheduler:
image: vimagick/airflow
command: scheduler
volumes:
- airflow_data:/opt/airflow
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure
depends_on:
- webserver
flower:
image: vimagick/airflow
command: flower
ports:
- "5555:5555"
volumes:
- airflow_data:/opt/airflow
deploy:
replicas: 1
placement:
constraints:
- node.role == manager
restart_policy:
condition: on-failure
depends_on:
- webserver
worker:
image: vimagick/airflow
command: worker
volumes:
- airflow_data:/opt/airflow
deploy:
replicas: 0
placement:
constraints:
- node.role == worker
restart_policy:
condition: on-failure
depends_on:
- webserver
volumes:
airflow_data:
driver: local
driver_opts:
type: nfs
o: "addr=10.0.0.1,nolock,soft,rw"
device: ":/export/airflow"

24
airflow/systemd/airflow Normal file
View File

@ -0,0 +1,24 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# This file is the environment file for Airflow. Put this file in /etc/sysconfig/airflow per default
# configuration of the systemd unit files.
#
AIRFLOW_HOME=/home/airflow/airflow
AIRFLOW_CONFIG=/home/airflow/airflow/airflow.cfg
PATH=/home/airflow/.virtualenvs/airflow/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games

16
airflow/systemd/airflow-flower.service Normal file
View File

@ -0,0 +1,16 @@
[Unit]
Description=Airflow celery flower
After=network.target postgresql.service redis-server.service
Wants=postgresql.service redis-server.service
[Service]
EnvironmentFile=/etc/default/airflow
User=airflow
Group=airflow
Type=simple
ExecStart=/home/airflow/.virtualenvs/airflow/bin/airflow flower --port=5555
Restart=on-failure
RestartSec=5s
[Install]
WantedBy=multi-user.target

16
airflow/systemd/airflow-scheduler.service Normal file
View File

@ -0,0 +1,16 @@
[Unit]
Description=Airflow scheduler daemon
After=network.target postgresql.service redis-server.service
Wants=postgresql.service redis-server.service
[Service]
EnvironmentFile=/etc/default/airflow
User=airflow
Group=airflow
Type=simple
ExecStart=/home/airflow/.virtualenvs/airflow/bin/airflow scheduler
Restart=always
RestartSec=5s
[Install]
WantedBy=multi-user.target

18
airflow/systemd/airflow-webserver.service Normal file
View File

@ -0,0 +1,18 @@
[Unit]
Description=Airflow webserver daemon
After=network.target postgresql.service redis-server.service
Wants=postgresql.service redis-server.service
[Service]
EnvironmentFile=/etc/default/airflow
User=airflow
Group=airflow
Type=simple
RuntimeDirectory=airflow
ExecStart=/home/airflow/.virtualenvs/airflow/bin/airflow webserver --pid /run/airflow/webserver.pid
Restart=on-failure
RestartSec=5s
PrivateTmp=true
[Install]
WantedBy=multi-user.target

16
airflow/systemd/airflow-worker.service Normal file
View File

@ -0,0 +1,16 @@
[Unit]
Description=Airflow celery worker daemon
After=network.target postgresql.service redis-server.service
Wants=postgresql.service redis-server.service
[Service]
EnvironmentFile=/etc/default/airflow
User=airflow
Group=airflow
Type=simple
ExecStart=/home/airflow/.virtualenvs/airflow/bin/airflow worker
Restart=on-failure
RestartSec=10s
[Install]
WantedBy=multi-user.target

29
ambari/Dockerfile Normal file
View File

@ -0,0 +1,29 @@
#
# Dockerfile for ambari
#
FROM ubuntu:18.04
MAINTAINER EasyPi Software Foundation
ENV AMBARI_VERSION=2.7.4.0
ENV AMBARI_SOURCE=http://public-repo-1.hortonworks.com/ambari/ubuntu18/2.x/updates/${AMBARI_VERSION}/ambari.list
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
RUN set -xe \
&& apt-get update \
&& apt-get install -y curl gnupg libpostgresql-jdbc-java openjdk-8-jdk openssh-client \
&& curl -sSL ${AMBARI_SOURCE} > /etc/apt/sources.list.d/ambari.list \
&& apt-key adv --recv-keys --keyserver keyserver.ubuntu.com B9733A7A07513CAD \
&& apt-get update \
&& apt-get install -y ambari-server \
&& ambari-server setup --jdbc-db=postgres --jdbc-driver=/usr/share/java/postgresql.jar \
&& rm -rf /var/lib/apt/lists/*
VOLUME /etc/ambari-server/conf
EXPOSE 8080
CMD set -ex \
&& ambari-server start \
&& sleep 5 \
&& tail --pid=$(cat /var/run/ambari-server/ambari-server.pid) -f /var/log/ambari-server/ambari-server.log

41
ambari/README.md Normal file
View File

@ -0,0 +1,41 @@
ambari
======
The [Apache Ambari][1] project is aimed at making Hadoop management simpler by
developing software for provisioning, managing, and monitoring Apache Hadoop
clusters. [Ambari][2] provides an intuitive, easy-to-use Hadoop management web UI
backed by its RESTful APIs.
![](https://docs.cloudera.com/common/img/product_icons/icon-ambari.png)
Make sure that ambari-server is reachable by ambari-agents via DNS.
## Up and Running
```bash
$ docker-compose run --rm server bash
>>> ambari-server setup
Customize user account for ambari-server daemon [y/n] (n)?
Do you want to change Oracle JDK [y/n] (n)?
Enter advanced database configuration [y/n] (n)? y
Enter choice (4):
Hostname (postgres):
Port (5432):
Database name (ambari):
Postgres schema (public):
Username (ambari):
Enter Database Password (ambari):
Proceed with configuring remote database connection properties [y/n] (y)?
>>> PGUSER=ambari PGPASSWORD=ambari psql -h postgres -d ambari -f /var/lib/ambari-server/resources/Ambari-DDL-Postgres-CREATE.sql
CREATE TABLE ...
CREATE INDEX ...
$ docker-compose up -d
Starting ambari_postgres_1 ... done
Starting ambari_server_1 ... done
$ curl http://localhost:8080/
```
[1]: https://ambari.apache.org/
[2]: https://docs.cloudera.com/HDPDocuments/Ambari/Ambari-2.7.4.0/index.html

118
ambari/data/ambari/ambari.properties Executable file
View File

@ -0,0 +1,118 @@
#
# Copyright 2011 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
agent.package.install.task.timeout=1800
agent.stack.retry.on_repo_unavailability=false
agent.stack.retry.tries=5
agent.task.timeout=900
agent.threadpool.size.max=25
ambari-server.user=root
ambari.python.wrap=ambari-python-wrap
bootstrap.dir=/var/run/ambari-server/bootstrap
bootstrap.script=/usr/lib/ambari-server/lib/ambari_server/bootstrap.py
bootstrap.setup_agent.script=/usr/lib/ambari-server/lib/ambari_server/setupAgent.py
client.threadpool.size.max=25
common.services.path=/var/lib/ambari-server/resources/common-services
custom.action.definitions=/var/lib/ambari-server/resources/custom_action_definitions
custom.postgres.jdbc.name=postgresql.jar
extensions.path=/var/lib/ambari-server/resources/extensions
gpl.license.accepted=true
http.cache-control=no-store
http.charset=utf-8
http.pragma=no-cache
http.strict-transport-security=max-age=31536000
http.x-content-type-options=nosniff
http.x-frame-options=DENY
http.x-xss-protection=1; mode=block
java.home=/usr/lib/jvm/java-8-openjdk-amd64
java.releases=jdk1.8
java.releases.ppc64le=
jce.download.supported=true
jdk.download.supported=true
jdk1.8.desc=Oracle JDK 1.8 + Java Cryptography Extension (JCE) Policy Files 8
jdk1.8.dest-file=jdk-8u112-linux-x64.tar.gz
jdk1.8.home=/usr/jdk64/
jdk1.8.jcpol-file=jce_policy-8.zip
jdk1.8.jcpol-url=http://public-repo-1.hortonworks.com/ARTIFACTS/jce_policy-8.zip
jdk1.8.re=(jdk.*)/jre
jdk1.8.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-8u112-linux-x64.tar.gz
kerberos.keytab.cache.dir=/var/lib/ambari-server/data/cache
kerberos.operation.verify.kdc.trust=true
local.database.user=postgres
metadata.path=/var/lib/ambari-server/resources/stacks
mpacks.staging.path=/var/lib/ambari-server/resources/mpacks
pid.dir=/var/run/ambari-server
recommendations.artifacts.lifetime=1w
recommendations.dir=/var/run/ambari-server/stack-recommendations
resources.dir=/var/lib/ambari-server/resources
rolling.upgrade.skip.packages.prefixes=
security.server.disabled.ciphers=TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384|TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384|TLS_RSA_WITH_AES_256_CBC_SHA256|TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384|TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384|TLS_DHE_RSA_WITH_AES_256_CBC_SHA256|TLS_DHE_DSS_WITH_AES_256_CBC_SHA256|TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA|TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA|TLS_RSA_WITH_AES_256_CBC_SHA|TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA|TLS_ECDH_RSA_WITH_AES_256_CBC_SHA|TLS_DHE_RSA_WITH_AES_256_CBC_SHA|TLS_DHE_DSS_WITH_AES_256_CBC_SHA|TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256|TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256|TLS_RSA_WITH_AES_128_CBC_SHA256|TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256|TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256|TLS_DHE_RSA_WITH_AES_128_CBC_SHA256|TLS_DHE_DSS_WITH_AES_128_CBC_SHA256|TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA|TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA|TLS_RSA_WITH_AES_128_CBC_SHA|TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA|TLS_ECDH_RSA_WITH_AES_128_CBC_SHA|TLS_DHE_RSA_WITH_AES_128_CBC_SHA|TLS_DHE_DSS_WITH_AES_128_CBC_SHA|TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA|TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA|TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA|TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA|SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA|SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA|TLS_EMPTY_RENEGOTIATION_INFO_SCSV|TLS_DH_anon_WITH_AES_256_CBC_SHA256|TLS_ECDH_anon_WITH_AES_256_CBC_SHA|TLS_DH_anon_WITH_AES_256_CBC_SHA|TLS_DH_anon_WITH_AES_128_CBC_SHA256|TLS_ECDH_anon_WITH_AES_128_CBC_SHA|TLS_DH_anon_WITH_AES_128_CBC_SHA|TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA|SSL_DH_anon_WITH_3DES_EDE_CBC_SHA|SSL_RSA_WITH_DES_CBC_SHA|SSL_DHE_RSA_WITH_DES_CBC_SHA|SSL_DHE_DSS_WITH_DES_CBC_SHA|SSL_DH_anon_WITH_DES_CBC_SHA|SSL_RSA_EXPORT_WITH_DES40_CBC_SHA|SSL_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA|SSL_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA|SSL_DH_anon_EXPORT_WITH_DES40_CBC_SHA|TLS_RSA_WITH_NULL_SHA256|TLS_ECDHE_ECDSA_WITH_NULL_SHA|TLS_ECDHE_RSA_WITH_NULL_SHA|SSL_RSA_WITH_NULL_SHA|TLS_ECDH_ECDSA_WITH_NULL_SHA|TLS_ECDH_RSA_WITH_NULL_SHA|TLS_ECDH_anon_WITH_NULL_SHA|SSL_RSA_WITH_NULL_MD5|TLS_KRB5_WITH_3DES_EDE_CBC_SHA|TLS_KRB5_WITH_3DES_EDE_CBC_MD5|TLS_KRB5_WITH_DES_CBC_SHA|TLS_KRB5_WITH_DES_CBC_MD5|TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA|TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5
security.server.keys_dir=/var/lib/ambari-server/keys
server.connection.max.idle.millis=900000
server.execution.scheduler.isClustered=false
server.execution.scheduler.maxDbConnections=5
server.execution.scheduler.maxThreads=5
server.execution.scheduler.misfire.toleration.minutes=480
server.fqdn.service.url=http://169.254.169.254/latest/meta-data/public-hostname
server.http.session.inactive_timeout=1800
server.jdbc.connection-pool=internal
server.jdbc.database=postgres
server.jdbc.database_name=ambari
server.jdbc.driver=org.postgresql.Driver
server.jdbc.hostname=postgres
server.jdbc.port=5432
server.jdbc.postgres.schema=public
server.jdbc.rca.driver=org.postgresql.Driver
server.jdbc.rca.url=jdbc:postgresql://postgres:5432/ambari
server.jdbc.rca.user.name=ambari
server.jdbc.rca.user.passwd=/etc/ambari-server/conf/password.dat
server.jdbc.url=jdbc:postgresql://postgres:5432/ambari
server.jdbc.user.name=ambari
server.jdbc.user.passwd=/etc/ambari-server/conf/password.dat
server.os_family=ubuntu18
server.os_type=ubuntu18
server.persistence.type=remote
server.python.log.level=INFO
server.python.log.name=ambari-server-command.log
server.stages.parallel=true
server.task.timeout=1200
server.tmp.dir=/var/lib/ambari-server/data/tmp
server.version.file=/var/lib/ambari-server/resources/version
shared.resources.dir=/usr/lib/ambari-server/lib/ambari_commons/resources
skip.service.checks=false
stack.java.home=/usr/lib/jvm/java-8-openjdk-amd64
stackadvisor.script=/var/lib/ambari-server/resources/scripts/stack_advisor.py
ulimit.open.files=65536
upgrade.parameter.convert.hive.tables.timeout=86400
upgrade.parameter.move.hive.tables.timeout=86400
user.inactivity.timeout.default=0
user.inactivity.timeout.role.readonly.default=0
views.ambari.request.connect.timeout.millis=30000
views.ambari.request.read.timeout.millis=45000
views.http.cache-control=no-store
views.http.charset=utf-8
views.http.pragma=no-cache
views.http.strict-transport-security=max-age=31536000
views.http.x-content-type-options=nosniff
views.http.x-frame-options=SAMEORIGIN
views.http.x-xss-protection=1; mode=block
views.request.connect.timeout.millis=5000
views.request.read.timeout.millis=10000
views.skip.home-directory-check.file-system.list=wasb,adls,adl
webapp.dir=/usr/lib/ambari-server/web

10
ambari/data/ambari/krb5JAASLogin.conf Executable file
View File

@ -0,0 +1,10 @@
com.sun.security.jgss.krb5.initiate {
com.sun.security.auth.module.Krb5LoginModule required
renewTGT=false
doNotPrompt=true
useKeyTab=true
keyTab="/etc/security/keytabs/ambari.keytab"
principal="ambari@EXAMPLE.COM"
storeKey=true
useTicketCache=false;
};

116
ambari/data/ambari/log4j.properties Executable file
View File

@ -0,0 +1,116 @@
# Copyright 2011 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Define some default values that can be overridden by system properties
# Root logger option
ambari.root.dir=
ambari.log.dir=${ambari.root.dir}/var/log/ambari-server
ambari.log.file=ambari-server.log
ambari.config-changes.file=ambari-config-changes.log
ambari.alerts.file=ambari-alerts.log
ambari.eclipselink.file=ambari-eclipselink.log
ambari.audit.file=ambari-audit.log
ambari.dbcheck.file=ambari-server-check-database.log
ambari.stackmerger.file=ambari-stack-merger.log
log4j.rootLogger=INFO,file
# Direct log messages to a log file
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=${ambari.log.dir}/${ambari.log.file}
log4j.appender.file.MaxFileSize=80MB
log4j.appender.file.MaxBackupIndex=60
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] %c{1}:%L - %m%n
# Log config changes
log4j.logger.configchange=INFO,configchange
log4j.additivity.configchange=false
log4j.appender.configchange=org.apache.log4j.FileAppender
log4j.appender.configchange.File=${ambari.log.dir}/${ambari.config-changes.file}
log4j.appender.configchange.layout=org.apache.log4j.PatternLayout
log4j.appender.configchange.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
# Log alert state changes
log4j.logger.alerts=INFO,alerts
log4j.additivity.alerts=false
log4j.appender.alerts=org.apache.log4j.FileAppender
log4j.appender.alerts.File=${ambari.log.dir}/${ambari.alerts.file}
log4j.appender.alerts.layout=org.apache.log4j.PatternLayout
log4j.appender.alerts.layout.ConversionPattern=%d{ISO8601} %m%n
# Log database check process
log4j.logger.org.apache.ambari.server.checks.DatabaseConsistencyChecker=INFO, dbcheck
log4j.additivity.org.apache.ambari.server.checks.DatabaseConsistencyChecker=false
log4j.appender.dbcheck=org.apache.log4j.FileAppender
log4j.appender.dbcheck.File=${ambari.log.dir}/${ambari.dbcheck.file}
log4j.appender.dbcheck.layout=org.apache.log4j.PatternLayout
log4j.appender.dbcheck.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
log4j.logger.org.apache.ambari.server.checks.DatabaseConsistencyCheckHelper=INFO, dbcheckhelper
log4j.additivity.org.apache.ambari.server.checks.DatabaseConsistencyCheckHelper=false
log4j.appender.dbcheckhelper=org.apache.log4j.FileAppender
log4j.appender.dbcheckhelper.File=${ambari.log.dir}/${ambari.dbcheck.file}
log4j.appender.dbcheckhelper.layout=org.apache.log4j.PatternLayout
log4j.appender.dbcheckhelper.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
# Log stack merger
log4j.logger.org.apache.ambari.server.stack.StackMerger=INFO,stackmerger,console-stackmerger
log4j.additivity.org.apache.ambari.server.stack.StackMerger=false
log4j.appender.console-stackmerger=org.apache.log4j.ConsoleAppender
log4j.appender.console-stackmerger.layout=org.apache.log4j.PatternLayout
log4j.appender.stackmerger=org.apache.log4j.FileAppender
log4j.appender.stackmerger.File=${ambari.log.dir}/${ambari.stackmerger.file}
log4j.appender.stackmerger.layout=org.apache.log4j.PatternLayout
log4j.appender.stackmerger.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
# EclipsLink -> slf4j bridge
log4j.logger.eclipselink=TRACE,eclipselink
log4j.additivity.eclipselink=false
log4j.appender.eclipselink=org.apache.log4j.RollingFileAppender
log4j.appender.eclipselink.File=${ambari.log.dir}/${ambari.eclipselink.file}
log4j.appender.eclipselink.MaxFileSize=50MB
log4j.appender.eclipselink.MaxBackupIndex=10
log4j.appender.eclipselink.layout=org.apache.log4j.PatternLayout
log4j.appender.eclipselink.layout.ConversionPattern=%m%n
# Jersey
log4j.logger.com.sun.jersey=WARN,file
log4j.logger.org.glassfish.jersey=WARN,file
# Jetty
log4j.logger.org.eclipse.jetty=WARN,file
# Audit logging
log4j.logger.audit=INFO,audit
log4j.additivity.audit=false
log4j.appender.audit=org.apache.log4j.rolling.RollingFileAppender
log4j.appender.audit.rollingPolicy=org.apache.log4j.rolling.FixedWindowRollingPolicy
log4j.appender.audit.rollingPolicy.ActiveFileName=${ambari.log.dir}/${ambari.audit.file}
log4j.appender.audit.rollingPolicy.FileNamePattern=${ambari.log.dir}/${ambari.audit.file}-%i.log.gz
log4j.appender.audit.rollingPolicy.maxIndex=13
log4j.appender.audit.triggeringPolicy=org.apache.log4j.rolling.SizeBasedTriggeringPolicy
log4j.appender.audit.triggeringPolicy.maxFileSize=50000000
log4j.appender.audit.layout=org.apache.log4j.PatternLayout
log4j.appender.audit.layout.ConversionPattern=%m%n
log4j.logger.org.apache.hadoop.yarn.client=WARN
log4j.logger.org.apache.ambari.server.security.authorization=WARN
log4j.logger.org.apache.ambari.server.security.authorization.AuthorizationHelper=INFO
log4j.logger.org.apache.ambari.server.security.authorization.AmbariLdapBindAuthenticator=INFO

65
ambari/data/ambari/metrics.properties Executable file
View File

@ -0,0 +1,65 @@
# Copyright 2011 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#################### Metrics Source Configs #####################
#Metric sources : jvm,database
metric.sources=jvm,event
#### JVM Source Configs ###
source.jvm.class=org.apache.ambari.server.metrics.system.impl.JvmMetricsSource
source.event.class=org.apache.ambari.server.metrics.system.impl.StompEventsMetricsSource
source.jvm.interval=10
#### Database Source Configs ###
# Note : To enable Database metrics source completely, add the following property to ambari.properties as well
# server.persistence.properties.eclipselink.profiler=org.apache.ambari.server.metrics.system.impl.AmbariPerformanceMonitor
source.database.class=org.apache.ambari.server.metrics.system.impl.DatabaseMetricsSource
# Ambari performance monitor granularity : NONE / NORMAL / HEAVY / ALL
source.database.performance.monitor.query.weight=HEAVY
# Collection interval in milliseconds
source.database.monitor.dumptime=60000
# Include entities to be tracked.
source.database.monitor.entities=Cluster(.*)Entity,Host(.*)Entity,ExecutionCommandEntity,ServiceComponentDesiredStateEntity,Alert(.*)Entity,StackEntity,StageEntity
# Include some metrics which have the keyword even if they are not part of requested Entities.
# Examples
# Query Types
# ReadAllQuery,ReadObjectQuery,UpdateObjectQuery,ReportQuery,InsertObjectQuery,ValueReadQuery,DataModifyQuery,DoesExistQuery
# Query stages
# ObjectBuilding,QueryPreparation,SqlPrepare,SqlGeneration,StatementExecute,RowFetch,ConnectCalls,UnitOfWorkCommit,ClientSessionReleases,ConnectionManagement,CacheHits
# Sequences
# host_role_command_id_seq,alert_history_id_seq
source.database.monitor.query.keywords.include=CacheMisses
################################################################
############## General Metrics Service Configs #################
#ambariserver.hostname.override=
################################################################

1
ambari/data/ambari/password.dat Normal file
View File

@ -0,0 +1 @@
ambari

11
ambari/data/ssh/config Normal file
View File

@ -0,0 +1,11 @@
Host ambari-agent1
HostName 172.16.1.101
User root
Host ambari-agent2
HostName 172.16.1.102
User root
Host ambari-agent3
HostName 172.16.1.102
User root

31
ambari/docker-compose.yml Normal file
View File

@ -0,0 +1,31 @@
version: "3.7"
services:
server:
image: vimagick/ambari
hostname: ambari-server
ports:
- "8080:8080"
- "8440:8440"
- "8441:8441"
volumes:
- ./data/ambari:/etc/ambari-server/conf
- ./data/log:/var/log/ambari-server
extra_hosts:
- ambari-agent1:172.16.1.101
- ambari-agent2:172.16.1.102
- ambari-agent3:172.16.1.103
depends_on:
- postgres
restart: unless-stopped
postgres:
image: postgres:alpine
volumes:
- ./data/postgres:/var/lib/postgresql/data
environment:
- POSTGRES_USER=ambari
- POSTGRES_PASSWORD=ambari
- POSTGRES_DB=ambari
restart: unless-stopped

40
apacheds/Dockerfile Normal file
View File

@ -0,0 +1,40 @@
#
# Dockerfile for apacheds
#
FROM debian:buster
MAINTAINER EasyPi Software Foundation
ENV GOSU_VERSION=1.11
ENV GOSU_URL=https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-amd64
ENV APACHEDS_VERSION=2.0.0.AM25
ENV APACHEDS_FILE=apacheds-${APACHEDS_VERSION}-amd64.deb
ENV APACHEDS_URL=http://mirrors.advancedhosters.com/apache/directory/apacheds/dist/${APACHEDS_VERSION}/${APACHEDS_FILE}
ENV APACHEDS_USER=apacheds
ENV APACHEDS_GROUP=apacheds
ENV APACHEDS_DATA=/var/lib/apacheds
ENV APACHEDS_INSTANCE=default
ENV APACHEDS_INSTANCE_DIR=${APACHEDS_DATA}/${APACHEDS_INSTANCE}
RUN set -xe \
&& apt update \
&& apt install -y \
apt-utils \
curl \
ldap-utils \
openjdk-11-jre-headless \
procps \
&& curl -sSL ${GOSU_URL} > /usr/local/bin/gosu \
&& chmod +x /usr/local/bin/gosu \
&& gosu nobody true \
&& curl -sSL ${APACHEDS_URL} > ${APACHEDS_FILE} \
&& dpkg -i ${APACHEDS_FILE} \
&& sed -i "/INSTANCES_DIRECTORY/s/-${APACHEDS_VERSION}//" /opt/apacheds-${APACHEDS_VERSION}/bin/apacheds \
&& rm -rf ${APACHEDS_FILE} /var/lib/apt/lists/*
EXPOSE 10389 10636
COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
ENTRYPOINT ["docker-entrypoint.sh"]

13
apacheds/README.md Normal file
View File

@ -0,0 +1,13 @@
apacheds
========
```yaml
apacheds:
image: vimagick/apacheds
ports:
- "10389:10389"
- "10636:10636"
volumes:
- ./data:/var/lib/apacheds
restart: unless-stopped
```

8
apacheds/docker-compose.yml Normal file
View File

@ -0,0 +1,8 @@
apacheds:
image: vimagick/apacheds
ports:
- "10389:10389"
- "10636:10636"
volumes:
- ./data:/var/lib/apacheds
restart: unless-stopped

12
apacheds/docker-entrypoint.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/sh
set -e
if ! [ -d ${APACHEDS_INSTANCE_DIR} ]; then
cp -r ${APACHEDS_DATA}-${APACHEDS_VERSION}/default ${APACHEDS_INSTANCE_DIR}
chown -R ${APACHEDS_USER}:${APACHEDS_GROUP} ${APACHEDS_INSTANCE_DIR}
fi
gosu ${APACHEDS_USER} /opt/apacheds-${APACHEDS_VERSION}/bin/apacheds start ${APACHEDS_INSTANCE}
tail -n 0 -f ${APACHEDS_INSTANCE_DIR}/log/apacheds.log

15
apm-server/README.md Normal file
View File

@ -0,0 +1,15 @@
apm-server
==========
The [APM Server][1] receives data from [APM agents][2] and transforms them into
Elasticsearch documents. It does this by exposing an HTTP server endpoint to
which agents stream the APM data they collect. After the APM Server has
validated and processed events from the APM agents, the server transforms the
data into Elasticsearch documents and stores them in corresponding
Elasticsearch indices.
The APM Server works in conjunction with APM agents, Elasticsearch, and Kibana.
Please view the APM Overview for details on how these components work together.
[1]: https://www.elastic.co/guide/en/apm/server/current/index.html
[2]: https://www.elastic.co/guide/en/apm/agent/index.html

841
apm-server/data/apm-server.yml Normal file
View File

@ -0,0 +1,841 @@
################### APM Server Configuration #########################
############################# APM Server ######################################
apm-server:
# Defines the host and port the server is listening on. use "unix:/path/to.sock" to listen on a unix domain socket.
host: "0.0.0.0:8200"
# Maximum permitted size in bytes of a request's header accepted by the server to be processed.
#max_header_size: 1048576
# Maximum permitted duration for reading an entire request.
#read_timeout: 30s
# Maximum permitted duration for writing a response.
#write_timeout: 30s
# Maximum duration in seconds before releasing resources when shutting down the server.
#shutdown_timeout: 5s
# Maximum allowed size in bytes of a single event
#max_event_size: 307200
#--
# Maximum number of new connections to accept simultaneously (0 means unlimited)
# max_connections: 0
# Authorization token to be checked. If a token is set here the agents must
# send their token in the following format: Authorization: Bearer <secret-token>.
# It is recommended to use an authorization token in combination with SSL enabled,
# and save the token in the beats keystore.
#secret_token:
#ssl.enabled: false
#ssl.certificate : "path/to/cert"
#ssl.key : "path/to/private_key"
# It is recommended to use the provided keystore instead of entering the passphrase in plain text.
#ssl.key_passphrase: ""
#rum:
# To enable real user monitoring (RUM) support set this to true.
#enabled: false
#event_rate:
# Defines the maximum amount of events allowed to be sent to the APM Server RUM
# endpoint per ip per second. Defaults to 300.
#limit: 300
# An LRU cache is used to keep a rate limit per IP for the most recently seen IPs.
# This setting defines the number of unique IPs that can be tracked in the cache.
# Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
#lru_size: 1000
#-- General RUM settings
# Comma separated list of permitted origins for real user monitoring.
# User-agents will send an origin header that will be validated against this list.
# An origin is made of a protocol scheme, host and port, without the url path.
# Allowed origins in this setting can have * to match anything (eg.: http://*.example.com)
# If an item in the list is a single '*', everything will be allowed
#allow_origins : ['*']
# Regexp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes.
# If the regexp matches, the stacktrace frame is considered to be a library frame.
#library_pattern: "node_modules|bower_components|~"
# Regexp to be matched against a stacktrace frame's `file_name`.
# If the regexp matches, the stacktrace frame is not used for calculating error groups.
# The default pattern excludes stacktrace frames that have a filename starting with '/webpack'
#exclude_from_grouping: "^/webpack"
# If a source map has previously been uploaded, source mapping is automatically applied
# to all error and transaction documents sent to the RUM endpoint.
#source_mapping:
# Source maps are always fetched from Elasticsearch, by default using the output.elasticsearch configuration.
# A different instance must be configured when using any other output.
# This setting only affects sourcemap reads - the output determines where sourcemaps are written.
#elasticsearch:
# Array of hosts to connect to.
# Scheme and port can be left out and will be set to the default (http and 9200)
# In case you specify and additional path, the scheme is required: http://localhost:9200/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
# hosts: ["localhost:9200"]
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
# The `cache.expiration` determines how long a source map should be cached before fetching it again from Elasticsearch.
# Note that values configured without a time unit will be interpreted as seconds.
#cache:
#expiration: 5m
# Source maps are stored in a seperate index.
# If the default index pattern for source maps at 'outputs.elasticsearch.indices'
# is changed, a matching index pattern needs to be specified here.
#index_pattern: "apm-*-sourcemap*"
# If set to true, APM Server augments data received by the agent with the original IP of the backend server,
# or the IP and User Agent of the real user (RUM requests). It defaults to true.
#capture_personal_data: true
# golang expvar support - https://golang.org/pkg/expvar/
#expvar:
# Set to true to Expose expvar
#enabled: false
# Url to expose expvar
#url: "/debug/vars"
# Instrumentation support for the server's HTTP endpoints and event publisher.
#instrumentation:
# Set to true to enable instrumentation of the APM server itself.
#enabled: false
# Environment in which the APM Server is running on (eg: staging, production, etc.)
#environment: ""
# Remote host to report instrumentation results to.
#hosts:
# - http://remote-apm-server:8200
# Remote apm-servers' secret_token
#secret_token:
# Metrics endpoint
#metrics:
# Set to false to disable the metrics endpoint
#enabled: true
# A pipeline is a definition of processors applied to documents when writing them to Elasticsearch.
# Using pipelines involves two steps:
# (1) registering a pipeline
# (2) applying a pipeline during data ingestion (see `output.elasticsearch.pipelines`)
#
# You can manually register pipelines, or use this configuration option to ensure
# pipelines are loaded and registered at the configured Elasticsearch instances.
# Automatic pipeline registration requires
# * `output.elasticsearch` to be enabled and configured.
# * having the required Elasticsearch Processor Plugins installed.
# APM Server default pipelines require you to have the `Ingest User Agent Plugin` installed.
# Find the default pipeline configuration at `ingest/pipeline/definition.json`.
#
#register.ingest.pipeline:
# Registers pipeline definitions in Elasticsearch on APM Server startup. Defaults to false.
#enabled: false
# Overwrites existing pipeline definitions in Elasticsearch. Defaults to true.
#overwrite: true
#================================ General ======================================
# Internal queue configuration for buffering events to be published.
#queue:
# Queue type by name (default 'mem')
# The memory queue will present all available events (up to the outputs
# bulk_max_size) to the output, the moment the output is ready to server
# another batch of events.
#mem:
# Max number of events the queue can buffer.
#events: 4096
# Hints the minimum number of events stored in the queue,
# before providing a batch of events to the outputs.
# A value of 0 (the default) ensures events are immediately available
# to be sent to the outputs.
#flush.min_events: 2048
# Maximum duration after which events are available to the outputs,
# if the number of events stored in the queue is < min_flush_events.
#flush.timeout: 1s
# Sets the maximum number of CPUs that can be executing simultaneously. The
# default is the number of logical CPUs available in the system.
#max_procs:
#============================== Template =====================================
# A template is used to set the mapping in Elasticsearch
# By default template loading is enabled and the template is loaded.
# These settings can be adjusted to load your own template or overwrite existing ones.
# Set to false to disable template loading.
#setup.template.enabled: true
# Template name. By default the template name is "apm-%{[beat.version]}"
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.name: "apm-%{[beat.version]}"
# Template pattern. By default the template pattern is "apm-%{[beat.version]}-*" to apply to the default index settings.
# The first part is the version of the beat and then -* is used to match all daily indices.
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.pattern: "apm-%{[beat.version]}-*"
# Path to fields.yml file to generate the template
#setup.template.fields: "${path.config}/fields.yml"
# Overwrite existing template
#setup.template.overwrite: false
# Elasticsearch template settings
#setup.template.settings:
# A dictionary of settings to place into the settings.index dictionary
# of the Elasticsearch template. For more details, please check
# https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
#index:
#number_of_shards: 1
#codec: best_compression
#number_of_routing_shards: 30
#mapping.total_fields.limit: 2000
# A dictionary of settings for the _source field. For more details, please check
# https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
#_source:
#enabled: false
#============================== Deprecated: Dashboards =====================================
#
# Deprecated: Loading dashboards from the APM Server into Kibana is deprecated from 6.4 on.
# We suggest to use the Kibana UI to load APM Server dashboards and index pattern instead.
#
# These settings control loading the sample dashboards to the Kibana index. Loading
# the dashboards are disabled by default and can be enabled either by setting the
# options here, or by using the `-setup` CLI flag or the `setup` command.
#setup.dashboards.enabled: false
# The directory from where to read the dashboards. The default is the `kibana`
# folder in the home path.
#setup.dashboards.directory: ${path.home}/kibana
# The URL from where to download the dashboards archive. It is used instead of
# the directory if it has a value.
#setup.dashboards.url:
# The file archive (zip file) from where to read the dashboards. It is used instead
# of the directory when it has a value.
#setup.dashboards.file:
# The name of the Kibana index to use for setting the configuration. Default is ".kibana"
#setup.dashboards.kibana_index: .kibana
# The Elasticsearch index name. This overwrites the index name defined in the
# dashboards and index pattern. Example: testbeat-*
# The dashboards.index needs to be changed in case the elasticsearch index pattern is modified.
#setup.dashboards.index:
# Always use the Kibana API for loading the dashboards instead of autodetecting
# how to install the dashboards by first querying Elasticsearch.
#setup.dashboards.always_kibana: false
# If true and Kibana is not reachable at the time when dashboards are loaded,
# it will retry to reconnect to Kibana instead of exiting with an error.
#setup.dashboards.retry.enabled: false
# Duration interval between Kibana connection retries.
#setup.dashboards.retry.interval: 1s
# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
#setup.dashboards.retry.maximum: 0
#============================== Deprecated: Kibana =====================================
# Deprecated: Starting with APM Server version 6.4, loading dashboards and index pattern
# from the APM Server into Kibana is deprecated.
# We suggest to use the Kibana UI to load APM Server dashboards and index pattern instead.
#
# Setting up a Kibana endpoint is not necessary when loading the index pattern and dashboards via the UI.
#setup.kibana:
# Kibana Host
# Scheme and port can be left out and will be set to the default (http and 5601)
# In case you specify and additional path, the scheme is required: http://localhost:5601/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
#host: "localhost:5601"
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
# Optional HTTP Path
#path: ""
# Use SSL settings for HTTPS. Default is false.
#ssl.enabled: false
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# SSL configuration. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
# It is recommended to use the provided keystore instead of entering the passphrase in plain text.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
#============================= Elastic Cloud ==================================
# These settings simplify using APM Server with the Elastic Cloud (https://cloud.elastic.co/).
# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
# [deprecated] `setup.kibana.host` options.
# You can find the `cloud.id` in the Elastic Cloud web UI.
#cloud.id:
# The cloud.auth setting overwrites the `output.elasticsearch.username` and
# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
#cloud.auth:
#================================ Outputs =====================================
# Configure what output to use when sending the data collected by the beat.
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
hosts: ["elasticsearch:9200"]
# Boolean flag to enable or disable the output module.
#enabled: true
# Set gzip compression level.
#compression_level: 0
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
# Dictionary of HTTP parameters to pass within the url with index operations.
#parameters:
#param1: value1
#param2: value2
# Number of workers per Elasticsearch host.
#worker: 1
# By using the configuration below, apm documents are stored to separate indices,
# depending on their `processor.event`:
# - error
# - transaction
# - span
# - sourcemap
#
# The indices are all prefixed with `apm-%{[beat.version]}`.
# To allow managing indices based on their age, all indices (except for sourcemaps)
# end with the information of the day they got indexed.
# e.g. "apm-6.3.0-transaction-2018.03.20"
#
# Be aware that you can only specify one Elasticsearch template and one Kibana Index Pattern,
# In case you modify the index patterns you must also update those configurations accordingly,
# as they need to be aligned:
# * `setup.template.name`
# * `setup.template.pattern`
# * `setup.dashboards.index`
#index: "apm-%{[beat.version]}-%{+yyyy.MM.dd}"
indices:
- index: "apm-%{[beat.version]}-sourcemap"
when.contains:
processor.event: "sourcemap"
- index: "apm-%{[beat.version]}-error-%{+yyyy.MM.dd}"
when.contains:
processor.event: "error"
- index: "apm-%{[beat.version]}-transaction-%{+yyyy.MM.dd}"
when.contains:
processor.event: "transaction"
- index: "apm-%{[beat.version]}-span-%{+yyyy.MM.dd}"
when.contains:
processor.event: "span"
- index: "apm-%{[beat.version]}-metric-%{+yyyy.MM.dd}"
when.contains:
processor.event: "metric"
- index: "apm-%{[beat.version]}-onboarding-%{+yyyy.MM.dd}"
when.contains:
processor.event: "onboarding"
# A pipeline is a definition of processors applied to documents when writing them to Elasticsearch.
# APM Server comes with a default pipeline definition, located at `ingets/pipeline/definition.json`.
# Pipelines are disabled by default. To make use of them you have to:
# (1) ensure pipelines are registered in Elasticsearch, see `apm-server.register.ingest.pipeline`
# (2) enable the following:
#pipelines:
#- pipeline: "apm_user_agent"
# Optional HTTP Path
#path: "/elasticsearch"
# Custom HTTP headers to add to each request
#headers:
# X-My-Header: Contents of the header
# Proxy server url
#proxy_url: http://proxy:3128
# The number of times a particular Elasticsearch index operation is attempted. If
# the indexing operation doesn't succeed after this many retries, the events are
# dropped. The default is 3.
#max_retries: 3
# The maximum number of events to bulk in a single Elasticsearch bulk API index request.
# The default is 50.
#bulk_max_size: 50
# The number of seconds to wait before trying to reconnect to Elasticsearch
# after a network error. After waiting backoff.init seconds, apm-server
# tries to reconnect. If the attempt fails, the backoff timer is increased
# exponentially up to backoff.max. After a successful connection, the backoff
# timer is reset. The default is 1s.
#backoff.init: 1s
# The maximum number of seconds to wait before attempting to connect to
# Elasticsearch after a network error. The default is 60s.
#backoff.max: 60s
# Configure http request timeout before failing an request to Elasticsearch.
#timeout: 90
# Use SSL settings for HTTPS. Default is false.
#ssl.enabled: false
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# SSL configuration. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
# It is recommended to use the provided keystore instead of entering the passphrase in plain text.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#----------------------------- Console output ---------------------------------
#output.console:
# Boolean flag to enable or disable the output module.
#enabled: false
# Pretty print json event
#pretty: false
#----------------------------- Logstash output ---------------------------------
#output.logstash:
# Boolean flag to enable or disable the output module.
#enabled: false
# The Logstash hosts
#hosts: ["localhost:5044"]
# Number of workers per Logstash host.
#worker: 1
# Set gzip compression level.
#compression_level: 3
# Configure escaping html symbols in strings.
#escape_html: true
# Optional maximum time to live for a connection to Logstash, after which the
# connection will be re-established. A value of `0s` (the default) will
# disable this feature.
#
# Not yet supported for async connections (i.e. with the "pipelining" option set)
#ttl: 30s
# Optional load balance the events between the Logstash hosts. Default is false.
#loadbalance: false
# Number of batches to be sent asynchronously to Logstash while processing
# new batches.
#pipelining: 2
# If enabled only a subset of events in a batch of events is transferred per
# group. The number of events to be sent increases up to `bulk_max_size`
# if no error is encountered.
#slow_start: false
# The number of seconds to wait before trying to reconnect to Logstash
# after a network error. After waiting backoff.init seconds, apm-server
# tries to reconnect. If the attempt fails, the backoff timer is increased
# exponentially up to backoff.max. After a successful connection, the backoff
# timer is reset. The default is 1s.
#backoff.init: 1s
# The maximum number of seconds to wait before attempting to connect to
# Logstash after a network error. The default is 60s.
#backoff.max: 60s
# Optional index name. The default index name is set to apm
# in all lowercase.
#index: 'apm'
# SOCKS5 proxy server URL
#proxy_url: socks5://user:password@socks5-server:2233
# Resolve names locally when using a proxy server. Defaults to false.
#proxy_use_local_resolver: false
# Enable SSL support. SSL is automatically enabled, if any SSL setting is set.
#ssl.enabled: true
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# Optional SSL configuration options. SSL is off by default.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#------------------------------- Kafka output ----------------------------------
#output.kafka:
# Boolean flag to enable or disable the output module.
#enabled: false
# The list of Kafka broker addresses from where to fetch the cluster metadata.
# The cluster metadata contain the actual Kafka brokers events are published
# to.
#hosts: ["localhost:9092"]
# The Kafka topic used for produced events. The setting can be a format string
# using any event field. To set the topic from document type use `%{[type]}`.
#topic: beats
# The Kafka event key setting. Use format string to create unique event key.
# By default no event key will be generated.
#key: ''
# The Kafka event partitioning strategy. Default hashing strategy is `hash`
# using the `output.kafka.key` setting or randomly distributes events if
# `output.kafka.key` is not configured.
#partition.hash:
# If enabled, events will only be published to partitions with reachable
# leaders. Default is false.
#reachable_only: false
# Configure alternative event field names used to compute the hash value.
# If empty `output.kafka.key` setting will be used.
# Default value is empty list.
#hash: []
# Authentication details. Password is required if username is set.
#username: ''
#password: ''
# Kafka version libbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
# Configure JSON encoding
#codec.json:
# Pretty print json event
#pretty: false
# Configure escaping html symbols in strings.
#escape_html: true
# Metadata update configuration. Metadata do contain leader information
# deciding which broker to use when publishing.
#metadata:
# Max metadata request retry attempts when cluster is in middle of leader
# election. Defaults to 3 retries.
#retry.max: 3
# Waiting time between retries during leader elections. Default is 250ms.
#retry.backoff: 250ms
# Refresh metadata interval. Defaults to every 10 minutes.
#refresh_frequency: 10m
# The number of concurrent load-balanced Kafka output workers.
#worker: 1
# The number of times to retry publishing an event after a publishing failure.
# After the specified number of retries, the events are typically dropped.
# Some Beats, such as Filebeat, ignore the max_retries setting and retry until
# all events are published. Set max_retries to a value less than 0 to retry
# until all events are published. The default is 3.
#max_retries: 3
# The maximum number of events to bulk in a single Kafka request. The default
# is 2048.
#bulk_max_size: 2048
# The number of seconds to wait for responses from the Kafka brokers before
# timing out. The default is 30s.
#timeout: 30s
# The maximum duration a broker will wait for number of required ACKs. The
# default is 10s.
#broker_timeout: 10s
# The number of messages buffered for each Kafka broker. The default is 256.
#channel_buffer_size: 256
# The keep-alive period for an active network connection. If 0s, keep-alives
# are disabled. The default is 0 seconds.
#keep_alive: 0
# Sets the output compression codec. Must be one of none, snappy and gzip. The
# default is gzip.
#compression: gzip
# The maximum permitted size of JSON-encoded messages. Bigger messages will be
# dropped. The default value is 1000000 (bytes). This value should be equal to
# or less than the broker's message.max.bytes.
#max_message_bytes: 1000000
# The ACK reliability level required from broker. 0=no response, 1=wait for
# local commit, -1=wait for all replicas to commit. The default is 1. Note:
# If set to 0, no ACKs are returned by Kafka. Messages might be lost silently
# on error.
#required_acks: 1
# The configurable ClientID used for logging, debugging, and auditing
# purposes. The default is "beats".
#client_id: beats
# Enable SSL support. SSL is automatically enabled, if any SSL setting is set.
#ssl.enabled: true
# Optional SSL configuration options. SSL is off by default.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#================================= Paths ======================================
# The home path for the apm-server installation. This is the default base path
# for all other path settings and for miscellaneous files that come with the
# distribution (for example, the sample dashboards).
# If not set by a CLI flag or in the configuration file, the default for the
# home path is the location of the binary.
#path.home:
# The configuration path for the apm-server installation. This is the default
# base path for configuration files, including the main YAML configuration file
# and the Elasticsearch template file. If not set by a CLI flag or in the
# configuration file, the default for the configuration path is the home path.
#path.config: ${path.home}
# The data path for the apm-server installation. This is the default base path
# for all the files in which apm-server needs to store its data. If not set by a
# CLI flag or in the configuration file, the default for the data path is a data
# subdirectory inside the home path.
#path.data: ${path.home}/data
# The logs path for a apm-server installation. This is the default location for
# the Beat's log files. If not set by a CLI flag or in the configuration file,
# the default for the logs path is a logs subdirectory inside the home path.
#path.logs: ${path.home}/logs
#================================ Logging ======================================
#
# There are three options for the log output: syslog, file, stderr.
# Under Windows systems, the log files are per default sent to the file output,
# under all other system per default to syslog.
# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
#logging.level: info
# Enable debug output for selected components. To enable all selectors use ["*"]
# Other available selectors are "beat", "publish", "service"
# Multiple selectors can be chained.
#logging.selectors: [ ]
# Send all logging output to syslog. The default is false.
#logging.to_syslog: true
# If enabled, apm-server periodically logs its internal metrics that have changed
# in the last period. For each metric that changed, the delta from the value at
# the beginning of the period is logged. Also, the total values for
# all non-zero internal metrics are logged on shutdown. The default is true.
#logging.metrics.enabled: false
# The period after which to log the internal metrics. The default is 30s.
#logging.metrics.period: 30s
# Logging to rotating files. Set logging.to_files to false to disable logging to
# files.
#logging.to_files: true
#logging.files:
# Configure the path where the logs are written. The default is the logs directory
# under the home path (the binary location).
#path: /var/log/apm-server
# The name of the files where the logs are written to.
#name: apm-server
# Configure log file size limit. If limit is reached, log file will be
# automatically rotated
#rotateeverybytes: 10485760 # = 10MB
# Number of rotated log files to keep. Oldest files will be deleted first.
#keepfiles: 7
# The permissions mask to apply when rotating log files. The default value is 0600.
# Must be a valid Unix-style file permissions mask expressed in octal notation.
#permissions: 0600
# Set to true to log messages in json format.
#logging.json: false
#================================ HTTP Endpoint ======================================
#
# Each beat can expose internal metrics through a HTTP endpoint. For security
# reasons the endpoint is disabled by default. This feature is currently experimental.
# Stats can be access through http://localhost:5066/stats . For pretty JSON output
# append ?pretty to the URL.
# Defines if the HTTP endpoint is enabled.
#http.enabled: false
# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
#http.host: localhost
# Port on which the HTTP endpoint will bind. Default is 5066.
#http.port: 5066
#============================== Xpack Monitoring ===============================
# APM server can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
# reporting is disabled by default.
# Set to true to enable the monitoring reporter.
#xpack.monitoring.enabled: false
# Uncomment to send the metrics to Elasticsearch. Most settings from the
# Elasticsearch output are accepted here as well. Any setting that is not set is
# automatically inherited from the Elasticsearch output configuration, so if you
# have the Elasticsearch output configured, you can simply uncomment the
# following line.
#xpack.monitoring.elasticsearch:
# username: "apm_system"
# password: ""

9
apm-server/docker-compose.yml Normal file
View File

@ -0,0 +1,9 @@
apm-server:
image: docker.elastic.co/apm/apm-server:6.5.4
ports:
- "8200:8200"
volumes:
- ./data:/usr/share/apm-server
extra_hosts:
- elasticsearch:1.2.3.4
restart: always

7
artifactory/README.md Normal file
View File

@ -0,0 +1,7 @@
artifactory
===========
[JFrog Artifactory][1] is the only Universal Repository Manager supporting all major
packaging formats, build tools and CI servers.
[1]: https://www.jfrog.com/confluence/display/RTF/Welcome+to+Artifactory

35
artifactory/docker-compose.yml Normal file
View File

@ -0,0 +1,35 @@
#
# https://www.jfrog.com/confluence/display/RTF/Installing+with+Docker
#
version: "3.7"
services:
postgres:
image: postgres:12-alpine
ports:
- "5432:5432"
volumes:
- ./data/postgres:/var/lib/postgresql/data
environment:
- POSTGRES_DB=artifactory
- POSTGRES_USER=artifactory
- POSTGRES_PASSWORD=artifactory
restart: always
artifactory:
image: docker.bintray.io/jfrog/artifactory-oss:6.13.1
ports:
- "8081:8081"
volumes:
- ./data/artifactory:/var/opt/jfrog/artifactory
environment:
- DB_TYPE=postgresql
- DB_HOST=postgres
- DB_USER=artifactory
- DB_PASSWORD=artifactory
- EXTRA_JAVA_OPTIONS=-Xms512m -Xmx2g -Xss256k -XX:+UseG1GC
depends_on:
- postgres
restart: always

9
awx/README.md Normal file
View File

@ -0,0 +1,9 @@
awx
===
[AWX][1] provides a web-based user interface, REST API, and task engine built
on top of Ansible. It is the upstream project for [Tower][2], a commercial
derivative of AWX.
[1]: https://github.com/ansible/awx
[2]: https://www.ansible.com/tower

1
awx/data/etc/SECRET_KEY Normal file
View File

@ -0,0 +1 @@
isasekrit

29
awx/data/etc/credentials.py Normal file
View File

@ -0,0 +1,29 @@
DATABASES = {
'default': {
'ATOMIC_REQUESTS': True,
'ENGINE': 'django.db.backends.postgresql',
'NAME': "awx",
'USER': "awx",
'PASSWORD': "awxpass",
'HOST': "postgres",
'PORT': "5432",
}
}
BROKER_URL = 'amqp://guest:guest@rabbitmq:5672/awx'
CHANNEL_LAYERS = {
'default': {'BACKEND': 'asgi_amqp.AMQPChannelLayer',
'ROUTING': 'awx.main.routing.channel_routing',
'CONFIG': {'url': BROKER_URL}}
}
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': 'memcached:11211'
},
'ephemeral': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
},
}

12
awx/data/etc/environment.sh Normal file
View File

@ -0,0 +1,12 @@
DATABASE_USER=awx
DATABASE_NAME=awx
DATABASE_HOST=postgres
DATABASE_PORT=5432
DATABASE_PASSWORD=awxpass
DATABASE_ADMIN_PASSWORD=postgrespass
MEMCACHED_HOST=memcached
MEMCACHED_PORT=11211
RABBITMQ_HOST=rabbitmq
RABBITMQ_PORT=5672
AWX_ADMIN_USER=admin
AWX_ADMIN_PASSWORD=password

69
awx/docker-compose.yml Normal file
View File

@ -0,0 +1,69 @@
#
# https://github.com/ansible/awx/raw/devel/installer/roles/local_docker/templates/docker-compose.yml.j2
#
version: '2'
services:
web:
image: ansible/awx_web
container_name: awx_web
hostname: awxweb
user: root
ports:
- "8052:8052"
volumes:
- ./data/etc/SECRET_KEY:/etc/tower/SECRET_KEY
- ./data/etc/environment.sh:/etc/tower/conf.d/environment.sh
- ./data/etc/credentials.py:/etc/tower/conf.d/credentials.py
- ./data/awx:/var/lib/awx/projects
depends_on:
- rabbitmq
- memcached
- postgres
restart: unless-stopped
task:
image: ansible/awx_task
container_name: awx_task
hostname: awx
user: root
volumes:
- ./data/etc/SECRET_KEY:/etc/tower/SECRET_KEY
- ./data/etc/environment.sh:/etc/tower/conf.d/environment.sh
- ./data/etc/credentials.py:/etc/tower/conf.d/credentials.py
- ./data/awx:/var/lib/awx/projects
depends_on:
- rabbitmq
- memcached
- web
- postgres
restart: unless-stopped
rabbitmq:
image: ansible/awx_rabbitmq
container_name: awx_rabbitmq
environment:
RABBITMQ_DEFAULT_VHOST: awx
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
RABBITMQ_ERLANG_COOKIE: cookiemonster
restart: unless-stopped
memcached:
image: memcached:alpine
container_name: awx_memcached
restart: unless-stopped
postgres:
image: postgres:12-alpine
container_name: awx_postgres
volumes:
- ./data/postgres:/var/lib/postgresql/data/pgdata:Z
environment:
POSTGRES_USER: awx
POSTGRES_PASSWORD: awxpass
POSTGRES_DB: awx
PGDATA: /var/lib/postgresql/data/pgdata
restart: unless-stopped

58
browserless/README.md Normal file
View File

@ -0,0 +1,58 @@
browserless
===========
[Browserless][1] makes it easy to run your puppeteer scripts in an optimized
way. It takes care of all the binaries and managing of Chrome so you don't have
to.
## docker-compose.yml
```yaml
browserless:
image: browserless/chrome
ports:
- "3000:3000"
environment:
- DEBUG=browserless/chrome
- MAX_CONCURRENT_SESSIONS=10
- CONNECTION_TIMEOUT=600000
- MAX_QUEUE_LENGTH=10
- ENABLE_CORS=true
- CHROME_REFRESH_TIME=3600000
shm_size: 2gb
restart: always
```
## screenshot.js
```javascript
'use strict';
const puppeteer = require('puppeteer');
(async() => {
const browser = await puppeteer.connect({browserWSEndpoint: 'ws://localhost:3000'});
const page = await browser.newPage();
await page.goto('https://www.google.com/', {waitUntil: 'networkidle2'});
await page.screenshot({path: 'google.png', fullPage: true});
await browser.close();
})();
```
## Up and Running
```bash
$ docker-compose up -d
$ PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install puppeteer
$ node screenshot.js
$ imgcat google.png
$ http http://127.0.0.1:3000/screenshot \
url=https://www.youtube.com \
options:='{"fullPage":true}' \
gotoOptions:='{"waitUntil":"networkidle2"}' > youtube.png
$ imgcat youtube.png
```
[1]: https://docs.browserless.io/

14
browserless/docker-compose.yml Normal file
View File

@ -0,0 +1,14 @@
browserless:
image: browserless/chrome
ports:
- "3000:3000"
environment:
- DEBUG=browserless/chrome
- MAX_CONCURRENT_SESSIONS=10
- CONNECTION_TIMEOUT=600000
- MAX_QUEUE_LENGTH=10
- CHROME_REFRESH_TIME=3600000
- ENABLE_CORS=true
- EXIT_ON_HEALTH_FAILURE=true
shm_size: 2gb
restart: always

25
browserless/docker-stack.yml Normal file
View File

@ -0,0 +1,25 @@
version: '3.5'
services:
browserless:
image: browserless/chrome:latest
ports:
- "3000:3000"
environment:
- DEBUG=browserless/chrome
- MAX_CONCURRENT_SESSIONS=10
- CONNECTION_TIMEOUT=300000
- MAX_QUEUE_LENGTH=10
- ENABLE_CORS=true
- EXIT_ON_HEALTH_FAILURE=true
deploy:
resources:
limits:
memory: 1024M
replicas: 56
restart_policy:
condition: on-failure
networks:
default:
ipam:
config:
- subnet: 192.168.128.0/24

7
code-server/README.md Normal file
View File

@ -0,0 +1,7 @@
code-server
===========
[code-server][1] is VS Code running on a remote server, accessible through the browser.
[1]: https://github.com/cdr/code-server

9
code-server/docker-compose.yml Normal file
View File

@ -0,0 +1,9 @@
code-server:
image: codercom/code-server
command: --allow-http --no-auth --disable-telemetry
ports:
- "8443:8443"
volumes:
- ./data:/home/coder/project
- /home/coder/local/share/code-server
restart: unless-stopped

2
dante/Dockerfile
View File

@ -8,7 +8,7 @@ MAINTAINER kev <noreply@easypi.pro>
ENV DANTE_VER 1.4.2
ENV DANTE_URL https://www.inet.no/dante/files/dante-$DANTE_VER.tar.gz
ENV DANTE_SHA baa25750633a7f9f37467ee43afdf7a95c80274394eddd7dcd4e1542aa75caad
ENV DANTE_SHA 4c97cff23e5c9b00ca1ec8a95ab22972813921d7fbf60fc453e3e06382fc38a7
ENV DANTE_FILE dante.tar.gz
ENV DANTE_TEMP dante
ENV DANTE_DEPS build-essential curl

6
docker-compose-arm/Dockerfile
View File

@ -2,16 +2,18 @@
# Dockerfile for building docker-compose-arm
#
FROM resin/rpi-raspbian:jessie
FROM arm32v7/debian:stretch
MAINTAINER EasyPi Software Foundation
ENV DOCKER_COMPOSE_VERSION 1.22.0
ENV DOCKER_COMPOSE_VERSION 1.24.1
ENV PYINSTALLER_VERSION 3.2.1
RUN set -xe \
&& apt-get update \
&& apt-get install -y build-essential \
curl \
libffi-dev \
libssl-dev \
python-dev \
zlib1g-dev \
&& curl https://bootstrap.pypa.io/get-pip.py | python \

68
drone/README.md
View File

@ -5,43 +5,36 @@ drone
## github
```yaml
drone:
image: drone/drone:1.5-linux-amd64
ports:
- "8080:80"
volumes:
- ./data:/data
environment:
- DRONE_AGENTS_ENABLED=true
- DRONE_SERVER_PROTO=http
- DRONE_SERVER_HOST=drone.easypi.pro
- DRONE_RPC_SECRET=secret
- DRONE_GITHUB_SERVER=https://github.com
- DRONE_GITHUB_CLIENT_ID=xxxxxx
- DRONE_GITHUB_CLIENT_SECRET=xxxxxx
restart: always
```
```yaml
#
# Github » Settings » Applications » Developer applications » Register new application
#
Application name: drone
Homepage URL: http://drone.easypi.info/
Homepage URL: http://drone.easypi.pro/
Application description: Drone is a Continuous Integration platform built on Docker, written in Go
Authorization callback URL: http://drone.easypi.info/authorize
Authorization callback URL: http://drone.easypi.pro/authorize
Client ID: ... (generated by github)
Client Secret: ... (generated by github)
```
## docker-compose.yml
```yaml
drone:
image: drone/drone
ports:
- "8000:8000"
volumes:
- ./drone:/var/lib/drone
- /var/run/docker.sock:/var/run/docker.sock
environment:
- REMOTE_DRIVER=github
- REMOTE_CONFIG=https://github.com?client_id=...&client_secret=...
# - REMOTE_DRIVER=gogs
# - REMOTE_CONFIG=https://git.easypi.info/?open=false
- DEBUG=false
restart: always
wall:
image: drone/drone-wall
ports:
- "8080:80"
restart: always
```
> Drone will register gogs webhooks automatically, you don't need to do it manually.
## nginx/sites-enabled/drone
@ -49,7 +42,7 @@ wall:
```
server {
listen 80;
server_name drone.easypi.info;
server_name drone.easypi.pro;
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
@ -57,7 +50,7 @@ server {
proxy_set_header Host $http_host;
proxy_set_header Origin "";
proxy_pass http://127.0.0.1:8000;
proxy_pass http://127.0.0.1:8080;
proxy_redirect off;
proxy_http_version 1.1;
proxy_buffering off;
@ -65,14 +58,6 @@ server {
chunked_transfer_encoding off;
}
}
server {
listen 80;
server_name wall.easypi.info;
location / {
proxy_pass http://127.0.0.1:8080;
}
}
```
## up and running
@ -81,13 +66,10 @@ server {
# server
$ cd ~/fig/drone/
$ docker-compose up -d
$ docker-compose logs
$ docker-compose logs -f
# client (login with remote driver credential)
$ firefox http://drone.easypi.info/
# dashboard
$ firefox http://wall.easypi.info/
$ firefox http://drone.easypi.pro/
```
[1]: http://readme.drone.io/usage/overview/
[1]: https://readme.drone.io

23
drone/arm/Dockerfile
View File

@ -1,23 +0,0 @@
#
# Dockerfile for drone-arm
#
FROM easypi/alpine-arm
MAINTAINER EasyPi Software Foundation
RUN apk add --no-cache ca-certificates
ADD drone /drone
ENV DATABASE_DRIVER=sqlite3 \
DATABASE_CONFIG=/var/lib/drone/drone.sqlite \
GODEBUG=netdns=go \
XDG_CACHE_HOME=/var/lib/drone
VOLUME $XDG_CACHE_HOME
WORKDIR $XDG_CACHE_HOME
EXPOSE 8000
ENTRYPOINT ["/drone"]
CMD ["server"]

18
drone/arm/README.md
View File

@ -1,24 +1,6 @@
drone-arm
=========
```yaml
drone:
image: armdrone/drone
ports:
- "8000:80"
volumes:
- ./data:/var/lib/drone
- /var/run/docker.sock:/var/run/docker.sock
environment:
- REMOTE_DRIVER=gogs
- REMOTE_CONFIG=http://git.easypi.info:3000/?open=false
- PLUGIN_FILTER=armdrone/*
- GIN_MODE=release
extra_hosts:
- git.easypi.info:192.168.31.231
restart: always
```
## Deploy Key Setup
```yaml

60
drone/arm/docker-compose.yml
View File

@ -1,45 +1,38 @@
version: '2'
version: '3.7'
services:
drone-server:
image: quay.io/armswarm/drone:0.8
drone:
image: drone/drone:1.6-linux-arm
ports:
- "8000:8000"
- "9000:9000"
- "8080:80"
volumes:
- ./data/drone:/var/lib/drone
- ./data/drone:/data
environment:
- DRONE_OPEN=true
- DRONE_HOST=https://drone.easypi.pro
- DRONE_DATABASE_DRIVER=mysql
- DRONE_DATABASE_DATASOURCE=root:root@tcp(mariadb:3306)/drone?parseTime=true
- DRONE_GOGS=true
- DRONE_GOGS_URL=https://gogs.easypi.pro
- DRONE_SECRET=0123456789
- DRONE_ADMIN=root
- DRONE_SERVER_HOST=drone
- DRONE_SERVER_PROTO=http
- DRONE_RPC_SECRET=secret
- DRONE_DATABASE_DRIVER=sqlite3
- DRONE_DATABASE_DATASOURCE=/data/database.sqlite
- DRONE_GOGS_SERVER=http://gogs:3000
- DRONE_AGENTS_ENABLED=true
depends_on:
- mariadb
- postgres
- gogs
restart: always
drone-agent:
image: drone/agent:linux-arm
agent:
image: drone/agent:1.6-linux-arm
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- DRONE_SERVER=drone-server:9000
- DRONE_SECRET=0123456789
- DRONE_RPC_PROTO=http
- DRONE_RPC_HOST=drone
- DRONE_RPC_SECRET=secret
- DRONE_RUNNER_NAME=agent
- DRONE_RUNNER_CAPACITY=2
depends_on:
- drone-server
restart: always
mariadb:
image: easypi/mariadb-arm
ports:
- "3306:3306"
environment:
- MYSQL_ROOT_PASSWORD=root
- MYSQL_DATABASE=drone
- drone
restart: always
gogs:
@ -50,12 +43,3 @@ services:
volumes:
- ./data/gogs:/data
restart: always
phpmyadmin:
image: easypi/phpmyadmin-arm
ports:
- "8080:80"
environment:
- PMA_HOST=mariadb
- PMA_PORT=3306
restart: always

BIN
drone/arm/drone
View File

Binary file not shown.

73
drone/docker-compose.yml
View File

@ -1,18 +1,57 @@
drone:
image: drone/drone
ports:
- "8000:80"
volumes:
- ./data:/var/lib/drone
- /var/run/docker.sock:/var/run/docker.sock
environment:
- REMOTE_DRIVER=gogs
- REMOTE_CONFIG=http://git.easypi.info:3000/?open=false
- DEBUG=false
restart: always
version: "3.7"
wall:
image: drone/drone-wall
ports:
- "8080:80"
restart: always
services:
drone:
image: drone/drone:1.6-linux-amd64
ports:
- "8080:80"
volumes:
- ./data/drone:/data
environment:
- DRONE_SERVER_HOST=drone
- DRONE_SERVER_PROTO=http
- DRONE_RPC_SECRET=secret
- DRONE_DATABASE_DRIVER=postgres
- DRONE_DATABASE_DATASOURCE=postgres://drone:drone@postgres:5432/drone?sslmode=disable
- DRONE_GOGS_SERVER=http://gogs:3000
- DRONE_AGENTS_ENABLED=true
depends_on:
- postgres
- gogs
restart: always
agent:
image: drone/agent:1.6-linux-amd64
volumes:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- DRONE_RPC_PROTO=http
- DRONE_RPC_HOST=drone
- DRONE_RPC_SECRET=secret
- DRONE_RUNNER_NAME=agent
- DRONE_RUNNER_CAPACITY=2
depends_on:
- drone
restart: always
postgres:
image: postgres:12-alpine
ports:
- "5432:5432"
volumes:
- ./data/postgres:/var/lib/postgresql/data
environment:
- POSTGRES_USER=drone
- POSTGRES_PASSWORD=drone
- POSTGRES_DB=drone
restart: always
gogs:
image: gogs/gogs
ports:
- "2222:22"
- "3000:3000"
volumes:
- ./data/gogs:/data
restart: always

11
elastalert/README.md
View File

@ -1,7 +1,10 @@
ElastAlert
==========
http://elastalert.readthedocs.io/en/latest/
[ElastAlert][1] is a simple framework for alerting on anomalies, spikes, or other
patterns of interest from data in Elasticsearch.
## up and running
```bash
$ docker-compose up -d
@ -9,5 +12,9 @@ $ docker-compose exec elastalert sh
>>> cd /opt/elastalert/rules
>>> elastalert-test-rule xxx.yaml
>>> exit
$ docker-compose restart
```
> ElastAlert will also load new rules, stop running missing rules, and restart
> modified rules as the files in this folder change.
[1]: http://elastalert.readthedocs.io/en/latest/

8
elastalert/docker-compose.yml
View File

@ -1,13 +1,13 @@
elastalert:
image: bitsensor/elastalert
image: bitsensor/elastalert:2.0.0
ports:
- "3030:3030"
- "3333:3333"
volumes:
- ./data/config.yaml:/opt/elastalert/config.yaml
- ./data/rules:/opt/elastalert/rules
environment:
- ES_HOST=elasticsearch
- ES_PORT=9200
external_links:
- elk_elasticsearch_1:elasticsearch
# extra_hosts:
# - elasticsearch:1.2.3.4
restart: always

11
elk/README.md
View File

@ -4,6 +4,7 @@ ELK
- Elasticsearch
- Logstash
- Kibana
- APM Server
## How it works
@ -21,6 +22,8 @@ $ docker-compose up -d
## Delete indices older than 7 days
File: delete-indices.yml
```yaml
---
actions:
@ -45,6 +48,14 @@ actions:
unit_count: 7
```
File: ~/.curator/curator.yml
```yaml
client:
hosts:
- 127.0.0.1
```
```bash
$ pip install elasticsearch-curator
$ curator delete-indices.yml

0
elk/delete-indices.yml → elk/curator/delete-indices.yml
View File

841
elk/data/apm-server.yml Normal file
View File

@ -0,0 +1,841 @@
################### APM Server Configuration #########################
############################# APM Server ######################################
apm-server:
# Defines the host and port the server is listening on. use "unix:/path/to.sock" to listen on a unix domain socket.
host: "0.0.0.0:8200"
# Maximum permitted size in bytes of a request's header accepted by the server to be processed.
#max_header_size: 1048576
# Maximum permitted duration for reading an entire request.
#read_timeout: 30s
# Maximum permitted duration for writing a response.
#write_timeout: 30s
# Maximum duration in seconds before releasing resources when shutting down the server.
#shutdown_timeout: 5s
# Maximum allowed size in bytes of a single event
#max_event_size: 307200
#--
# Maximum number of new connections to accept simultaneously (0 means unlimited)
# max_connections: 0
# Authorization token to be checked. If a token is set here the agents must
# send their token in the following format: Authorization: Bearer <secret-token>.
# It is recommended to use an authorization token in combination with SSL enabled,
# and save the token in the beats keystore.
secret_token: "1870d518-6b62-450f-82b7-44fda593963f"
#ssl.enabled: false
#ssl.certificate : "path/to/cert"
#ssl.key : "path/to/private_key"
# It is recommended to use the provided keystore instead of entering the passphrase in plain text.
#ssl.key_passphrase: ""
#rum:
# To enable real user monitoring (RUM) support set this to true.
#enabled: false
#event_rate:
# Defines the maximum amount of events allowed to be sent to the APM Server RUM
# endpoint per ip per second. Defaults to 300.
#limit: 300
# An LRU cache is used to keep a rate limit per IP for the most recently seen IPs.
# This setting defines the number of unique IPs that can be tracked in the cache.
# Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
#lru_size: 1000
#-- General RUM settings
# Comma separated list of permitted origins for real user monitoring.
# User-agents will send an origin header that will be validated against this list.
# An origin is made of a protocol scheme, host and port, without the url path.
# Allowed origins in this setting can have * to match anything (eg.: http://*.example.com)
# If an item in the list is a single '*', everything will be allowed
#allow_origins : ['*']
# Regexp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes.
# If the regexp matches, the stacktrace frame is considered to be a library frame.
#library_pattern: "node_modules|bower_components|~"
# Regexp to be matched against a stacktrace frame's `file_name`.
# If the regexp matches, the stacktrace frame is not used for calculating error groups.
# The default pattern excludes stacktrace frames that have a filename starting with '/webpack'
#exclude_from_grouping: "^/webpack"
# If a source map has previously been uploaded, source mapping is automatically applied
# to all error and transaction documents sent to the RUM endpoint.
#source_mapping:
# Source maps are always fetched from Elasticsearch, by default using the output.elasticsearch configuration.
# A different instance must be configured when using any other output.
# This setting only affects sourcemap reads - the output determines where sourcemaps are written.
#elasticsearch:
# Array of hosts to connect to.
# Scheme and port can be left out and will be set to the default (http and 9200)
# In case you specify and additional path, the scheme is required: http://localhost:9200/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
# hosts: ["localhost:9200"]
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
# The `cache.expiration` determines how long a source map should be cached before fetching it again from Elasticsearch.
# Note that values configured without a time unit will be interpreted as seconds.
#cache:
#expiration: 5m
# Source maps are stored in a seperate index.
# If the default index pattern for source maps at 'outputs.elasticsearch.indices'
# is changed, a matching index pattern needs to be specified here.
#index_pattern: "apm-*-sourcemap*"
# If set to true, APM Server augments data received by the agent with the original IP of the backend server,
# or the IP and User Agent of the real user (RUM requests). It defaults to true.
#capture_personal_data: true
# golang expvar support - https://golang.org/pkg/expvar/
#expvar:
# Set to true to Expose expvar
#enabled: false
# Url to expose expvar
#url: "/debug/vars"
# Instrumentation support for the server's HTTP endpoints and event publisher.
#instrumentation:
# Set to true to enable instrumentation of the APM server itself.
#enabled: false
# Environment in which the APM Server is running on (eg: staging, production, etc.)
#environment: ""
# Remote host to report instrumentation results to.
#hosts:
# - http://remote-apm-server:8200
# Remote apm-servers' secret_token
#secret_token:
# Metrics endpoint
#metrics:
# Set to false to disable the metrics endpoint
#enabled: true
# A pipeline is a definition of processors applied to documents when writing them to Elasticsearch.
# Using pipelines involves two steps:
# (1) registering a pipeline
# (2) applying a pipeline during data ingestion (see `output.elasticsearch.pipelines`)
#
# You can manually register pipelines, or use this configuration option to ensure
# pipelines are loaded and registered at the configured Elasticsearch instances.
# Automatic pipeline registration requires
# * `output.elasticsearch` to be enabled and configured.
# * having the required Elasticsearch Processor Plugins installed.
# APM Server default pipelines require you to have the `Ingest User Agent Plugin` installed.
# Find the default pipeline configuration at `ingest/pipeline/definition.json`.
#
#register.ingest.pipeline:
# Registers pipeline definitions in Elasticsearch on APM Server startup. Defaults to false.
#enabled: false
# Overwrites existing pipeline definitions in Elasticsearch. Defaults to true.
#overwrite: true
#================================ General ======================================
# Internal queue configuration for buffering events to be published.
#queue:
# Queue type by name (default 'mem')
# The memory queue will present all available events (up to the outputs
# bulk_max_size) to the output, the moment the output is ready to server
# another batch of events.
#mem:
# Max number of events the queue can buffer.
#events: 4096
# Hints the minimum number of events stored in the queue,
# before providing a batch of events to the outputs.
# A value of 0 (the default) ensures events are immediately available
# to be sent to the outputs.
#flush.min_events: 2048
# Maximum duration after which events are available to the outputs,
# if the number of events stored in the queue is < min_flush_events.
#flush.timeout: 1s
# Sets the maximum number of CPUs that can be executing simultaneously. The
# default is the number of logical CPUs available in the system.
#max_procs:
#============================== Template =====================================
# A template is used to set the mapping in Elasticsearch
# By default template loading is enabled and the template is loaded.
# These settings can be adjusted to load your own template or overwrite existing ones.
# Set to false to disable template loading.
#setup.template.enabled: true
# Template name. By default the template name is "apm-%{[beat.version]}"
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.name: "apm-%{[beat.version]}"
# Template pattern. By default the template pattern is "apm-%{[beat.version]}-*" to apply to the default index settings.
# The first part is the version of the beat and then -* is used to match all daily indices.
# The template name and pattern has to be set in case the elasticsearch index pattern is modified.
#setup.template.pattern: "apm-%{[beat.version]}-*"
# Path to fields.yml file to generate the template
#setup.template.fields: "${path.config}/fields.yml"
# Overwrite existing template
#setup.template.overwrite: false
# Elasticsearch template settings
#setup.template.settings:
# A dictionary of settings to place into the settings.index dictionary
# of the Elasticsearch template. For more details, please check
# https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
#index:
#number_of_shards: 1
#codec: best_compression
#number_of_routing_shards: 30
#mapping.total_fields.limit: 2000
# A dictionary of settings for the _source field. For more details, please check
# https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html
#_source:
#enabled: false
#============================== Deprecated: Dashboards =====================================
#
# Deprecated: Loading dashboards from the APM Server into Kibana is deprecated from 6.4 on.
# We suggest to use the Kibana UI to load APM Server dashboards and index pattern instead.
#
# These settings control loading the sample dashboards to the Kibana index. Loading
# the dashboards are disabled by default and can be enabled either by setting the
# options here, or by using the `-setup` CLI flag or the `setup` command.
#setup.dashboards.enabled: false
# The directory from where to read the dashboards. The default is the `kibana`
# folder in the home path.
#setup.dashboards.directory: ${path.home}/kibana
# The URL from where to download the dashboards archive. It is used instead of
# the directory if it has a value.
#setup.dashboards.url:
# The file archive (zip file) from where to read the dashboards. It is used instead
# of the directory when it has a value.
#setup.dashboards.file:
# The name of the Kibana index to use for setting the configuration. Default is ".kibana"
#setup.dashboards.kibana_index: .kibana
# The Elasticsearch index name. This overwrites the index name defined in the
# dashboards and index pattern. Example: testbeat-*
# The dashboards.index needs to be changed in case the elasticsearch index pattern is modified.
#setup.dashboards.index:
# Always use the Kibana API for loading the dashboards instead of autodetecting
# how to install the dashboards by first querying Elasticsearch.
#setup.dashboards.always_kibana: false
# If true and Kibana is not reachable at the time when dashboards are loaded,
# it will retry to reconnect to Kibana instead of exiting with an error.
#setup.dashboards.retry.enabled: false
# Duration interval between Kibana connection retries.
#setup.dashboards.retry.interval: 1s
# Maximum number of retries before exiting with an error, 0 for unlimited retrying.
#setup.dashboards.retry.maximum: 0
#============================== Deprecated: Kibana =====================================
# Deprecated: Starting with APM Server version 6.4, loading dashboards and index pattern
# from the APM Server into Kibana is deprecated.
# We suggest to use the Kibana UI to load APM Server dashboards and index pattern instead.
#
# Setting up a Kibana endpoint is not necessary when loading the index pattern and dashboards via the UI.
#setup.kibana:
# Kibana Host
# Scheme and port can be left out and will be set to the default (http and 5601)
# In case you specify and additional path, the scheme is required: http://localhost:5601/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
#host: "localhost:5601"
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
# Optional HTTP Path
#path: ""
# Use SSL settings for HTTPS. Default is false.
#ssl.enabled: false
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# SSL configuration. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
# It is recommended to use the provided keystore instead of entering the passphrase in plain text.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
#============================= Elastic Cloud ==================================
# These settings simplify using APM Server with the Elastic Cloud (https://cloud.elastic.co/).
# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
# [deprecated] `setup.kibana.host` options.
# You can find the `cloud.id` in the Elastic Cloud web UI.
#cloud.id:
# The cloud.auth setting overwrites the `output.elasticsearch.username` and
# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
#cloud.auth:
#================================ Outputs =====================================
# Configure what output to use when sending the data collected by the beat.
#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
hosts: ["elasticsearch:9200"]
# Boolean flag to enable or disable the output module.
#enabled: true
# Set gzip compression level.
#compression_level: 0
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
# Dictionary of HTTP parameters to pass within the url with index operations.
#parameters:
#param1: value1
#param2: value2
# Number of workers per Elasticsearch host.
#worker: 1
# By using the configuration below, apm documents are stored to separate indices,
# depending on their `processor.event`:
# - error
# - transaction
# - span
# - sourcemap
#
# The indices are all prefixed with `apm-%{[beat.version]}`.
# To allow managing indices based on their age, all indices (except for sourcemaps)
# end with the information of the day they got indexed.
# e.g. "apm-6.3.0-transaction-2018.03.20"
#
# Be aware that you can only specify one Elasticsearch template and one Kibana Index Pattern,
# In case you modify the index patterns you must also update those configurations accordingly,
# as they need to be aligned:
# * `setup.template.name`
# * `setup.template.pattern`
# * `setup.dashboards.index`
#index: "apm-%{[beat.version]}-%{+yyyy.MM.dd}"
indices:
- index: "apm-%{[beat.version]}-sourcemap"
when.contains:
processor.event: "sourcemap"
- index: "apm-%{[beat.version]}-error-%{+yyyy.MM.dd}"
when.contains:
processor.event: "error"
- index: "apm-%{[beat.version]}-transaction-%{+yyyy.MM.dd}"
when.contains:
processor.event: "transaction"
- index: "apm-%{[beat.version]}-span-%{+yyyy.MM.dd}"
when.contains:
processor.event: "span"
- index: "apm-%{[beat.version]}-metric-%{+yyyy.MM.dd}"
when.contains:
processor.event: "metric"
- index: "apm-%{[beat.version]}-onboarding-%{+yyyy.MM.dd}"
when.contains:
processor.event: "onboarding"
# A pipeline is a definition of processors applied to documents when writing them to Elasticsearch.
# APM Server comes with a default pipeline definition, located at `ingets/pipeline/definition.json`.
# Pipelines are disabled by default. To make use of them you have to:
# (1) ensure pipelines are registered in Elasticsearch, see `apm-server.register.ingest.pipeline`
# (2) enable the following:
#pipelines:
#- pipeline: "apm_user_agent"
# Optional HTTP Path
#path: "/elasticsearch"
# Custom HTTP headers to add to each request
#headers:
# X-My-Header: Contents of the header
# Proxy server url
#proxy_url: http://proxy:3128
# The number of times a particular Elasticsearch index operation is attempted. If
# the indexing operation doesn't succeed after this many retries, the events are
# dropped. The default is 3.
#max_retries: 3
# The maximum number of events to bulk in a single Elasticsearch bulk API index request.
# The default is 50.
#bulk_max_size: 50
# The number of seconds to wait before trying to reconnect to Elasticsearch
# after a network error. After waiting backoff.init seconds, apm-server
# tries to reconnect. If the attempt fails, the backoff timer is increased
# exponentially up to backoff.max. After a successful connection, the backoff
# timer is reset. The default is 1s.
#backoff.init: 1s
# The maximum number of seconds to wait before attempting to connect to
# Elasticsearch after a network error. The default is 60s.
#backoff.max: 60s
# Configure http request timeout before failing an request to Elasticsearch.
#timeout: 90
# Use SSL settings for HTTPS. Default is false.
#ssl.enabled: false
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# SSL configuration. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
# It is recommended to use the provided keystore instead of entering the passphrase in plain text.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#----------------------------- Console output ---------------------------------
#output.console:
# Boolean flag to enable or disable the output module.
#enabled: false
# Pretty print json event
#pretty: false
#----------------------------- Logstash output ---------------------------------
#output.logstash:
# Boolean flag to enable or disable the output module.
#enabled: false
# The Logstash hosts
#hosts: ["localhost:5044"]
# Number of workers per Logstash host.
#worker: 1
# Set gzip compression level.
#compression_level: 3
# Configure escaping html symbols in strings.
#escape_html: true
# Optional maximum time to live for a connection to Logstash, after which the
# connection will be re-established. A value of `0s` (the default) will
# disable this feature.
#
# Not yet supported for async connections (i.e. with the "pipelining" option set)
#ttl: 30s
# Optional load balance the events between the Logstash hosts. Default is false.
#loadbalance: false
# Number of batches to be sent asynchronously to Logstash while processing
# new batches.
#pipelining: 2
# If enabled only a subset of events in a batch of events is transferred per
# group. The number of events to be sent increases up to `bulk_max_size`
# if no error is encountered.
#slow_start: false
# The number of seconds to wait before trying to reconnect to Logstash
# after a network error. After waiting backoff.init seconds, apm-server
# tries to reconnect. If the attempt fails, the backoff timer is increased
# exponentially up to backoff.max. After a successful connection, the backoff
# timer is reset. The default is 1s.
#backoff.init: 1s
# The maximum number of seconds to wait before attempting to connect to
# Logstash after a network error. The default is 60s.
#backoff.max: 60s
# Optional index name. The default index name is set to apm
# in all lowercase.
#index: 'apm'
# SOCKS5 proxy server URL
#proxy_url: socks5://user:password@socks5-server:2233
# Resolve names locally when using a proxy server. Defaults to false.
#proxy_use_local_resolver: false
# Enable SSL support. SSL is automatically enabled, if any SSL setting is set.
#ssl.enabled: true
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# Optional SSL configuration options. SSL is off by default.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#------------------------------- Kafka output ----------------------------------
#output.kafka:
# Boolean flag to enable or disable the output module.
#enabled: false
# The list of Kafka broker addresses from where to fetch the cluster metadata.
# The cluster metadata contain the actual Kafka brokers events are published
# to.
#hosts: ["localhost:9092"]
# The Kafka topic used for produced events. The setting can be a format string
# using any event field. To set the topic from document type use `%{[type]}`.
#topic: beats
# The Kafka event key setting. Use format string to create unique event key.
# By default no event key will be generated.
#key: ''
# The Kafka event partitioning strategy. Default hashing strategy is `hash`
# using the `output.kafka.key` setting or randomly distributes events if
# `output.kafka.key` is not configured.
#partition.hash:
# If enabled, events will only be published to partitions with reachable
# leaders. Default is false.
#reachable_only: false
# Configure alternative event field names used to compute the hash value.
# If empty `output.kafka.key` setting will be used.
# Default value is empty list.
#hash: []
# Authentication details. Password is required if username is set.
#username: ''
#password: ''
# Kafka version libbeat is assumed to run against. Defaults to the "1.0.0".
#version: '1.0.0'
# Configure JSON encoding
#codec.json:
# Pretty print json event
#pretty: false
# Configure escaping html symbols in strings.
#escape_html: true
# Metadata update configuration. Metadata do contain leader information
# deciding which broker to use when publishing.
#metadata:
# Max metadata request retry attempts when cluster is in middle of leader
# election. Defaults to 3 retries.
#retry.max: 3
# Waiting time between retries during leader elections. Default is 250ms.
#retry.backoff: 250ms
# Refresh metadata interval. Defaults to every 10 minutes.
#refresh_frequency: 10m
# The number of concurrent load-balanced Kafka output workers.
#worker: 1
# The number of times to retry publishing an event after a publishing failure.
# After the specified number of retries, the events are typically dropped.
# Some Beats, such as Filebeat, ignore the max_retries setting and retry until
# all events are published. Set max_retries to a value less than 0 to retry
# until all events are published. The default is 3.
#max_retries: 3
# The maximum number of events to bulk in a single Kafka request. The default
# is 2048.
#bulk_max_size: 2048
# The number of seconds to wait for responses from the Kafka brokers before
# timing out. The default is 30s.
#timeout: 30s
# The maximum duration a broker will wait for number of required ACKs. The
# default is 10s.
#broker_timeout: 10s
# The number of messages buffered for each Kafka broker. The default is 256.
#channel_buffer_size: 256
# The keep-alive period for an active network connection. If 0s, keep-alives
# are disabled. The default is 0 seconds.
#keep_alive: 0
# Sets the output compression codec. Must be one of none, snappy and gzip. The
# default is gzip.
#compression: gzip
# The maximum permitted size of JSON-encoded messages. Bigger messages will be
# dropped. The default value is 1000000 (bytes). This value should be equal to
# or less than the broker's message.max.bytes.
#max_message_bytes: 1000000
# The ACK reliability level required from broker. 0=no response, 1=wait for
# local commit, -1=wait for all replicas to commit. The default is 1. Note:
# If set to 0, no ACKs are returned by Kafka. Messages might be lost silently
# on error.
#required_acks: 1
# The configurable ClientID used for logging, debugging, and auditing
# purposes. The default is "beats".
#client_id: beats
# Enable SSL support. SSL is automatically enabled, if any SSL setting is set.
#ssl.enabled: true
# Optional SSL configuration options. SSL is off by default.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Configure SSL verification mode. If `none` is configured, all server hosts
# and certificates will be accepted. In this mode, SSL based connections are
# susceptible to man-in-the-middle attacks. Use only for testing. Default is
# `full`.
#ssl.verification_mode: full
# List of supported/valid TLS versions. By default all TLS versions 1.0 up to
# 1.2 are enabled.
#ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
# Optional passphrase for decrypting the Certificate Key.
#ssl.key_passphrase: ''
# Configure cipher suites to be used for SSL connections
#ssl.cipher_suites: []
# Configure curve types for ECDHE based cipher suites
#ssl.curve_types: []
# Configure what types of renegotiation are supported. Valid options are
# never, once, and freely. Default is never.
#ssl.renegotiation: never
#================================= Paths ======================================
# The home path for the apm-server installation. This is the default base path
# for all other path settings and for miscellaneous files that come with the
# distribution (for example, the sample dashboards).
# If not set by a CLI flag or in the configuration file, the default for the
# home path is the location of the binary.
#path.home:
# The configuration path for the apm-server installation. This is the default
# base path for configuration files, including the main YAML configuration file
# and the Elasticsearch template file. If not set by a CLI flag or in the
# configuration file, the default for the configuration path is the home path.
#path.config: ${path.home}
# The data path for the apm-server installation. This is the default base path
# for all the files in which apm-server needs to store its data. If not set by a
# CLI flag or in the configuration file, the default for the data path is a data
# subdirectory inside the home path.
#path.data: ${path.home}/data
# The logs path for a apm-server installation. This is the default location for
# the Beat's log files. If not set by a CLI flag or in the configuration file,
# the default for the logs path is a logs subdirectory inside the home path.
#path.logs: ${path.home}/logs
#================================ Logging ======================================
#
# There are three options for the log output: syslog, file, stderr.
# Under Windows systems, the log files are per default sent to the file output,
# under all other system per default to syslog.
# Sets log level. The default log level is info.
# Available log levels are: error, warning, info, debug
#logging.level: info
# Enable debug output for selected components. To enable all selectors use ["*"]
# Other available selectors are "beat", "publish", "service"
# Multiple selectors can be chained.
#logging.selectors: [ ]
# Send all logging output to syslog. The default is false.
#logging.to_syslog: true
# If enabled, apm-server periodically logs its internal metrics that have changed
# in the last period. For each metric that changed, the delta from the value at
# the beginning of the period is logged. Also, the total values for
# all non-zero internal metrics are logged on shutdown. The default is true.
#logging.metrics.enabled: false
# The period after which to log the internal metrics. The default is 30s.
#logging.metrics.period: 30s
# Logging to rotating files. Set logging.to_files to false to disable logging to
# files.
#logging.to_files: true
#logging.files:
# Configure the path where the logs are written. The default is the logs directory
# under the home path (the binary location).
#path: /var/log/apm-server
# The name of the files where the logs are written to.
#name: apm-server
# Configure log file size limit. If limit is reached, log file will be
# automatically rotated
#rotateeverybytes: 10485760 # = 10MB
# Number of rotated log files to keep. Oldest files will be deleted first.
#keepfiles: 7
# The permissions mask to apply when rotating log files. The default value is 0600.
# Must be a valid Unix-style file permissions mask expressed in octal notation.
#permissions: 0600
# Set to true to log messages in json format.
#logging.json: false
#================================ HTTP Endpoint ======================================
#
# Each beat can expose internal metrics through a HTTP endpoint. For security
# reasons the endpoint is disabled by default. This feature is currently experimental.
# Stats can be access through http://localhost:5066/stats . For pretty JSON output
# append ?pretty to the URL.
# Defines if the HTTP endpoint is enabled.
#http.enabled: false
# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost.
#http.host: localhost
# Port on which the HTTP endpoint will bind. Default is 5066.
#http.port: 5066
#============================== Xpack Monitoring ===============================
# APM server can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
# reporting is disabled by default.
# Set to true to enable the monitoring reporter.
#xpack.monitoring.enabled: false
# Uncomment to send the metrics to Elasticsearch. Most settings from the
# Elasticsearch output are accepted here as well. Any setting that is not set is
# automatically inherited from the Elasticsearch output configuration, so if you
# have the Elasticsearch output configured, you can simply uncomment the
# following line.
#xpack.monitoring.elasticsearch:
# username: "apm_system"
# password: ""

5
elk/data/logstash.yml
View File

@ -1,6 +1,7 @@
http.host: "0.0.0.0"
path.config: /usr/share/logstash/pipeline
#xpack.monitoring.enabled: false
xpack.monitoring.elasticsearch.url: http://elasticsearch:9200
#xpack.monitoring.enabled: true
#xpack.monitoring.elasticsearch.hosts:
#- http://elasticsearch:9200
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: changeme

99
elk/docker-compose.yml
View File

@ -1,45 +1,58 @@
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.1.2
ports:
- "9200:9200"
- "9300:9300"
volumes:
- ./data:/usr/share/elasticsearch/data
environment:
- node.name=es1
- cluster.name=elk
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms1g -Xmx1g"
ulimits:
memlock:
soft: -1
hard: -1
restart: unless-stopped
version: '3.5'
logstash:
image: docker.elastic.co/logstash/logstash:6.1.2
ports:
- "5044:5044"
- "9600:9600"
- "12201:12201/udp"
links:
- elasticsearch
volumes:
- ./data/logstash.yml:/usr/share/logstash/config/logstash.yml
- ./data/pipeline:/usr/share/logstash/pipeline
- /usr/share/logstash/vendor/bundle
environment:
- node.name=ls1
- "LS_JAVA_OPTS=-Xms1g -Xmx1g"
restart: unless-stopped
services:
kibana:
image: docker.elastic.co/kibana/kibana:6.1.2
ports:
- "5601:5601"
links:
- elasticsearch
environment:
SERVER_NAME: kibana
ELASTICSEARCH_URL: http://elasticsearch:9200
restart: unless-stopped
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.3.0
ports:
- "9200:9200"
volumes:
- ./data:/usr/share/elasticsearch/data
environment:
- node.name=es1
- cluster.name=docker-cluster
- cluster.initial_master_nodes=es1
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms16g -Xmx16g"
ulimits:
memlock:
soft: -1
hard: -1
restart: unless-stopped
logstash:
image: docker.elastic.co/logstash/logstash:7.3.0
ports:
- "5044:5044"
- "9600:9600"
- "12201:12201/udp"
depends_on:
- elasticsearch
volumes:
- ./data/logstash.yml:/usr/share/logstash/config/logstash.yml
- ./data/pipeline:/usr/share/logstash/pipeline
environment:
- node.name=ls1
- "LS_JAVA_OPTS=-Xms4g -Xmx4g"
restart: unless-stopped
kibana:
image: docker.elastic.co/kibana/kibana:7.3.0
ports:
- "5601:5601"
depends_on:
- elasticsearch
environment:
SERVER_NAME: kibana
ELASTICSEARCH_URL: http://elasticsearch:9200
restart: unless-stopped
apm-server:
image: docker.elastic.co/apm/apm-server:7.3.0
ports:
- "8200:8200"
depends_on:
- elasticsearch
volumes:
- ./data/apm-server.yml:/usr/share/apm-server/apm-server.yml
restart: unless-stopped

6
flogo/docker-compose.yml Normal file
View File

@ -0,0 +1,6 @@
flogo:
image: flogo/flogo-docker
command: eula-accept
ports:
- "3303:3303"
restart: always

22
flower/README.md Normal file
View File

@ -0,0 +1,22 @@
flower
======
[Flower][1] is a web based tool for monitoring and administrating Celery clusters.
## docker-compose.yml
```yaml
flower:
image: mher/flower
ports:
- "5555:5555"
environment:
- CELERY_BROKER_URL=redis://redis:6379/0
- FLOWER_PORT=5555
- FLOWER_BASIC_AUTH=username:password
extra_hosts:
- redis:x.x.x.x
restart: always
```
[1]: https://flower.readthedocs.io/en/latest/

11
flower/docker-compose.yml Normal file
View File

@ -0,0 +1,11 @@
flower:
image: mher/flower
ports:
- "5555:5555"
environment:
- CELERY_BROKER_URL=redis://redis:6379/0
- FLOWER_PORT=5555
- FLOWER_BASIC_AUTH=username:password
extra_hosts:
- redis:x.x.x.x
restart: always

22
freegeoip/Dockerfile Normal file
View File

@ -0,0 +1,22 @@
#
# Dockerfile for freegeoip
#
FROM alpine
MAINTAINER EasyPi Software Foundation
ENV FREEGEOIP_VERSION 3.4.1
ENV FREEGEOIP_FILE freegeoip-${FREEGEOIP_VERSION}-linux-amd64.tar.gz
ENV FREEGEOIP_URL https://github.com/fiorix/freegeoip/releases/download/v${FREEGEOIP_VERSION}/${FREEGEOIP_FILE}
WORKDIR /opt/freegeoip
RUN set -xe \
&& apk add --no-cache curl tar \
&& curl -sSL ${FREEGEOIP_URL} | tar xz --strip 1 \
&& apk del curl tar
EXPOSE 8080 8888
ENTRYPOINT ["./freegeoip"]
CMD ["-public", "public", "-http", ":8080", "-internal-server", ":8888"]

46
freegeoip/README.md Normal file
View File

@ -0,0 +1,46 @@
freegeoip
=========
## docker-compose.yml
```yaml
freegeoip:
image: vimagick/freegeoip
ports:
- "8080:8080"
- "8888:8888"
restart: always
```
## up and running
```bash
$ docker-compose up -d
$ curl -s http://localhost:8080/json/8.8.8.8 | jq .
{
"ip": "8.8.8.8",
"country_code": "US",
"country_name": "United States",
"region_code": "",
"region_name": "",
"city": "",
"zip_code": "",
"time_zone": "",
"latitude": 37.751,
"longitude": -97.822,
"metro_code": 0
}
$ curl -s http://localhost:8888/metrics
freegeoip_client_connections{proto="http"} 0
freegeoip_client_country_code_total{country_code="unknown"} 7
freegeoip_client_ipproto_version_total{ip="4"} 7
freegeoip_db_events_total{event="loaded"} 1
go_gc_duration_seconds{quantile="0"} 5.9754e-05
go_gc_duration_seconds{quantile="0.25"} 7.0367e-05
go_gc_duration_seconds{quantile="0.5"} 9.6169e-05
go_gc_duration_seconds{quantile="0.75"} 0.000112867
go_gc_duration_seconds{quantile="1"} 0.000260533
go_gc_duration_seconds_sum 0.001055739
```

6
freegeoip/docker-compose.yml Normal file
View File

@ -0,0 +1,6 @@
freegeoip:
image: vimagick/freegeoip
ports:
- "8080:8080"
- "8888:8888"
restart: always

12
freegeoip/freegeoip.service Normal file
View File

@ -0,0 +1,12 @@
[Unit]
Description=IP Geolocation Web Server
After=network-online.target
[Service]
WorkingDirectory=/opt/freegeoip
ExecStart=/opt/freegeoip/freegeoip -public public -http :8080 -internal-server :8888
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target

2
frp/Dockerfile
View File

@ -5,7 +5,7 @@
FROM alpine
MAINTAINER kev <noreply@easypi.pro>
ENV FRP_VERSION 0.20.0
ENV FRP_VERSION 0.29.0
ENV FRP_URL https://github.com/fatedier/frp/releases/download/v${FRP_VERSION}/frp_${FRP_VERSION}_linux_amd64.tar.gz
WORKDIR /opt/frp

4
frp/arm/Dockerfile
View File

@ -2,10 +2,10 @@
# Dockerfile for frp-arm
#
FROM alpine
FROM arm32v7/alpine:3
MAINTAINER EasyPi Software Foundation
ENV FRP_VERSION 0.20.0
ENV FRP_VERSION 0.29.0
ENV FRP_URL https://github.com/fatedier/frp/releases/download/v${FRP_VERSION}/frp_${FRP_VERSION}_linux_arm.tar.gz
WORKDIR /opt/frp

7
frp/arm/docker-compose.yml
View File

@ -1,6 +1,7 @@
frps:
frpc:
image: easypi/frp-arm
command: ./frpc -c frpc.ini
volumes:
- ./data/frps.ini:/opt/frp/frps.ini
- ./data/frpc.ini:/opt/frp/frpc.ini
net: host
restart: always
restart: unless-stopped

3
frp/data/frpc.ini
View File

@ -2,7 +2,8 @@
server_addr = 1.2.3.4
server_port = 7000
protocol = kcp
privilege_token = 12345678
token = 12345678
admin_port = 7400
[ssh]
type = tcp

2
frp/openwrt/frpc Normal file → Executable file
View File

@ -1,5 +1,5 @@
#!/bin/sh /etc/rc.common
# Copyright (C) 2018 EasyPi Software Foundation
# Copyright (C) 2019 EasyPi Software Foundation
START=99
USE_PROCD=1

2
frp/openwrt/frps Normal file → Executable file
View File

@ -1,5 +1,5 @@
#!/bin/sh /etc/rc.common
# Copyright (C) 2018 EasyPi Software Foundation
# Copyright (C) 2019 EasyPi Software Foundation
START=99
USE_PROCD=1

3
frp/systemd/frpc.service
View File

@ -3,7 +3,10 @@ Description=frp client
After=network.target
[Service]
User=nobody
ExecStart=/usr/local/bin/frpc -c /etc/frp/frpc.ini
ExecReload=/usr/local/bin/frpc reload -c /etc/frp/frpc.ini
LimitNOFILE=65536
RestartSec=5
Restart=always

14
frp/systemd/frpc@.service Normal file
View File

@ -0,0 +1,14 @@
[Unit]
Description=Frp Client Service
After=network.target
[Service]
User=nobody
ExecStart=/usr/local/bin/frpc -c /etc/frp/%i.ini
ExecReload=/usr/local/bin/frpc reload -c /etc/frp/%i.ini
LimitNOFILE=65536
RestartSec=5
Restart=always
[Install]
WantedBy=multi-user.target

2
frp/systemd/frps.service
View File

@ -3,7 +3,9 @@ Description=frp server
After=network.target
[Service]
User=nobody
ExecStart=/usr/local/bin/frps -c /etc/frp/frps.ini
LimitNOFILE=65536
RestartSec=5
Restart=always

4
ghost/arm/Dockerfile
View File

@ -5,10 +5,10 @@
FROM easypi/alpine-arm
MAINTAINER EasyPi Software Foundation
ENV GHOST_VERSION 1.24.9
ENV GHOST_VERSION 2.15.0
ENV GHOST_INSTALL /var/lib/ghost
ENV GHOST_CONTENT /var/lib/ghost/content
ENV GHOST_CLI_VERSION 1.8.1
ENV GHOST_CLI_VERSION 1.9.9
ENV NPM_CONFIG_LOGLEVEL warn
ENV NODE_ENV production

7
gitea/README.md Normal file
View File

@ -0,0 +1,7 @@
gitea
=====
[Gitea][1] is a community managed lightweight code hosting solution written in
Go. It is published under the MIT license.
[1]: https://gitea.io/

34
gitea/docker-compose.yml Normal file
View File

@ -0,0 +1,34 @@
version: "3.7"
services:
server:
image: gitea/gitea
ports:
- "2222:22"
- "3000:3000"
volumes:
- ./data/gitea:/data
environment:
- USER_UID=1000
- USER_GID=1000
- DB_TYPE=postgres
- DB_HOST=postgres:5432
- DB_NAME=gitea
- DB_USER=gitea
- DB_PASSWD=gitea
depends_on:
- postgres
restart: always
postgres:
image: postgres:12-alpine
ports:
- "5432:5432"
volumes:
- ./data/postgres:/var/lib/postgresql/data
environment:
- POSTGRES_USER=gitea
- POSTGRES_PASSWORD=gitea
- POSTGRES_DB=gitea
restart: always

3
gogs/README.md
View File

@ -38,4 +38,7 @@ $ tree -FL 3 ./data/git/
└── user2/
```
Please read [this][2] to learn more about `app.ini`.
[1]: https://gogs.io/
[2]: https://gogs.io/docs/advanced/configuration_cheat_sheet.html

34
gogs/arm/Dockerfile
View File

@ -2,21 +2,23 @@
# Dockerfile for gogs-arm
#
FROM easypi/alpine-arm
FROM arm32v7/alpine:3
MAINTAINER EasyPi Software Foundation
ENV GOGS_VERSION 0.11.53
ENV GOSU_VERSION 1.11
ENV GOGS_VERSION 0.11.91
ENV GOGS_CUSTOM /data/gogs
RUN apk add --no-cache bash \
ca-certificates \
curl \
git \
linux-pam \
openssh \
s6 \
socat \
tar
RUN apk add --no-cache \
bash \
ca-certificates \
curl \
git \
linux-pam \
openssh \
s6 \
socat \
tar
RUN set -xe \
&& adduser -H -D -g 'Gogs Git User' -h /data/git -s /bin/bash git \
@ -24,16 +26,16 @@ RUN set -xe \
&& echo "export GOGS_CUSTOM=${GOGS_CUSTOM}" >> /etc/profile
RUN set -xe \
&& curl -L https://github.com/tianon/gosu/releases/download/1.10/gosu-armhf > /usr/sbin/gosu \
&& curl -L https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-armhf > /usr/sbin/gosu \
&& chmod +x /usr/sbin/gosu
RUN set -xe \
&& mkdir /app/ \
&& cd /app/ \
&& curl -LO https://github.com/gogits/gogs/releases/download/v${GOGS_VERSION}/raspi2_armv6.zip \
&& unzip raspi2_armv6.zip \
&& rm raspi2_armv6.zip \
&& ln -s /lib/libc.musl-armhf.so.1 /lib/ld-linux-armhf.so.3
&& curl -LO https://github.com/gogits/gogs/releases/download/v${GOGS_VERSION}/raspi_armv7.zip \
&& unzip raspi_armv7.zip \
&& rm raspi_armv7.zip \
&& ln -s /lib/libc.musl-armv7.so.1 /lib/ld-linux-armhf.so.3
RUN set -xe \
&& cd /app/gogs/ \

14
gostatsd/data/gostatsd.toml Normal file
View File

@ -0,0 +1,14 @@
[graphite]
address = "graphite:2003"
dial_timeout = '5s'
write_timeout = '30s'
mode = 'tags'
global_prefix = 'stats'
global_suffix = ''
prefix_counter = 'counters'
prefix_timer = 'timers'
prefix_gauge = 'gauges'
prefix_sets = 'sets'

12
gostatsd/docker-compose.yml Normal file
View File

@ -0,0 +1,12 @@
gostatsd:
image: atlassianlabs/gostatsd
command: gostatsd --backends=graphite --config-path=/etc/gostatsd/gostatsd.toml
ports:
- "8125:8125/udp"
- "8126:8126/tcp"
- "8181:8181/tcp"
volumes:
- ./data:/etc/gostatsd
external_links:
- graphite_graphite_1:graphite
restart: unless-stopped

1
grafana/README.md
View File

@ -27,6 +27,7 @@ grafana:
- GF_SMTP_FROM_ADDRESS=grafana@example.com
- GF_SMTP_PASSWORD=******
- GF_USERS_ALLOW_SIGN_UP=false
- GF_ALERTING_ERROR_OR_TIMEOUT=keep_state
restart: always
```

1
grafana/docker-compose.yml
View File

@ -14,4 +14,5 @@ grafana:
- GF_SMTP_FROM_ADDRESS=grafana@example.com
- GF_SMTP_PASSWORD=******
- GF_USERS_ALLOW_SIGN_UP=false
- GF_ALERTING_ERROR_OR_TIMEOUT=keep_state
restart: always

10
graphite/Dockerfile
View File

@ -5,7 +5,7 @@
FROM alpine
MAINTAINER EasyPi Software Foundation
ENV GRAPHITE_VERSION=1.1.3
ENV GRAPHITE_VERSION=1.1.5
ENV GRAPHITE_CONF_DIR=/opt/graphite/conf
ENV GRAPHITE_STORAGE_DIR=/opt/graphite/storage
ENV PATH=$PATH:/opt/graphite/bin
@ -34,7 +34,8 @@ RUN set -xe \
&& echo "SECRET_KEY = '$(head -c 16 /dev/urandom | base64)'" > graphite/local_settings.py \
&& curl -sSL https://github.com/graphite-project/graphite-web/raw/master/webapp/manage.py > manage.py \
&& chmod +x manage.py \
&& ./manage.py migrate --run-syncdb --noinput \
&& ./manage.py collectstatic --noinput --settings=graphite.settings \
&& ./manage.py migrate --noinput --run-syncdb \
&& apk del build-base \
curl \
git \
@ -48,8 +49,11 @@ COPY supervisord.conf /etc/supervisor/
VOLUME $GRAPHITE_CONF_DIR \
$GRAPHITE_STORAGE_DIR
EXPOSE 2003 \
EXPOSE 2003/udp \
2003 \
2004 \
2023 \
2024 \
7002 \
8080 \
9001

12
graphite/README.md
View File

@ -14,7 +14,10 @@ graphite:
image: vimagick/graphite
ports:
- "2003:2003"
- "2003:2003/udp"
- "2004:2004"
- "2023:2023"
- "2024:2024"
- "7002:7002"
- "8080:8080"
- "9001:9001"
@ -32,14 +35,21 @@ $ mkdir -p data/storage/log/webapp
$ docker-compose up -d
$ docker-compose exec graphite sh
>>> vi conf/storage-schemas.conf
>>> python webapp/manage.py migrate --run-syncdb --noinput
>>> python webapp/manage.py migrate --noinput --run-syncdb
>>> python webapp/manage.py createsuperuser
>>> python webapp/manage.py changepassword
>>> supervisorctl restart
>>> supervisorctl status
carbon-aggregator RUNNING pid 9, uptime 0:00:13
carbon-cache RUNNING pid 8, uptime 0:00:22
graphite-webapp RUNNING pid 7, uptime 0:00:24
>>> exit
$ tree -F -L 4
├── data/
│   ├── conf/
│   │   ├── aggregation-rules.conf
│   │   ├── carbon.conf
│   │   ├── rewrite-rules.conf
│   │   └── storage-schemas.conf
│   └── storage/
│   ├── carbon-cache-a.pid

43
graphite/data/conf/aggregation-rules.conf Normal file
View File

@ -0,0 +1,43 @@
# The form of each line in this file should be as follows:
#
# output_template (frequency) = method input_pattern
#
# This will capture any received metrics that match 'input_pattern'
# for calculating an aggregate metric. The calculation will occur
# every 'frequency' seconds and the 'method' can specify 'sum' or
# 'avg'. The name of the aggregate metric will be derived from
# 'output_template' filling in any captured fields from 'input_pattern'.
#
# For example, if you're metric naming scheme is:
#
# <env>.applications.<app>.<server>.<metric>
#
# You could configure some aggregations like so:
#
# <env>.applications.<app>.all.requests (60) = sum <env>.applications.<app>.*.requests
# <env>.applications.<app>.all.latency (60) = avg <env>.applications.<app>.*.latency
#
# As an example, if the following metrics are received:
#
# prod.applications.apache.www01.requests
# prod.applications.apache.www01.requests
#
# They would all go into the same aggregation buffer and after 60 seconds the
# aggregate metric 'prod.applications.apache.all.requests' would be calculated
# by summing their values.
#
# Template components such as <env> will match everything up to the next dot.
# To match metric multiple components including the dots, use <<metric>> in the
# input template:
#
# <env>.applications.<app>.all.<app_metric> (60) = sum <env>.applications.<app>.*.<<app_metric>>
#
# It is also possible to use regular expressions. Following the example above
# when using:
#
# <env>.applications.<app>.<domain>.requests (60) = sum <env>.applications.<app>.<domain>\d{2}.requests
#
# You will end up with 'prod.applications.apache.www.requests' instead of
# 'prod.applications.apache.all.requests'.
#
# Note that any time this file is modified, it will be re-read automatically.

8
graphite/data/conf/carbon.conf
View File

@ -505,13 +505,13 @@ PICKLE_RECEIVER_PORT = 2024
# If set true, metric received will be forwarded to DESTINATIONS in addition to
# the output of the aggregation rules. If set false the carbon-aggregator will
# only ever send the output of aggregation.
FORWARD_ALL = True
FORWARD_ALL = False
# Filenames of the configuration files to use for this instance of aggregator.
# Filenames are relative to CONF_DIR.
#
# AGGREGATION_RULES = aggregation-rules.conf
# REWRITE_RULES = rewrite-rules.conf
AGGREGATION_RULES = aggregation-rules.conf
REWRITE_RULES = rewrite-rules.conf
# This is a list of carbon daemons we will send any relayed or
# generated metrics to. The default provided would send to a single

18
graphite/data/conf/rewrite-rules.conf Normal file
View File

@ -0,0 +1,18 @@
# This file defines regular expression patterns that can be used to
# rewrite metric names in a search & replace fashion. It consists of two
# sections, [pre] and [post]. The rules in the pre section are applied to
# metric names as soon as they are received. The post rules are applied
# after aggregation has taken place.
#
# The general form of each rule is as follows:
#
# regex-pattern = replacement-text
#
# For example:
#
# [post]
# _sum$ =
# _avg$ =
#
# These rules would strip off a suffix of _sum or _avg from any metric names
# after aggregation.

0
graphite/data/storage/log/webapp/.gitkeep Normal file
View File

4
graphite/docker-compose.yml
View File

@ -1,9 +1,11 @@
graphite:
image: vimagick/graphite
ports:
- "2003:2003"
- "2003:2003/udp"
- "2003:2003"
- "2004:2004"
- "2023:2023"
- "2024:2024"
- "7002:7002"
- "8080:8080"
- "9001:9001"

5
graphite/supervisord.conf
View File

@ -15,6 +15,11 @@ command = carbon-cache.py --debug start
redirect_stderr = true
autorestart = true
[program:carbon-aggregator]
command = carbon-aggregator.py --debug start
redirect_stderr = true
autorestart = true
[program:graphite-webapp]
command = gunicorn -b :8080 graphite.wsgi:application
directory = /opt/graphite/webapp

2
hass/Dockerfile
View File

@ -9,7 +9,9 @@ RUN set -xe \
&& apk update \
&& apk add --no-cache ca-certificates \
build-base \
libffi-dev \
linux-headers \
openssl-dev \
python3 \
python3-dev \
&& pip3 install --no-cache-dir homeassistant \

23
hass/arm/Dockerfile
View File

@ -2,20 +2,25 @@
# Dockerfile for hass-arm (Home Assistant)
#
FROM easypi/alpine-arm
FROM arm32v7/alpine:3
MAINTAINER EasyPi Software Foundation
ENV HASS_VERSION=0.100.2
ENV HASS_CLI_VERSION=3.1.0
RUN set -xe \
&& apk update \
&& apk add --no-cache ca-certificates \
build-base \
linux-headers \
python3 \
python3-dev \
&& pip3 install --no-cache-dir homeassistant \
&& wget https://github.com/home-assistant/hassio-cli/releases/download/1.3.1/hassio_armhf -O /usr/local/bin/hassio
&& apk add --no-cache \
ca-certificates \
build-base \
libffi-dev \
linux-headers \
openssl-dev \
python3 \
python3-dev \
&& pip3 install --no-cache-dir homeassistant==${HASS_VERSION} \
&& wget https://github.com/home-assistant/hassio-cli/releases/download/${HASS_CLI_VERSION}/hassio_armhf -O /usr/local/bin/hassio
VOLUME /etc/hass
EXPOSE 8123
ENTRYPOINT ["hass", "--config", "/etc/hass"]

55
hass/arm/Dockerfile.debian
View File

@ -2,37 +2,42 @@
# Dockerfile for hass-arm (Home Assistant)
#
FROM resin/rpi-raspbian:jessie
FROM balenalib/rpi-raspbian:buster
MAINTAINER EasyPi Software Foundation
ENV HASS_VERSION=0.100.2
ENV HASS_CLI_VERSION=3.1.0
RUN set -xe \
&& apt-get update \
&& apt-get install -y build-essential \
bluez \
curl \
libbluetooth3 \
libbluetooth-dev \
libboost-python-dev \
libboost-thread-dev \
libglib2.0 \
libglib2.0-dev \
pkg-config \
python-dev \
python3-dev \
&& apt-get install -y \
build-essential \
bluez \
curl \
libbluetooth3 \
libbluetooth-dev \
libboost-python-dev \
libboost-thread-dev \
libglib2.0 \
libglib2.0-dev \
pkg-config \
python-dev \
python3-dev \
&& curl https://bootstrap.pypa.io/get-pip.py | python3 \
&& pip3 install --no-cache-dir gattlib \
homeassistant \
pybluez \
&& setcap 'cap_net_raw,cap_net_admin+eip' /usr/bin/python3.4 \
&& apt-get remove -y curl \
libbluetooth-dev \
libboost-python-dev \
libboost-thread-dev \
libglib2.0-dev \
pkg-config \
python-dev
&& pip3 install --no-cache-dir \
gattlib \
homeassistant \
pybluez \
&& setcap 'cap_net_raw,cap_net_admin+eip' /usr/bin/python3.7 \
&& apt-get remove -y \
curl \
libbluetooth-dev \
libboost-python-dev \
libboost-thread-dev \
libglib2.0-dev \
pkg-config \
python-dev
VOLUME /etc/hass
EXPOSE 8123
ENTRYPOINT ["hass", "--config", "/etc/hass"]

Some files were not shown because too many files have changed in this diff Show More