From 81e2d4c3a1eebd6e655b8cd710f266317eb8f167 Mon Sep 17 00:00:00 2001 From: kev Date: Wed, 25 Jul 2018 18:37:35 +0800 Subject: [PATCH] update airflow --- airflow/README.md | 30 ++++++++++++++++++++++++++++++ airflow/docker-compose-master.yml | 26 +++++++++++++++----------- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/airflow/README.md b/airflow/README.md index 46615b4..531e131 100644 --- a/airflow/README.md +++ b/airflow/README.md @@ -1,2 +1,32 @@ airflow ======= + +## How It Works + +``` + +---------+ + +--> | Worker1 | + redis | +---------+ + postgres | ++--------+ | +---------+ +| Master | <--+--> | Worker2 | ++--------+ | +---------+ + webserver | + scheduler | +---------+ + flower +--> | Worker3 | + +---------+ +``` + +## Quick Start + +```bash +# On Master +$ docker-compose up -d +$ chmod 777 data/airflow/dags +$ docker-compose exec webserver cp -r /usr/local/lib/python3.6/site-packages/airflow/example_dags dags + +# On Workers +$ docker-compose up -d +$ chmod 777 data/airflow/dags +$ docker-compose exec worker cp -r /usr/local/lib/python3.6/site-packages/airflow/example_dags dags +``` diff --git a/airflow/docker-compose-master.yml b/airflow/docker-compose-master.yml index d933263..f3a863e 100644 --- a/airflow/docker-compose-master.yml +++ b/airflow/docker-compose-master.yml @@ -44,17 +44,10 @@ services: interval: 30s timeout: 30s retries: 3 - restart: always - - flower: - image: puckel/docker-airflow:1.9.0-4 - command: flower - ports: - - "5555:5555" - environment: - - EXECUTOR=Celery - depends_on: - - redis + extra_hosts: + - "worker1:10.99.0.4" + - "worker2:10.99.0.5" + - "worker3:10.99.0.6" restart: always scheduler: @@ -70,3 +63,14 @@ services: depends_on: - webserver restart: always + + flower: + image: puckel/docker-airflow:1.9.0-4 + command: flower + ports: + - "5555:5555" + environment: + - EXECUTOR=Celery + depends_on: + - webserver + restart: always