update airflow

airflow/README.md

@@ -1,2 +1,32 @@
 airflow
 =======
+
+## How It Works
+
+```
+                   +---------+
+              +--> | Worker1 |
+ redis        |    +---------+
+ postgres     |
++--------+    |    +---------+
+| Master | <--+--> | Worker2 |
++--------+    |    +---------+
+ webserver    |
+ scheduler    |    +---------+
+ flower       +--> | Worker3 |
+                   +---------+
+```
+
+## Quick Start
+
+```bash
+# On Master
+$ docker-compose up -d
+$ chmod 777 data/airflow/dags
+$ docker-compose exec webserver cp -r /usr/local/lib/python3.6/site-packages/airflow/example_dags dags
+
+# On Workers
+$ docker-compose up -d
+$ chmod 777 data/airflow/dags
+$ docker-compose exec worker cp -r /usr/local/lib/python3.6/site-packages/airflow/example_dags dags
+```

airflow/docker-compose-master.yml

@@ -44,17 +44,10 @@ services:
       interval: 30s
       timeout: 30s
       retries: 3
-    restart: always
-  
-  flower:
-    image: puckel/docker-airflow:1.9.0-4
-    command: flower
-    ports:
-      - "5555:5555"
-    environment:
-      - EXECUTOR=Celery
-    depends_on:
-      - redis
+    extra_hosts:
+      - "worker1:10.99.0.4"
+      - "worker2:10.99.0.5"
+      - "worker3:10.99.0.6"
     restart: always
   
   scheduler:
@@ -70,3 +63,14 @@ services:
     depends_on:
       - webserver
     restart: always
+  
+  flower:
+    image: puckel/docker-airflow:1.9.0-4
+    command: flower
+    ports:
+      - "5555:5555"
+    environment:
+      - EXECUTOR=Celery
+    depends_on:
+      - webserver
+    restart: always