add youtube-dl

README.md

@@ -24,6 +24,8 @@ https://hub.docker.com/u/vimagick/
 - [x] collectd
 - [x] dante
 - [x] dnscrypt
+    - [x] dnscrypt-proxy
+    - [x] dnscrypt-wrapper
 - [x] dokuwiki
 - [x] ferm
 - [x] fteproxy
@@ -43,6 +45,8 @@ https://hub.docker.com/u/vimagick/
 - [x] pure-ftpd
 - [x] scrapyd
 - [x] shadowsocks
+    - [x] shadowsocks
+    - [x] shadowsocks-libev
 - [x] shadowvpn
 - [x] splash
 - [x] taskd
@@ -51,7 +55,9 @@ https://hub.docker.com/u/vimagick/
 - [x] tor
 - [x] urlwatch
 - [x] webkit
-- [x] youtube-dl :+1:
+- [x] youtube
+    - [x] youtube-dl
+    - [x] youtube-worker
 
 ## 3rd-party
 

youtube/youtube-worker/Dockerfile

@@ -0,0 +1,16 @@
+#
+# Dockerfile for youtube-worker
+#
+
+FROM alpine
+MAINTAINER kev <noreplay@datageek.info>
+
+RUN apk add -U ca-certificates py-pip \
+    && rm -rf /var/cache/apk/* \
+    && pip install redis youtube-dl
+
+COPY worker.py /code/
+VOLUME /data
+WORKDIR /data
+
+CMD ["python", "/code/worker.py"]

youtube/youtube-worker/README.md

@@ -0,0 +1,55 @@
+youtube-worker
+==============
+
+youtube-worker = youtube-dl + redis
+
+## docker-compose.yml
+
+```
+worker:
+  image: vimagick/youtube-worker
+  links:
+    - redis
+  volumes:
+    - data:/data
+  environment:
+    - PASSWORD=secret-passwd
+  restart: always
+
+redis:
+  image: redis
+  command: redis-server --requirepass 'secret-passwd'
+  ports:
+    - "6379:6379"
+  restart: always
+```
+
+## server
+
+```
+$ cd ~/fig/youtube/
+
+$ docker-compose up -d
+
+$ docker-compose logs
+Attaching to youtube_worker_1
+worker_1 | 2015-07-12T16:07:07 [INFO] connect redis
+worker_1 | 2015-07-12T16:07:26 [INFO] process: %s
+worker_1 | [youtube] os6U77Hhm_s: Downloading webpage
+worker_1 | [youtube] os6U77Hhm_s: Downloading video info webpage
+worker_1 | [youtube] os6U77Hhm_s: Extracting video information
+worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest
+worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest
+worker_1 | [download] Destination: Shia LaBeouf TED Talk-os6U77Hhm_s.mp4
+[download] 100% of 11.03MiB in 00:0297MiB/s ETA 00:00known ETA
+```
+
+## client
+
+```
+$ redis-cli -h server -a 'secret-passwd'
+server> lpush urls 'https://www.youtube.com/watch?v=os6U77Hhm_s'
+server> quit
+
+$ rsync -ahP user@server:fig/youtube/data ~/Movies/youtube
+```

youtube/youtube-worker/docker-compose.yml

@@ -0,0 +1,16 @@
+worker:
+  image: vimagick/youtube-worker
+  links:
+    - redis
+  volumes:
+    - data:/data
+  environment:
+    - PASSWORD=secret-passwd
+  restart: always
+
+redis:
+  image: redis
+  command: redis-server --requirepass 'secret-passwd'
+  ports:
+    - "6379:6379"
+  restart: always

youtube/youtube-worker/worker.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# youtube_dl worker
+#
+
+import logging
+import os
+import redis
+import youtube_dl
+
+
+def download(url):
+
+    with youtube_dl.YoutubeDL() as ydl:
+        ydl.download([url])
+
+
+if __name__ == '__main__':
+
+    logging.basicConfig(format='%(asctime)s [%(levelname)s] %(msg)s', datefmt='%FT%T', level='INFO')
+    logging.info('connect redis')
+    rdb = redis.StrictRedis(host='redis', password=os.getenv('PASSWORD'))
+    rdb.ping()
+
+    while True:
+        try:
+            _, url = rdb.brpop('urls')
+            logging.info('process: %s', url)
+            download(url)
+        except Exception as ex:
+            logging.error('error: %s', ex)
+