self-hosted/dockerfiles
1
0
Fork 0
mirror of https://github.com/vimagick/dockerfiles.git synced 2025-05-29 22:57:35 +02:00
Code Issues Releases Activity

update

Browse Source
This commit is contained in:
kev 2015-07-13 01:58:37 +08:00
parent 6ec51642c7
commit d54936a41e
2 changed files with 21 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
youtube/youtube-worker/README.md
View File

@ -33,23 +33,23 @@ $ docker-compose up -d
$ docker-compose logs $ docker-compose logs
Attaching to youtube_worker_1 Attaching to youtube_worker_1
worker_1 | 2015-07-12T16:07:07 [INFO] connect redis worker_1 | 2015-07-12T17:50:02 [INFO] connect redis
worker_1 | 2015-07-12T16:07:26 [INFO] process: os6U77Hhm_s worker_1 | 2015-07-12T17:50:06 [INFO] process: os6U77Hhm_s
worker_1 | [youtube] os6U77Hhm_s: Downloading webpage worker_1 | [youtube] os6U77Hhm_s: Downloading webpage
worker_1 | [youtube] os6U77Hhm_s: Downloading video info webpage worker_1 | [youtube] os6U77Hhm_s: Downloading video info webpage
worker_1 | [youtube] os6U77Hhm_s: Extracting video information worker_1 | [youtube] os6U77Hhm_s: Extracting video information
worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest
worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest worker_1 | [youtube] os6U77Hhm_s: Downloading DASH manifest
worker_1 | [download] Destination: Shia LaBeouf TED Talk-os6U77Hhm_s.mp4 worker_1 | [download] Destination: Shia LaBeouf TED Talk-os6U77Hhm_s.mp4
[download] 100% of 11.03MiB in 00:0297MiB/s ETA 00:00known ETA [download] 100% of 11.03MiB in 00:0014MiB/s ETA 00:001nown ETA
worker_1 | 2015-07-12T16:07:28 [INFO] success: True worker_1 | 2015-07-12T17:50:07 [INFO] success: True
``` ```
## client ## client
``` ```
$ redis-cli -h server -a 'secret-passwd' $ redis-cli -h server -a 'secret-passwd'
server> lpush urls os6U77Hhm_s server> lpush pending os6U77Hhm_s
server> keys * server> keys *
1) "pending" 1) "pending"
server> keys * server> keys *
@ -61,9 +61,9 @@ server> quit
$ rsync -ahP user@server:fig/youtube/data $ rsync -ahP user@server:fig/youtube/data
receiving file list ... receiving file list ...
2 files to consider 2 files to consider
drwxr-xr-x 74 2015/07/13 00:52:25 data drwxr-xr-x 74 2015/07/13 01:50:07 data
-rw-r--r-- 0 2015/07/13 01:04:40 data/Shia LaBeouf TED Talk-os6U77Hhm_s.mp4 -rw-r--r-- 11569834 2015/06/15 17:19:16 data/Shia LaBeouf TED Talk-os6U77Hhm_s.mp4
sent 16 bytes received 116 bytes 29.33 bytes/sec sent 16 bytes received 116 bytes 29.33 bytes/sec
total size is 0 speedup is 0.00 total size is 11.57M speedup is 87650.26
``` ```

21
youtube/youtube-worker/worker.py
View File

@ -19,7 +19,7 @@ def download(url):
if status == 'downloading': if status == 'downloading':
rdb.zadd('running', now, url) rdb.zadd('running', now, url)
elif info['status'] == 'error': elif status == 'error':
rdb.zrem('running', url) rdb.zrem('running', url)
rdb.zadd('error', now, url) rdb.zadd('error', now, url)
elif status == 'finished': elif status == 'finished':
@ -29,13 +29,18 @@ def download(url):
if rdb.zrank('finished', url) != None: if rdb.zrank('finished', url) != None:
return False return False
opts = { try:
'format': os.getenv('FORMAT', 'best'), opts = {
'progress_hooks': [hook], 'format': os.getenv('FORMAT', 'best'),
} 'progress_hooks': [hook],
}
with youtube_dl.YoutubeDL(opts) as ydl: with youtube_dl.YoutubeDL(opts) as ydl:
ydl.download([url]) ydl.download([url])
except Exception as ex:
logging.error('error: %s', ex)
rdb.zrem('running', url)
rdb.zadd('error', int(time.time()), url)
return False
return True return True