1
0
mirror of https://github.com/vimagick/dockerfiles.git synced 2024-12-25 02:04:06 +02:00

add scrapyrt tutorial for scrapyd

This commit is contained in:
kev 2018-01-25 15:01:53 +08:00
parent c349530e41
commit 299d84de53
2 changed files with 29 additions and 1 deletions

View File

@ -18,7 +18,7 @@ utility which allows you to deploy your project to a Scrapyd server.
[pillow][6] is the Python Imaging Library to support the ImagesPipeline.
This image is based on `debian:jessie`, 6 latest python packages are installed:
This image is based on `debian:stretch`, 6 latest python packages are installed:
- `scrapy`: git+https://github.com/scrapy/scrapy.git
- `scrapyd`: git+https://github.com/scrapy/scrapyd.git
@ -48,6 +48,16 @@ scrapy:
- .:/code
working_dir: /code
restart: always
scrapyrt:
image: vimagick/scrapyd
command: scrapyrt -i 0.0.0.0 -p 9080
ports:
- "9080:9080"
volumes:
- .:/code
working_dir: /code
restart: always
```
## Run it as background-daemon for scrapyd
@ -122,6 +132,14 @@ $ docker-compose run --rm scrapy
>>> exit
```
## Run it as realtime crawler for scrapyrt
```bash
$ git clone https://github.com/scrapy/quotesbot.git .
$ docker-compose up -d scrapyrt
$ curl -s 'http://localhost:9080/crawl.json?spider_name=toscrape-css&callback=parse&url=http://quotes.toscrape.com/&max_requests=5' | jq -c '.items[]'
```
[1]: https://github.com/scrapy/scrapy
[2]: https://github.com/scrapy/scrapyd
[3]: https://github.com/scrapy/scrapyd-client

View File

@ -14,3 +14,13 @@ scrapy:
- .:/code
working_dir: /code
restart: always
scrapyrt:
image: vimagick/scrapyd
command: scrapyrt -i 0.0.0.0 -p 9080
ports:
- "9080:9080"
volumes:
- .:/code
working_dir: /code
restart: always