mirror of
https://github.com/vimagick/dockerfiles.git
synced 2024-12-25 02:04:06 +02:00
add scrapyrt tutorial for scrapyd
This commit is contained in:
parent
c349530e41
commit
299d84de53
@ -18,7 +18,7 @@ utility which allows you to deploy your project to a Scrapyd server.
|
||||
|
||||
[pillow][6] is the Python Imaging Library to support the ImagesPipeline.
|
||||
|
||||
This image is based on `debian:jessie`, 6 latest python packages are installed:
|
||||
This image is based on `debian:stretch`, 6 latest python packages are installed:
|
||||
|
||||
- `scrapy`: git+https://github.com/scrapy/scrapy.git
|
||||
- `scrapyd`: git+https://github.com/scrapy/scrapyd.git
|
||||
@ -48,6 +48,16 @@ scrapy:
|
||||
- .:/code
|
||||
working_dir: /code
|
||||
restart: always
|
||||
|
||||
scrapyrt:
|
||||
image: vimagick/scrapyd
|
||||
command: scrapyrt -i 0.0.0.0 -p 9080
|
||||
ports:
|
||||
- "9080:9080"
|
||||
volumes:
|
||||
- .:/code
|
||||
working_dir: /code
|
||||
restart: always
|
||||
```
|
||||
|
||||
## Run it as background-daemon for scrapyd
|
||||
@ -122,6 +132,14 @@ $ docker-compose run --rm scrapy
|
||||
>>> exit
|
||||
```
|
||||
|
||||
## Run it as realtime crawler for scrapyrt
|
||||
|
||||
```bash
|
||||
$ git clone https://github.com/scrapy/quotesbot.git .
|
||||
$ docker-compose up -d scrapyrt
|
||||
$ curl -s 'http://localhost:9080/crawl.json?spider_name=toscrape-css&callback=parse&url=http://quotes.toscrape.com/&max_requests=5' | jq -c '.items[]'
|
||||
```
|
||||
|
||||
[1]: https://github.com/scrapy/scrapy
|
||||
[2]: https://github.com/scrapy/scrapyd
|
||||
[3]: https://github.com/scrapy/scrapyd-client
|
||||
|
@ -14,3 +14,13 @@ scrapy:
|
||||
- .:/code
|
||||
working_dir: /code
|
||||
restart: always
|
||||
|
||||
scrapyrt:
|
||||
image: vimagick/scrapyd
|
||||
command: scrapyrt -i 0.0.0.0 -p 9080
|
||||
ports:
|
||||
- "9080:9080"
|
||||
volumes:
|
||||
- .:/code
|
||||
working_dir: /code
|
||||
restart: always
|
||||
|
Loading…
Reference in New Issue
Block a user