mirror of
https://github.com/vimagick/dockerfiles.git
synced 2025-04-21 12:07:00 +02:00
update urlwatch
This commit is contained in:
kev
parent
fb57740c81
commit
01bc6ecd52
@ -5,29 +5,32 @@
|
|||||||
FROM alpine
|
FROM alpine
|
||||||
MAINTAINER kev <noreply@easypi.pro>
|
MAINTAINER kev <noreply@easypi.pro>
|
||||||
|
|
||||||
RUN apk add -U ca-certificates \
|
RUN set -xe \
|
||||||
|
&& apk add --no-cache ca-certificates \
|
||||||
build-base \
|
build-base \
|
||||||
|
libffi-dev \
|
||||||
libxml2 \
|
libxml2 \
|
||||||
libxml2-dev \
|
libxml2-dev \
|
||||||
libxslt \
|
libxslt \
|
||||||
libxslt-dev \
|
libxslt-dev \
|
||||||
make \
|
openssl-dev \
|
||||||
python-dev \
|
python3 \
|
||||||
py-pip \
|
python3-dev \
|
||||||
&& pip install cssselect \
|
&& python3 -m pip install appdirs \
|
||||||
futures \
|
cssselect \
|
||||||
keyring \
|
keyring \
|
||||||
lxml \
|
lxml \
|
||||||
urlwatch \
|
minidb \
|
||||||
pyyaml \
|
pyyaml \
|
||||||
|
requests \
|
||||||
&& apk del build-base \
|
&& apk del build-base \
|
||||||
|
libffi-dev \
|
||||||
libxml2-dev \
|
libxml2-dev \
|
||||||
libxslt-dev \
|
libxslt-dev \
|
||||||
python-dev \
|
openssl-dev \
|
||||||
&& rm -rf /var/cache/apk/* \
|
python3-dev \
|
||||||
&& echo '*/15 * * * * cd ~/.urlwatch && make' | crontab -
|
&& echo '*/30 * * * * urlwatch' | crontab -
|
||||||
|
|
||||||
COPY urlwatch /root/.urlwatch
|
|
||||||
VOLUME /root/.urlwatch
|
VOLUME /root/.urlwatch
|
||||||
WORKDIR /root/.urlwatch
|
WORKDIR /root/.urlwatch
|
||||||
|
|
||||||
|
|||||||
@ -3,82 +3,14 @@ urlwatch
|
|||||||
|
|
||||||
[urlwatch][1] is a tool for monitoring webpages for updates.
|
[urlwatch][1] is a tool for monitoring webpages for updates.
|
||||||
|
|
||||||
```
|
|
||||||
cron: triggered every 15 minutes
|
|
||||||
-> make: generate urls.txt from urls.yml
|
|
||||||
-> urlwatch: fetch webpages
|
|
||||||
-> hooks.py: extract info
|
|
||||||
-> email: send via smtp
|
|
||||||
-> (^_^)
|
|
||||||
```
|
|
||||||
|
|
||||||
## docker-compose.yml
|
## docker-compose.yml
|
||||||
|
|
||||||
```
|
```yaml
|
||||||
urlwatch:
|
urlwatch:
|
||||||
image: vimagick/urlwatch
|
image: vimagick/urlwatch
|
||||||
volumes:
|
volumes:
|
||||||
- urlwatch/Makefile:/root/.urlwatch/Makefile
|
- ./data:/root/.urlwatch
|
||||||
- urlwatch/urls.yml:/root/.urlwatch/urls.yml
|
|
||||||
restart: always
|
restart: always
|
||||||
```
|
```
|
||||||
|
|
||||||
## Makefile
|
[1]: https://thp.io/2008/urlwatch/
|
||||||
|
|
||||||
```
|
|
||||||
SHELL = /bin/sh
|
|
||||||
PATH := /usr/local/bin:$(PATH)
|
|
||||||
|
|
||||||
SMTP = smtp.easypi.info:587
|
|
||||||
FROM = urlwatch@easypi.info
|
|
||||||
PASS = password
|
|
||||||
TO = noreply@easypi.info
|
|
||||||
|
|
||||||
all: setup urls.txt
|
|
||||||
urlwatch -s $(SMTP) -f $(FROM) -t $(TO) -A -T
|
|
||||||
|
|
||||||
urls.txt: urls.yml
|
|
||||||
python lib/hooks.py
|
|
||||||
|
|
||||||
setup:
|
|
||||||
python -c 'import keyring; keyring.set_password("$(SMTP)", "$(FROM)", "$(PASS)")'
|
|
||||||
```
|
|
||||||
|
|
||||||
> Please change `STMP`/`FROM`/`PASS`/`TO` to correct value.
|
|
||||||
|
|
||||||
## urls.yml
|
|
||||||
|
|
||||||
```
|
|
||||||
python:
|
|
||||||
url: https://www.python.org/downloads/
|
|
||||||
exp: //div[@class="download-unknown"]/p[@class]/a[1]/text()
|
|
||||||
|
|
||||||
ubuntu:
|
|
||||||
url: http://www.ubuntu.com/download/server
|
|
||||||
exp: //div[contains(@class, "row-hero")]//h2/text()
|
|
||||||
|
|
||||||
coreos:
|
|
||||||
url: https://coreos.com
|
|
||||||
exp: //div[@class="co-p-homepage-release-text"]/text()
|
|
||||||
|
|
||||||
urlwatch:
|
|
||||||
url: https://github.com/thp/urlwatch/releases
|
|
||||||
exp: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
|
|
||||||
```
|
|
||||||
|
|
||||||
## Email alert
|
|
||||||
|
|
||||||
```
|
|
||||||
***************************************************************************
|
|
||||||
CHANGED: https://coreos.com#coreos
|
|
||||||
***************************************************************************
|
|
||||||
--- @ Tue, 07 Jul 2015 17:15:01 +0000
|
|
||||||
+++ @ Tue, 07 Jul 2015 20:00:01 +0000
|
|
||||||
@@ -1 +1 @@
|
|
||||||
-coreos: 723.1.0
|
|
||||||
+coreos: 735.0.0
|
|
||||||
|
|
||||||
***************************************************************************
|
|
||||||
```
|
|
||||||
|
|
||||||
[1]: thp.io/2008/urlwatch/
|
|
||||||
|
|||||||
4
urlwatch/data/urls.yaml
Normal file
4
urlwatch/data/urls.yaml
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
- url: "https://github.com/thp/urlwatch/releases/latest"
|
||||||
|
filter:
|
||||||
|
- xpath: '(//div[contains(@class,"release-timeline-tags")]//h4)[1]/a'
|
||||||
|
- html2text: re
|
||||||
4
urlwatch/data/urlwatch.yaml
Normal file
4
urlwatch/data/urlwatch.yaml
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
report:
|
||||||
|
slack:
|
||||||
|
webhook_url: https://hooks.slack.com/services/xxxxxx
|
||||||
|
enabled: true
|
||||||
@ -1,6 +1,5 @@
|
|||||||
urlwatch:
|
urlwatch:
|
||||||
image: vimagick/urlwatch
|
image: vimagick/urlwatch
|
||||||
volumes:
|
volumes:
|
||||||
- ./urlwatch/Makefile:/root/.urlwatch/Makefile
|
- ./data:/root/.urlwatch
|
||||||
- ./urlwatch/urls.yml:/root/.urlwatch/urls.yml
|
restart: unless-stopped
|
||||||
restart: always
|
|
||||||
|
|||||||
@ -1,16 +0,0 @@
|
|||||||
SHELL = /bin/sh
|
|
||||||
PATH := /usr/local/bin:$(PATH)
|
|
||||||
|
|
||||||
SMTP = smtp.easypi.info:587
|
|
||||||
FROM = urlwatch@easypi.info
|
|
||||||
PASS = password
|
|
||||||
TO = noreply@easypi.info
|
|
||||||
|
|
||||||
all: setup urls.txt
|
|
||||||
urlwatch -s $(SMTP) -f $(FROM) -t $(TO) -A -T
|
|
||||||
|
|
||||||
urls.txt: urls.yml
|
|
||||||
python lib/hooks.py
|
|
||||||
|
|
||||||
setup:
|
|
||||||
python -c 'import keyring; keyring.set_password("$(SMTP)", "$(FROM)", "$(PASS)")'
|
|
||||||
@ -1,59 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
#
|
|
||||||
# version detector for urlwatch
|
|
||||||
#
|
|
||||||
|
|
||||||
import cssselect.xpath
|
|
||||||
import lxml.html
|
|
||||||
import os.path
|
|
||||||
import re
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
|
|
||||||
urlwatch_dir = os.path.expanduser(os.path.join('~', '.urlwatch'))
|
|
||||||
urls_txt = os.path.join(urlwatch_dir, 'urls.txt')
|
|
||||||
urls_yml = os.path.join(urlwatch_dir, 'urls.yml')
|
|
||||||
config = yaml.load(open(urls_yml))
|
|
||||||
tr = cssselect.xpath.HTMLTranslator()
|
|
||||||
|
|
||||||
|
|
||||||
def filter(url, data):
|
|
||||||
|
|
||||||
try:
|
|
||||||
|
|
||||||
for key, cfg in config.items():
|
|
||||||
if url.endswith('#' + key):
|
|
||||||
if not url.startswith(cfg['url']):
|
|
||||||
raise Exception('url mismatch')
|
|
||||||
if 'xpath' in cfg:
|
|
||||||
xpath = cfg['xpath']
|
|
||||||
elif 'css' in cfg:
|
|
||||||
xpath = tr.css_to_xpath(cfg['css'])
|
|
||||||
else:
|
|
||||||
raise Exception('xpath/css required')
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
return data
|
|
||||||
|
|
||||||
dom = lxml.html.fromstring(data.encode('utf-8'))
|
|
||||||
txt = dom.xpath(xpath)[0]
|
|
||||||
ver = re.search(r'(?i)([0-9]+\.)*[0-9]+(-?(alpha|beta|rc)[0-9]+)?', txt).group(0).lower()
|
|
||||||
return '{}: {}\n'.format(key, ver)
|
|
||||||
|
|
||||||
except:
|
|
||||||
|
|
||||||
return '{}: {}\n'.format(key, 'unknown')
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
|
|
||||||
print 'Extract urls from <{}> ({})'.format(urls_yml, len(config))
|
|
||||||
urls = ['{0[url]}#{1}\n'.format(cfg, key) for key, cfg in config.items()]
|
|
||||||
|
|
||||||
print 'Write urls to <{}> ({})'.format(urls_txt, len(urls))
|
|
||||||
with open(urls_txt, 'w') as f:
|
|
||||||
f.writelines(urls)
|
|
||||||
|
|
||||||
print 'Everything is OK!'
|
|
||||||
|
|
||||||
@ -1,145 +0,0 @@
|
|||||||
###### OS ######
|
|
||||||
|
|
||||||
alpine:
|
|
||||||
url: http://alpinelinux.org/
|
|
||||||
xpath: //span[@class="icon-download"]/following-sibling::p/a/text()
|
|
||||||
|
|
||||||
archlinux:
|
|
||||||
url: https://www.archlinux.org/download/
|
|
||||||
xpath: //li[strong="Current Release:"]/text()
|
|
||||||
|
|
||||||
centos:
|
|
||||||
url: http://www.centos.org/download/
|
|
||||||
xpath: //div[@class="downloadbutton"]/a[.="DVD ISO"]/@href
|
|
||||||
|
|
||||||
coreos:
|
|
||||||
url: https://coreos.com
|
|
||||||
xpath: //div[@class="co-p-homepage-release-text"]/text()
|
|
||||||
|
|
||||||
debian:
|
|
||||||
url: https://www.debian.org/releases/
|
|
||||||
xpath: //li[contains(., "current stable release")]/a/text()
|
|
||||||
|
|
||||||
fedora:
|
|
||||||
url: https://getfedora.org/en/server/download/
|
|
||||||
xpath: //h2[starts-with(., "Download")]/text()
|
|
||||||
|
|
||||||
freebsd:
|
|
||||||
url: http://www.freebsd.org/releases/
|
|
||||||
xpath: //div[@id="sidenav"]//li[contains(text(), "Production Release:")][1]/a/text()
|
|
||||||
|
|
||||||
kali:
|
|
||||||
url: https://www.kali.org/kali-linux-releases/
|
|
||||||
xpath: //h2[.="Kali Linux Release History"]/following-sibling::ul/li[1]/strong/text()
|
|
||||||
|
|
||||||
raspbian:
|
|
||||||
url: https://www.raspberrypi.org/downloads/
|
|
||||||
xpath: //div[h3="Raspbian"]//span[.="Kernel version:"]/following-sibling::strong/text()
|
|
||||||
|
|
||||||
redhat:
|
|
||||||
url: https://access.redhat.com/products/red-hat-enterprise-linux/evaluation
|
|
||||||
xpath: //a[contains(@class, "btn-download")]/small/text()
|
|
||||||
|
|
||||||
smartos:
|
|
||||||
url: https://github.com/joyent/smartos-live/releases
|
|
||||||
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
|
|
||||||
|
|
||||||
ubuntu:
|
|
||||||
url: http://www.ubuntu.com/download/server
|
|
||||||
xpath: //div[contains(@class, "row-hero")]//h2/text()
|
|
||||||
|
|
||||||
###### LANG ######
|
|
||||||
|
|
||||||
go:
|
|
||||||
url: https://golang.org/dl/
|
|
||||||
xpath: //h3[.="Stable versions"]/following-sibling::div[1]/@id
|
|
||||||
|
|
||||||
nodejs:
|
|
||||||
url: https://nodejs.org/
|
|
||||||
xpath: //div[@id="intro"]/p[@class="version"]/text()
|
|
||||||
|
|
||||||
python:
|
|
||||||
url: https://www.python.org/downloads/
|
|
||||||
xpath: //div[@class="download-unknown"]/p[@class]/a[1]/text()
|
|
||||||
|
|
||||||
###### DB ######
|
|
||||||
|
|
||||||
mongodb:
|
|
||||||
url: https://www.mongodb.org/downloads
|
|
||||||
xpath: //div[@id="production-release-header"]/h2/@data-production-version
|
|
||||||
|
|
||||||
redis:
|
|
||||||
url: http://redis.io
|
|
||||||
xpath: //a[contains(@href, "/releases/")]/text()
|
|
||||||
|
|
||||||
influxdb:
|
|
||||||
url: https://influxdb.com/download/index.html
|
|
||||||
xpath: //h2[contains(., "Stable")]/text()
|
|
||||||
|
|
||||||
###### TOOL ######
|
|
||||||
|
|
||||||
vagrant:
|
|
||||||
url: https://www.vagrantup.com/downloads.html
|
|
||||||
xpath: //h2[@class="os-name" and .="Mac OS X"]/following-sibling::ul/li/a/@href
|
|
||||||
|
|
||||||
git:
|
|
||||||
url: http://git-scm.com
|
|
||||||
xpath: //span[@class="version"]/text()
|
|
||||||
|
|
||||||
docker:
|
|
||||||
url: https://github.com/docker/docker/releases
|
|
||||||
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
|
|
||||||
|
|
||||||
phantomjs:
|
|
||||||
url: http://phantomjs.org
|
|
||||||
xpath: //span[@class="version"]/text()
|
|
||||||
|
|
||||||
firefox:
|
|
||||||
url: https://www.mozilla.org/en-US/firefox/new/
|
|
||||||
xpath: //li[@class="os_osx"]/a[@class="download-link"]/@href
|
|
||||||
|
|
||||||
jquery:
|
|
||||||
url: http://jquery.com
|
|
||||||
xpath: //span[@class="download"]/following-sibling::span/text()
|
|
||||||
|
|
||||||
vim:
|
|
||||||
url: http://en.wikipedia.org/wiki/Vim_%28text_editor%29
|
|
||||||
xpath: //th[a="Stable release"]/following-sibling::td/text()
|
|
||||||
|
|
||||||
scrapy:
|
|
||||||
url: https://github.com/scrapy/scrapy/releases
|
|
||||||
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
|
|
||||||
|
|
||||||
scrapyd:
|
|
||||||
url: https://github.com/scrapy/scrapyd/releases
|
|
||||||
xpath: //div[contains(@class, "label-latest")]//h1[@class="release-title"]/a/text()
|
|
||||||
|
|
||||||
urlwatch:
|
|
||||||
url: https://github.com/thp/urlwatch/releases
|
|
||||||
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
|
|
||||||
|
|
||||||
monit:
|
|
||||||
url: http://mmonit.com/monit/changes/
|
|
||||||
xpath: //div[@class="container"]/div[@class="row"][1]//h3/a/text()
|
|
||||||
|
|
||||||
virtualbox:
|
|
||||||
url: https://www.virtualbox.org/wiki/Downloads
|
|
||||||
xpath: //strong/following-sibling::a[@class="ext-link" and contains(@href, "dmg")]/@href
|
|
||||||
|
|
||||||
###### VPN ######
|
|
||||||
|
|
||||||
openvpn:
|
|
||||||
url: https://openvpn.net/index.php/open-source/downloads.html
|
|
||||||
xpath: //h2[contains(., "released on")]/text()
|
|
||||||
|
|
||||||
tinc:
|
|
||||||
url: http://tinc-vpn.org/
|
|
||||||
xpath: //h3/a[contains(@href, "download")]/text()
|
|
||||||
|
|
||||||
shadowsocks:
|
|
||||||
url: https://github.com/shadowsocks/shadowsocks/releases
|
|
||||||
xpath: //ul[@class="release-timeline-tags"]/li[1]//span[@class="tag-name"]/text()
|
|
||||||
|
|
||||||
shadowsocks-libev:
|
|
||||||
url: https://github.com/shadowsocks/shadowsocks-libev/releases
|
|
||||||
xpath: //div[contains(@class, "label-latest")]//h1[@class="release-title"]/a/text()
|
|
||||||
Loading…
x
Reference in New Issue
Block a user