diff --git a/prestodb/Dockerfile.alpine b/prestodb/Dockerfile.alpine index 67b81e3..f95f70e 100644 --- a/prestodb/Dockerfile.alpine +++ b/prestodb/Dockerfile.alpine @@ -14,7 +14,7 @@ ENV PATH=${PRESTO_HOME}/bin:${PATH} WORKDIR $PRESTO_HOME RUN set -xe \ - && apk add --no-cache curl python tar \ + && apk add --no-cache curl less python tar \ && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \ && curl -sSL $PRESTO_CLI_URL > ./bin/presto \ && chmod +x ./bin/presto \ diff --git a/prestodb/README.md b/prestodb/README.md index 85ebde5..406b58c 100644 --- a/prestodb/README.md +++ b/prestodb/README.md @@ -27,12 +27,12 @@ $ docker stack deploy -c docker-stack.yml prestodb $ docker service update --replicas-max-per-node=1 prestodb_worker -$ docker service update --replicas 10 prestodb_worker +$ docker service update --replicas=10 prestodb_worker $ docker ps | grep prestodb_coordinator | awk '{print $1}' 4cc5c6c420d7 -$ docker exec -it 4cc5c6c420d7 prestodb --server localhost:8080 --catalog tpch +$ docker exec -it 4cc5c6c420d7 presto --server localhost:8080 --catalog tpch >>> show schemas; >>> show tables from tiny; >>> select * from tiny.customer limit 10; diff --git a/prestodb/docker-stack.yml b/prestodb/docker-stack.yml index 0a12b56..87030f2 100644 --- a/prestodb/docker-stack.yml +++ b/prestodb/docker-stack.yml @@ -14,12 +14,12 @@ services: placement: constraints: - node.role == manager - - node.hostname == presto-coordinator restart_policy: condition: on-failure worker: image: vimagick/prestodb:alpine + entrypoint: sh -c 'launcher run -Dnode.id=$$HOSTNAME' volumes: - /data:/data - prestodb_worker_conf:/opt/presto/etc diff --git a/prestosql/Dockerfile.alpine b/prestosql/Dockerfile.alpine index 15feed6..d119dde 100644 --- a/prestosql/Dockerfile.alpine +++ b/prestosql/Dockerfile.alpine @@ -14,7 +14,7 @@ ENV PATH=${PRESTO_HOME}/bin:${PATH} WORKDIR $PRESTO_HOME RUN set -xe \ - && apk add --no-cache curl python tar \ + && apk add --no-cache curl less python tar \ && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \ && curl -sSL $PRESTO_CLI_URL > ./bin/presto \ && chmod +x ./bin/presto \ diff --git a/prestosql/Dockerfile.debian b/prestosql/Dockerfile.debian index a5a7fd3..b192aed 100644 --- a/prestosql/Dockerfile.debian +++ b/prestosql/Dockerfile.debian @@ -1,5 +1,5 @@ # -# Dockerfile for presto-ce +# Dockerfile for prestosql # FROM openjdk:8-jre-slim-buster diff --git a/prestosql/README.md b/prestosql/README.md index ee0af6b..a3a562d 100644 --- a/prestosql/README.md +++ b/prestosql/README.md @@ -1,6 +1,46 @@ prestosql -========= +======== [Presto][1] is a high performance, distributed SQL query engine for big data. -[1]: https://prestosql.io +## standalone mode + +```yaml +prestosql: + image: vimagick/prestosql:alpine + ports: + - "8080:8080" + volumes: + - ./conf/standalone:/opt/presto/etc:ro + - /data:/data + restart: unless-stopped +``` + +## cluster mode + +```bash +$ ansible all -a 'mkdir -p /data' + +$ docker stack deploy -c docker-stack.yml prestosql + +$ docker service update --replicas-max-per-node=1 prestosql_worker + +$ docker service update --replicas=10 prestosql_worker + +$ docker ps | grep prestosql_coordinator | awk '{print $1}' +4cc5c6c420d7 + +$ docker exec -it 4cc5c6c420d7 presto --server localhost:8080 --catalog tpch +>>> show schemas; +>>> show tables from tiny; +>>> select * from tiny.customer limit 10; +>>> quit + +$ curl http://localhost:8080/ui/ + +$ curl http://localhost:8080/v1/service/presto/general +``` + +> :warning: If volume settings are changed, you need to remove them manually on all nodes. + +[1]: https://prestosql.io/ diff --git a/prestosql/conf/coordinator/catalog/jmx.properties b/prestosql/conf/coordinator/catalog/jmx.properties new file mode 100644 index 0000000..b6e0372 --- /dev/null +++ b/prestosql/conf/coordinator/catalog/jmx.properties @@ -0,0 +1 @@ +connector.name=jmx diff --git a/prestosql/conf/coordinator/catalog/tpcds.properties b/prestosql/conf/coordinator/catalog/tpcds.properties new file mode 100644 index 0000000..ba8147d --- /dev/null +++ b/prestosql/conf/coordinator/catalog/tpcds.properties @@ -0,0 +1 @@ +connector.name=tpcds diff --git a/prestosql/conf/coordinator/catalog/tpch.properties b/prestosql/conf/coordinator/catalog/tpch.properties new file mode 100644 index 0000000..75110c5 --- /dev/null +++ b/prestosql/conf/coordinator/catalog/tpch.properties @@ -0,0 +1 @@ +connector.name=tpch diff --git a/prestosql/conf/coordinator/config.properties b/prestosql/conf/coordinator/config.properties new file mode 100644 index 0000000..bb11096 --- /dev/null +++ b/prestosql/conf/coordinator/config.properties @@ -0,0 +1,9 @@ +coordinator=true +node-scheduler.include-coordinator=false +http-server.http.port=8080 +query.max-memory=50GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=2GB +discovery-server.enabled=true +discovery.uri=http://127.0.0.1:8080 +node.internal-address=coordinator diff --git a/prestosql/conf/coordinator/jvm.config b/prestosql/conf/coordinator/jvm.config new file mode 100644 index 0000000..afd8e8a --- /dev/null +++ b/prestosql/conf/coordinator/jvm.config @@ -0,0 +1,9 @@ +-server +-Xmx16G +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-XX:OnOutOfMemoryError=kill -9 %p diff --git a/prestosql/conf/coordinator/log.properties b/prestosql/conf/coordinator/log.properties new file mode 100644 index 0000000..1c52627 --- /dev/null +++ b/prestosql/conf/coordinator/log.properties @@ -0,0 +1 @@ +com.facebook.presto=INFO diff --git a/prestosql/conf/coordinator/node.properties b/prestosql/conf/coordinator/node.properties new file mode 100644 index 0000000..24ee579 --- /dev/null +++ b/prestosql/conf/coordinator/node.properties @@ -0,0 +1,3 @@ +node.environment=production +node.data-dir=/data +node.id=coordinator diff --git a/prestosql/conf/standalone/catalog/jmx.properties b/prestosql/conf/standalone/catalog/jmx.properties new file mode 100644 index 0000000..b6e0372 --- /dev/null +++ b/prestosql/conf/standalone/catalog/jmx.properties @@ -0,0 +1 @@ +connector.name=jmx diff --git a/prestosql/conf/standalone/catalog/tpcds.properties b/prestosql/conf/standalone/catalog/tpcds.properties new file mode 100644 index 0000000..ba8147d --- /dev/null +++ b/prestosql/conf/standalone/catalog/tpcds.properties @@ -0,0 +1 @@ +connector.name=tpcds diff --git a/prestosql/conf/standalone/catalog/tpch.properties b/prestosql/conf/standalone/catalog/tpch.properties new file mode 100644 index 0000000..75110c5 --- /dev/null +++ b/prestosql/conf/standalone/catalog/tpch.properties @@ -0,0 +1 @@ +connector.name=tpch diff --git a/prestosql/conf/standalone/config.properties b/prestosql/conf/standalone/config.properties new file mode 100644 index 0000000..d54e2a1 --- /dev/null +++ b/prestosql/conf/standalone/config.properties @@ -0,0 +1,8 @@ +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=5GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=2GB +discovery-server.enabled=true +discovery.uri=http://127.0.0.1:8080 diff --git a/prestosql/conf/standalone/jvm.config b/prestosql/conf/standalone/jvm.config new file mode 100644 index 0000000..afd8e8a --- /dev/null +++ b/prestosql/conf/standalone/jvm.config @@ -0,0 +1,9 @@ +-server +-Xmx16G +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-XX:OnOutOfMemoryError=kill -9 %p diff --git a/prestosql/conf/standalone/log.properties b/prestosql/conf/standalone/log.properties new file mode 100644 index 0000000..1c52627 --- /dev/null +++ b/prestosql/conf/standalone/log.properties @@ -0,0 +1 @@ +com.facebook.presto=INFO diff --git a/prestosql/conf/standalone/node.properties b/prestosql/conf/standalone/node.properties new file mode 100644 index 0000000..f2cf0e0 --- /dev/null +++ b/prestosql/conf/standalone/node.properties @@ -0,0 +1,3 @@ +node.environment=production +node.data-dir=/data +node.id=standalone diff --git a/prestosql/conf/worker/catalog/jmx.properties b/prestosql/conf/worker/catalog/jmx.properties new file mode 100644 index 0000000..b6e0372 --- /dev/null +++ b/prestosql/conf/worker/catalog/jmx.properties @@ -0,0 +1 @@ +connector.name=jmx diff --git a/prestosql/conf/worker/catalog/tpcds.properties b/prestosql/conf/worker/catalog/tpcds.properties new file mode 100644 index 0000000..ba8147d --- /dev/null +++ b/prestosql/conf/worker/catalog/tpcds.properties @@ -0,0 +1 @@ +connector.name=tpcds diff --git a/prestosql/conf/worker/catalog/tpch.properties b/prestosql/conf/worker/catalog/tpch.properties new file mode 100644 index 0000000..75110c5 --- /dev/null +++ b/prestosql/conf/worker/catalog/tpch.properties @@ -0,0 +1 @@ +connector.name=tpch diff --git a/prestosql/conf/worker/config.properties b/prestosql/conf/worker/config.properties new file mode 100644 index 0000000..2be3f48 --- /dev/null +++ b/prestosql/conf/worker/config.properties @@ -0,0 +1,7 @@ +coordinator=false +http-server.http.port=8080 +query.max-memory=50GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=2GB +discovery.uri=http://coordinator:8080 +#node.internal-address=worker1 diff --git a/prestosql/conf/worker/jvm.config b/prestosql/conf/worker/jvm.config new file mode 100644 index 0000000..afd8e8a --- /dev/null +++ b/prestosql/conf/worker/jvm.config @@ -0,0 +1,9 @@ +-server +-Xmx16G +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-XX:OnOutOfMemoryError=kill -9 %p diff --git a/prestosql/conf/worker/log.properties b/prestosql/conf/worker/log.properties new file mode 100644 index 0000000..1c52627 --- /dev/null +++ b/prestosql/conf/worker/log.properties @@ -0,0 +1 @@ +com.facebook.presto=INFO diff --git a/prestosql/conf/worker/node.properties b/prestosql/conf/worker/node.properties new file mode 100644 index 0000000..2a9d1e8 --- /dev/null +++ b/prestosql/conf/worker/node.properties @@ -0,0 +1,3 @@ +node.environment=production +node.data-dir=/data +#node.id=worker1 diff --git a/prestosql/docker-stack.yml b/prestosql/docker-stack.yml index 4f76563..cdd1540 100644 --- a/prestosql/docker-stack.yml +++ b/prestosql/docker-stack.yml @@ -14,12 +14,12 @@ services: placement: constraints: - node.role == manager - - node.hostname == presto-coordinator restart_policy: condition: on-failure worker: image: vimagick/prestosql:alpine + entrypoint: sh -c 'launcher run -Dnode.id=$$HOSTNAME' volumes: - /data:/data - prestosql_worker_conf:/opt/presto/etc