split presto to prestodb+prestosql

2025-11-25 22:22:29 +02:00 · 2019-10-21 11:36:26 +08:00
parent 46eb7659a1
commit b48eefc547
32 changed files with 121 additions and 63 deletions
--- a/prestodb/Dockerfile.alpine
+++ b/prestodb/Dockerfile.alpine
@@ -0,0 +1,28 @@
+#
+# Dockerfile for prestodb
+#
+
+FROM openjdk:8-jre-alpine
+MAINTAINER kev <noreply@easypi.pro>
+
+ENV PRESTO_VERSION=0.227
+ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz
+ENV PRESTO_CLI_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar
+ENV PRESTO_HOME=/opt/presto
+ENV PATH=${PRESTO_HOME}/bin:${PATH}
+
+WORKDIR $PRESTO_HOME
+
+RUN set -xe \
+    && apk add --no-cache curl python tar \
+    && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \
+    && curl -sSL $PRESTO_CLI_URL > ./bin/presto \
+    && chmod +x ./bin/presto \
+    && apk del curl tar
+
+VOLUME /data
+
+EXPOSE 8080
+
+ENTRYPOINT ["launcher"]
+CMD ["run"]
--- a/prestodb/Dockerfile.debian
+++ b/prestodb/Dockerfile.debian
@@ -0,0 +1,30 @@
+#
+# Dockerfile for prestodb
+#
+
+FROM openjdk:8-jre-slim-buster
+MAINTAINER kev <noreply@easypi.pro>
+
+ENV PRESTO_VERSION=0.227
+ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz
+ENV PRESTO_CLI_URL=https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar
+ENV PRESTO_HOME=/opt/presto
+ENV PATH=${PRESTO_HOME}/bin:${PATH}
+
+WORKDIR $PRESTO_HOME
+
+RUN set -xe \
+    && apt-get update \
+    && apt-get install -y curl less python \
+    && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \
+    && curl -sSL $PRESTO_CLI_URL > ./bin/presto \
+    && chmod +x ./bin/presto \
+    && apt-get remove -y curl \
+    && rm -rf /var/lib/apt/lists/*
+
+VOLUME /data
+
+EXPOSE 8080
+
+ENTRYPOINT ["launcher"]
+CMD ["run"]
--- a/prestodb/README.md
+++ b/prestodb/README.md
@@ -0,0 +1,48 @@
+prestodb
+========
+
+[Presto][1] is an open source distributed SQL query engine for running
+interactive analytic queries against data sources of all sizes ranging from
+gigabytes to petabytes.
+
+## standalone mode
+
+```yaml
+prestodb:
+  image: vimagick/prestodb
+  ports:
+    - "8080:8080"
+  volumes:
+    - ./conf/standalone:/opt/presto/etc:ro
+    - /data:/data
+  restart: unless-stopped
+```
+
+## cluster mode
+
+```bash
+$ ansible all -a 'mkdir -p /data'
+
+$ docker stack deploy -c docker-stack.yml prestodb
+
+$ docker service update --replicas-max-per-node=1 prestodb_worker
+
+$ docker service update --replicas 10 prestodb_worker
+
+$ docker ps | grep prestodb_coordinator | awk '{print $1}'
+4cc5c6c420d7
+
+$ docker exec -it 4cc5c6c420d7 prestodb --server localhost:8080 --catalog tpch
+>>> show schemas;
+>>> show tables from tiny;
+>>> select * from tiny.customer limit 10;
+>>> quit
+
+$ curl http://localhost:8080/ui/
+
+$ curl http://localhost:8080/v1/service/presto/general
+```
+
+> :warning: If volume settings are changed, you need to remove them manually on all nodes.
+
+[1]: https://prestodb.github.io/
--- a/prestodb/conf/coordinator/catalog/jmx.properties
+++ b/prestodb/conf/coordinator/catalog/jmx.properties
@@ -0,0 +1 @@
+connector.name=jmx
--- a/prestodb/conf/coordinator/catalog/tpcds.properties
+++ b/prestodb/conf/coordinator/catalog/tpcds.properties
@@ -0,0 +1 @@
+connector.name=tpcds
--- a/prestodb/conf/coordinator/catalog/tpch.properties
+++ b/prestodb/conf/coordinator/catalog/tpch.properties
@@ -0,0 +1 @@
+connector.name=tpch
--- a/prestodb/conf/coordinator/config.properties
+++ b/prestodb/conf/coordinator/config.properties
@@ -0,0 +1,9 @@
+coordinator=true
+node-scheduler.include-coordinator=false
+http-server.http.port=8080
+query.max-memory=50GB
+query.max-memory-per-node=1GB
+query.max-total-memory-per-node=2GB
+discovery-server.enabled=true
+discovery.uri=http://127.0.0.1:8080
+node.internal-address=coordinator
--- a/prestodb/conf/coordinator/jvm.config
+++ b/prestodb/conf/coordinator/jvm.config
@@ -0,0 +1,9 @@
+-server
+-Xmx16G
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+UseGCOverheadLimit
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
+-XX:OnOutOfMemoryError=kill -9 %p
--- a/prestodb/conf/coordinator/log.properties
+++ b/prestodb/conf/coordinator/log.properties
@@ -0,0 +1 @@
+com.facebook.presto=INFO
--- a/prestodb/conf/coordinator/node.properties
+++ b/prestodb/conf/coordinator/node.properties
@@ -0,0 +1,3 @@
+node.environment=production
+node.data-dir=/data
+node.id=coordinator
--- a/prestodb/conf/standalone/catalog/jmx.properties
+++ b/prestodb/conf/standalone/catalog/jmx.properties
@@ -0,0 +1 @@
+connector.name=jmx
--- a/prestodb/conf/standalone/catalog/tpcds.properties
+++ b/prestodb/conf/standalone/catalog/tpcds.properties
@@ -0,0 +1 @@
+connector.name=tpcds
--- a/prestodb/conf/standalone/catalog/tpch.properties
+++ b/prestodb/conf/standalone/catalog/tpch.properties
@@ -0,0 +1 @@
+connector.name=tpch
--- a/prestodb/conf/standalone/config.properties
+++ b/prestodb/conf/standalone/config.properties
@@ -0,0 +1,8 @@
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+query.max-memory=5GB
+query.max-memory-per-node=1GB
+query.max-total-memory-per-node=2GB
+discovery-server.enabled=true
+discovery.uri=http://127.0.0.1:8080
--- a/prestodb/conf/standalone/jvm.config
+++ b/prestodb/conf/standalone/jvm.config
@@ -0,0 +1,9 @@
+-server
+-Xmx16G
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+UseGCOverheadLimit
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
+-XX:OnOutOfMemoryError=kill -9 %p
--- a/prestodb/conf/standalone/log.properties
+++ b/prestodb/conf/standalone/log.properties
@@ -0,0 +1 @@
+com.facebook.presto=INFO
--- a/prestodb/conf/standalone/node.properties
+++ b/prestodb/conf/standalone/node.properties
@@ -0,0 +1,3 @@
+node.environment=production
+node.data-dir=/data
+node.id=standalone
--- a/prestodb/conf/worker/catalog/jmx.properties
+++ b/prestodb/conf/worker/catalog/jmx.properties
@@ -0,0 +1 @@
+connector.name=jmx
--- a/prestodb/conf/worker/catalog/tpcds.properties
+++ b/prestodb/conf/worker/catalog/tpcds.properties
@@ -0,0 +1 @@
+connector.name=tpcds
--- a/prestodb/conf/worker/catalog/tpch.properties
+++ b/prestodb/conf/worker/catalog/tpch.properties
@@ -0,0 +1 @@
+connector.name=tpch
--- a/prestodb/conf/worker/config.properties
+++ b/prestodb/conf/worker/config.properties
@@ -0,0 +1,7 @@
+coordinator=false
+http-server.http.port=8080
+query.max-memory=50GB
+query.max-memory-per-node=1GB
+query.max-total-memory-per-node=2GB
+discovery.uri=http://coordinator:8080
+#node.internal-address=worker1
--- a/prestodb/conf/worker/jvm.config
+++ b/prestodb/conf/worker/jvm.config
@@ -0,0 +1,9 @@
+-server
+-Xmx16G
+-XX:+UseG1GC
+-XX:G1HeapRegionSize=32M
+-XX:+UseGCOverheadLimit
+-XX:+ExplicitGCInvokesConcurrent
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
+-XX:OnOutOfMemoryError=kill -9 %p
--- a/prestodb/conf/worker/log.properties
+++ b/prestodb/conf/worker/log.properties
@@ -0,0 +1 @@
+com.facebook.presto=INFO
--- a/prestodb/conf/worker/node.properties
+++ b/prestodb/conf/worker/node.properties
@@ -0,0 +1,3 @@
+node.environment=production
+node.data-dir=/data
+#node.id=worker1
--- a/prestodb/docker-compose.yml
+++ b/prestodb/docker-compose.yml
@@ -0,0 +1,20 @@
+nfs:
+  image: erichough/nfs-server
+  ports:
+    - "111:111/tcp"
+    - "111:111/udp"
+    - "2049:2049/tcp"
+    - "2049:2049/udp"
+    - "32765:32765/tcp"
+    - "32765:32765/udp"
+    - "32767:32767/tcp"
+    - "32767:32767/udp"
+  volumes:
+    - ./conf:/export/presto:ro
+    - /lib/modules:/lib/modules:ro
+  environment:
+    - NFS_LOG_LEVEL=DEBUG
+    - NFS_EXPORT_0=/export/prestodb/coordinator *(ro,no_subtree_check)
+    - NFS_EXPORT_1=/export/prestodb/worker1     *(ro,no_subtree_check)
+  privileged: true
+  restart: unless-stopped
--- a/prestodb/docker-stack.yml
+++ b/prestodb/docker-stack.yml
@@ -0,0 +1,50 @@
+version: "3.7"
+
+services:
+
+  coordinator:
+    image: vimagick/prestodb
+    ports:
+      - "8080:8080"
+    volumes:
+      - /data:/data
+      - prestodb_coordinator_conf:/opt/presto/etc
+    deploy:
+      replicas: 1
+      placement:
+        constraints:
+          - node.role == manager
+          - node.hostname == presto-coordinator
+      restart_policy:
+        condition: on-failure
+  
+  worker:
+    image: vimagick/prestodb
+    volumes:
+      - /data:/data
+      - prestodb_worker_conf:/opt/presto/etc
+    deploy:
+      replicas: 0
+      placement:
+        constraints:
+          - node.role == worker
+      restart_policy:
+        condition: on-failure
+    depends_on:
+      - coordinator
+
+volumes:
+
+  prestodb_coordinator_conf:
+    driver: local
+    driver_opts:
+      type: nfs
+      o: "addr=10.0.0.1,nolock,soft,ro"
+      device: ":/export/prestodb/coordinator"
+
+  prestodb_worker_conf:
+    driver: local
+    driver_opts:
+      type: nfs
+      o: "addr=10.0.0.1,nolock,soft,ro"
+      device: ":/export/prestodb/worker"