From 69f348aa24a0c8a2aba3da5f7c106df8c92fe308 Mon Sep 17 00:00:00 2001 From: Richard Yen Date: Mon, 2 Jul 2018 21:41:49 +0000 Subject: [PATCH 1/7] Clean up README.md formatting --- README.md | 99 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index fcda625..3099b1c 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ Setup There are three ways to install your own copy: -* Using Vagrant [ VirtualBox machine, fully automated ] -* Calling `puppet apply` -* Manually +1. Using Vagrant [ VirtualBox machine, fully automated ] +1. Calling `puppet apply` +1. Manually First get the source code: @@ -16,7 +16,8 @@ First get the source code: ## Vagrant setup -WARNING: first call `vagrant up` fetches ~400MB of vbox image from the Internet. +*WARNING:* +The first call of `vagrant up` fetches a ~400MB vbox image from the Internet. 1. Call `vagrant up` @@ -25,23 +26,23 @@ Point your browser on http://192.168.44.55 (or use `/etc/hosts` entry explain.de ## Puppet setup 1. Install puppet on your machine, e.g. Debian installation described [here](https://docs.puppetlabs.com/guides/install_puppet/install_debian_ubuntu.html). -2. Open explain.pp and fix line 5 with correct project dir value -3. Call `sudo puppet apply --logdest console explain.pp` on your working copy directory. +1. Open `explain.pp` and fix line 5 with correct project dir value +1. Call `sudo puppet apply --logdest console explain.pp` on your working copy directory. ## Manual setup -1) Mojolicious - -You have to have Mojolicious installed on your server. Mojolicious is a web framework for Perl. +### 1) Mojolicious +You have to have Mojolicious installed on your server. +Mojolicious is a web framework for Perl. http://mojolicio.us/ Installation can be accomplished with one command line: curl -L cpanmin.us | perl - -n Mojolicious -See the "Installation" section at http://mojolicio.us/ for details. +See the `Installation` section at http://mojolicio.us/ for details. -2) Perl Dependencies: +### 2) Perl Dependencies: You will need the following packages installed in your version of Perl: DBD::Pg @@ -51,61 +52,77 @@ You will need the following packages installed in your version of Perl: Email::Valid -Install the above packages using "cpan" then "i Date::Simple", "i Mail::Sender" &etc. +Install the above packages using `cpan` then `-i Date::Simple`, `-i Mail::Sender`, &etc. Note that in case of most current Linux distributions, you can install most of these from binary package repositories. For example, in case of Ubuntu and Debian, you can: -apt-get install libmojolicious-perl libmail-sender-perl libdate-simple-perl libemail-valid-perl + apt-get install libmojolicious-perl \ + libmail-sender-perl \ + libdate-simple-perl \ + libemail-valid-perl \ + libxml-simple-perl \ + libdbd-pg-perl -And then only add Pg::Explain via cpan. +And then only add `Pg::Explain` via CPAN. -3) PostgreSQL +### 3) PostgreSQL +You'll need to have PostgreSQL installed in order to record all the explain +plans into history. Consult the [PostgreSQL Wiki](https://wiki.postgresql.org/wiki/Detailed_installation_guides) +for more information -A) Create a new database "explain". This will be were the explain server will store the "users" -and "plans" tables in the default schema "public". +#### 3A) Create a new database `explain` +This will be were the explain server will store the `users` and `plans` tables +in the default schema `public`. -B) Run SQL scripts. Log into postgres as postgres. Switch to the "explain" database. -Execute the SQL scripts located in the "sql" directory in the following order: +#### 3B) Run SQL scripts +Log in to the `explain` database and execute the SQL scripts located in this +project's `sql` directory in the following order: \i create.sql \i patch-001.sql \i patch-002.sql -The "create.sql" will create tables in the explain database "public" schema, i.e. "plans" and "users". +The `create.sql` will create tables in the `explain` database `public` schema, +i.e. `plans` and `users`. -B) Create a user role. I use "explaind" [explain daemon] and remember to provides it a password and then configure -the explain.json file to reflect this new role and password. +#### 3C) Create a user role +I use `explaind` [explain daemon], for example. Remember to provide it a +password and then configure the `explain.json` file to reflect this new role and +password. -C) Grant all rights to the tables in "explain" to your role "explaind": + CREATE USER explaind WITH PASSWORD 'explain'; - grant all on plans, users to explaind; +#### 3D) Grant all rights to the tables in `explain` to your role: -D) modify /etc/postgresql-9.3/pg_hba.conf so that it has the server as "127.0.0.1" + GRANT ALL ON plans, users TO explaind; + +#### 3E) modify `pg_hba.conf` so that it has the server as "127.0.0.1" local all all 127.0.0.1 trust -If you do not want to alter /etc/postgresql-9.3/pg_hba.conf, then you might be able -to modify the explain.json.dsn setting to specify the name of the value i -n the server column from your /etc/postgresql-9.3/pg_hba.conf file, e.g. "localhost". This -alternative approach has not been tested. It would be desirable to install this project with -the minimal amount of configuration changes, so I encourage someone to come up with a solution -that removes this step "D". +If you do not want to alter `pg_hba.conf`, then you might be able to modify the +`explain.json.dsn` setting to specify the name of the value in the server column +from your `pg_hba.conf` file, e.g. "localhost". This alternative approach has +not been tested. It would be desirable to install this project with the minimal +amount of configuration changes, so I encourage someone to come up with a +solution that removes this step "E". -4) Alter configuration file explain.json making sure you have the correct values for your database connection. -See companion documentation file explain-json-notes.txt +### 4) Configure `explain.json` +Make sure you have the correct values for your database connection. +See companion documentation file `explain-json-notes.txt` -5) Shell - -The explain server runs on port 3000. Make sure port 3000 is available and not in use by another process. -You may have to specify a different port, e.g. 3200. I do not know where you do that, probably in Mojolicious. +### 5) Shell +The explain server runs on port 3000. Make sure port 3000 is available and not +in use by another process. You may have to specify a different port (e.g. 3200). +I do not know where you do that, probably in Mojolicious. In the trunk directory for this project, run in a shell: - ./explain daemon + ./explain.pl daemon -Then access the web page http://localhost:3000 - -6) when you access the web page, remember to login and create an account for yourself so that your explain plans +### 6) Browser +Then access the web page `http://localhost:3000` When you access the web page, +remember to login and create an account for yourself so that your explain plans will be associated with your account. From 3d1b6fcf91b3021b5ca42752bb3a8d77a87c766a Mon Sep 17 00:00:00 2001 From: Richard Yen Date: Mon, 2 Jul 2018 15:20:47 -0700 Subject: [PATCH 2/7] Make Vagrantfile usable for non-Puppet users --- Vagrantfile | 18 +++++++++++------- vagrant_bootstrap.sh | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 vagrant_bootstrap.sh diff --git a/Vagrantfile b/Vagrantfile index cf575fd..0eeab35 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -12,12 +12,16 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| vb.customize ["modifyvm", :id, "--memory", "1024"] end - config.vm.provision "puppet" do |puppet| - puppet.options = "--verbose --debug" - puppet.manifests_path = "." - puppet.manifest_file = "explain.pp" - puppet.facter = { - "use_vagrant" => true - } + if Vagrant.has_plugin?("vagrant-puppet-install") + config.vm.provision "puppet" do |puppet| + puppet.options = "--verbose --debug" + puppet.manifests_path = "." + puppet.manifest_file = "explain.pp" + puppet.facter = { + "use_vagrant" => true + } + end + else + config.vm.provision "shell", path: "vagrant_bootstrap.sh" end end diff --git a/vagrant_bootstrap.sh b/vagrant_bootstrap.sh new file mode 100644 index 0000000..60cb83f --- /dev/null +++ b/vagrant_bootstrap.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +PGDATA="/etc/postgresql/9.3/main" + +curl -s -L cpanmin.us | perl - -n Mojolicious +apt-get -y -qq install cpanminus libmojolicious-perl libmail-sender-perl libdate-simple-perl libemail-valid-perl libxml-simple-perl libdbd-pg-perl libxml-simple-perl +cpanm -q Pg::Explain +apt-get -y -qq install postgresql postgresql-contrib +sed -i -e "s/md5/trust/" -e "s/peer/trust/" ${PGDATA}/pg_hba.conf +sudo -u postgres psql -tc "select pg_reload_conf()" + +createdb -U postgres explain +psql -qU postgres explain < /vagrant/sql/create.sql +psql -qU postgres explain < /vagrant/sql/patch-001.sql +psql -qU postgres explain < /vagrant/sql/patch-002.sql +psql -qU postgres explain -c "CREATE USER explain WITH PASSWORD 'explain'" +psql -qU postgres explain -c "GRANT ALL ON plans, users TO explain" +echo "Your Vagrant box is all set up. Use 'vagrant ssh' to log in, then call '/vagrant/explain.pl daemon' to start the service" From 441fc9802e492ff21afd99a87afba5cd3ea8669a Mon Sep 17 00:00:00 2001 From: Richard Yen Date: Tue, 3 Jul 2018 00:13:49 -0700 Subject: [PATCH 3/7] Ignore Vagrant metadata --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c65550f..6dc687e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ log explain.production.json +.vagrant From 334dc3b710ac05d97cfe5483f87e77d2af9c85f4 Mon Sep 17 00:00:00 2001 From: Richard Yen Date: Thu, 5 Jul 2018 14:21:03 -0700 Subject: [PATCH 4/7] Apply all patches in `sql/` folder Upgrade to newer version of PG (support for FILTER syntax found in `patch-008.sql` begins in 9.4) Fix up some variable names --- vagrant_bootstrap.sh | 47 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/vagrant_bootstrap.sh b/vagrant_bootstrap.sh index 60cb83f..0013216 100644 --- a/vagrant_bootstrap.sh +++ b/vagrant_bootstrap.sh @@ -1,18 +1,47 @@ #!/bin/bash -PGDATA="/etc/postgresql/9.3/main" +export PGDATA="/etc/postgresql/10/main" +export APP_USER="explaind" +export APP_PASS="explainpass" +# Set username in explain.json +sed -i "s/\"username\" : \"explain\"/\"username\" : \"${APP_USER}\"/" /vagrant/explain.json +sed -i "s/\"password\" : \"explain\"/\"password\" : \"${APP_PASS}\"/" /vagrant/explain.json + +# Install dependencies curl -s -L cpanmin.us | perl - -n Mojolicious -apt-get -y -qq install cpanminus libmojolicious-perl libmail-sender-perl libdate-simple-perl libemail-valid-perl libxml-simple-perl libdbd-pg-perl libxml-simple-perl +apt-get -y -qq install wget ca-certificates cpanminus libmojolicious-perl libmail-sender-perl libdate-simple-perl libemail-valid-perl libxml-simple-perl libdbd-pg-perl libxml-simple-perl cpanm -q Pg::Explain -apt-get -y -qq install postgresql postgresql-contrib -sed -i -e "s/md5/trust/" -e "s/peer/trust/" ${PGDATA}/pg_hba.conf + +# Install Postgres +echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - +sudo apt-get -y -qq update +sudo apt-get -y -qq upgrade +sudo apt-get -y -qq install postgresql-10 +sed -i -e "s/peer/trust/" ${PGDATA}/pg_hba.conf sudo -u postgres psql -tc "select pg_reload_conf()" +# Create database createdb -U postgres explain -psql -qU postgres explain < /vagrant/sql/create.sql -psql -qU postgres explain < /vagrant/sql/patch-001.sql -psql -qU postgres explain < /vagrant/sql/patch-002.sql -psql -qU postgres explain -c "CREATE USER explain WITH PASSWORD 'explain'" -psql -qU postgres explain -c "GRANT ALL ON plans, users TO explain" + +# Create user +psql -qU postgres explain -c "CREATE USER ${APP_USER} WITH PASSWORD '${APP_PASS}'" + +# Apply patches +psql -q -f /vagrant/sql/create.sql -U postgres explain +for i in `seq -f "%03g" 1 10` +do + psql -q -f /vagrant/sql/patch-${i}.sql -U postgres explain +done + +# Apply grants +psql -qU postgres explain -c "GRANT ALL ON plans, users TO ${APP_USER}" +psql -qU postgres explain -c "GRANT ALL ON SCHEMA plans TO ${APP_USER}" +for x in `echo {0..9} {A..Z} {a..z}` +do + psql -qU postgres explain -c "GRANT ALL PRIVILEGES ON plans.\"part_${x}\" TO ${APP_USER}" +done + +# Done echo "Your Vagrant box is all set up. Use 'vagrant ssh' to log in, then call '/vagrant/explain.pl daemon' to start the service" From a7e087d63a73849ec23973a9097a090931b4d2d1 Mon Sep 17 00:00:00 2001 From: Richard Yen Date: Thu, 5 Jul 2018 14:23:09 -0700 Subject: [PATCH 5/7] Replace testing username/pass to defaults --- vagrant_bootstrap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vagrant_bootstrap.sh b/vagrant_bootstrap.sh index 0013216..c6ea13c 100644 --- a/vagrant_bootstrap.sh +++ b/vagrant_bootstrap.sh @@ -1,8 +1,8 @@ #!/bin/bash export PGDATA="/etc/postgresql/10/main" -export APP_USER="explaind" -export APP_PASS="explainpass" +export APP_USER="explain" +export APP_PASS="explain" # Set username in explain.json sed -i "s/\"username\" : \"explain\"/\"username\" : \"${APP_USER}\"/" /vagrant/explain.json From 22808904ebc7661dba4039b620a7f91b2b176c7f Mon Sep 17 00:00:00 2001 From: Richard Yen Date: Wed, 11 Jul 2018 09:55:59 -0700 Subject: [PATCH 6/7] Add steps for setting up on Docker --- vagrant_bootstrap.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vagrant_bootstrap.sh b/vagrant_bootstrap.sh index c6ea13c..5fbea37 100644 --- a/vagrant_bootstrap.sh +++ b/vagrant_bootstrap.sh @@ -11,6 +11,13 @@ sed -i "s/\"password\" : \"explain\"/\"password\" : \"${APP_PASS}\"/" /vagrant/e # Install dependencies curl -s -L cpanmin.us | perl - -n Mojolicious apt-get -y -qq install wget ca-certificates cpanminus libmojolicious-perl libmail-sender-perl libdate-simple-perl libemail-valid-perl libxml-simple-perl libdbd-pg-perl libxml-simple-perl + +### This script *can* be run on a Docker Ubuntu VM if desired. +### To do so, additional commands need to be run: +# apt-get install make curl libclone-perl +# sed -i "s/exit.*/exit 0/" /usr/sbin/policy-rc.d + +# Install Pg::Explain cpanm -q Pg::Explain # Install Postgres From 970e2778c8d37a3bf47af3b09b2be01ae4d0ed93 Mon Sep 17 00:00:00 2001 From: Richard Yen Date: Tue, 31 Jul 2018 15:23:50 -0700 Subject: [PATCH 7/7] Rearrange steps, make Docker-ization simpler --- vagrant_bootstrap.sh | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/vagrant_bootstrap.sh b/vagrant_bootstrap.sh index 5fbea37..9937512 100644 --- a/vagrant_bootstrap.sh +++ b/vagrant_bootstrap.sh @@ -3,19 +3,22 @@ export PGDATA="/etc/postgresql/10/main" export APP_USER="explain" export APP_PASS="explain" - -# Set username in explain.json -sed -i "s/\"username\" : \"explain\"/\"username\" : \"${APP_USER}\"/" /vagrant/explain.json -sed -i "s/\"password\" : \"explain\"/\"password\" : \"${APP_PASS}\"/" /vagrant/explain.json - -# Install dependencies -curl -s -L cpanmin.us | perl - -n Mojolicious -apt-get -y -qq install wget ca-certificates cpanminus libmojolicious-perl libmail-sender-perl libdate-simple-perl libemail-valid-perl libxml-simple-perl libdbd-pg-perl libxml-simple-perl +export BASEDIR="/vagrant" ### This script *can* be run on a Docker Ubuntu VM if desired. ### To do so, additional commands need to be run: -# apt-get install make curl libclone-perl +# apt-get update +# apt-get -y -qq install make curl libclone-perl lsb-release sudo # sed -i "s/exit.*/exit 0/" /usr/sbin/policy-rc.d +# export BASEDIR="/explain" + +# Set username in explain.json +sed -i "s/\"username\" : \"explain\"/\"username\" : \"${APP_USER}\"/" ${BASEDIR}/explain.json +sed -i "s/\"password\" : \"explain\"/\"password\" : \"${APP_PASS}\"/" ${BASEDIR}/explain.json + +# Install dependencies +curl -s -L cpanmin.us | perl - -n Mojolicious +apt-get -y -qq install wget ca-certificates cpanminus libmojolicious-perl libmail-sender-perl libdate-simple-perl libemail-valid-perl libxml-simple-perl libdbd-pg-perl # Install Pg::Explain cpanm -q Pg::Explain @@ -36,10 +39,10 @@ createdb -U postgres explain psql -qU postgres explain -c "CREATE USER ${APP_USER} WITH PASSWORD '${APP_PASS}'" # Apply patches -psql -q -f /vagrant/sql/create.sql -U postgres explain +psql -q -f ${BASEDIR}/sql/create.sql -U postgres explain for i in `seq -f "%03g" 1 10` do - psql -q -f /vagrant/sql/patch-${i}.sql -U postgres explain + psql -q -f ${BASEDIR}/sql/patch-${i}.sql -U postgres explain done # Apply grants