2017-06-20 12:57:23 +02:00
import os
2017-05-03 13:14:48 +02:00
import unittest
2017-06-27 07:42:52 +02:00
from . helpers . ptrack_helpers import ProbackupTest , idx_ptrack
2017-05-03 13:14:48 +02:00
2017-07-12 16:28:28 +02:00
module_name = ' ptrack_vacuum_bits_frozen '
2017-05-03 13:14:48 +02:00
class SimpleTest ( ProbackupTest , unittest . TestCase ) :
2017-05-22 13:17:43 +02:00
# @unittest.skip("skip")
# @unittest.expectedFailure
2017-05-03 13:14:48 +02:00
def test_ptrack_vacuum_bits_frozen ( self ) :
2017-05-17 11:46:38 +02:00
fname = self . id ( ) . split ( ' . ' ) [ 3 ]
2017-07-12 16:28:28 +02:00
node = self . make_simple_node ( base_dir = " {0} / {1} /node " . format ( module_name , fname ) ,
2017-05-03 13:14:48 +02:00
set_replication = True ,
2017-06-20 12:57:23 +02:00
initdb_params = [ ' --data-checksums ' ] ,
2017-05-03 13:14:48 +02:00
pg_options = { ' ptrack_enable ' : ' on ' , ' wal_level ' : ' replica ' , ' max_wal_senders ' : ' 2 ' } )
2017-07-12 16:28:28 +02:00
backup_dir = os . path . join ( self . tmp_path , module_name , fname , ' backup ' )
2017-06-20 12:57:23 +02:00
self . init_pb ( backup_dir )
self . add_instance ( backup_dir , ' node ' , node )
2017-05-03 13:14:48 +02:00
node . start ( )
2017-06-20 12:57:23 +02:00
2017-05-03 13:14:48 +02:00
self . create_tblspace_in_node ( node , ' somedata ' )
# Create table and indexes
2017-07-12 16:28:28 +02:00
res = node . safe_psql (
" postgres " ,
" create sequence t_seq; create table t_heap tablespace somedata as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i " )
for i in idx_ptrack :
if idx_ptrack [ i ] [ ' type ' ] != ' heap ' and idx_ptrack [ i ] [ ' type ' ] != ' seq ' :
node . safe_psql ( " postgres " , " create index {0} on {1} using {2} ( {3} ) tablespace somedata " . format (
i , idx_ptrack [ i ] [ ' relation ' ] , idx_ptrack [ i ] [ ' type ' ] , idx_ptrack [ i ] [ ' column ' ] ) )
node . safe_psql ( ' postgres ' , ' checkpoint ' )
for i in idx_ptrack :
# get size of heap and indexes. size calculated in pages
idx_ptrack [ i ] [ ' old_size ' ] = self . get_fork_size ( node , i )
# get path to heap and index files
idx_ptrack [ i ] [ ' path ' ] = self . get_fork_path ( node , i )
# calculate md5sums of pages
idx_ptrack [ i ] [ ' old_pages ' ] = self . get_md5_per_page_for_fork (
idx_ptrack [ i ] [ ' path ' ] , idx_ptrack [ i ] [ ' old_size ' ] )
2018-07-14 00:17:04 +02:00
self . backup_node ( backup_dir , ' node ' , node , options = [ ' -j10 ' , ' --stream ' ] )
2017-07-12 16:28:28 +02:00
node . safe_psql ( ' postgres ' , ' vacuum freeze t_heap ' )
node . safe_psql ( ' postgres ' , ' checkpoint ' )
for i in idx_ptrack :
# get new size of heap and indexes. size calculated in pages
idx_ptrack [ i ] [ ' new_size ' ] = self . get_fork_size ( node , i )
# update path to heap and index files in case they`ve changed
idx_ptrack [ i ] [ ' path ' ] = self . get_fork_path ( node , i )
# calculate new md5sums for pages
idx_ptrack [ i ] [ ' new_pages ' ] = self . get_md5_per_page_for_fork (
idx_ptrack [ i ] [ ' path ' ] , idx_ptrack [ i ] [ ' new_size ' ] )
# get ptrack for every idx
idx_ptrack [ i ] [ ' ptrack ' ] = self . get_ptrack_bits_per_page_for_fork (
node , idx_ptrack [ i ] [ ' path ' ] , [ idx_ptrack [ i ] [ ' old_size ' ] , idx_ptrack [ i ] [ ' new_size ' ] ] )
# compare pages and check ptrack sanity
self . check_ptrack_sanity ( idx_ptrack [ i ] )
# Clean after yourself
self . del_test_dir ( module_name , fname )
2017-09-28 09:32:06 +02:00
# @unittest.skip("skip")
2017-07-12 16:28:28 +02:00
def test_ptrack_vacuum_bits_frozen_replica ( self ) :
fname = self . id ( ) . split ( ' . ' ) [ 3 ]
master = self . make_simple_node ( base_dir = " {0} / {1} /master " . format ( module_name , fname ) ,
set_replication = True ,
initdb_params = [ ' --data-checksums ' ] ,
pg_options = { ' ptrack_enable ' : ' on ' , ' wal_level ' : ' replica ' , ' max_wal_senders ' : ' 2 ' } )
backup_dir = os . path . join ( self . tmp_path , module_name , fname , ' backup ' )
self . init_pb ( backup_dir )
self . add_instance ( backup_dir , ' master ' , master )
master . start ( )
self . backup_node ( backup_dir , ' master ' , master , options = [ ' --stream ' ] )
2017-09-28 09:32:06 +02:00
replica = self . make_simple_node ( base_dir = " {0} / {1} /replica " . format ( module_name , fname ) )
replica . cleanup ( )
2017-07-12 16:28:28 +02:00
2017-09-28 09:32:06 +02:00
self . restore_node ( backup_dir , ' master ' , replica )
2017-07-12 16:28:28 +02:00
2017-09-28 09:32:06 +02:00
self . add_instance ( backup_dir , ' replica ' , replica )
self . set_replica ( master , replica , synchronous = True )
self . set_archiving ( backup_dir , ' replica ' , replica , replica = True )
replica . start ( )
2017-07-12 16:28:28 +02:00
# Create table and indexes
2017-09-28 09:32:06 +02:00
master . safe_psql (
2017-05-03 13:14:48 +02:00
" postgres " ,
2017-09-28 09:32:06 +02:00
" create sequence t_seq; create table t_heap as select i as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,256) i " )
2017-05-03 13:14:48 +02:00
for i in idx_ptrack :
2017-07-12 16:28:28 +02:00
if idx_ptrack [ i ] [ ' type ' ] != ' heap ' and idx_ptrack [ i ] [ ' type ' ] != ' seq ' :
2017-09-28 09:32:06 +02:00
master . safe_psql ( " postgres " , " create index {0} on {1} using {2} ( {3} ) " . format (
2017-07-12 16:28:28 +02:00
i , idx_ptrack [ i ] [ ' relation ' ] , idx_ptrack [ i ] [ ' type ' ] , idx_ptrack [ i ] [ ' column ' ] ) )
2017-05-03 13:14:48 +02:00
2017-09-28 09:32:06 +02:00
master . safe_psql ( ' postgres ' , ' checkpoint ' )
2017-05-03 13:14:48 +02:00
for i in idx_ptrack :
# get size of heap and indexes. size calculated in pages
2017-09-28 09:32:06 +02:00
idx_ptrack [ i ] [ ' old_size ' ] = self . get_fork_size ( replica , i )
2017-05-03 13:14:48 +02:00
# get path to heap and index files
2017-09-28 09:32:06 +02:00
idx_ptrack [ i ] [ ' path ' ] = self . get_fork_path ( replica , i )
2017-05-03 13:14:48 +02:00
# calculate md5sums of pages
idx_ptrack [ i ] [ ' old_pages ' ] = self . get_md5_per_page_for_fork (
2017-05-03 17:05:19 +02:00
idx_ptrack [ i ] [ ' path ' ] , idx_ptrack [ i ] [ ' old_size ' ] )
2017-05-03 13:14:48 +02:00
2017-09-28 09:32:06 +02:00
# Take PTRACK backup to clean every ptrack
2018-07-14 00:17:04 +02:00
self . backup_node ( backup_dir , ' replica ' , replica , options = [ ' -j10 ' ,
2017-09-28 09:32:06 +02:00
' --master-host=localhost ' , ' --master-db=postgres ' , ' --master-port= {0} ' . format ( master . port ) ] )
2017-05-03 13:14:48 +02:00
2017-09-28 09:32:06 +02:00
master . safe_psql ( ' postgres ' , ' vacuum freeze t_heap ' )
master . safe_psql ( ' postgres ' , ' checkpoint ' )
2017-05-03 13:14:48 +02:00
for i in idx_ptrack :
# get new size of heap and indexes. size calculated in pages
2017-09-28 09:32:06 +02:00
idx_ptrack [ i ] [ ' new_size ' ] = self . get_fork_size ( replica , i )
2017-05-03 13:14:48 +02:00
# update path to heap and index files in case they`ve changed
2017-09-28 09:32:06 +02:00
idx_ptrack [ i ] [ ' path ' ] = self . get_fork_path ( replica , i )
2017-05-03 13:14:48 +02:00
# calculate new md5sums for pages
idx_ptrack [ i ] [ ' new_pages ' ] = self . get_md5_per_page_for_fork (
2017-05-03 17:05:19 +02:00
idx_ptrack [ i ] [ ' path ' ] , idx_ptrack [ i ] [ ' new_size ' ] )
2017-05-03 13:14:48 +02:00
# get ptrack for every idx
2017-05-05 15:21:49 +02:00
idx_ptrack [ i ] [ ' ptrack ' ] = self . get_ptrack_bits_per_page_for_fork (
2017-09-28 09:32:06 +02:00
replica , idx_ptrack [ i ] [ ' path ' ] , [ idx_ptrack [ i ] [ ' old_size ' ] , idx_ptrack [ i ] [ ' new_size ' ] ] )
2017-05-03 13:14:48 +02:00
# compare pages and check ptrack sanity
self . check_ptrack_sanity ( idx_ptrack [ i ] )
2017-06-27 07:42:52 +02:00
# Clean after yourself
2017-07-12 16:28:28 +02:00
self . del_test_dir ( module_name , fname )