diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index b980bc629c4..64aceee7e1e 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -858,7 +858,18 @@ class ModifyConfSetting(Command): def __init__(self, name, file, optName, optVal, optType='string', ctxt=LOCAL, remoteHost=None): cmdStr = None if optType == 'number': - cmdStr = "perl -p -i.bak -e 's/^%s[ ]*=[ ]*\\d+/%s=%d/' %s" % (optName, optName, optVal, file) + # Replace the whole value: valid integers can be signed, octal, + # hexadecimal, or quoted scientific notation. Matching only the + # initial digits can leave a suffix that corrupts the config. + # Keep trailing whitespace/comments (including CRLF line endings) + # and require a setting-name + # separator so similarly named parameters are not changed. + # \x27 is a quote inside the shell's single-quoted Perl program. + cmdStr = ( + "perl -p -i.bak -e 's/^[ \\t]*%s(?:[ \\t]*=[ \\t]*|[ \\t]+)" + "(?:\\x27[^\\x27]*\\x27|[^\\s\\x27#]+)" + "(?=[ \\t]*(?:#|\\r?$))/%s=%d/i' %s" + ) % (optName, optName, optVal, file) elif optType == 'string': cmdStr = "perl -i -p -e \"s/^%s[ ]*=[ ]*'[^']*'/%s='%s'/\" %s" % (optName, optName, optVal, file) else: @@ -1670,6 +1681,22 @@ def createTempDirectoryName(coordinatorDataDirectory, tempDirPrefix): datetime.datetime.now().strftime('%m%d%Y'), os.getpid()) +# Cherry-picked from Greenplum Database (greenplum-db/gpdb-archive @ 482967c1b4), +# gpMgmt/bin/gppylib/commands/gp.py. +def is_gprecoverseg_running(): + gprecoverseg_pidfile = os.path.join(get_coordinatordatadir(), 'gprecoverseg.lock', 'PID') + try: + with open(gprecoverseg_pidfile) as pidfile: + # The file can still be empty while the lock owner writes its PID. + gprecoverseg_pid = int(pidfile.read()) + except (OSError, ValueError): + return False + + if gprecoverseg_pid <= 0: + return False + + return check_pid(gprecoverseg_pid) + #------------------------------------------------------------------------- class GpRecoverSeg(Command): """ diff --git a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_gp.py b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_gp.py index 4ac75c69685..9ba728d1a8e 100644 --- a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_gp.py +++ b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_gp.py @@ -4,9 +4,9 @@ # from gppylib.commands.base import CommandResult -from mock import patch +from mock import patch, mock_open -from gppylib.commands.gp import is_pid_postmaster, get_postmaster_pid_locally +from gppylib.commands.gp import is_pid_postmaster, get_postmaster_pid_locally, is_gprecoverseg_running from test.unit.gp_unittest import GpTestCase, run_tests @@ -158,5 +158,38 @@ def test_get_postmaster_pid_locally_error(self, mock1): def test_get_postmaster_pid_locally_empty(self, mock1): self.assertEqual(get_postmaster_pid_locally('/tmp'), -1) + @patch('gppylib.commands.gp.check_pid', return_value=True) + @patch('gppylib.commands.gp.get_coordinatordatadir', return_value='/data/coordinator') + @patch("builtins.open", new_callable=mock_open, read_data="123") + def test_is_gprecoverseg_running_succeeds(self, mock_file, mock1, mock2): + result = is_gprecoverseg_running() + mock_file.assert_called_once_with('/data/coordinator/gprecoverseg.lock/PID') + mock2.assert_called_once_with(123) + self.assertTrue(result) + + @patch('gppylib.commands.gp.check_pid') + @patch('gppylib.commands.gp.get_coordinatordatadir', return_value='/data/coordinator') + def test_is_gprecoverseg_running_with_invalid_pid(self, mock_datadir, mock_check_pid): + for contents in ['', ' \n', 'not-a-pid', '0', '-1']: + with self.subTest(contents=contents): + with patch('builtins.open', mock_open(read_data=contents)): + self.assertFalse(is_gprecoverseg_running()) + mock_check_pid.assert_not_called() + + @patch('gppylib.commands.gp.check_pid', return_value=False) + @patch('gppylib.commands.gp.get_coordinatordatadir', return_value='/data/coordinator') + @patch('builtins.open', new_callable=mock_open, read_data='123') + def test_is_gprecoverseg_running_with_dead_pid(self, mock_file, mock_datadir, mock_check_pid): + self.assertFalse(is_gprecoverseg_running()) + mock_check_pid.assert_called_once_with(123) + + @patch('gppylib.commands.gp.check_pid') + @patch('gppylib.commands.gp.get_coordinatordatadir', return_value='/invalid/path/') + def test_is_gprecoverseg_running_when_pidfile_does_not_exists(self, mock1, mock2): + result = is_gprecoverseg_running() + self.assertFalse(result) + self.assertFalse(mock2.called) + + if __name__ == '__main__': run_tests() diff --git a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py index 869a37d312b..1c5997833ae 100644 --- a/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py +++ b/gpMgmt/bin/gppylib/operations/buildMirrorSegments.py @@ -240,6 +240,9 @@ def __build_mirrors(self, actionName, gpEnv, gpArray): recovery_info_by_host = recoveryinfo.build_recovery_info(self.__mirrorsToBuild) + # Remove any existing progress files for segments to be recovered + self.remove_existing_progress_files(recovery_info_by_host) + self._run_setup_recovery(actionName, recovery_info_by_host) backout_map = self._update_config(recovery_info_by_host, gpArray) @@ -293,13 +296,17 @@ def _update_config(self, recovery_info_by_host, gpArray): return backout_map - def _remove_progress_files(self, recovery_info_by_host, recovery_results): + # Remove any existing progress file of segments that will be recovered by + # current gprecoverseg execution. The files written by this run are left in + # place: gpstate reads them to report progress, and they are what an + # operator looks at after a recovery that went wrong. + def remove_existing_progress_files(self, recovery_info_by_host): remove_progress_file_cmds = [] for hostName, recovery_info_list in recovery_info_by_host.items(): for ri in recovery_info_list: - if recovery_results.was_bb_rewind_successful(ri.target_segment_dbid): - remove_progress_file_cmds.append(self._get_remove_cmd(ri.progress_file, hostName)) - self.__runWaitAndCheckWorkerPoolForErrorsAndClear(remove_progress_file_cmds, suppressErrorCheck=False) + remove_progress_file_cmds.append(self._get_remove_cmd("*dbid{}.out".format(ri.target_segment_dbid), + hostName)) + self.__runWaitAndCheckWorkerPoolForErrorsAndClear(remove_progress_file_cmds, suppressErrorCheck=True) def _revert_config_update(self, recovery_results, backout_map): if len(backout_map) == 0: @@ -447,8 +454,12 @@ def _get_progress_cmd(self, progressFile, targetSegmentDbId, targetHostname): remoteHost=targetHostname) return None + # remove_file is a name pattern (e.g. "*dbid2.out"), matched in the log + # directory the progress files are written to. def _get_remove_cmd(self, remove_file, target_host): - return base.Command("remove file", "rm -f {}".format(shlex.quote(remove_file)), ctxt=base.REMOTE, remoteHost=target_host) + return base.Command("remove file", "find {} -name {} -delete".format(shlex.quote(gplog.get_logger_dir()), + shlex.quote(remove_file)), + ctxt=base.REMOTE, remoteHost=target_host) def __runWaitAndCheckWorkerPoolForErrorsAndClear(self, cmds, suppressErrorCheck=False, progressCmds=[]): for cmd in cmds: @@ -486,7 +497,6 @@ def _run_recovery(self, action_name, recovery_info_by_host, gpEnv): recovery_results = RecoveryResult(action_name, completed_recovery_results, self.__logger) recovery_results.print_bb_rewind_and_start_errors() - self._remove_progress_files(recovery_info_by_host, recovery_results) return recovery_results def _do_recovery(self, recovery_info_by_host, gpEnv): diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_buildMirrorSegments.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_buildMirrorSegments.py index 61a88c21539..0154b83d139 100644 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_buildMirrorSegments.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_buildMirrorSegments.py @@ -522,6 +522,24 @@ def test_run_setup_recovery_empty_errors(self): self._assert_setup_recovery() + def test_remove_existing_progress_files(self): + self._setup_recovery_mocks() + + recovery_info = OrderedDict() + recovery_info['host1'] = [self.recovery_info1] + recovery_info['host2'] = [self.recovery_info2, self.recovery_info3] + + self.default_build_mirrors_obj.remove_existing_progress_files(recovery_info) + + self.assertEqual([call(ANY, suppressErrorCheck=True)], + self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) + rm_progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[0][0][0] + self.assertEqual(['host1', 'host2', 'host2'], [cmd.remoteHost for cmd in rm_progress_cmds]) + self.assertEqual(["find /tmp/logdir -name '*dbid2.out' -delete", + "find /tmp/logdir -name '*dbid3.out' -delete", + "find /tmp/logdir -name '*dbid4.out' -delete"], + [cmd.cmdStr for cmd in rm_progress_cmds]) + def test_run_recovery_no_errors(self): self._setup_recovery_mocks() @@ -531,12 +549,11 @@ def test_run_recovery_no_errors(self): self.default_build_mirrors_obj._run_recovery(self.default_action_name, recovery_info, self.gpEnv) - self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True), call(ANY, suppressErrorCheck=False)], + self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True)], self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) seg_recovery_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[0][0][0] progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[0][1]['progressCmds'] - rm_progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[1][0][0] #TODO fix formatting expected_recovery_cmd_strs = ["""$GPHOME/sbin/gpsegrecovery.py -c '[{"target_datadir": "/datadir2", "target_port": 7001, "target_segment_dbid": 2, "source_hostname": "source_host_for_dbid2", "source_port": 7002, "is_full_recovery": true, "progress_file": "/tmp/progress_file2"}]' -l /tmp/logdir -b 64 --era=dummy_era""", """$GPHOME/sbin/gpsegrecovery.py -c '[{"target_datadir": "/datadir3", "target_port": 7003, "target_segment_dbid": 3, "source_hostname": "source_host_for_dbid3", "source_port": 7004, "is_full_recovery": true, "progress_file": "/tmp/progress_file3"}, {"target_datadir": "/datadir4", "target_port": 7005, "target_segment_dbid": 4, "source_hostname": "source_host_for_dbid4", "source_port": 7006, "is_full_recovery": true, "progress_file": "/tmp/progress_file4"}]' -l /tmp/logdir -b 64 --era=dummy_era"""] @@ -558,13 +575,6 @@ def test_run_recovery_no_errors(self): progress_cmds[2].cmdStr) self.assertEqual(1, self.mock_gp_era.call_count) - self.assertEqual(3, len(rm_progress_cmds)) - self.assertEqual("host1", rm_progress_cmds[0].remoteHost) - self.assertEqual("rm -f /tmp/progress_file2", rm_progress_cmds[0].cmdStr) - self.assertEqual("host2", rm_progress_cmds[1].remoteHost) - self.assertEqual("rm -f /tmp/progress_file3", rm_progress_cmds[1].cmdStr) - self.assertEqual("host2", rm_progress_cmds[2].remoteHost) - self.assertEqual("rm -f /tmp/progress_file4", rm_progress_cmds[2].cmdStr) self._assert_run_recovery() @@ -584,11 +594,9 @@ def test_run_recovery_all_dbids_fail_only_bb_rewind_errors(self): self._setup_recovery_mocks([host1_error, host2_error]) self.default_build_mirrors_obj._run_recovery(self.default_action_name, recovery_info, self.gpEnv) - self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True), call(ANY, suppressErrorCheck=False)], + self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True)], self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) - rm_progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[1][0][0] - self.assertEqual(0, len(rm_progress_cmds)) self._assert_run_recovery() @@ -599,13 +607,9 @@ def test_run_recovery_some_dbids_fail_all_bb_rewind_errors(self): self._setup_recovery_mocks([host1_error, host2_error]) self.default_build_mirrors_obj._run_recovery(self.default_action_name, recovery_info, self.gpEnv) - self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True), call(ANY, suppressErrorCheck=False)], + self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True)], self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) - rm_progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[1][0][0] - self.assertEqual(1, len(rm_progress_cmds)) - self.assertEqual("host2", rm_progress_cmds[0].remoteHost) - self.assertEqual("rm -f /tmp/progress_file3", rm_progress_cmds[0].cmdStr) self._assert_run_recovery() @@ -616,11 +620,8 @@ def test_run_recovery_all_dbids_fail_all_start_errors(self): host2_error = '[{"error_type": "start", "error_msg":"some error for dbid 4", "dbid": 4, "datadir": "/datadir4", "port": 7005, "progress_file": "/tmp/progress4"}]' self._setup_recovery_mocks([host1_error, host2_error]) self.default_build_mirrors_obj._run_recovery(self.default_action_name, recovery_info, self.gpEnv) - self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True), call(ANY, suppressErrorCheck=False)], + self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True)], self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) - rm_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[1][0][0] - - self.assertEqual(3, len(rm_cmds)) self._assert_run_recovery() def test_run_recovery_some_dbids_fail_all_start_errors(self): @@ -631,10 +632,8 @@ def test_run_recovery_some_dbids_fail_all_start_errors(self): self._setup_recovery_mocks([host1_error, host2_error]) self.default_build_mirrors_obj._run_recovery(self.default_action_name, recovery_info, self.gpEnv) - self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True), call(ANY, suppressErrorCheck=False)], + self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True)], self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) - rm_progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[1][0][0] - self.assertEqual(3, len(rm_progress_cmds)) self._assert_run_recovery() @@ -647,14 +646,10 @@ def test_run_recovery_all_dbids_fail_bb_rewind_and_start_errors(self): self.default_build_mirrors_obj._run_recovery(self.default_action_name, recovery_info, self.gpEnv) - self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True), call(ANY, suppressErrorCheck=False)], + self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True)], self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) - rm_progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[1][0][0] # Since start passed for dbid:3, we should have deleted it's progress file - self.assertEqual(1, len(rm_progress_cmds)) - self.assertEqual("host1", rm_progress_cmds[0].remoteHost) - self.assertEqual("rm -f /tmp/progress_file3", rm_progress_cmds[0].cmdStr) self._assert_run_recovery() @@ -671,15 +666,9 @@ def test_run_recovery_all_dbids_fail_bb_rewind_and_default_errors(self): self.default_build_mirrors_obj._run_recovery(self.default_action_name, recovery_info, self.gpEnv) - self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True), call(ANY, suppressErrorCheck=False)], + self.assertEqual([call(ANY, progressCmds=ANY, suppressErrorCheck=True)], self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list) - rm_progress_cmds = self.default_build_mirrors_obj._GpMirrorListToBuild__runWaitAndCheckWorkerPoolForErrorsAndClear.call_args_list[1][0][0] - self.assertEqual(2, len(rm_progress_cmds)) - self.assertEqual("host1", rm_progress_cmds[0].remoteHost) - self.assertEqual("host2", rm_progress_cmds[1].remoteHost) - self.assertEqual("rm -f /tmp/progress_file2", rm_progress_cmds[0].cmdStr) - self.assertEqual("rm -f /tmp/progress_file3", rm_progress_cmds[1].cmdStr) self._assert_run_recovery() diff --git a/gpMgmt/bin/gppylib/programs/clsRecoverSegment.py b/gpMgmt/bin/gppylib/programs/clsRecoverSegment.py index 81377136f83..503360368a4 100644 --- a/gpMgmt/bin/gppylib/programs/clsRecoverSegment.py +++ b/gpMgmt/bin/gppylib/programs/clsRecoverSegment.py @@ -262,6 +262,25 @@ def run(self): if optionCnt > 1: raise ProgramArgumentValidationException("Only one of -i, -p, and -r may be specified") + # -o only writes out a sample configuration file describing what would + # be recovered; it does not recover anything, so combining it with an + # option that says where to recover to is a contradiction rather than a + # refinement. + if self.__options.outputSampleConfigFile: + if self.__options.recoveryConfigFile is not None: + raise ProgramArgumentValidationException("Invalid -i provided with -o argument") + if self.__options.rebalanceSegments: + raise ProgramArgumentValidationException("Invalid -r provided with -o argument") + + # A full resynchronisation rebuilds the mirror from its primary, which + # is neither what -r (rebalance back to preferred roles) nor -p + # (recover onto a different host) asks for. + if self.__options.forceFullResynchronization: + if self.__options.rebalanceSegments: + raise ProgramArgumentValidationException("-F option is not supported with -r option") + if self.__options.newRecoverHosts is not None: + raise ProgramArgumentValidationException("-F option is not supported with -p option") + faultProberInterface.getFaultProber().initializeProber(gpEnv.getCoordinatorPort()) confProvider = configInterface.getConfigurationProvider().initializeProvider(gpEnv.getCoordinatorPort()) @@ -378,12 +397,16 @@ def signal_handler(sig, frame): if not mirrorBuilder.recover_mirrors(gpEnv, gpArray): if self.termination_requested: self.logger.error("gprecoverseg process was interrupted by the user.") + self.logger.error("gprecoverseg failed. Please check the output for more details.") sys.exit(1) if self.termination_requested: self.logger.info("Not able to terminate the recovery process since it has been completed successfully.") self.logger.info("********************************") + self.logger.info("Future gprecoverseg executions might remove the currently created pg_basebackup/pg_rewind/rsync " + "progress files, please save these files if needed.") + self.logger.info("********************************") self.logger.info("Segments successfully recovered.") self.logger.info("********************************") diff --git a/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py b/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py index 34b6a12ff2e..823a3dbc8e1 100644 --- a/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py +++ b/gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py @@ -1,11 +1,102 @@ import abc from typing import List +from contextlib import closing +from gppylib.db import dbconn +from gppylib import gplog from gppylib.mainUtils import ExceptionNoStackTraceNeeded from gppylib.operations.detect_unreachable_hosts import get_unreachable_segment_hosts from gppylib.parseutils import line_reader, check_values, canonicalize_address -from gppylib.utils import checkNotNone, normalizeAndValidateInputPath +from gppylib.utils import checkNotNone, normalizeAndValidateInputPath, validateHostnameAddress from gppylib.gparray import GpArray, Segment +from gppylib.commands.gp import RECOVERY_REWIND_APPNAME + +logger = gplog.get_default_logger() + + +# cherry-pick from Greenplum Database +# greenplum-db/gpdb-archive @ 482967c1b4, +# gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py +def get_segments_with_running_basebackup(coordinator_port): + """ + Returns a list of contentIds of source segments of running pg_basebackup processes + At present gp_stat_replication table does not contain any info about datadir and dbid of the target of running pg_basebackup + + coordinator_port is passed in because DbURL() on its own uses PGPORT, or + 5432 when that is unset, and neither need be the coordinator's port. + """ + + sql = "select gp_segment_id from gp_stat_replication where application_name = 'pg_basebackup'" + + try: + with closing(dbconn.connect(dbconn.DbURL(port=coordinator_port))) as conn: + res = dbconn.query(conn, sql) + rows = res.fetchall() + except Exception as e: + raise Exception("Failed to query gp_stat_replication: %s" %str(e)) + + segments_with_running_basebackup = {row[0] for row in rows} + + if len(segments_with_running_basebackup) == 0: + logger.debug("No basebackup running") + return segments_with_running_basebackup + + +# cherry-pick from Greenplum Database +# greenplum-db/gpdb-archive @ 482967c1b4, +# gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py +def is_pg_rewind_running(hostname, port): + """ + Returns true if a pg_rewind process is running for the given segment + """ + logger.debug( + "Checking for running instances of pg_rewind with host {} and port {} as source server".format(hostname, port)) + + """ + Reasons to depend on pg_stat_activity table: + * pg_rewind is invoked using --source-server connection string as it needs libpq connection + with source server, which will be remote to the target server and --source-pgdata can not + be used in that case. Thus pg_stat_activity will always contain entry for active pg_rewind. + * pg_rewind keeps a connection open throughout it's lifecycle, so pg_stat_activity will contain + entries for active pg_rewind process till the end of execution. + * gpstate uses the above mentioned approach (pg_stat_activity entry check) to check for + incremental recoveries in progress.Thus, using the same approach will make it consistent + everywhere. + """ + + sql = "SELECT count(*) FROM pg_stat_activity WHERE application_name = '{}'".format(RECOVERY_REWIND_APPNAME) + try: + url = dbconn.DbURL(hostname=hostname, port=port, dbname='template1') + with closing(dbconn.connect(url, utility=True)) as conn: + res = dbconn.querySingleton(conn, sql) + return res > 0 + + except Exception as e: + raise Exception("Failed to query pg_stat_activity for segment hostname: {}, port: {}, error: {}".format( + hostname, str(port), str(e))) + + +# cherry-pick from Greenplum Database +# greenplum-db/gpdb-archive @ 482967c1b4, +# gpMgmt/bin/gppylib/programs/clsRecoverSegment_triples.py +def extract_recovery_config_info(parts): + """ + Extracts relevant recovery configuration information from a list of parts. + + Three parts is the historical form, "
||", and the + hostname is taken to be the address. Four parts puts an explicit hostname + first; it is then cross-checked against the address, and against the + hostname the segment is registered under. + """ + address, port, datadir = parts[-3:] + hostname_check_required = False + hostname = address + + if len(parts) == 4: + hostname = parts[0] + hostname_check_required = True + + return hostname, address, port, datadir, hostname_check_required class RecoveryTriplet: @@ -84,10 +175,12 @@ def validate(failed, live, failover): class RecoveryTripletRequest: - def __init__(self, failed, failover_host=None, failover_port=None, failover_datadir=None, is_new_host=False): + def __init__(self, failed, failover_host_name=None, failover_host_address=None, failover_port=None, + failover_datadir=None, is_new_host=False): self.failed = failed - self.failover_host = failover_host + self.failover_host_name = failover_host_name + self.failover_host_address = failover_host_address self.failover_port = failover_port self.failover_datadir = failover_datadir self.failover_to_new_host = is_new_host @@ -142,12 +235,56 @@ def _convert_requests_to_triplets(self, requests: List[RecoveryTripletRequest]) triplets = [] dbIdToPeerMap = self.gpArray.getDbIdToPeerMap() + + failed_segments_with_running_basebackup = [] + failed_segments_with_running_pgrewind = [] + segments_with_running_basebackup = get_segments_with_running_basebackup( + self.gpArray.coordinator.getSegmentPort()) + for req in requests: + """ + When running gprecoverseg (any sort of recovery full/incremental), if the pg_rewind, pg_basebackup is + already running for a segment, that segment should be skipped from the recovery. The reason being that + there should be only one writer per target segment at a time. Having several writers to a target will + eventually make the segment inconsistent and in a weird state. + + Although technically we could allow user to run a full recovery to a new host even if there is a + pg_rewind/pg_basebackup running for that segment. This is a pretty rare scenario and we have decided not + to over complicates the code just to support this scenario. + """ + failed_segment_dbid = req.failed.getSegmentDbId() + peer = dbIdToPeerMap.get(failed_segment_dbid) + if peer is None: + raise Exception("No peer found for dbid {}. liveSegment is None".format(failed_segment_dbid)) + peer_contentid = peer.getSegmentContentId() + + # Every reason to skip this segment is settled before the failover + # block below, which rewrites the segment registered in gpArray. + # updateSystemConfig() writes every segment changed there to the + # catalog, so a skipped segment would be recorded at a location + # nothing was recovered to. + if req.failed.unreachable and not req.failover_to_new_host: + continue + + # Asked only after the skip above. is_pg_rewind_running() connects + # to the peer, and on a double fault -- failed segment and peer + # both down -- that connection raises. Asked first, it would abort + # the whole gprecoverseg run rather than skip the one segment, + # which is the opposite of what is wanted when a cluster is already + # degraded. + if peer_contentid in segments_with_running_basebackup: + failed_segments_with_running_basebackup.append(peer_contentid) + continue + + if is_pg_rewind_running(peer.getSegmentHostName(), peer.getSegmentPort()): + failed_segments_with_running_pgrewind.append(peer_contentid) + continue + # TODO: These 2 cases have different behavior which might be confusing to the user. # "|| ||" does full recovery # "||" does incremental recovery failover = None - if req.failover_host: + if req.failover_host_address: # these two lines make it so that failover points to the object that is registered in gparray # as the failed segment(!). @@ -155,20 +292,32 @@ def _convert_requests_to_triplets(self, requests: List[RecoveryTripletRequest]) req.failed = failover.copy() # now update values in failover segment - failover.setSegmentAddress(req.failover_host) - failover.setSegmentHostName(req.failover_host) + if req.failover_host_name != req.failover_host_address: + # Validate if the hostname and address are of the same host + if not validateHostnameAddress(req.failover_host_name, req.failover_host_address): + logger.warning( + "Not able to co-relate hostname:{0} with address:{1}. " + "Skipping recovery for segments with contentId {2}" + .format(req.failover_host_name, req.failover_host_address, peer_contentid)) + continue + + failover.setSegmentHostName(req.failover_host_name) + failover.setSegmentAddress(req.failover_host_address) failover.setSegmentPort(int(req.failover_port)) failover.setSegmentDataDirectory(req.failover_datadir) failover.unreachable = False if req.failover_to_new_host else failover.unreachable - # this must come AFTER the if check above because failedSegment can be adjusted to - # point to a different object - if req.failed.unreachable and not req.failover_to_new_host: - continue + triplets.append(RecoveryTriplet(req.failed, peer, failover)) - peer = dbIdToPeerMap.get(req.failed.getSegmentDbId()) + if len(failed_segments_with_running_basebackup) > 0: + logger.warning( + "Found pg_basebackup running for segments with contentIds %s, skipping recovery of these segments" % ( + failed_segments_with_running_basebackup)) - triplets.append(RecoveryTriplet(req.failed, peer, failover)) + if len(failed_segments_with_running_pgrewind) > 0: + logger.warning( + "Found pg_rewind running for segments with contentIds %s, skipping recovery of these segments" % ( + failed_segments_with_running_pgrewind)) return triplets @@ -222,7 +371,11 @@ def _check_new_hosts(): for failedHost, failoverHost in zip(sorted(failedSegments.keys()), self.newHosts): for failed in failedSegments[failedHost]: failoverPort = self.portAssigner.findAndReservePort(failoverHost, failoverHost) - req = RecoveryTripletRequest(failed, failoverHost, failoverPort, failed.getSegmentDataDirectory(), True) + req = RecoveryTripletRequest(failed, failover_host_name=failoverHost, + failover_host_address=failoverHost, + failover_port=failoverPort, + failover_datadir=failed.getSegmentDataDirectory(), + is_new_host=True) requests.append(req) return self._convert_requests_to_triplets(requests) @@ -285,6 +438,11 @@ def getTriplets(self): def _find_failed_from_row(): failed = None for segment in self.gpArray.getDbList(): + # When the input configuration file carries an explicit hostname + # it has to match the one the segment is registered under, or + # this is not the segment the operator meant. + if row['hostname_check_required'] and segment.getSegmentHostName() != row['failedHostname']: + continue if (segment.getSegmentAddress() == row['failedAddress'] and str(segment.getSegmentPort()) == row['failedPort'] and segment.getSegmentDataDirectory() == row['failedDataDirectory']): @@ -292,15 +450,32 @@ def _find_failed_from_row(): break if failed is None: - raise Exception("A segment to recover was not found in configuration. " \ - "This segment is described by address|port|directory '%s|%s|%s'" % - (row['failedAddress'], row['failedPort'], row['failedDataDirectory'])) + # Keep Cloudberry's wording -- the two spaces and the names of + # the fields -- rather than upstream's terser form. It tells + # the operator which field is which, and the scenario that + # covers this asserts on it literally. + msg = "A segment to recover was not found in configuration. " \ + "This segment is described by " + if row['hostname_check_required']: + msg += "hostname|address|port|directory '{}|{}|{}|{}'".format( + row['failedHostname'], row['failedAddress'], + row['failedPort'], row['failedDataDirectory']) + else: + msg += "address|port|directory '{}|{}|{}'".format( + row['failedAddress'], row['failedPort'], + row['failedDataDirectory']) + + raise Exception(msg) return failed requests = [] for row in self.rows: - req = RecoveryTripletRequest(_find_failed_from_row(), row.get('newAddress'), row.get('newPort'), row.get('newDataDirectory')) + req = RecoveryTripletRequest(_find_failed_from_row(), + failover_host_name=row.get('newHostname'), + failover_host_address=row.get('newAddress'), + failover_port=row.get('newPort'), + failover_datadir=row.get('newDataDirectory')) requests.append(req) return self._convert_requests_to_triplets(requests) @@ -322,31 +497,43 @@ def _parseConfigFile(config_file): msg = "line %d of file %s: expected 1 or 2 groups but found %d" % (lineno, config_file, len(groups)) raise ExceptionNoStackTraceNeeded(msg) parts = groups[0].split('|') - if len(parts) != 3: - msg = "line %d of file %s: expected 3 parts on failed segment group, obtained %d" % ( + + if len(parts) not in {3, 4}: + msg = "line {0} of file {1}: expected 3 or 4 parts on failed segment group, obtained {2}" .format( lineno, config_file, len(parts)) raise ExceptionNoStackTraceNeeded(msg) - address, port, datadir = parts - check_values(lineno, address=address, port=port, datadir=datadir) + + hostname, address, port, datadir, hostname_check_required = extract_recovery_config_info(parts) + + check_values(lineno, hostname=hostname, address=address, port=port, datadir=datadir) datadir = normalizeAndValidateInputPath(datadir, f.name, lineno) row = { + 'failedHostname': hostname, 'failedAddress': address, 'failedPort': port, 'failedDataDirectory': datadir, - 'lineno': lineno + 'lineno': lineno, + 'hostname_check_required': hostname_check_required } if len(groups) == 2: parts2 = groups[1].split('|') - if len(parts2) != 3: - msg = "line %d of file %s: expected 3 parts on new segment group, obtained %d" % ( - lineno, config_file, len(parts2)) + if len(parts2) not in [3, 4] or len(parts) != len(parts2): + msg = "line {0} of file {1}: expected equal parts, either 3 or 4 on both segment group, obtained {2} on " \ + "group1 and {3} on group2" .format( + lineno, config_file, len(parts), len(parts2)) raise ExceptionNoStackTraceNeeded(msg) - address2, port2, datadir2 = parts2 - check_values(lineno, address=address2, port=port2, datadir=datadir2) + if len(parts2) == 4: + hostname2, address2, port2, datadir2 = parts2 + else: + address2, port2, datadir2 = parts2 + hostname2 = address2 + + check_values(lineno, hostname=hostname2, address=address2, port=port2, datadir=datadir2) datadir2 = normalizeAndValidateInputPath(datadir2, f.name, lineno) row.update({ + 'newHostname': hostname2, 'newAddress': address2, 'newPort': port2, 'newDataDirectory': datadir2 diff --git a/gpMgmt/bin/gppylib/programs/clsSystemState.py b/gpMgmt/bin/gppylib/programs/clsSystemState.py index effcaddd440..e21bc495378 100644 --- a/gpMgmt/bin/gppylib/programs/clsSystemState.py +++ b/gpMgmt/bin/gppylib/programs/clsSystemState.py @@ -22,7 +22,6 @@ from gppylib.system import configurationInterface as configInterface from gppylib.system.environment import GpCoordinatorEnvironment from gppylib.utils import TableLogger -from gppylib.commands.gp import get_coordinatordatadir logger = gplog.get_default_logger() @@ -682,15 +681,18 @@ def logSegments(segments, logAsPairs, additionalFieldsToLog=[]): self.__addClusterDownWarning(gpArray, data) - recovery_progress_file = get_recovery_progress_file(gplog) - segments_under_recovery = self._parse_recovery_progress_data(data, recovery_progress_file, gpArray) - gprecoverseg_lock_dir = os.path.join(get_coordinatordatadir() + '/gprecoverseg.lock') - if segments_under_recovery and os.path.exists(gprecoverseg_lock_dir): - logger.info("----------------------------------------------------") - logger.info("Segments in recovery") - logSegments(segments_under_recovery, False, [VALUE_RECOVERY_TYPE, VALUE_RECOVERY_COMPLETED_BYTES, VALUE_RECOVERY_TOTAL_BYTES, - VALUE_RECOVERY_PERCENTAGE]) - exitCode = 1 + # Check if gprecoverseg process is running. We do not want to + # show the progress if the gprecoverseg process is not running. + if gp.is_gprecoverseg_running(): + recovery_progress_file = get_recovery_progress_file(gplog) + segments_under_recovery = self._parse_recovery_progress_data(data, recovery_progress_file, gpArray) + + if segments_under_recovery: + logger.info("----------------------------------------------------") + logger.info("Segments in recovery") + logSegments(segments_under_recovery, False, [VALUE_RECOVERY_TYPE, VALUE_RECOVERY_COMPLETED_BYTES, + VALUE_RECOVERY_TOTAL_BYTES, VALUE_RECOVERY_PERCENTAGE]) + exitCode = 1 # final output -- no errors, then log this message if exitCode == 0: diff --git a/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py b/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py index e2bf74bdf67..719fbd912ce 100644 --- a/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py +++ b/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py @@ -2,29 +2,44 @@ import copy import io -from mock import Mock, patch, MagicMock +import itertools +import os +from mock import Mock, patch, MagicMock, call import tempfile import gppylib from gparray import Segment, GpArray -from gppylib.programs.clsRecoverSegment_triples import RecoveryTripletsUserConfigFile, RecoveryTripletsFactory, RecoveryTriplet -from test.unit.gp_unittest import GpTestCase +from gppylib.programs.clsRecoverSegment_triples import RecoveryTripletsUserConfigFile, RecoveryTripletsFactory, \ + RecoveryTriplet, get_segments_with_running_basebackup, is_pg_rewind_running, extract_recovery_config_info +from test.unit.gp_unittest import GpTestCase, FakeCursor class RecoveryTripletsFactoryTestCase(GpTestCase): def setUp(self): # Set maxDiff to None to see the entire diff on the console in case of failure self.maxDiff = None + mock_logger = Mock(spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal']) + + self.apply_patches([ + patch('gppylib.programs.clsRecoverSegment_triples.logger', return_value=mock_logger), + ]) + + self.mock_logger = self.get_mock_from_apply_patch('logger') def run_single_ConfigFile_test(self, test): with tempfile.NamedTemporaryFile() as f: f.write(test["config"].encode("utf-8")) f.flush() - return self._run_single_FromGpArray_test(test["gparray"], f.name, None, None, test.get("unreachable_existing_hosts")) + return self._run_single_FromGpArray_test(test["gparray"], f.name, None, None, + test.get("is_pgrewind_running", itertools.repeat(False)), + test.get("segments_with_running_basebackup", set()), + test.get("unreachable_existing_hosts")) def run_single_GpArray_test(self, test): return self._run_single_FromGpArray_test(test["gparray"], None, test["new_hosts"], test.get("unreachable_hosts"), - test.get("unreachable_existing_hosts")) + test.get("is_pgrewind_running", itertools.repeat(False)), + test.get("segments_with_running_basebackup", set()), + test.get("unreachable_existing_hosts")) #TODO: do we want new hosts here? We do not officially support new hosts with "-i" def test_RecoveryTripletsUserConfigFile_getMirrorTriples_should_pass(self): @@ -100,7 +115,91 @@ def test_RecoveryTripletsUserConfigFile_getMirrorTriples_should_pass(self): "config": "sdw2|21000|/mirror/gpseg0", "unreachable_existing_hosts": ['sdw2'], "expected": [] - } + }, + # A skipped segment that the config file moves must keep its + # location in gpArray; run_pass_tests() checks that only the + # expected triplets changed it. + { + "name": "one_mirror_inconfig_has_running_basebackup", + "gparray": self.all_up_gparray_str, + "config": """sdw2|21000|/mirror/gpseg0 sdw2|21000|/mirror/gpseg0_new + sdw1|21000|/mirror/gpseg6 sdw1|21000|/mirror/gpseg6_new""", + "segments_with_running_basebackup": {0}, + "expected": [self._triplet('16|6|m|m|s|u|sdw1|sdw1|21000|/mirror/gpseg6', + '8|6|p|p|s|u|sdw4|sdw4|20000|/primary/gpseg6', + '16|6|m|m|s|u|sdw1|sdw1|21000|/mirror/gpseg6_new')] + }, + { + "name": "some_mirrors_inconfig_have_running_basebackup", + "gparray": self.all_up_gparray_str, + "config": """sdw2|21000|/mirror/gpseg0 sdw2|21000|/mirror/gpseg0_new + sdw2|21001|/mirror/gpseg1 sdw2|21000|/mirror/gpseg1_new + sdw1|21000|/mirror/gpseg6 sdw1|21000|/mirror/gpseg6_new""", + "segments_with_running_basebackup": {0, 1}, + "expected": [self._triplet('16|6|m|m|s|u|sdw1|sdw1|21000|/mirror/gpseg6', + '8|6|p|p|s|u|sdw4|sdw4|20000|/primary/gpseg6', + '16|6|m|m|s|u|sdw1|sdw1|21000|/mirror/gpseg6_new')] + }, + { + "name": "all_mirrors_inconfig_have_running_basebackup", + "gparray": self.all_up_gparray_str, + "config": """sdw2|21000|/mirror/gpseg0 sdw2|21000|/mirror/gpseg0_new + sdw2|21001|/mirror/gpseg1 sdw2|21000|/mirror/gpseg1_new + sdw1|21000|/mirror/gpseg6 sdw1|21000|/mirror/gpseg6_new""", + "segments_with_running_basebackup": {0, 1, 6}, + "expected": [] + }, + { + "name": "one_moved_mirror_inconfig_has_running_pgrewind", + "gparray": self.all_up_gparray_str, + "config": """sdw2|21000|/mirror/gpseg0 sdw2|21000|/mirror/gpseg0_new + sdw1|21000|/mirror/gpseg6 sdw1|21000|/mirror/gpseg6_new""", + "is_pgrewind_running": [True, False], + "expected": [self._triplet('16|6|m|m|s|u|sdw1|sdw1|21000|/mirror/gpseg6', + '8|6|p|p|s|u|sdw4|sdw4|20000|/primary/gpseg6', + '16|6|m|m|s|u|sdw1|sdw1|21000|/mirror/gpseg6_new')] + }, + { + "name": "one_mirror_inconfig_has_running_pgrewind", + "gparray": self.three_failedover_segs_gparray_str, + "config": """sdw1|20000|/primary/gpseg0 + sdw1|20001|/primary/gpseg1 + sdw2|20000|/primary/gpseg2""", + "is_pgrewind_running": [True, False, False], + "expected": [self._triplet('3|1|m|p|s|d|sdw1|sdw1|20001|/primary/gpseg1', + '7|1|p|m|s|u|sdw2|sdw2|21001|/mirror/gpseg1', + None), + self._triplet('4|2|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg2', + '8|2|p|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2', + None)] + }, + { + "name": "mirrors_inconfig_have_running_pgrewind_and_pgbasebackup", + "gparray": self.three_failedover_segs_gparray_str, + "config": """sdw1|20000|/primary/gpseg0 + sdw1|20001|/primary/gpseg1 + sdw2|20000|/primary/gpseg2""", + "segments_with_running_basebackup": {1, 2}, + "is_pgrewind_running": [True, False, False], + "expected": [] + }, + { + "name": "all_mirrors_inconfig_have_running_pgrewind", + "gparray": self.three_failedover_segs_gparray_str, + "config": """sdw1|20000|/primary/gpseg0 + sdw1|20001|/primary/gpseg1 + sdw2|20000|/primary/gpseg2""", + "is_pgrewind_running": [True, True, True], + "expected": [] + }, + { + "name": "in_place_1_part_with_4_parameter", + "gparray": self.all_up_gparray_str, + "config": "sdw2|sdw2|21000|/mirror/gpseg0", + "expected": [self._triplet('10|0|m|m|s|u|sdw2|sdw2|21000|/mirror/gpseg0', + '2|0|p|p|s|u|sdw1|sdw1|20000|/primary/gpseg0', + None)] + }, ] self.run_pass_tests(tests, self.run_single_ConfigFile_test) @@ -269,6 +368,70 @@ def test_RecoveryTripletsInPlaceAndNewHosts_getMirrorTriples_should_pass(self): '6|0|p|m|s|u|sdw2|sdw2|21000|/mirror/gpseg0', '2|0|m|p|s|d|new_1|new_1|20000|/primary/gpseg0', failed_unreachable=True)] }, + { + "name": "one_failed_segments_has_running_basebackup", + "gparray": self.three_failedover_segs_gparray_str, + "new_hosts": [], + "segments_with_running_basebackup": {0}, + "expected": [self._triplet('3|1|m|p|s|d|sdw1|sdw1|20001|/primary/gpseg1', + '7|1|p|m|s|u|sdw2|sdw2|21001|/mirror/gpseg1', + None), + self._triplet('4|2|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg2', + '8|2|p|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2', + None)] + }, + { + "name": "all_failed_segments_have_running_basebackup", + "gparray": self.three_failedover_segs_gparray_str, + "new_hosts": [], + "segments_with_running_basebackup": {0, 1, 2}, + "expected": [] + }, + { + "name": "failed_segments_have_running_pgrewind_or_pgbasebackup", + "gparray": self.three_failedover_segs_gparray_str, + "new_hosts": [], + "segments_with_running_basebackup": {1}, + "is_pgrewind_running": [True, False, False], + "expected": [self._triplet('4|2|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg2', + '8|2|p|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2', + None)] + }, + { + "name": "some_failed_segments_have_running_pgrewind", + "gparray": self.three_failedover_segs_gparray_str, + "new_hosts": [], + "is_pgrewind_running": [True, False, True], + "expected": [self._triplet('3|1|m|p|s|d|sdw1|sdw1|20001|/primary/gpseg1', + '7|1|p|m|s|u|sdw2|sdw2|21001|/mirror/gpseg1', + None)] + }, + # Moving to new hosts rewrites gpArray too; a skipped segment + # must keep its old host. + { + "name": "new_hosts_one_failed_segment_has_running_basebackup", + "gparray": self.three_failedover_segs_gparray_str, + "new_hosts": ['new_1', 'new_2'], + "segments_with_running_basebackup": {0}, + "expected": [self._triplet('3|1|m|p|s|d|sdw1|sdw1|20001|/primary/gpseg1', + '7|1|p|m|s|u|sdw2|sdw2|21001|/mirror/gpseg1', + '3|1|m|p|s|d|new_1|new_1|20001|/primary/gpseg1'), + self._triplet('4|2|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg2', + '8|2|p|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2', + '4|2|m|p|s|d|new_2|new_2|20000|/primary/gpseg2')] + }, + { + "name": "new_hosts_one_failed_segment_has_running_pgrewind", + "gparray": self.three_failedover_segs_gparray_str, + "new_hosts": ['new_1', 'new_2'], + "is_pgrewind_running": [False, True, False], + "expected": [self._triplet('2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0', + '6|0|p|m|s|u|sdw2|sdw2|21000|/mirror/gpseg0', + '2|0|m|p|s|d|new_1|new_1|20000|/primary/gpseg0'), + self._triplet('4|2|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg2', + '8|2|p|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2', + '4|2|m|p|s|d|new_2|new_2|20000|/primary/gpseg2')] + }, ] self.run_pass_tests(tests, self.run_single_GpArray_test) @@ -348,6 +511,68 @@ def test_RecoveryTripletsInPlaceAndNewHosts_getMirrorTriples_should_fail(self): ] self.run_fail_tests(tests, self.run_single_GpArray_test) + @patch('gppylib.db.dbconn.connect', side_effect=Exception()) + def test_get_segments_with_running_basebackup_conn_exception(self, mock1): + with self.assertRaises(Exception) as ex: + get_segments_with_running_basebackup(15432) + + self.assertTrue('Failed to query gp_stat_replication' in str(ex.exception)) + + @patch('gppylib.db.dbconn.connect', autospec=True) + @patch('gppylib.db.dbconn.query', side_effect=Exception()) + def test_get_segments_with_running_basebackup_query_exception(self, mock1, mock2): + with self.assertRaises(Exception) as ex: + get_segments_with_running_basebackup(15432) + + self.assertTrue('Failed to query gp_stat_replication:' in str(ex.exception)) + + @patch('gppylib.db.dbconn.connect', autospec=True) + @patch('gppylib.db.dbconn.query', return_value=FakeCursor(my_list=[])) + def test_get_segments_with_running_basebackup_no_basebackup(self, mock1, mock2): + segments_with_running_basebackup = get_segments_with_running_basebackup(15432) + self.assertEqual(list(segments_with_running_basebackup), []) + self.assertEqual([call("No basebackup running")], self.mock_logger.debug.call_args_list) + + @patch('gppylib.db.dbconn.connect', autospec=True) + @patch('gppylib.db.dbconn.query', return_value=FakeCursor(my_list=[[1]])) + def test_get_segments_with_running_basebackup_has_basebackup(self, mock1, mock2): + segments_with_running_basebackup = get_segments_with_running_basebackup(15432) + self.assertEqual(list(segments_with_running_basebackup), [1]) + self.assertEqual(0, self.mock_logger.debug.call_count) + + # PGPORT need not be set, or set to the coordinator's port. + @patch.dict(os.environ, {'PGPORT': '5432'}) + @patch('gppylib.db.dbconn.connect', autospec=True) + @patch('gppylib.db.dbconn.query', return_value=FakeCursor(my_list=[])) + def test_get_segments_with_running_basebackup_uses_coordinator_port(self, mock_query, mock_connect): + get_segments_with_running_basebackup(15432) + self.assertEqual(15432, mock_connect.call_args[0][0].pgport) + + def test_triplets_check_basebackup_on_coordinator_port(self): + self.run_single_GpArray_test({ + "gparray": self.all_up_gparray_str.replace('|mdw|mdw|5432|', '|mdw|mdw|15432|'), + "new_hosts": [], + }) + gppylib.programs.clsRecoverSegment_triples.get_segments_with_running_basebackup.assert_called_once_with(15432) + + @patch('gppylib.db.dbconn.connect', side_effect=Exception()) + def test_is_pg_rewind_running_conn_exception(self, mock1): + with self.assertRaises(Exception) as ex: + is_pg_rewind_running("sdw1", 6001) + self.assertEqual('Failed to query pg_stat_activity for segment hostname: sdw1, port: 6001, error: ', str(ex.exception)) + + @patch('gppylib.db.dbconn.connect', autospec=True) + @patch('gppylib.db.dbconn.querySingleton', return_value=2) + def test_is_pg_rewind_running_returns_true_for_query_result_greater_than_one(self, mock1, mock2): + self.assertEqual(is_pg_rewind_running("sdw1", 6001), True) + + @patch('gppylib.db.dbconn.connect', autospec=True) + @patch('gppylib.db.dbconn.querySingleton', return_value=0) + def test_is_pg_rewind_running_returns_false(self, mock1, mock2): + self.assertEqual(is_pg_rewind_running("sdw1", 6001), False) + self.assertEqual([call("Checking for running instances of pg_rewind with host sdw1 and port 6001 as source " + "server")], self.mock_logger.debug.call_args_list) + def run_pass_tests(self, tests, fn_to_test): for test in tests: with self.subTest(msg=test["name"]): @@ -438,10 +663,13 @@ def __init__(self, arg): 4|2|p|p|s|u|sdw2|sdw2|20000|/primary/gpseg2 5|3|p|p|s|u|sdw2|sdw2|20001|/primary/gpseg3''' - def _run_single_FromGpArray_test(self, gparray_str, config_file, new_hosts, unreachable_hosts, - unreachable_existing_hosts=None): + def _run_single_FromGpArray_test(self, gparray_str, config_file, new_hosts, unreachable_hosts, is_pgrewind_running, + segments_with_running_basebackup, unreachable_existing_hosts=None): unreachable_hosts = unreachable_hosts if unreachable_hosts else [] gppylib.programs.clsRecoverSegment_triples.get_unreachable_segment_hosts = Mock(return_value=unreachable_hosts) + gppylib.programs.clsRecoverSegment_triples.get_segments_with_running_basebackup = Mock( + return_value=segments_with_running_basebackup) + gppylib.programs.clsRecoverSegment_triples.is_pg_rewind_running = Mock(side_effect=is_pgrewind_running) initial_gparray = self.get_gp_array(gparray_str, unreachable_existing_hosts) mutated_gparray = self.get_gp_array(gparray_str, unreachable_existing_hosts) @@ -540,7 +768,9 @@ def run_single_parser_test(test): "config": """sdw1|20000|/primary/gpseg0 sdw3|20001|/primary/gpseg5 sdw1|20001|/primary/gpseg1 sdw1|40001|/primary/gpseg_new sdw3|20000|/primary/gpseg4 - sdw4|20000|/primary/gpseg6 sdw4|20000|/primary/gpseg6""" + sdw4|20000|/primary/gpseg6 sdw4|20000|/primary/gpseg6 + sdw5|sdw5|20000|/primary/gpseg0 sdw3|10.0.34.5|20001|/primary/gpseg5 + sdw6|sdw6|20000|/primary/gpseg4""" }, { "name": "6X_web_doc", @@ -582,7 +812,7 @@ def test_parsing_should_pass(self): """sdw1|20000|/mirror/gpseg0 sdw3|20001|/mirror/gpseg5 sdw1|20000 sdw3|20001|/mirror/gpseg5""", "expected": - "line 2 of file .*: expected 3 parts on failed segment group, obtained 2" + "line 2 of file .*: expected 3 or 4 parts on failed segment group, obtained 2" }, { "name": @@ -590,7 +820,7 @@ def test_parsing_should_pass(self): "config": """sdw1|20000|/mirror/gpseg0 sdw3|20001|/mirror/gpseg5 sdw2|50001|/data2/mirror/gpseg1 sdw4|50001""", "expected": - "line 2 of file .*: expected 3 parts on new segment group, obtained 2" + "line 2 of file .*: expected equal parts, either 3 or 4 on both segment group, obtained 3 on group1 and 2 on group2" }, { "name": @@ -697,7 +927,25 @@ def test_parsing_should_pass(self): "name": "new_port_invalid", "config": """sdw2|50001|/data2/mirror/gpseg1 sdw4|new_invalid_port|relative/new/mirror/gpseg1""", "expected": "Invalid port on line 1" - } + }, + { + "name": + "invalid_parts_present_in_group_1", + "config": + """sdw1|10.0.34.2|20000|/primary/gpseg0 sdw3|10.0.34.5|20001|/primary/gpseg5 + sdw1|20000|/primary/gpseg0 sdw3|10.0.34.5|20001|/primary/gpseg5""", + "expected": + "line 2 of file .*: expected equal parts, either 3 or 4 on both segment group, obtained 3 on group1 and 4 on group2" + }, + { + "name": + "invalid_parts_present_in_group_2", + "config": + """sdw1|10.0.34.2|20000|/primary/gpseg0 sdw3|10.0.34.5|20001|/primary/gpseg5 + sdw1|10.0.34.2|20000|/primary/gpseg0 sdw3|20001|/primary/gpseg5""", + "expected": + "line 2 of file .*: expected equal parts, either 3 or 4 on both segment group, obtained 4 on group1 and 3 on group2" + }, ] def test_parsing_should_fail(self): @@ -717,21 +965,30 @@ def _get_expected_config(config_str): for line in config_str.splitlines(): lineno += 1 groups = line.split() + parts = groups[0].split('|') + hostname, address, port, datadir, hostname_check_required = extract_recovery_config_info(parts) - address, port, datadir = groups[0].split('|') row = { + 'failedHostname': hostname, 'failedAddress': address, 'failedPort': port, 'failedDataDirectory': datadir, - 'lineno': lineno + 'lineno': lineno, + 'hostname_check_required': hostname_check_required } if len(groups) > 1: - address, port, datadir = groups[1].split('|') + parts2 = groups[1].split('|') + if len(parts2) == 4: + hostname2, address2, port2, datadir2 = parts2 + else: + address2, port2, datadir2 = parts2 + hostname2 = address2 row.update({ - 'newAddress': address, - 'newPort': port, - 'newDataDirectory': datadir + 'newHostname': hostname2, + 'newAddress': address2, + 'newPort': port2, + 'newDataDirectory': datadir2 }) rows.append(row) diff --git a/gpMgmt/bin/gppylib/recoveryinfo.py b/gpMgmt/bin/gppylib/recoveryinfo.py index f13ea81394b..b4530bc2194 100644 --- a/gpMgmt/bin/gppylib/recoveryinfo.py +++ b/gpMgmt/bin/gppylib/recoveryinfo.py @@ -1,8 +1,10 @@ from collections import defaultdict import datetime import json +import shlex from gppylib import gplog +from gppylib.commands.base import Command, REMOTE class RecoveryInfo(object): @@ -169,8 +171,36 @@ def print_setup_recovery_errors(self): self._logger.error(setup_recovery_error_pattern.format(hostname, error.port, error.error_msg)) self._print_invalid_errors() + # Cherry-picked from Greenplum Database (greenplum-db/gpdb-archive @ 482967c1b4), + # gpMgmt/bin/gppylib/recoveryinfo.py, with the line it filters out + # changed: pg_rewind reports its own failure differently since + # PostgreSQL 13. See the comment in the body. + def get_error_from_logfile(self, progress_file, hostname): + """ + Traverse the error file to fetch last registered occurrence of 'error, panic or fatal'. + pg_rewind logs a line of its own after every failed attempt, which is + never the reason for the failure -- the reason is whatever the backend + wrote just above it -- so that line is excluded. Upstream excludes + PostgreSQL 12's wording of it, "fatal: postgres single-user mode of + target instance failed for command"; since 13 pg_rewind writes + "error: postgres single-user mode in target cluster failed" followed by + a separate "detail: Command was: ...", and only the first of those two + carries a word this grep looks for. + To ensure grep doesn't return a non-zero exit code, it is ORed with true. + """ + cmdStr = r'set -o pipefail; cat {} | (grep -i "ERROR\|PANIC\|FATAL" | grep -v "error: postgres single-user mode in target cluster failed" || true) | tail -1'.format( + shlex.quote(progress_file)) + cmd = Command(name="Parse logfile for errors on a remote host", cmdStr=cmdStr, ctxt=REMOTE, remoteHost=hostname) + cmd.run() + + if cmd.get_results().rc != 0: + self._logger.debug("Failed while parsing the logfile %s" % progress_file) + return None + + return cmd.get_results().stdout.strip() + def print_bb_rewind_and_start_errors(self): - bb_rewind_error_pattern = " hostname: {}; port: {}; logfile: {}; recoverytype: {}" + bb_rewind_error_pattern = " hostname: {}; port: {}; logfile: {}; recoverytype: {}; error: {}" if len(self._bb_errors) > 0 or len(self._rewind_errors) > 0: self._logger.info("----------------------------------------------------------") if len(self._rewind_errors) > 0: @@ -181,11 +211,15 @@ def print_bb_rewind_and_start_errors(self): for hostname, errors in self._rewind_errors.items(): for error in errors: self._logger.info(bb_rewind_error_pattern.format(hostname, error.port, error.progress_file, - error.error_type)) + error.error_type, + self.get_error_from_logfile(error.progress_file, + hostname))) for hostname, errors in self._bb_errors.items(): for error in errors: self._logger.info(bb_rewind_error_pattern.format(hostname, error.port, error.progress_file, - error.error_type)) + error.error_type, + self.get_error_from_logfile(error.progress_file, + hostname))) start_error_pattern = " hostname: {}; port: {}; datadir: {}" if len(self._start_errors) > 0: diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpsegrecovery.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpsegrecovery.py index 983b5f25a9a..afc7438496c 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpsegrecovery.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpsegrecovery.py @@ -1,7 +1,10 @@ from contextlib import redirect_stderr from mock import call, Mock, patch, ANY import io +import os +import shutil import sys +import tempfile from .gp_unittest import GpTestCase import gpsegrecovery @@ -18,11 +21,13 @@ def setUp(self): self.mock_logger = Mock() self.apply_patches([ patch('gpsegrecovery.start_segment', return_value=Mock()), + patch('gpsegrecovery.ModifyConfSetting', return_value=Mock()), patch('gppylib.commands.pg.PgRewind.__init__', return_value=None), patch('gppylib.commands.pg.PgRewind.run') ]) self.mock_pgrewind_run = self.get_mock_from_apply_patch('run') self.mock_pgrewind_init = self.get_mock_from_apply_patch('__init__') + self.mock_modify_conf = self.get_mock_from_apply_patch('ModifyConfSetting') p = Segment.initFromString("1|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0") m = Segment.initFromString("2|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0") @@ -56,8 +61,13 @@ def test_incremental_run_passes(self): self.assertEqual(1, self.mock_pgrewind_run.call_count) self.assertEqual(call(validateAfter=True), self.mock_pgrewind_run.call_args) logger_call_args = [call('Running pg_rewind with progress output temporarily in /tmp/test_progress_file'), - call('Successfully ran pg_rewind for dbid: 2')] + call('Successfully ran pg_rewind for dbid: 2'), + call('Updating /data/mirror0/postgresql.conf')] self.assertEqual(logger_call_args, self.mock_logger.info.call_args_list) + self.mock_modify_conf.assert_called_once_with('Updating /data/mirror0/postgresql.conf', + '/data/mirror0/postgresql.conf', 'port', 50000, + optType='number') + self.mock_modify_conf.return_value.run.assert_called_once_with(validateAfter=True) gpsegrecovery.start_segment.assert_called_once_with(self.seg_recovery_info, self.mock_logger, self.era) def test_incremental_run_exception(self): @@ -71,6 +81,7 @@ def test_incremental_run_exception(self): self.assertEqual(call(validateAfter=True), self.mock_pgrewind_run.call_args) self.assertEqual([call('Running pg_rewind with progress output temporarily in /tmp/test_progress_file')], self.mock_logger.info.call_args_list) + self.assertEqual(0, self.mock_modify_conf.call_count) self.assertEqual(0, gpsegrecovery.start_segment.call_count) self._assert_cmd_failed('{"error_type": "incremental", "error_msg": "pg_rewind failed", "dbid": 2, ' \ '"datadir": "/data/mirror0", "port": 50000, "progress_file": "/tmp/test_progress_file"}') @@ -92,7 +103,8 @@ def test_incremental_start_segment_exception(self): self.assertEqual(1, self.mock_pgrewind_init.call_count) self.assertEqual(1, self.mock_pgrewind_run.call_count) logger_call_args = [call('Running pg_rewind with progress output temporarily in /tmp/test_progress_file'), - call('Successfully ran pg_rewind for dbid: 2')] + call('Successfully ran pg_rewind for dbid: 2'), + call('Updating /data/mirror0/postgresql.conf')] self.assertEqual(logger_call_args, self.mock_logger.info.call_args_list) gpsegrecovery.start_segment.assert_called_once_with(self.seg_recovery_info, self.mock_logger, self.era) self._assert_cmd_failed('{"error_type": "start", "error_msg": "pg_ctl start failed", "dbid": 2, ' \ @@ -107,11 +119,13 @@ def setUp(self): self.mock_logger = Mock(spec=['log', 'info', 'debug', 'error', 'warn', 'exception']) self.apply_patches([ patch('gpsegrecovery.start_segment', return_value=Mock()), + patch('gpsegrecovery.ModifyConfSetting', return_value=Mock()), patch('gpsegrecovery.PgBaseBackup.__init__', return_value=None), patch('gpsegrecovery.PgBaseBackup.run') ]) self.mock_pgbasebackup_run = self.get_mock_from_apply_patch('run') self.mock_pgbasebackup_init = self.get_mock_from_apply_patch('__init__') + self.mock_modify_conf = self.get_mock_from_apply_patch('ModifyConfSetting') p = Segment.initFromString("1|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0") m = Segment.initFromString("2|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0") @@ -135,8 +149,13 @@ def _assert_basebackup_runs(self, expected_init_args): self.assertEqual(1, self.mock_pgbasebackup_run.call_count) self.assertEqual(call(validateAfter=True), self.mock_pgbasebackup_run.call_args) expected_logger_info_args = [call('Running pg_basebackup with progress output temporarily in /tmp/test_progress_file'), - call("Successfully ran pg_basebackup for dbid: 2")] + call("Successfully ran pg_basebackup for dbid: 2"), + call('Updating /data/mirror0/postgresql.conf')] self.assertEqual(expected_logger_info_args, self.mock_logger.info.call_args_list) + self.mock_modify_conf.assert_called_once_with('Updating /data/mirror0/postgresql.conf', + '/data/mirror0/postgresql.conf', 'port', 50000, + optType='number') + self.mock_modify_conf.return_value.run.assert_called_once_with(validateAfter=True) gpsegrecovery.start_segment.assert_called_once_with(self.seg_recovery_info, self.mock_logger, self.era) def _assert_cmd_passed(self): @@ -277,6 +296,7 @@ def setUp(self): self.apply_patches([ patch('gpsegrecovery.SegmentStart.__init__', return_value=None), patch('gpsegrecovery.SegmentStart.run'), + patch('gpsegrecovery.ModifyConfSetting', return_value=Mock()), ]) self.mock_segment_start_init = self.get_mock_from_apply_patch('__init__') @@ -396,3 +416,69 @@ def test_start_segment_passes(self, mock_run, mock_init): mock_init.assert_called_once() self.assertEqual(1, self.mock_logger.info.call_count) mock_run.assert_called_once_with(validateAfter=True) + + +class UpdatePortInConfTestCase(GpTestCase): + """ + Runs the real rewrite, not a mock of it: the rewrite reports success + whether or not it matched a line, so only the file shows whether the port + was changed. + """ + def setUp(self): + self.datadir = tempfile.mkdtemp() + self.recovery_info = RecoveryInfo(self.datadir, 50000, 2, 'sdw1', 40000, False, '/tmp/test_progress_file') + + def tearDown(self): + shutil.rmtree(self.datadir) + super(UpdatePortInConfTestCase, self).tearDown() + + def _rewrite(self, line): + conf = os.path.join(self.datadir, 'postgresql.conf') + with open(conf, 'w') as f: + f.write(line + '\n') + gpsegrecovery.update_port_in_conf(self.recovery_info, Mock()) + with open(conf) as f: + return f.read().rstrip('\n') + + def test_every_form_postgresql_conf_accepts_is_rewritten(self): + forms = ['port=40000', 'port = 40000', ' port = 40000', 'port\t=40000', '\tport=40000', + 'port 40000', "port = '40000'", 'PORT = 40000', 'Port=40000'] + for form in forms: + with self.subTest(form=form): + self.assertEqual('port=50000', self._rewrite(form)) + + def test_trailing_comment_is_kept(self): + self.assertEqual('port=50000 # sets the listener port', + self._rewrite('port = 40000 # sets the listener port')) + + def test_commented_out_and_other_settings_are_left_alone(self): + for line in ['#port = 5432', '#PORT = 5432', 'port_range = 40000', 'PORT_RANGE = 40000']: + with self.subTest(line=line): + self.assertEqual(line, self._rewrite(line)) + + def test_alternative_numeric_values_are_replaced_completely(self): + values = ['+40000', "'+40000'", '0116100', "'0116100'", + '0x9c40', "'0x9c40'", '40000.0', "'40000.0'", "'4e4'"] + for value in values: + with self.subTest(value=value): + self.assertEqual('port=50000', self._rewrite('port = ' + value)) + self.assertEqual('port=50000 # source port', + self._rewrite('PORT ' + value + ' # source port')) + + def test_last_port_assignment_is_also_rewritten(self): + self.assertEqual('port=50000\nport=50000 # override', + self._rewrite("port=40000\nPORT = '0x9c40' # override")) + + def test_rewrite_preserves_adjacent_lines(self): + self.assertEqual("# source config\n\nport=50000\nport_range = 40000", + self._rewrite("# source config\n\nport = '4e4'\nport_range = 40000")) + + def test_crlf_lines_are_rewritten_and_line_endings_preserved(self): + conf = os.path.join(self.datadir, 'postgresql.conf') + with open(conf, 'wb') as f: + f.write(b"# source config\r\nport = 40000\r\nPORT = '40000' \t\r\n" + b"port = 40000 # override\r\nport_range = 40000\r\n") + gpsegrecovery.update_port_in_conf(self.recovery_info, Mock()) + with open(conf, 'rb') as f: + self.assertEqual(b"# source config\r\nport=50000\r\nport=50000 \t\r\n" + b"port=50000 # override\r\nport_range = 40000\r\n", f.read()) diff --git a/gpMgmt/bin/gppylib/utils.py b/gpMgmt/bin/gppylib/utils.py index 42fca86592e..bc5af671fa5 100644 --- a/gpMgmt/bin/gppylib/utils.py +++ b/gpMgmt/bin/gppylib/utils.py @@ -612,3 +612,45 @@ def formatInsertValuesList(row, starelid, inclHLL): rowVals.append('\t{0}::{1}'.format(val, typ)) return rowVals + + +def _resolve_all_addresses(name): + """ + Every address a name resolves to, across address families. + + Not gethostbyaddr(): it answers for one family only, so on a dual-stack + host "localhost" can come back as ::1 while an address that names the same + machine comes back as 127.0.0.1. Comparing those two lists says the names + are different hosts when they are not. + """ + return {info[4][0] for info in socket.getaddrinfo(name, None)} + + +# cherry-pick from Greenplum Database +# greenplum-db/gpdb-archive @ 482967c1b4, gpMgmt/bin/gppylib/utils.py +def validateHostnameAddress(hostname, address): + """ + validateHostnameAddress : validates that given hostname and address are for the same host + Address can be hostname/alias also. Resolves hostname and address both to get the associated addresses. + + @param hostname Name of the host to be validated + @param address Address of the host to be validated. Can be hostname/alias also + @return if the address and hostname are of the same host + """ + try: + resolved_address_list = _resolve_all_addresses(hostname) + resolved_address_list_2 = _resolve_all_addresses(address) + except Exception as e: + # This means given hostname or address is not reachable + logger.warning( + "Could not resolve hostname:{0}." + .format(hostname)) + return False + + # Resolved address and hostname should have at least one IP address common if they are of same host + if not bool(resolved_address_list.intersection(resolved_address_list_2)): + logger.warning( + "Given address:{0} not present in resolved hostname:{1} address list we got:{2}".format( + address, hostname, sorted(resolved_address_list))) + return False + return True diff --git a/gpMgmt/sbin/gpsegrecovery.py b/gpMgmt/sbin/gpsegrecovery.py index 811a73ccbb5..a1a6891ee35 100644 --- a/gpMgmt/sbin/gpsegrecovery.py +++ b/gpMgmt/sbin/gpsegrecovery.py @@ -7,7 +7,7 @@ from gppylib.commands.pg import PgBaseBackup, PgRewind from recovery_base import RecoveryBase, set_recovery_cmd_results from gppylib.commands.base import Command -from gppylib.commands.gp import SegmentStart +from gppylib.commands.gp import SegmentStart, ModifyConfSetting from gppylib.gparray import Segment from gppylib.commands.unix import terminate_proc_tree @@ -63,6 +63,10 @@ def run(self): self.error_type = RecoveryErrorType.DEFAULT_ERROR self.logger.info("Successfully ran pg_basebackup for dbid: {}".format( self.recovery_info.target_segment_dbid)) + + # Updating port number on conf after recovery + update_port_in_conf(self.recovery_info, self.logger) + self.error_type = RecoveryErrorType.START_ERROR start_segment(self.recovery_info, self.logger, self.era) @@ -87,10 +91,27 @@ def run(self): cmd.run(validateAfter=True) self.logger.info("Successfully ran pg_rewind for dbid: {}".format(self.recovery_info.target_segment_dbid)) + # Updating port number on conf after recovery + update_port_in_conf(self.recovery_info, self.logger) + self.error_type = RecoveryErrorType.START_ERROR start_segment(self.recovery_info, self.logger, self.era) +def update_port_in_conf(recovery_info, logger): + """ + pg_basebackup and pg_rewind both copy the source segment's + postgresql.conf, which carries the *source* port. Point it back at this + segment before it is started, or the mirror comes up on its primary's + port. + """ + logger.info("Updating %s/postgresql.conf" % recovery_info.target_datadir) + modifyConfCmd = ModifyConfSetting('Updating %s/postgresql.conf' % recovery_info.target_datadir, + "{}/{}".format(recovery_info.target_datadir, 'postgresql.conf'), + 'port', recovery_info.target_port, optType='number') + modifyConfCmd.run(validateAfter=True) + + def start_segment(recovery_info, logger, era): seg = Segment(None, None, None, None, None, None, None, None, recovery_info.target_port, recovery_info.target_datadir) diff --git a/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature b/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature index b63b279caae..9817183b579 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature @@ -38,8 +38,6 @@ Feature: Tests for gpaddmirrors | -B 2 -b 1 | 2 | 1 | | -B 1 -b 2 | 1 | 2 | - # needs_fix: a recovered mirror keeps its primary's port in postgresql.conf (#1639 a3bfe47d5c3) - @needs_fix Scenario: gpaddmirrors fails for recovery setup errors Given the cluster is generated with "3" primaries only When gpaddmirrors adds 3 mirrors with one mirror's datadir not empty @@ -155,8 +153,6 @@ Feature: Tests for gpaddmirrors | all | None | 0,1,2 | - # needs_fix: a recovered mirror keeps its primary's port in postgresql.conf (#1639 a3bfe47d5c3) - @needs_fix Scenario: gpaddmirrors can add mirrors and display progress in gpstate Given the cluster is generated with "3" primaries only And all files in gpAdminLogs directory are deleted diff --git a/gpMgmt/test/behave/mgmt_utils/gpmovemirrors.feature b/gpMgmt/test/behave/mgmt_utils/gpmovemirrors.feature index 5f527af6035..06aa918bb8f 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpmovemirrors.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpmovemirrors.feature @@ -22,8 +22,6 @@ Feature: Tests for gpmovemirrors When the user runs gpmovemirrors with additional args "--invalid-option" Then gpmovemirrors should return a return code of 2 - # needs_fix: gprecoverseg/gppylib fixes (#1639 a3bfe47d5c3 / 51955ac3d8f / 7056e02e306) - @needs_fix Scenario: gpmovemirrors can change the location of mirrors within a single host Given a standard local demo cluster is created And a gpmovemirrors directory under '/tmp/gpmovemirrors' with mode '0700' is created @@ -39,8 +37,6 @@ Feature: Tests for gpmovemirrors And check segment conf: postgresql.conf And verify that mirrors are recognized after a restart - # needs_fix: gprecoverseg/gppylib fixes (#1639 a3bfe47d5c3 / 51955ac3d8f / 7056e02e306) - @needs_fix Scenario: gpmovemirrors can change the port of mirrors within a single host Given a standard local demo cluster is created And a gpmovemirrors directory under '/tmp/gpmovemirrors' with mode '0700' is created @@ -117,8 +113,6 @@ Feature: Tests for gpmovemirrors assert that gp_seg_config wasn't updated """ @skip_cleanup - # needs_fix: gprecoverseg/gppylib fixes (#1639 a3bfe47d5c3 / 51955ac3d8f / 7056e02e306) - @needs_fix Scenario Outline: user can if mirrors failed to move initially Given the database is running And all the segments are running @@ -163,8 +157,6 @@ Feature: Tests for gpmovemirrors | run gprecoverseg | all | None | 0,1,2 | running in place full recovery for all failed contents | @skip_cleanup - # needs_fix: gprecoverseg/gppylib fixes (#1639 a3bfe47d5c3 / 51955ac3d8f / 7056e02e306) - @needs_fix Scenario: gpmovemirrors can move mirrors even if start fails for some mirrors Given the database is running And all the segments are running @@ -251,8 +243,6 @@ Feature: Tests for gpmovemirrors @demo_cluster @concourse_cluster @skip_cleanup - # needs_fix: skip segments that already have a recovery running (#1639 7056e02e306) - @needs_fix Scenario: gpmovemirrors gives warning if pg_basebackup is already running for one of the mirrors to be moved Given the database is running And all the segments are running @@ -292,8 +282,6 @@ Feature: Tests for gpmovemirrors @demo_cluster @concourse_cluster @skip_cleanup - # needs_fix: skip segments that already have a recovery running (#1639 7056e02e306) - @needs_fix Scenario: gpmovemirrors gives warning if pg_basebackup is already running for some of the mirrors to be moved Given the database is running And all the segments are running @@ -334,8 +322,6 @@ Feature: Tests for gpmovemirrors @demo_cluster @concourse_cluster @skip_cleanup - # needs_fix: skip segments that already have a recovery running (#1639 7056e02e306) - @needs_fix Scenario: gpmovemirrors gives warning if pg_basebackup is already running for all mirrors to be moved Given the database is running And all the segments are running diff --git a/gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature b/gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature index 68d7be4b053..151d9f2f0dc 100644 --- a/gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature +++ b/gpMgmt/test/behave/mgmt_utils/gprecoverseg.feature @@ -1,7 +1,4 @@ -# needs_fix: the gprecoverseg fixes from #1639 (a3bfe47d5c3, 879b5638026, -# 7056e02e306, 0aa26662c2c, 51955ac3d8f); without them a scenario hangs and -# the job reaches its timeout. -@gprecoverseg @needs_fix +@gprecoverseg Feature: gprecoverseg tests @differential @@ -816,8 +813,6 @@ Feature: gprecoverseg tests @demo_cluster @concourse_cluster - # needs_fix: skip segments that already have a recovery running (#1639 7056e02e306) - @needs_fix Scenario: gprecoverseg gives warning if pg_basebackup already running for one of the failed segments Given the database is running And all the segments are running @@ -850,8 +845,6 @@ Feature: gprecoverseg tests @demo_cluster @concourse_cluster - # needs_fix: skip segments that already have a recovery running (#1639 7056e02e306) - @needs_fix Scenario: gprecoverseg gives warning if pg_basebackup already running for some of the failed segments Given the database is running And all the segments are running @@ -886,8 +879,6 @@ Feature: gprecoverseg tests @demo_cluster @concourse_cluster - # needs_fix: skip segments that already have a recovery running (#1639 7056e02e306) - @needs_fix Scenario: gprecoverseg -aF gives warning if pg_basebackup already running for all of the failed segments Given the database is running And all the segments are running @@ -921,8 +912,6 @@ Feature: gprecoverseg tests @demo_cluster @concourse_cluster - # needs_fix: skip segments that already have a recovery running (#1639 7056e02e306) - @needs_fix Scenario: gprecoverseg -i gives warning if pg_basebackup already running for all failed segments Given the database is running And all the segments are running @@ -1787,8 +1776,6 @@ Feature: gprecoverseg tests @concourse_cluster - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix Scenario: gprecoverseg recovery to new host populates hostname and address from the config file correctly Given the database is running And all the segments are running @@ -1806,8 +1793,6 @@ Feature: gprecoverseg tests And the segments are synchronized @concourse_cluster - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix Scenario: gprecoverseg recovery to same host (full inplace) populates hostname and address from the config file correctly Given the database is running And all the segments are running @@ -1826,8 +1811,6 @@ Feature: gprecoverseg tests @concourse_cluster - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix Scenario: gprecoverseg recovery with invalid format with hostname in config file Given the database is running And all the segments are running @@ -1847,8 +1830,6 @@ Feature: gprecoverseg tests @concourse_cluster - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix Scenario: gprecoverseg incremental recovery populates hostname and address from the config file correctly Given the database is running And all the segments are running @@ -1866,8 +1847,6 @@ Feature: gprecoverseg tests And the segments are synchronized @concourse_cluster - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix Scenario: gprecoverseg recovery with and without hostname parameter in config file Given the database is running And all the segments are running @@ -1882,8 +1861,6 @@ Feature: gprecoverseg tests And the segments are synchronized @concourse_cluster - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix Scenario: gprecoverseg throws warning and skips recovery if provided hostname and address can not be resolved to same host Given the database is running And all the segments are running @@ -1902,8 +1879,6 @@ Feature: gprecoverseg tests And the segments are synchronized @concourse_cluster - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix Scenario: gprecoverseg incremental recovery fails if config file contains wrong hostname of failed segment Given the database is running And all the segments are running @@ -1921,8 +1896,6 @@ Feature: gprecoverseg tests And gprecoverseg should return a return code of 0 And the cluster is rebalanced - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix @demo_cluster Scenario: gprecoverseg recovers segment when config file contains hostname on demo cluster Given the database is running @@ -1941,8 +1914,6 @@ Feature: gprecoverseg tests And the cluster configuration has no segments where "content=0 and status='d'" Then the cluster is rebalanced - # needs_fix: hostname in the -i recovery config file (#1639 0aa26662c2c) - @needs_fix @demo_cluster Scenario: gprecoverseg skips recovery when config file contains invalid hostname on demo cluster Given the database is running