diff -r 66cc9e0a6768 mysql-test/lib/mtr_cases.pl --- a/mysql-test/lib/mtr_cases.pl Thu Dec 04 21:37:12 2008 -0800 +++ b/mysql-test/lib/mtr_cases.pl Thu Dec 04 21:46:15 2008 -0800 @@ -334,6 +334,10 @@ $tinfo->{'slave_num'}= 1; # Default for rpl* tests, use one slave + if ( $tname eq 'rpl_mirror_binlog' ) + { + $tinfo->{'slave_num'}= 3; + } } if ( defined mtr_match_prefix($tname,"federated") ) @@ -344,15 +348,20 @@ my $master_opt_file= "$testdir/$tname-master.opt"; my $slave_opt_file= "$testdir/$tname-slave.opt"; - my $slave_mi_file= "$testdir/$tname.slave-mi"; + my $slave_mi_files= ["$testdir/$tname.slave-mi", + "$testdir/$tname.1.slave-mi", + "$testdir/$tname.2.slave-mi"]; my $master_sh= "$testdir/$tname-master.sh"; my $slave_sh= "$testdir/$tname-slave.sh"; my $disabled_file= "$testdir/$tname.disabled"; my $im_opt_file= "$testdir/$tname-im.opt"; - $tinfo->{'master_opt'}= []; - $tinfo->{'slave_opt'}= []; - $tinfo->{'slave_mi'}= []; + $tinfo->{'master_opt'}= []; + $tinfo->{'slave_opt'}= []; + $tinfo->{'slave_mi'}= {}; + $tinfo->{'slave_mi'}{0}= []; + $tinfo->{'slave_mi'}{1}= []; + $tinfo->{'slave_mi'}{2}= []; if ( -f $master_opt_file ) { @@ -427,9 +436,14 @@ push(@{$tinfo->{'slave_opt'}}, @$slave_opt); } - if ( -f $slave_mi_file ) + my $mi_idx= 0; + foreach my $slave_mi_file ( @$slave_mi_files ) { - $tinfo->{'slave_mi'}= mtr_get_opts_from_file($slave_mi_file); + if ( -f $slave_mi_file ) + { + $tinfo->{'slave_mi'}{$mi_idx}= mtr_get_opts_from_file($slave_mi_file); + } + $mi_idx+= 1; } if ( -f $master_sh ) diff -r 66cc9e0a6768 mysql-test/mysql-test-run.pl --- a/mysql-test/mysql-test-run.pl Thu Dec 04 21:37:12 2008 -0800 +++ b/mysql-test/mysql-test-run.pl Thu Dec 04 21:46:15 2008 -0800 @@ -275,6 +275,7 @@ our $opt_stress_test_file= ""; our $opt_warnings; +our $opt_slave_innodb= 0; our $opt_skip_ndbcluster= 0; our $opt_skip_ndbcluster_slave= 0; @@ -299,6 +300,8 @@ our $used_binlog_format; our $used_default_engine; our $debug_compiled_binaries; + +our $current_testname= ""; our %mysqld_variables; @@ -645,6 +648,7 @@ 'testcase-timeout=i' => \$opt_testcase_timeout, 'suite-timeout=i' => \$opt_suite_timeout, 'warnings|log-warnings' => \$opt_warnings, + 'slave-innodb' => \$opt_slave_innodb, # Options which are no longer used (map { $_ => \&warn_about_removed_option } @removed_options), @@ -1001,6 +1005,14 @@ { $ENV{'BIG_TEST'}= 1; } + + # -------------------------------------------------------------------------- + # Big test flags + # -------------------------------------------------------------------------- + if ( $opt_big_test ) + { + $ENV{'BIG_TEST'}= 1; + } # -------------------------------------------------------------------------- # Gcov flag @@ -1885,7 +1897,9 @@ $ENV{'SLAVE_MYSOCK'}= $slave->[0]->{'path_sock'}; $ENV{'SLAVE_MYPORT'}= $slave->[0]->{'port'}; $ENV{'SLAVE_MYPORT1'}= $slave->[1]->{'port'}; + $ENV{'SLAVE_MYSOCK1'}= $slave->[1]->{'path_sock'}; $ENV{'SLAVE_MYPORT2'}= $slave->[2]->{'port'}; + $ENV{'SLAVE_MYSOCK2'}= $slave->[2]->{'path_sock'}; $ENV{'MYSQL_TCP_PORT'}= $mysqld_variables{'port'}; $ENV{'DEFAULT_MASTER_PORT'}= $mysqld_variables{'master-port'}; @@ -2375,6 +2389,8 @@ if ( ! $glob_win32 ) { symlink("$glob_mysql_test_dir/std_data", "$opt_vardir/std_data_ln"); + my @a = ("chmod", "-R", "o+r", "$glob_mysql_test_dir/std_data"); + system(@a) == 0 or die "system @ failed: $?" } else { @@ -3466,6 +3482,8 @@ $ENV{'TZ'}= $tinfo->{'timezone'}; mtr_verbose("Setting timezone: $tinfo->{'timezone'}"); + $current_testname= $tinfo->{'name'}; + my $master_restart= run_testcase_need_master_restart($tinfo); my $slave_restart= run_testcase_need_slave_restart($tinfo); @@ -3881,7 +3899,8 @@ unless $mysqld->{'type'} eq 'slave'; mtr_add_arg($args, "%s--init-rpl-role=slave", $prefix); - if (! ( $opt_skip_slave_binlog || $skip_binlog )) + + if (! ($opt_skip_slave_binlog or ($current_testname eq 'rpl_mirror_binlog')) ) { mtr_add_arg($args, "%s--log-bin=%s/log/slave%s-bin", $prefix, $opt_vardir, $sidx); # FIXME use own dir for binlogs @@ -4568,7 +4587,7 @@ if ( ! $slave->[$idx]->{'pid'} ) { mysqld_start($slave->[$idx],$tinfo->{'slave_opt'}, - $tinfo->{'slave_mi'}); + $tinfo->{'slave_mi'}{$idx}); } } @@ -4580,7 +4599,6 @@ # Wait for clusters to start foreach my $cluster (@{$clusters}) { - next if !$cluster->{'pid'}; if (ndbcluster_wait_started($cluster, "")) @@ -5179,6 +5197,7 @@ skip-im Don't start IM, and skip the IM test cases big-test Set the environment variable BIG_TEST, which can be checked from test cases. + Options that specify ports diff -r 66cc9e0a6768 mysql-test/r/rpl_mirror_binlog.result --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mysql-test/r/rpl_mirror_binlog.result Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,441 @@ +stop slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +reset master; +reset slave; +drop table if exists t1,t2,t3,t4,t5,t6,t7,t8,t9; +start slave; +drop table if exists t1; +create table t1(n int) engine = InnoDB; +insert into t1 values (300); +insert into t1 values (299); +insert into t1 values (298); +insert into t1 values (297); +insert into t1 values (296); +insert into t1 values (295); +insert into t1 values (294); +insert into t1 values (293); +insert into t1 values (292); +insert into t1 values (291); +insert into t1 values (290); +insert into t1 values (289); +insert into t1 values (288); +insert into t1 values (287); +insert into t1 values (286); +insert into t1 values (285); +insert into t1 values (284); +insert into t1 values (283); +insert into t1 values (282); +insert into t1 values (281); +insert into t1 values (280); +insert into t1 values (279); +insert into t1 values (278); +insert into t1 values (277); +insert into t1 values (276); +insert into t1 values (275); +insert into t1 values (274); +insert into t1 values (273); +insert into t1 values (272); +insert into t1 values (271); +insert into t1 values (270); +insert into t1 values (269); +insert into t1 values (268); +insert into t1 values (267); +insert into t1 values (266); +insert into t1 values (265); +insert into t1 values (264); +insert into t1 values (263); +insert into t1 values (262); +insert into t1 values (261); +insert into t1 values (260); +insert into t1 values (259); +insert into t1 values (258); +insert into t1 values (257); +insert into t1 values (256); +insert into t1 values (255); +insert into t1 values (254); +insert into t1 values (253); +insert into t1 values (252); +insert into t1 values (251); +insert into t1 values (250); +insert into t1 values (249); +insert into t1 values (248); +insert into t1 values (247); +insert into t1 values (246); +insert into t1 values (245); +insert into t1 values (244); +insert into t1 values (243); +insert into t1 values (242); +insert into t1 values (241); +insert into t1 values (240); +insert into t1 values (239); +insert into t1 values (238); +insert into t1 values (237); +insert into t1 values (236); +insert into t1 values (235); +insert into t1 values (234); +insert into t1 values (233); +insert into t1 values (232); +insert into t1 values (231); +insert into t1 values (230); +insert into t1 values (229); +insert into t1 values (228); +insert into t1 values (227); +insert into t1 values (226); +insert into t1 values (225); +insert into t1 values (224); +insert into t1 values (223); +insert into t1 values (222); +insert into t1 values (221); +insert into t1 values (220); +insert into t1 values (219); +insert into t1 values (218); +insert into t1 values (217); +insert into t1 values (216); +insert into t1 values (215); +insert into t1 values (214); +insert into t1 values (213); +insert into t1 values (212); +insert into t1 values (211); +insert into t1 values (210); +insert into t1 values (209); +insert into t1 values (208); +insert into t1 values (207); +insert into t1 values (206); +insert into t1 values (205); +insert into t1 values (204); +insert into t1 values (203); +insert into t1 values (202); +insert into t1 values (201); +insert into t1 values (200); +insert into t1 values (199); +insert into t1 values (198); +insert into t1 values (197); +insert into t1 values (196); +insert into t1 values (195); +insert into t1 values (194); +insert into t1 values (193); +insert into t1 values (192); +insert into t1 values (191); +insert into t1 values (190); +insert into t1 values (189); +insert into t1 values (188); +insert into t1 values (187); +insert into t1 values (186); +insert into t1 values (185); +insert into t1 values (184); +insert into t1 values (183); +insert into t1 values (182); +insert into t1 values (181); +insert into t1 values (180); +insert into t1 values (179); +insert into t1 values (178); +insert into t1 values (177); +insert into t1 values (176); +insert into t1 values (175); +insert into t1 values (174); +insert into t1 values (173); +insert into t1 values (172); +insert into t1 values (171); +insert into t1 values (170); +insert into t1 values (169); +insert into t1 values (168); +insert into t1 values (167); +insert into t1 values (166); +insert into t1 values (165); +insert into t1 values (164); +insert into t1 values (163); +insert into t1 values (162); +insert into t1 values (161); +insert into t1 values (160); +insert into t1 values (159); +insert into t1 values (158); +insert into t1 values (157); +insert into t1 values (156); +insert into t1 values (155); +insert into t1 values (154); +insert into t1 values (153); +insert into t1 values (152); +insert into t1 values (151); +insert into t1 values (150); +insert into t1 values (149); +insert into t1 values (148); +insert into t1 values (147); +insert into t1 values (146); +insert into t1 values (145); +insert into t1 values (144); +insert into t1 values (143); +insert into t1 values (142); +insert into t1 values (141); +insert into t1 values (140); +insert into t1 values (139); +insert into t1 values (138); +insert into t1 values (137); +insert into t1 values (136); +insert into t1 values (135); +insert into t1 values (134); +insert into t1 values (133); +insert into t1 values (132); +insert into t1 values (131); +insert into t1 values (130); +insert into t1 values (129); +insert into t1 values (128); +insert into t1 values (127); +insert into t1 values (126); +insert into t1 values (125); +insert into t1 values (124); +insert into t1 values (123); +insert into t1 values (122); +insert into t1 values (121); +insert into t1 values (120); +insert into t1 values (119); +insert into t1 values (118); +insert into t1 values (117); +insert into t1 values (116); +insert into t1 values (115); +insert into t1 values (114); +insert into t1 values (113); +insert into t1 values (112); +insert into t1 values (111); +insert into t1 values (110); +insert into t1 values (109); +insert into t1 values (108); +insert into t1 values (107); +insert into t1 values (106); +insert into t1 values (105); +insert into t1 values (104); +insert into t1 values (103); +insert into t1 values (102); +insert into t1 values (101); +insert into t1 values (100); +insert into t1 values (99); +insert into t1 values (98); +insert into t1 values (97); +insert into t1 values (96); +insert into t1 values (95); +insert into t1 values (94); +insert into t1 values (93); +insert into t1 values (92); +insert into t1 values (91); +insert into t1 values (90); +insert into t1 values (89); +insert into t1 values (88); +insert into t1 values (87); +insert into t1 values (86); +insert into t1 values (85); +insert into t1 values (84); +insert into t1 values (83); +insert into t1 values (82); +insert into t1 values (81); +insert into t1 values (80); +insert into t1 values (79); +insert into t1 values (78); +insert into t1 values (77); +insert into t1 values (76); +insert into t1 values (75); +insert into t1 values (74); +insert into t1 values (73); +insert into t1 values (72); +insert into t1 values (71); +insert into t1 values (70); +insert into t1 values (69); +insert into t1 values (68); +insert into t1 values (67); +insert into t1 values (66); +insert into t1 values (65); +insert into t1 values (64); +insert into t1 values (63); +insert into t1 values (62); +insert into t1 values (61); +insert into t1 values (60); +insert into t1 values (59); +insert into t1 values (58); +insert into t1 values (57); +insert into t1 values (56); +insert into t1 values (55); +insert into t1 values (54); +insert into t1 values (53); +insert into t1 values (52); +insert into t1 values (51); +insert into t1 values (50); +insert into t1 values (49); +insert into t1 values (48); +insert into t1 values (47); +insert into t1 values (46); +insert into t1 values (45); +insert into t1 values (44); +insert into t1 values (43); +insert into t1 values (42); +insert into t1 values (41); +insert into t1 values (40); +insert into t1 values (39); +insert into t1 values (38); +insert into t1 values (37); +insert into t1 values (36); +insert into t1 values (35); +insert into t1 values (34); +insert into t1 values (33); +insert into t1 values (32); +insert into t1 values (31); +insert into t1 values (30); +insert into t1 values (29); +insert into t1 values (28); +insert into t1 values (27); +insert into t1 values (26); +insert into t1 values (25); +insert into t1 values (24); +insert into t1 values (23); +insert into t1 values (22); +insert into t1 values (21); +insert into t1 values (20); +insert into t1 values (19); +insert into t1 values (18); +insert into t1 values (17); +insert into t1 values (16); +insert into t1 values (15); +insert into t1 values (14); +insert into t1 values (13); +insert into t1 values (12); +insert into t1 values (11); +insert into t1 values (10); +insert into t1 values (9); +insert into t1 values (8); +insert into t1 values (7); +insert into t1 values (6); +insert into t1 values (5); +insert into t1 values (4); +insert into t1 values (3); +insert into t1 values (2); +insert into t1 values (1); +"The following are SLAVE." +select count(distinct n) from t1; +count(distinct n) +300 +select min(n) from t1; +min(n) +1 +select max(n) from t1; +max(n) +300 +show slave status; +Slave_IO_State Master_Host Master_User Master_Port Connect_Retry Master_Log_File Read_Master_Log_Pos Relay_Log_File Relay_Log_Pos Relay_Master_Log_File Slave_IO_Running Slave_SQL_Running Replicate_Do_DB Replicate_Ignore_DB Replicate_Do_Table Replicate_Ignore_Table Replicate_Wild_Do_Table Replicate_Wild_Ignore_Table Last_Errno Last_Error Skip_Counter Exec_Master_Log_Pos Relay_Log_Space Until_Condition Until_Log_File Until_Log_Pos Master_SSL_Allowed Master_SSL_CA_File Master_SSL_CA_Path Master_SSL_Cert Master_SSL_Cipher Master_SSL_Key Seconds_Behind_Master +Waiting for master to send event 127.0.0.1 root 9306 1 master-bin.000014 2849 # # master-bin.000014 Yes Yes # 0 0 2849 # None 0 No # +show master status; +File Position Binlog_Do_DB Binlog_Ignore_DB +master-bin.000014 2849 +"The following are SLAVE1." +start slave; +select count(distinct n) from t1; +count(distinct n) +300 +select min(n) from t1; +min(n) +1 +select max(n) from t1; +max(n) +300 +show slave status; +Slave_IO_State Master_Host Master_User Master_Port Connect_Retry Master_Log_File Read_Master_Log_Pos Relay_Log_File Relay_Log_Pos Relay_Master_Log_File Slave_IO_Running Slave_SQL_Running Replicate_Do_DB Replicate_Ignore_DB Replicate_Do_Table Replicate_Ignore_Table Replicate_Wild_Do_Table Replicate_Wild_Ignore_Table Last_Errno Last_Error Skip_Counter Exec_Master_Log_Pos Relay_Log_Space Until_Condition Until_Log_File Until_Log_Pos Master_SSL_Allowed Master_SSL_CA_File Master_SSL_CA_Path Master_SSL_Cert Master_SSL_Cipher Master_SSL_Key Seconds_Behind_Master +Waiting for master to send event 127.0.0.1 root 9308 1 master-bin.000014 2849 # # master-bin.000014 Yes Yes # 0 0 2849 # None 0 No # +"The following are SLAVE." +MAKE MASTER MASTER_LOG_FILE='master-bin', +MASTER_SERVER_ID=2, +INDEX='replication-log'; +ERROR HY000: Could not initialize master info structure; more error messages can be found in the MySQL error log +stop slave; +MAKE MASTER MASTER_LOG_FILE='master-bin', +MASTER_SERVER_ID=2, +INDEX='replication_log'; +ERROR HY000: Could not initialize master info structure; more error messages can be found in the MySQL error log +MAKE MASTER REVOKE SESSION WITH KILL; +MAKE MASTER MASTER_LOG_FILE='master-bin', +MASTER_SERVER_ID=2, +INDEX='replication_log' + WITH BINLOG; +MAKE MASTER GRANT SESSION; +delete from t1 where n > 250; +select count(distinct n) from t1; +count(distinct n) +250 +"The following are SLAVE1." +select count(distinct n) from t1; +count(distinct n) +250 +select min(n) from t1; +min(n) +1 +select max(n) from t1; +max(n) +250 +"The following are SLAVE2." +start slave; +select count(distinct n) from t1; +count(distinct n) +250 +select min(n) from t1; +min(n) +1 +select max(n) from t1; +max(n) +250 +show slave status; +Slave_IO_State Master_Host Master_User Master_Port Connect_Retry Master_Log_File Read_Master_Log_Pos Relay_Log_File Relay_Log_Pos Relay_Master_Log_File Slave_IO_Running Slave_SQL_Running Replicate_Do_DB Replicate_Ignore_DB Replicate_Do_Table Replicate_Ignore_Table Replicate_Wild_Do_Table Replicate_Wild_Ignore_Table Last_Errno Last_Error Skip_Counter Exec_Master_Log_Pos Relay_Log_Space Until_Condition Until_Log_File Until_Log_Pos Master_SSL_Allowed Master_SSL_CA_File Master_SSL_CA_Path Master_SSL_Cert Master_SSL_Cipher Master_SSL_Key Seconds_Behind_Master +Waiting for master to send event 127.0.0.1 root 9308 1 master-bin.000015 189 # # master-bin.000015 Yes Yes # 0 0 189 # None 0 No # +drop table t1; +drop table t1; +"The following are SLAVE." +show master logs; +Log_name File_size +master-bin.000001 4214 +master-bin.000002 4212 +master-bin.000003 4212 +master-bin.000004 4212 +master-bin.000005 4212 +master-bin.000006 4212 +master-bin.000007 4212 +master-bin.000008 4212 +master-bin.000009 4212 +master-bin.000010 4194 +master-bin.000011 4190 +master-bin.000012 4190 +master-bin.000013 4190 +master-bin.000014 2849 +master-bin.000015 265 +show master status; +File Position Binlog_Do_DB Binlog_Ignore_DB +master-bin.000015 265 +"The following are SLAVE2." +show master logs; +Log_name File_size +master-bin.000001 4214 +master-bin.000002 4212 +master-bin.000003 4212 +master-bin.000004 4212 +master-bin.000005 4212 +master-bin.000006 4212 +master-bin.000007 4212 +master-bin.000008 4212 +master-bin.000009 4212 +master-bin.000010 4194 +master-bin.000011 4190 +master-bin.000012 4190 +master-bin.000013 4190 +master-bin.000014 2849 +master-bin.000015 265 +show master status; +File Position Binlog_Do_DB Binlog_Ignore_DB +master-bin.000015 265 +purge master logs to 'master-bin.000006'; +show master logs; +Log_name File_size +master-bin.000006 4212 +master-bin.000007 4212 +master-bin.000008 4212 +master-bin.000009 4212 +master-bin.000010 4194 +master-bin.000011 4190 +master-bin.000012 4190 +master-bin.000013 4190 +master-bin.000014 2849 +master-bin.000015 265 +reset master; +ERROR HY000: Binlog closed, cannot RESET MASTER diff -r 66cc9e0a6768 mysql-test/t/rpl_mirror_binlog-master.opt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mysql-test/t/rpl_mirror_binlog-master.opt Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,1 @@ +-O max_binlog_size=4096 diff -r 66cc9e0a6768 mysql-test/t/rpl_mirror_binlog-slave.opt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mysql-test/t/rpl_mirror_binlog-slave.opt Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,1 @@ +--rpl_mirror_binlog_enabled=1 --log-bin-index=replication_log diff -r 66cc9e0a6768 mysql-test/t/rpl_mirror_binlog.1.slave-mi --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mysql-test/t/rpl_mirror_binlog.1.slave-mi Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,1 @@ +--master-user=root --master-connect-retry=1 --master-host=127.0.0.1 --master-password="" --master-port=9308 --server-id=3 diff -r 66cc9e0a6768 mysql-test/t/rpl_mirror_binlog.2.slave-mi --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mysql-test/t/rpl_mirror_binlog.2.slave-mi Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,1 @@ +--master-user=root --master-connect-retry=1 --master-host=127.0.0.1 --master-password="" --master-port=9308 --server-id=4 diff -r 66cc9e0a6768 mysql-test/t/rpl_mirror_binlog.test --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mysql-test/t/rpl_mirror_binlog.test Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,119 @@ +-- source include/master-slave.inc +-- source include/have_innodb.inc +connect (slave_sec,localhost,root,,test,$SLAVE_MYPORT1,$SLAVE_MYSOCK1); +connect (slave_ter,localhost,root,,test,$SLAVE_MYPORT2,$SLAVE_MYSOCK2); + +connection master; +--disable_warnings +drop table if exists t1; +--enable_warnings +create table t1(n int) engine = InnoDB; + +let $i=300; +while ($i) +{ + eval insert into t1 values ($i); + dec $i; +} + +save_master_pos; + +connection slave; +sync_with_master; + +echo "The following are SLAVE."; +select count(distinct n) from t1; +select min(n) from t1; +select max(n) from t1; +--replace_column 8 # 9 # 18 # 23 # 33 # +show slave status; +show master status; + +connection slave_sec; +echo "The following are SLAVE1."; +start slave; +sync_with_master; + +select count(distinct n) from t1; +select min(n) from t1; +select max(n) from t1; +--replace_column 8 # 9 # 18 # 23 # 33 # +show slave status; + +# make the slave the new master +connection slave; +echo "The following are SLAVE."; + +# The first 1201 error is caused by running slave. +--error 1201 +MAKE MASTER MASTER_LOG_FILE='master-bin', + MASTER_SERVER_ID=2, + INDEX='replication-log'; +stop slave; + +# The second 1201 error is caused by failover mode. +--error 1201 +MAKE MASTER MASTER_LOG_FILE='master-bin', + MASTER_SERVER_ID=2, + INDEX='replication_log'; + +MAKE MASTER REVOKE SESSION WITH KILL; +MAKE MASTER MASTER_LOG_FILE='master-bin', + MASTER_SERVER_ID=2, + INDEX='replication_log' + WITH BINLOG; + +MAKE MASTER GRANT SESSION; + +delete from t1 where n > 250; +save_master_pos; + +select count(distinct n) from t1; + +connection slave_sec; +echo "The following are SLAVE1."; + +sync_with_master; +select count(distinct n) from t1; +select min(n) from t1; +select max(n) from t1; + +connection slave_ter; +echo "The following are SLAVE2."; +start slave; +sync_with_master; + +select count(distinct n) from t1; +select min(n) from t1; +select max(n) from t1; + +--replace_column 8 # 9 # 18 # 23 # 33 # +show slave status; + +connection master; +drop table t1; + +connection slave; +drop table t1; +save_master_pos; + +connection slave_sec; +sync_with_master; + +connection slave; +echo "The following are SLAVE."; + +show master logs; +show master status; + + +connection slave_ter; +echo "The following are SLAVE2."; +sync_with_master; + +show master logs; +show master status; +purge master logs to 'master-bin.000006'; +show master logs; +--error 1186 +reset master; diff -r 66cc9e0a6768 patch_info/mirror_binlog.info --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/patch_info/mirror_binlog.info Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,6 @@ +File=mirror_binlog.patch +Name=Mirroring binary logs on slave +Version=V1 +Author=Google +License=GPL +Comment=contains FastMaster promotion patch diff -r 66cc9e0a6768 sql/Makefile.am --- a/sql/Makefile.am Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/Makefile.am Thu Dec 04 21:46:15 2008 -0800 @@ -68,7 +68,7 @@ sql_array.h sql_cursor.h \ examples/ha_example.h ha_archive.h \ examples/ha_tina.h ha_blackhole.h \ - ha_federated.h + ha_federated.h repl_mule.h mysqld_SOURCES = sql_lex.cc sql_handler.cc \ item.cc item_sum.cc item_buff.cc item_func.cc \ item_cmpfunc.cc item_strfunc.cc item_timefunc.cc \ @@ -105,7 +105,7 @@ sp_cache.cc parse_file.cc sql_trigger.cc \ examples/ha_example.cc ha_archive.cc \ examples/ha_tina.cc ha_blackhole.cc \ - ha_federated.cc + ha_federated.cc repl_mule.cc gen_lex_hash_SOURCES = gen_lex_hash.cc gen_lex_hash_LDADD = $(LDADD) $(CXXLDFLAGS) diff -r 66cc9e0a6768 sql/Makefile.in --- a/sql/Makefile.in Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/Makefile.in Thu Dec 04 21:46:15 2008 -0800 @@ -152,7 +152,7 @@ sp_rcontext.$(OBJEXT) sp.$(OBJEXT) sp_cache.$(OBJEXT) \ parse_file.$(OBJEXT) sql_trigger.$(OBJEXT) \ ha_example.$(OBJEXT) ha_archive.$(OBJEXT) ha_tina.$(OBJEXT) \ - ha_blackhole.$(OBJEXT) ha_federated.$(OBJEXT) + ha_blackhole.$(OBJEXT) ha_federated.$(OBJEXT) repl_mule.$(OBJEXT) mysqld_OBJECTS = $(am_mysqld_OBJECTS) mysqld_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_2) \ $(am__DEPENDENCIES_2) $(am__DEPENDENCIES_2) \ @@ -516,7 +516,7 @@ sql_array.h sql_cursor.h \ examples/ha_example.h ha_archive.h \ examples/ha_tina.h ha_blackhole.h \ - ha_federated.h + ha_federated.h repl_mule.h mysqld_SOURCES = sql_lex.cc sql_handler.cc \ item.cc item_sum.cc item_buff.cc item_func.cc \ @@ -554,7 +554,7 @@ sp_cache.cc parse_file.cc sql_trigger.cc \ examples/ha_example.cc ha_archive.cc \ examples/ha_tina.cc ha_blackhole.cc \ - ha_federated.cc + ha_federated.cc repl_mule.cc gen_lex_hash_SOURCES = gen_lex_hash.cc gen_lex_hash_LDADD = $(LDADD) $(CXXLDFLAGS) @@ -748,6 +748,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/protocol.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/records.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/repl_failsafe.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/repl_mule.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/set_var.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/slave.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sp.Po@am__quote@ diff -r 66cc9e0a6768 sql/lex.h --- a/sql/lex.h Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/lex.h Thu Dec 04 21:46:15 2008 -0800 @@ -292,6 +292,7 @@ { "LONGTEXT", SYM(LONGTEXT)}, { "LOOP", SYM(LOOP_SYM)}, { "LOW_PRIORITY", SYM(LOW_PRIORITY)}, + { "MAKE", SYM(MAKE_SYM)}, { "MASTER", SYM(MASTER_SYM)}, { "MASTER_CONNECT_RETRY", SYM(MASTER_CONNECT_RETRY_SYM)}, { "MASTER_HOST", SYM(MASTER_HOST_SYM)}, diff -r 66cc9e0a6768 sql/log.cc --- a/sql/log.cc Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/log.cc Thu Dec 04 21:46:15 2008 -0800 @@ -79,7 +79,9 @@ bool binlog_init() { - return !opt_bin_log; + if (!opt_bin_log) + binlog_hton.prepare = NULL; + return 0; /* return !opt_bin_log; */ } static int binlog_close_connection(THD *thd) @@ -406,6 +408,7 @@ :bytes_written(0), last_time(0), query_start(0), name(0), prepared_xids(0), log_type(LOG_CLOSED), file_id(1), open_count(1), write_error(FALSE), inited(FALSE), need_start_event(TRUE), + mule_binlog_(0), description_event_for_exec(0), description_event_for_queue(0) { /* @@ -506,7 +509,10 @@ const char *log_name) { File index_file_nr= -1; - DBUG_ASSERT(!my_b_inited(&index_file)); + + /* If the index is already opened, do not open it again. */ + if (my_b_inited(&index_file)) + return FALSE; /* First open of this class instance @@ -750,7 +756,7 @@ if (file >= 0) my_close(file,MYF(0)); end_io_cache(&log_file); - end_io_cache(&index_file); + close_index_file(); safeFree(name); log_type= LOG_CLOSED; DBUG_RETURN(1); @@ -768,7 +774,10 @@ int MYSQL_LOG::raw_get_current_log(LOG_INFO* linfo) { strmake(linfo->log_file_name, log_file_name, sizeof(linfo->log_file_name)-1); - linfo->pos = my_b_tell(&log_file); + if (!mule_binlog_) + linfo->pos = my_b_tell(&log_file); + else + linfo->pos = my_b_filelength(&log_file); return 0; } @@ -935,6 +944,11 @@ if (need_lock) pthread_mutex_lock(&LOCK_index); safe_mutex_assert_owner(&LOCK_index); + + if (open_index_file(index_file_name, NULL) != 0) { + error = -1; + goto err; + } /* As the file is flushed, we can't get an error here */ (void) reinit_io_cache(&index_file, READ_CACHE, linfo->index_file_offset, 0, @@ -1446,18 +1460,19 @@ SYNOPSIS new_file() need_lock Set to 1 if caller has not locked LOCK_log + logfile_name the specified log filename. NOTE The new file name is stored last in the index file */ -void MYSQL_LOG::new_file(bool need_lock) +void MYSQL_LOG::new_file(bool need_lock, const char* log_filename) { char new_name[FN_REFLEN], *new_name_ptr, *old_name; enum_log_type save_log_type; DBUG_ENTER("MYSQL_LOG::new_file"); - if (!is_open()) + if (!is_log_open()) { DBUG_PRINT("info",("log is closed")); DBUG_VOID_RETURN; @@ -1496,7 +1511,9 @@ We have to do this here and not in open as we want to store the new file name in the current binary log file. */ - if (generate_new_name(new_name, name)) + if (log_filename) { + fn_format(new_name,log_filename,mysql_data_home,"",4); + } else if (generate_new_name(new_name, name)) goto end; new_name_ptr=new_name; @@ -1571,7 +1588,7 @@ bytes_written+= ev->data_written; DBUG_PRINT("info",("max_size: %lu",max_size)); if ((uint) my_b_append_tell(&log_file) > max_size) - new_file(0); + new_file(0); err: pthread_mutex_unlock(&LOCK_log); @@ -1600,8 +1617,14 @@ bytes_written += len; } while ((buf=va_arg(args,const char*)) && (len=va_arg(args,uint))); DBUG_PRINT("info",("max_size: %lu",max_size)); - if ((uint) my_b_append_tell(&log_file) > max_size) - new_file(0); + + /* If max_size is BINLOG_NOSWITCH_SIZE, binlog would not switch because + * of file size limit. + */ + if (max_size != BINLOG_NOSWITCH_SIZE && + (uint) my_b_append_tell(&log_file) > max_size) { + new_file(0); + } err: if (!error) @@ -2492,6 +2515,17 @@ DBUG_VOID_RETURN; } +int MYSQL_LOG::flush_log_file() { + return flush_io_cache(&log_file); +} + +int MYSQL_LOG::close_index_file() { + if (my_b_inited(&index_file)) { + end_io_cache(&index_file); + my_close(index_file.file, MYF(0)); + } + return 0; +} /* Check if a string is a valid number diff -r 66cc9e0a6768 sql/log_event.h --- a/sql/log_event.h Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/log_event.h Thu Dec 04 21:46:15 2008 -0800 @@ -94,6 +94,14 @@ #define LINE_TERM_EMPTY 0x4 #define LINE_START_EMPTY 0x8 #define ESCAPED_EMPTY 0x10 + +/* This server-id value is used to indicate a special master-info event + * in relay-log. + * We will enforce in database that replication can not set this value + * as the server-id. + */ +#define MASTER_INFO_SERVER_ID 0xffffffff + /***************************************************************************** diff -r 66cc9e0a6768 sql/mysql_priv.h --- a/sql/mysql_priv.h Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/mysql_priv.h Thu Dec 04 21:46:15 2008 -0800 @@ -462,6 +462,7 @@ /* BINLOG_DUMP options */ #define BINLOG_DUMP_NON_BLOCK 1 +#define BINLOG_MIRROR_CLIENT 0x0004 /* sql_show.cc:show_log_files() */ #define SHOW_LOG_STATUS_FREE "FREE" @@ -1374,6 +1375,7 @@ extern const char **errmesg; /* Error messages */ extern const char *myisam_recover_options_str; extern const char *in_left_expr_name, *in_additional_cond, *in_having_cond; +extern char *opt_binlog_index_name; extern const char * const triggers_file_ext; extern const char * const trigname_file_ext; extern Eq_creator eq_creator; @@ -1875,6 +1877,10 @@ extern "C" void unireg_abort(int exit_code); void kill_delayed_threads(void); bool check_stack_overrun(THD *thd, long margin, char *dummy); +extern my_bool rpl_mirror_binlog_enabled; +extern ulong sync_mirror_binlog_period; +extern my_bool rpl_mirror_binlog_no_replicate; +extern ulong rpl_mirror_binlog_clients, rpl_mirror_binlog_status; #else #define unireg_abort(exit_code) DBUG_RETURN(exit_code) inline void kill_delayed_threads(void) {} diff -r 66cc9e0a6768 sql/mysqld.cc --- a/sql/mysqld.cc Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/mysqld.cc Thu Dec 04 21:46:15 2008 -0800 @@ -555,6 +555,7 @@ pthread_mutex_t LOCK_global_user_client_stats; pthread_mutex_t LOCK_global_table_stats; pthread_mutex_t LOCK_global_index_stats; +pthread_mutex_t LOCK_failover_master; /* The below lock protects access to two global server variables: max_prepared_stmt_count and prepared_stmt_count. These variables @@ -584,13 +585,15 @@ char *master_ssl_key, *master_ssl_cert; char *master_ssl_ca, *master_ssl_capath, *master_ssl_cipher; +char *opt_binlog_index_name; + /* Static variables */ static bool kill_in_progress, segfaulted; static my_bool opt_do_pstack, opt_bootstrap, opt_myisam_log; static int cleanup_done; static ulong opt_specialflag, opt_myisam_block_size; -static char *opt_logname, *opt_update_logname, *opt_binlog_index_name; +static char *opt_logname, *opt_update_logname; static char *opt_tc_heuristic_recover; static char *mysql_home_ptr, *pidfile_name_ptr; static char **defaults_argv; @@ -598,6 +601,32 @@ static my_socket unix_sock,ip_sock; struct rand_struct sql_rand; // used by sql_class.cc:THD::THD() + +/* When set, we are inside a failover slave and deny all non-super access */ +bool failover_deny_access= 0; + +/* When set, binlog will be mirrored on the replica. */ +my_bool rpl_mirror_binlog_enabled; + +/* Sync the mirrored binlog to disk after every #th event. */ +ulong sync_mirror_binlog_period; + +/* The fixed size for replication event buffer. Replication event can exceed + * the size. + */ +//ulong rpl_event_buffer_size; + +/* This is a mirror binlog status variable on the primary to indicate how many + * mirror binlog servers are connecting. + */ +ulong rpl_mirror_binlog_clients = 0; + +/* This indicates whether mirror binlog is working on a replica database. It + * requires: + * . rpl_mirror_binlog_enabled = 1 + * . the slave I/O thread is running and mirror binlog is also dumped + */ +ulong rpl_mirror_binlog_status = 0; /* OS specific variables */ @@ -1315,6 +1344,7 @@ (void) pthread_cond_destroy(&COND_flush_thread_cache); (void) pthread_cond_destroy(&COND_manager); (void) pthread_mutex_destroy(&LOCK_stats); + (void) pthread_mutex_destroy(&LOCK_failover_master); (void) pthread_mutex_destroy(&LOCK_global_user_client_stats); (void) pthread_mutex_destroy(&LOCK_global_table_stats); (void) pthread_mutex_destroy(&LOCK_global_index_stats); @@ -3164,6 +3194,7 @@ (void) pthread_cond_init(&COND_rpl_status, NULL); #endif (void) pthread_mutex_init(&LOCK_stats, MY_MUTEX_INIT_FAST); + (void) pthread_mutex_init(&LOCK_failover_master, MY_MUTEX_INIT_FAST); (void) pthread_mutex_init(&LOCK_global_user_client_stats, MY_MUTEX_INIT_FAST); (void) pthread_mutex_init(&LOCK_global_table_stats, MY_MUTEX_INIT_FAST); (void) pthread_mutex_init(&LOCK_global_index_stats, MY_MUTEX_INIT_FAST); @@ -3398,39 +3429,8 @@ if (opt_bin_log) { - char buf[FN_REFLEN]; - const char *ln; - ln= mysql_bin_log.generate_name(opt_bin_logname, "-bin", 1, buf); - if (!opt_bin_logname && !opt_binlog_index_name) - { - /* - User didn't give us info to name the binlog index file. - Picking `hostname`-bin.index like did in 4.x, causes replication to - fail if the hostname is changed later. So, we would like to instead - require a name. But as we don't want to break many existing setups, we - only give warning, not error. - */ - sql_print_warning("No argument was provided to --log-bin, and " - "--log-bin-index was not used; so replication " - "may break when this MySQL server acts as a " - "master and has his hostname changed!! Please " - "use '--log-bin=%s' to avoid this problem.", ln); - } - if (ln == buf) - { - my_free(opt_bin_logname, MYF(MY_ALLOW_ZERO_PTR)); - opt_bin_logname=my_strdup(buf, MYF(0)); - } - if (mysql_bin_log.open_index_file(opt_binlog_index_name, ln)) - { - unireg_abort(1); - } - - /* - Used to specify which type of lock we need to use for queries of type - INSERT ... SELECT. This will change when we have row level logging. - */ - using_update_log=1; + if (make_master_open_index(&opt_bin_logname, opt_binlog_index_name) != 0) + unireg_abort(1); } if (xid_cache_init()) @@ -3480,9 +3480,10 @@ unireg_abort(1); } - if (opt_bin_log && mysql_bin_log.open(opt_bin_logname, LOG_BIN, 0, - WRITE_CACHE, 0, max_binlog_size, 0)) - unireg_abort(1); + if (opt_bin_log && + make_master(NULL, opt_bin_logname, opt_binlog_index_name, NULL) != 0) { + unireg_abort(1); + } #ifdef HAVE_REPLICATION if (opt_bin_log && expire_logs_days) @@ -5098,6 +5098,8 @@ OPT_INNODB_READ_IO_THREADS, OPT_INNODB_WRITE_IO_THREADS, OPT_INNODB_ADAPTIVE_HASH_INDEX, + OPT_RPL_MIRROR_BINLOG, + OPT_SYNC_MIRROR_BINLOG, OPT_FEDERATED, OPT_INNODB_USE_LEGACY_CARDINALITY_ALGORITHM }; @@ -5725,6 +5728,11 @@ {"rpl-recovery-rank", OPT_RPL_RECOVERY_RANK, "Undocumented.", (gptr*) &rpl_recovery_rank, (gptr*) &rpl_recovery_rank, 0, GET_ULONG, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"rpl_mirror_binlog_enabled", OPT_RPL_MIRROR_BINLOG, + "1 = support mirroring binlogs. 0 = disable mirroring binlogs", + (gptr*) &rpl_mirror_binlog_enabled, + (gptr*) &rpl_mirror_binlog_enabled, 0, GET_BOOL, NO_ARG, + 0, 0, 1, 0, 1, 0}, {"safe-mode", OPT_SAFE, "Skip some optimize stages (for testing).", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef TO_BE_DELETED @@ -5849,6 +5857,11 @@ {"symbolic-links", 's', "Enable symbolic link support.", (gptr*) &my_use_symdir, (gptr*) &my_use_symdir, 0, GET_BOOL, NO_ARG, IF_PURIFY(0,1), 0, 0, 0, 0, 0}, + {"sync-mirror-binlog", OPT_SYNC_MIRROR_BINLOG, + "Sync the mirrored binlog to disk after every #th event. " + "#=0 (the default) does no sync. Syncing slows MySQL down", + (gptr*) &sync_mirror_binlog_period, + (gptr*) &sync_mirror_binlog_period, 0, GET_ULONG, REQUIRED_ARG, 0, 0, ~0L, 0, 1, 0}, {"sysdate-is-now", OPT_SYSDATE_IS_NOW, "Non-default option to alias SYSDATE() to NOW() to make it safe-replicable. Since 5.0, SYSDATE() returns a `dynamic' value different for different invocations, even within the same statement.", (gptr*) &global_system_variables.sysdate_is_now, @@ -6625,6 +6638,7 @@ {"Delayed_errors", (char*) &delayed_insert_errors, SHOW_LONG}, {"Delayed_insert_threads", (char*) &delayed_insert_threads, SHOW_LONG_CONST}, {"Delayed_writes", (char*) &delayed_insert_writes, SHOW_LONG}, + {"Failover_deny_access", (char*) &failover_deny_access, SHOW_LONG}, {"Flush_commands", (char*) &refresh_version, SHOW_LONG_CONST}, {"Handler_commit", (char*) offsetof(STATUS_VAR, ha_commit_count), SHOW_LONG_STATUS}, {"Handler_delete", (char*) offsetof(STATUS_VAR, ha_delete_count), SHOW_LONG_STATUS}, diff -r 66cc9e0a6768 sql/repl_mule.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/repl_mule.cc Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,466 @@ +/* + Copyright (C) 2007 Google Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#include "mysql_priv.h" +#include +#include "slave.h" +#include "repl_mule.h" + +/* max log size: 2GB */ +#define MAX_LOG_SIZE BINLOG_NOSWITCH_SIZE + +ReplMule::ReplMule(THD* thd, MASTER_INFO *mi, RelayStatus status, + my_off_t file_size, const char *binlog_indexname, + MYSQL_LOG *binlog, ulong sync_period) + : desc_event_(new Format_description_log_event(BINLOG_VERSION)), + io_thd_(thd), mi_(mi), status_(status), dump_position_(0L), + file_size_(file_size), mule_log_(binlog), + mule_log_sync_period_(sync_period), mule_log_event_counter_(0) { + char llbuf1[22], llbuf2[22]; + + DBUG_ENTER("ReplMule::ReplMule"); + + /* Indicate that we are in replication mule mode. */ + mule_log_->set_mule_mode(); + + strmake(curr_log_filename_, mi->master_log_name, + sizeof(curr_log_filename_)-1); + strmake(mule_indexname_, binlog_indexname, sizeof(mule_indexname_)-1); + + /* Open the mule log file */ + if (!mule_log_->is_log_open()) { + /* Do not open binlog file when master_log_name is not specified. We + * are at the I/O thread initialization time and we do not know what + * filename we are going to dump. + * We wait for the next rotation event to indicate the filename. + */ + if (strlen(curr_log_filename_) > 0 && + mule_log_->open(curr_log_filename_, LOG_BIN, NULL, + SEQ_READ_APPEND, true, MAX_LOG_SIZE, 0) != 0) { + sql_print_error("ReplMule: open binlog failed: %s", + curr_log_filename_); + status_ = MULE_ERROR; + DBUG_VOID_RETURN; + } + } + + switch (status_) { + case MULE_BEHIND: + dump_position_ = mi->master_log_pos; + mi->master_log_pos = file_size_; + sql_print_information("ReplicationMule: MULE_BEHIND - new(%s), old(%s)", + llstr(mi->master_log_pos, llbuf1), + llstr(dump_position_, llbuf2)); + break; + case RELAY_MATCH_MULE: + case RELAY_MATCH_MULE_RUN: + dump_position_ = mi->master_log_pos; + sql_print_information("ReplicationMule: RELAY_MATCH_MULE."); + break; + case MULE_VERIFY: + case MULE_VERIFY_RELAY_BEHIND: + dump_position_ = mi->master_log_pos; + mi->master_log_pos = BIN_LOG_HEADER_SIZE; + sql_print_information( + "ReplicationMule: MULE_VERIFY - old(%s), file_size(%s)", + llstr(dump_position_, llbuf1), llstr(file_size_, llbuf2)); + + /* seek to the beginning of the file for verification */ + seekToPosition(BIN_LOG_HEADER_SIZE); + break; + } + + DBUG_VOID_RETURN; +} + +ReplMule::~ReplMule() { + DBUG_ENTER("ReplMule::~ReplMule"); + + if (mule_log_->is_log_open()) + mule_log_->close(LOG_CLOSE_INDEX); + mule_log_->clear_mule_mode(); + + /* If we are still in MULE_BEHIND or MULE_VERIFY state and we exit from + * I/O thread, it means we encountered some errors. + * mi->master_log_pos might be used by later slave start. It is being + * changed here to do event dumping or event verification. So, we should + * restore it to its original value. + */ + switch (status_) { + case MULE_BEHIND: + case MULE_VERIFY: + if (mi_->master_log_pos < dump_position_) + mi_->master_log_pos = dump_position_; + break; + } + + delete desc_event_; + + DBUG_VOID_RETURN; +} + +ReplMule::WriteStatus ReplMule::writeEvent(const char* buf, ulong event_len) { + WriteStatus dump_status = WRITE_RELAY; + char llbuf1[22], llbuf2[22], llbuf3[22]; + char *verify_event; + bool verified = false; + bool skip_event = false; + + DBUG_ENTER("ReplMule::dumpEvent"); + switch (status_) { + case MULE_VERIFY: + case MULE_VERIFY_RELAY_BEHIND: + if (buf[EVENT_TYPE_OFFSET] == ROTATE_EVENT && + IsFakeRotation(buf, event_len)) { + /* Do not verify the faked rotate event */ + if (status_ == MULE_VERIFY) + dump_status = SKIP_RELAY; + break; + } + verify_event = new char[event_len]; + if (verify_event == NULL) { + sql_print_error( + "ReplMule::dumpEvent - insufficient memory in verification, " + "position(%s), event_len(%d).", + llstr(mi_->master_log_pos, llbuf1), event_len); + dump_status = WRITE_ERROR; + break; + } + if (my_b_read(mule_log_->get_log_file(), (byte*) verify_event, + event_len) != 0) { + sql_print_error( + "ReplMule::dumpEvent - read log error in verification, " + "position(%s), event_len(%d).", + llstr(mi_->master_log_pos, llbuf1), event_len); + dump_status = WRITE_ERROR; + delete verify_event; + break; + } + verified = (memcmp(buf, verify_event, event_len) == 0); + delete verify_event; + if (!verified) { + sql_print_error( + "ReplMule::dumpEvent - event does not match at position(%s)", + llstr(mi_->master_log_pos, llbuf1)); + dump_status = WRITE_ERROR; + break; + } + /* fall through */ + case MULE_BEHIND: + dump_status = SKIP_RELAY; + if (status_ == MULE_BEHIND && + queueEvent(buf, event_len, &skip_event) != 0) { + dump_status = WRITE_ERROR; + break; + } + + /* Skip faked rotation event */ + if (!skip_event) + mi_->master_log_pos += event_len; + + if (mi_->master_log_pos == dump_position_) { + if (dump_position_ < file_size_) { + status_ = MULE_VERIFY_RELAY_BEHIND; + } else { + status_ = RELAY_MATCH_MULE; + } + sql_print_information( + "ReplMule::dumpEvent - new status(%d) " + "master_log_pos(%s), dump_pos(%s), file_size(%s)", status_, + llstr(mi_->master_log_pos, llbuf1), llstr(dump_position_, llbuf2), + llstr(file_size_, llbuf3)); + } else if (mi_->master_log_pos == file_size_) { + if (dump_position_ > file_size_) { + status_ = MULE_BEHIND; + } else { + status_ = RELAY_MATCH_MULE; + } + sql_print_information( + "ReplMule::dumpEvent - new status(%d) " + "master_log_pos(%s), dump_pos(%s), file_size(%s)", status_, + llstr(mi_->master_log_pos, llbuf1), llstr(dump_position_, llbuf2), + llstr(file_size_, llbuf3)); + } else if (status_ != MULE_VERIFY_RELAY_BEHIND && + mi_->master_log_pos > dump_position_) { + sql_print_error( + "ReplMule::dumpEvent - mule position(%s) does not match " + "relay-log position(%s).", + llstr(mi_->master_log_pos, llbuf1), llstr(dump_position_, llbuf2)); + dump_status = WRITE_ERROR; + } + break; + case RELAY_MATCH_MULE_RUN: + if (buf[EVENT_TYPE_OFFSET] == FORMAT_DESCRIPTION_EVENT) { + sql_print_information(" RELAY_MATCH_MULE event %d", buf[EVENT_TYPE_OFFSET] ); + /* Do not write format description record if size is the same */ + break; + } + case RELAY_MATCH_MULE: + if (queueEvent(buf, event_len, &skip_event) != 0) + dump_status = WRITE_ERROR; + break; + } + + DBUG_RETURN(dump_status); +} + +int ReplMule::appendEvent(const char* buf, ulong event_len) { + char llbuf1[22]; + int error; + + DBUG_ENTER("ReplMule::appendEvent"); + + error = mule_log_->appendv(buf,event_len,0); + if (error != 0) { + sql_print_error("ReplMule::appendEvent - append error at %s(%s)", + mi_->master_log_name, + llstr(mi_->master_log_pos, llbuf1)); + } else if (mule_log_->flush_log_file() != 0) { + sql_print_error("ReplMule::appendEvent - flush error at %s(%s)", + mi_->master_log_name, + llstr(mi_->master_log_pos, llbuf1)); + error = -1; + } else if (mule_log_sync_period_ > 0) { + mule_log_event_counter_++; + if (mule_log_event_counter_ >= mule_log_sync_period_) { + mule_log_event_counter_ = 0; + error = my_sync(mule_log_->get_log_file()->file, MYF(MY_WME)); + if (error != 0) + sql_print_error("ReplMule::appendEvent - sync error at %s(%s)", + mi_->master_log_name, + llstr(mi_->master_log_pos, llbuf1)); + } + } + + DBUG_RETURN(error); +} + +int ReplMule::queueEvent(const char* buf, ulong event_len, bool *skip_event) { + int error = 0; + + DBUG_ENTER("ReplMule::queueEvent"); + + *skip_event = false; + + mule_log_->lock_log(); + if (buf[EVENT_TYPE_OFFSET] == ROTATE_EVENT) { + Rotate_log_event rev(buf, event_len, desc_event_); + + /* If this is a faked rotate event and the specified filename is + * the same as the current binlog filename, ignore the event. + */ + if (IsFakeRotation(rev)) { + *skip_event = true; + DBUG_PRINT("info",("skipped faked rotation event")); + } else { + /* Only append real events. */ + if (rev.when != 0) + error = appendEvent(buf, event_len); + + /* Only rotate file when append succeeds. */ + if (error == 0) { + /* Create a new file: lock both index and log. */ + if (strlen(curr_log_filename_) == 0) { + /* If curr_log_filename_ is not specified, then this is the first + * valid rotation event to indicate the filename. + */ + error = mule_log_->open(rev.new_log_ident, LOG_BIN, NULL, + SEQ_READ_APPEND, true, MAX_LOG_SIZE, 0); + } else { + mule_log_->new_file(0, rev.new_log_ident); + } + + strmake(curr_log_filename_, rev.new_log_ident, + strlen(rev.new_log_ident)); + + DBUG_PRINT("info",("rotate file: %s", rev.new_log_ident)); + } + } + } else { + error = appendEvent(buf, event_len); + } + mule_log_->unlock_log(); + + DBUG_RETURN(error); +} + +void ReplMule::seekToPosition(my_off_t pos) { + DBUG_ENTER("ReplMule::seekToPosition"); + DBUG_PRINT("enter",("seek_pos: %ld", (ulong) pos)); + + my_b_seek(mule_log_->get_log_file(), pos); + DBUG_VOID_RETURN; +} + +bool ReplMule::IsFakeRotation(const char* buf, ulong event_len) { + DBUG_ENTER("ReplMule::IsFakeRotation"); + + Rotate_log_event rev(buf, event_len, desc_event_); + DBUG_RETURN(IsFakeRotation(rev)); +} + +bool ReplMule::IsFakeRotation(const Rotate_log_event& rev) { + DBUG_ENTER("ReplMule::IsFakeRotation"); + DBUG_RETURN(rev.when == 0 && + rev.ident_len == strlen(curr_log_filename_) && + strcmp(rev.new_log_ident, curr_log_filename_) == 0); +} + +/* createReplicationMule: + * Create a mule that relays master's replication binlog and + * generate an exact same copy on the local filesystem. + * + * Code flow: + * last_mulelog = scan the existing mule log index to find it + * if (mulelog index is not created or there is no mule log inside it) + * old_mule_log <- requested dumping position + * requested dumping position <- 0 in the file + * else + * check whether the mule log matches the requested dump + * (whether the last mule log name/size matches) + * if the mule log name does not match + * exit with an error + * if (the mule log size does not match the requested dump position) + * request the dump from position 0 and read all events + * verify all events with the corresponding events in mule log + * if (the verification succeeds) + * continue the dump + * else + * exit with an error + */ +ReplMule* ReplMule::createReplicationMule( + THD* thd, MASTER_INFO *mi, const char *binlog_indexname, + MYSQL_LOG *binlog) { + ReplMule *mule = NULL; + LOG_INFO linfo; + bool index_opened = false; + + DBUG_ENTER("ReplMule::createReplicationMule"); + + /* binlog_indexname must be set to some real value. */ + DBUG_ASSERT(binlog_indexname); + + /* Lock binlog index for all binlog operations */ + binlog->lock_index(); + index_opened = binlog->open_index_file(binlog_indexname, NULL); + DBUG_PRINT("info",("open index file succeed: %d", index_opened)); + sql_print_information("createReplicationMule"); + + /* Scan the existing binlog index to find the last relayed binlog */ + if (index_opened || + binlog->find_log_pos(&linfo, NullS, false) != 0) { + /* binlog index is not created or has no log file inside: + * . old_relay_binlog <- requested dumping position + * . requested dumping position <- 0 in the file + */ + if (mi->master_log_pos == BIN_LOG_HEADER_SIZE) { + mule = new ReplMule(thd, mi, RELAY_MATCH_MULE, BIN_LOG_HEADER_SIZE, + binlog_indexname, binlog, sync_mirror_binlog_period); + } else { + mule = new ReplMule(thd, mi, MULE_BEHIND, BIN_LOG_HEADER_SIZE, + binlog_indexname, binlog, sync_mirror_binlog_period); + } + + if (mule == NULL) { + sql_print_error("Mule malloc operation failed."); + } + } else { + IO_CACHE* log_file; + MY_STAT stat; + char last_binlog_name[FN_REFLEN]; + + /* Find the last log file from the binlog index. + * Check whether the last binlog matches the requested dump for both + * binlog name and binlog size. + */ + for (;;) { + strmake(last_binlog_name, linfo.log_file_name, FN_REFLEN); + last_binlog_name[FN_REFLEN - 1] = '\0'; + if (binlog->find_next_log(&linfo, false)) + break; + } + DBUG_PRINT("info",("the last binlog: %s", last_binlog_name)); + + /* if the binlog name does not match, exit with an error. */ + if (strcmp(last_binlog_name+dirname_length(last_binlog_name), + mi->master_log_name) != 0) { + sql_print_error("Mule binlog(%s) does not match new relay-binlog(%s)", + last_binlog_name, mi->master_log_name); + } /* Open the last binlog. */ + else if (binlog->open(last_binlog_name, LOG_BIN, NULL, + SEQ_READ_APPEND, true, MAX_LOG_SIZE, 0) != 0) { + sql_print_error("Mule open last binlog failed: %s", last_binlog_name); + } else { + bool valid_file_size = true; + + /* Get the binlog size. */ + log_file = binlog->get_log_file(); + if (my_fstat(log_file->file, &stat, MYF(0)) == 0) { + /* If the binlog size does not match the requested dump position, then + * request the dump from position 0 and verify all events, we need to + * verify events because the mule log might be used for serving during + * anytime. We must be sure that they are correct. + */ + sql_print_information("Binglog size %d", stat.st_size); + if (stat.st_size == mi->master_log_pos) { + mule = new ReplMule(thd, mi, RELAY_MATCH_MULE_RUN, stat.st_size, + binlog_indexname, binlog, + sync_mirror_binlog_period); + } else if (stat.st_size > BIN_LOG_HEADER_SIZE) { + mule = new ReplMule(thd, mi, MULE_VERIFY, stat.st_size, + binlog_indexname, binlog, + sync_mirror_binlog_period); + } else if (stat.st_size == BIN_LOG_HEADER_SIZE) { + mule = new ReplMule(thd, mi, MULE_BEHIND, BIN_LOG_HEADER_SIZE, + binlog_indexname, binlog, + sync_mirror_binlog_period); + } else { + char llbuf[22]; + valid_file_size = false; + sql_print_error("Mule binlog file(%s) invalid size: %s", + last_binlog_name, llstr(stat.st_size, llbuf)); + } + } else { + valid_file_size = false; + sql_print_error("Mule binlog file(%s): fstat failed.", + last_binlog_name); + } + + if (valid_file_size) { + if (mule == NULL) { + sql_print_error("Mule malloc operation failed."); + } else if (mule->status_ == MULE_ERROR) { + /* If mule creation fails, indicate the error. */ + delete mule; + mule = NULL; + } + } + } + } + + /* Clear the mule binlog mode if there are errors. */ + if (mule == NULL) { + binlog->clear_mule_mode(); + binlog->close_index_file(); + } + + /* Unlock binlog index */ + binlog->unlock_index(); + + DBUG_RETURN(mule); +} diff -r 66cc9e0a6768 sql/repl_mule.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/sql/repl_mule.h Thu Dec 04 21:46:15 2008 -0800 @@ -0,0 +1,166 @@ +/* + Copyright (C) 2007 Google Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +*/ + +#ifndef SQL_REPL_MULE_H__ +#define SQL_REPL_MULE_H__ + +/* Replication Mule is the class that is responsible for generating + * an exact copy of the binlog from a master database. We call this feature + * mirror binlog and it can be enabled by setting rpl_mirror_binlog. We + * need to keep the same copy for the following purposes: + * . The replica can serve the binlog transparently as if they are the + * master database. This can relieve master connection overhead. + * . During failover, the replica can become the new master and serve + * old binlogs transparently. + * (The Mule name comes from the popular P2P software eMule.) + * + * Internally, we call the mirrored binlog mule log. + */ + +class THD; +class Rotate_log_event; +class Format_description_log_event; +typedef struct st_master_info MASTER_INFO; + +class ReplMule { + public: + /* Because I/O thread also creates relay-binlog, instead of an exact + * copy of the original master's binlog, we have two resources that + * might get out of sync. + * This enum indicates the status: + * MULE_BEHIND - the mule's header is behind: + * (mule is activated for the first time) + * RELAY_MATCH_MULE - mule matches relay-log + * RELAY_MATCH_MULE_RUN - mule matches relay-log and it was not empty binlog + * MULE_VERIFY - mule has more events than the relay-log and needs + * verification; we can not verify based on relay-log + * events because events might get changed a little; + * verification starts with downloading all events in + * the last binlog from the master and compare with + * all events in the mule log; + * MULE_VERIFY_RELAY_BEHIND - mule has more events than the relay-log + * and relay-log needs to write events + * MULE_ERROR - mule detects errors in event duplicate + * + * When the mule mirrors binlogs, it writes an event into the mule log + * first. Then, I/O thread writes the event into the relay log. + */ + enum RelayStatus { + MULE_BEHIND = 1, + RELAY_MATCH_MULE = 2, + RELAY_MATCH_MULE_RUN = 7, + MULE_VERIFY = 3, + MULE_VERIFY_RELAY_BEHIND = 4, + MULE_ERROR = 5, + }; + + enum WriteStatus { + WRITE_RELAY = 1, + WRITE_ERROR = 2, + SKIP_RELAY = 3, + }; + + private: + const Format_description_log_event *desc_event_; + THD *io_thd_; + MASTER_INFO *mi_; + + /* + * I/O thread will write both mule log for mirror binlog and relay log + * for SQL thread. + * The variable indicates whether the two are in sync. + */ + RelayStatus status_; + + /* The starting event writing position. */ + my_off_t dump_position_; + + /* During the initial setup, the last mule log's file size. */ + my_off_t file_size_; + + /* Internally, we call the mirrored binlog mule log. */ + MYSQL_LOG *mule_log_; + + /* Sync the mule log to disk for every #N events. */ + ulong mule_log_sync_period_; + ulong mule_log_event_counter_; + + /* mule log's index filename */ + char mule_indexname_[FN_REFLEN]; + + /* the current mule log's filename */ + char curr_log_filename_[FN_REFLEN]; + + ReplMule(THD* thd, MASTER_INFO *mi, RelayStatus status, + my_off_t file_size, const char *binlog_indexname, + MYSQL_LOG *binlog, ulong sync_period); + + /* + * Queue the event into the current mule log. If it is a rotation + * event, generate a new mule log file. + * Indicate whether the event is skipped because it is an fake event. + * A fake event is generated by the master to indicate the current + * reading position. + */ + int queueEvent(const char* buf, ulong event_len, bool *skip_event); + + /* Append the event to the current mule log. */ + int appendEvent(const char* buf, ulong event_len); + + bool IsFakeRotation(const char* buf, ulong event_len); + bool IsFakeRotation(const Rotate_log_event& rev); + + /* Seek to the specified position in the current open mule log. */ + void seekToPosition(my_off_t pos); + + public: + + ~ReplMule(); + + /* Dump the event into mule binlog. + * Input: + * buf (IN) - replication event buffer + * event_len (IN) - the event length + * + * Return: + * . WRITE_RELAY: the relay log needs to writing the event + * . WRITE_ERROR: the writing encountered errors + * . SKIP_RELAY: the relay log should skip the event + */ + WriteStatus writeEvent(const char* buf, ulong event_len); + + /* createReplicationMule: + * Create a mule that relays master's replication binlog and + * generate an exact same copy on the local filesystem. + * + * Input: + * thd (IN) - replication I/O thread + * mi (IN) - master info struct for I/O thread's progress + * binlog_indexname (IN) - filename for binlog's index + * binlog (IN) - replication binlog + * + * Return: + * . a replication mule if success + * . NULL if there are any errors + */ + static ReplMule *createReplicationMule(THD* thd, MASTER_INFO *mi, + const char *binlog_indexname, + MYSQL_LOG *binlog); +}; + +#endif /* SQL_REPL_MULE_H__ */ diff -r 66cc9e0a6768 sql/set_var.cc --- a/sql/set_var.cc Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/set_var.cc Thu Dec 04 21:46:15 2008 -0800 @@ -345,6 +345,8 @@ slog_verb); sys_var_long_ptr sys_rpl_recovery_rank("rpl_recovery_rank", &rpl_recovery_rank); +sys_var_bool_ptr sys_rpl_mirror_binlog_enabled("rpl_mirror_binlog_enabled", + &rpl_mirror_binlog_enabled); sys_var_long_ptr sys_query_cache_size("query_cache_size", &query_cache_size, fix_query_cache_size); @@ -364,6 +366,9 @@ sys_var_thd_ulong sys_trans_prealloc_size("transaction_prealloc_size", &SV::trans_prealloc_size, 0, fix_trans_mem_root); +sys_var_long_ptr sys_sync_mirror_binlog_period( + "sync_mirror_binlog_period", + &sync_mirror_binlog_period); #ifdef HAVE_QUERY_CACHE sys_var_long_ptr sys_query_cache_limit("query_cache_limit", @@ -774,6 +779,7 @@ &sys_relay_log_purge, #endif &sys_rpl_recovery_rank, + &sys_rpl_mirror_binlog_enabled, &sys_safe_updates, &sys_secure_auth, &sys_secure_file_priv, @@ -1113,6 +1119,8 @@ {"relay_log_space_limit", (char*) &relay_log_space_limit, SHOW_LONGLONG}, #endif {sys_rpl_recovery_rank.name,(char*) &sys_rpl_recovery_rank, SHOW_SYS}, + {sys_rpl_mirror_binlog_enabled.name, + (char *) &sys_rpl_mirror_binlog_enabled, SHOW_SYS}, {"secure_auth", (char*) &sys_secure_auth, SHOW_SYS}, {"secure_file_priv", (char*) &sys_secure_file_priv, SHOW_SYS}, #ifdef HAVE_SMEM diff -r 66cc9e0a6768 sql/slave.cc --- a/sql/slave.cc Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/slave.cc Thu Dec 04 21:46:15 2008 -0800 @@ -25,6 +25,7 @@ #include #include #include +#include "repl_mule.h" #include #include @@ -3527,6 +3528,7 @@ RELAY_LOG_INFO *rli= &mi->rli; char llbuff[22]; uint retry_count; + ReplMule *mule = NULL; // needs to call my_thread_init(), otherwise we get a coredump in DBUG_ stuff my_thread_init(); @@ -3609,6 +3611,23 @@ if (get_master_version_and_clock(mysql, mi)) goto err; + if (rpl_mirror_binlog_enabled && !mule) { + if (opt_binlog_index_name == NULL) { + sql_print_error("\"log-bin-index\" must be set in mirror binlog."); + goto err; + } + + /* Create the mule to generate the exact copy of the binlog */ + mule = ReplMule::createReplicationMule( + thd, mi, opt_binlog_index_name, &mysql_bin_log); + + /* If we could not create the mule, we stop the I/O thread and report + * an error. + */ + if (mule == NULL) + goto err; + } + if (mi->rli.relay_log.description_event_for_queue->binlog_version > 1) { /* @@ -3624,6 +3643,7 @@ DBUG_PRINT("info",("Starting reading binary log from master")); while (!io_slave_killed(thd,mi)) { + const char* event_buf; bool suppress_warnings= 0; thd_proc_info(thd, "Requesting binlog dump"); if (request_dump(mysql, mi, &suppress_warnings)) @@ -3754,10 +3774,25 @@ goto connected; } // if (event_len == packet_error) + event_buf = (const char*)mysql->net.read_pos + 1; + + if (mule) { + ReplMule::WriteStatus d_status = + mule->writeEvent(event_buf, event_len); + switch (d_status) { + case ReplMule::WRITE_RELAY: + break; + case ReplMule::SKIP_RELAY: + /* Skip writing relay event; go back to read the next event */ + continue; + case ReplMule::WRITE_ERROR: + goto err; + } + } + retry_count=0; // ok event, reset retry counter thd_proc_info(thd, "Queueing master event to the relay log"); - if (queue_event(mi,(const char*)mysql->net.read_pos + 1, - event_len)) + if (queue_event(mi, event_buf, event_len)) { sql_print_error("Slave I/O thread could not queue event from master"); goto err; @@ -3847,6 +3882,7 @@ change_rpl_status(RPL_ACTIVE_SLAVE,RPL_IDLE_SLAVE); DBUG_ASSERT(thd->net.buff != 0); net_end(&thd->net); // destructor will not free it, because net.vio is 0 + delete mule; close_thread_tables(thd, 0); pthread_mutex_lock(&LOCK_thread_count); THD_CHECK_SENTRY(thd); diff -r 66cc9e0a6768 sql/sql_class.h --- a/sql/sql_class.h Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/sql_class.h Thu Dec 04 21:46:15 2008 -0800 @@ -152,6 +152,12 @@ #define LOG_INFO_FATAL -7 #define LOG_INFO_IN_USE -8 +/* If the maximum size is equal to this value, binlog would not rotate on + * size limit. + */ +#define BINLOG_NOSWITCH_SIZE ((ulong) -1) + + /* bitmap to SQL_LOG::close() */ #define LOG_CLOSE_INDEX 1 #define LOG_CLOSE_TO_BE_OPENED 2 @@ -245,6 +251,9 @@ bool no_auto_events; friend class Log_event; + /* mule replication mode */ + bool mule_binlog_; + public: /* These describe the log's format. This is used only for relay logs. @@ -317,7 +326,8 @@ } bool open_index_file(const char *index_file_name_arg, const char *log_name); - void new_file(bool need_lock); + int close_index_file(); + void new_file(bool need_lock= 1, const char* log_filename= NULL); bool write(THD *thd, enum enum_server_command command, const char *format, ...) ATTRIBUTE_FORMAT(printf, 4, 5); bool write(THD *thd, const char *query, uint query_length, @@ -357,7 +367,27 @@ int get_current_log(LOG_INFO* linfo); int raw_get_current_log(LOG_INFO* linfo); uint next_file_id(); - inline bool is_open() { return log_type != LOG_CLOSED; } + + /* Because mysql use is_open() to check whether replication is on, + * we will let the check fail during binlog mule mode. Mule replication + * and normal master replication can not be on at the same time. + * + * is_log_open(): the binlog file is open for either purpose + * + * is_open(): the binlog is open for master replication. + * is_mule_open(): the binlog is open for mirror binlog or for + * replication mule; refer repl_mule.h for details + */ + bool is_log_open() { + return log_type != LOG_CLOSED; + } + bool is_open() { + return (!mule_binlog_) && is_log_open(); + } + bool is_mule_open() { + return (mule_binlog_) && is_log_open(); + } + inline char* get_index_fname() { return index_file_name;} inline char* get_log_fname() { return log_file_name; } inline char* get_name() { return name; } @@ -366,8 +396,18 @@ inline void lock_index() { pthread_mutex_lock(&LOCK_index);} inline void unlock_index() { pthread_mutex_unlock(&LOCK_index);} + inline void lock_log() { pthread_mutex_lock(&LOCK_log);} + inline void unlock_log() { pthread_mutex_unlock(&LOCK_log);} inline IO_CACHE *get_index_file() { return &index_file;} inline uint32 get_open_count() { return open_count; } + /* Look in file repl_mule.h for the definition of mule. */ + void set_mule_mode() { + mule_binlog_ = 1; + } + void clear_mule_mode() { + mule_binlog_ = 0; + } + int flush_log_file(); }; /* diff -r 66cc9e0a6768 sql/sql_lex.h --- a/sql/sql_lex.h Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/sql_lex.h Thu Dec 04 21:46:15 2008 -0800 @@ -104,6 +104,7 @@ // TODO(mcallaghan): update status_vars in mysqld to export these SQLCOM_SHOW_USER_STATS, SQLCOM_SHOW_TABLE_STATS, SQLCOM_SHOW_INDEX_STATS, SQLCOM_SHOW_CLIENT_STATS, + SQLCOM_MAKE_MASTER, /* This should be the last !!! */ SQLCOM_END }; @@ -171,6 +172,12 @@ char *ssl_key, *ssl_cert, *ssl_ca, *ssl_capath, *ssl_cipher; char *relay_log_name; ulong relay_log_pos; + + /* the following fields are used for make master command */ + char *log_index_name; + bool in_failover; + bool kill_session; + bool with_old_binlog; } LEX_MASTER_INFO; diff -r 66cc9e0a6768 sql/sql_parse.cc --- a/sql/sql_parse.cc Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/sql_parse.cc Thu Dec 04 21:46:15 2008 -0800 @@ -402,6 +402,15 @@ passwd_len ? "yes": "no", thd->main_security_ctx.master_access, (thd->db ? thd->db : "*none*"))); + + /* If we are in failover mode, reject all non-super user connections. */ + if (is_in_failover() && + !(thd->main_security_ctx.master_access & SUPER_ACL)) { + net_send_error(thd, ER_SPECIFIC_ACCESS_DENIED_ERROR, + "super-user only during failover"); + DBUG_RETURN(-1); + } + if (check_count) { @@ -3470,6 +3479,22 @@ else res = load_master_data(thd); break; + + case SQLCOM_MAKE_MASTER: + { + thd_proc_info(thd, "Making master"); + + if (check_global_access(thd, SUPER_ACL)) + goto error; + res = make_master(thd, NULL, NULL, &lex->mi); + if (res == 0) { + // TODO -- wei is this OK, setting it to NULL? + thd_proc_info(thd, 0); + send_ok(thd); + } + break; + } + #endif /* HAVE_REPLICATION */ #ifdef HAVE_NDBCLUSTER_DB case SQLCOM_SHOW_NDBCLUSTER_STATUS: diff -r 66cc9e0a6768 sql/sql_repl.cc --- a/sql/sql_repl.cc Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/sql_repl.cc Thu Dec 04 21:46:15 2008 -0800 @@ -20,11 +20,19 @@ #include "log_event.h" #include +extern pthread_mutex_t LOCK_failover_master; +extern bool failover_deny_access; + int max_binlog_dump_events = 0; // unlimited my_bool opt_sporadic_binlog_dump_fail = 0; #ifndef DBUG_OFF static int binlog_dump_count = 0; #endif + +static int make_master_open_log(MYSQL_LOG *log, const char *opt_name, + bool no_auto_events, ulong max_size); +static int set_in_failover(bool kill_session); +static void clear_in_failover(void); /* fake_rotate_event() builds a fake (=which does not exist physically in any @@ -255,7 +263,7 @@ bool purge_master_logs(THD* thd, const char* to_log) { char search_file_name[FN_REFLEN]; - if (!mysql_bin_log.is_open()) + if (!mysql_bin_log.is_log_open()) { send_ok(thd); return FALSE; @@ -308,6 +316,44 @@ return error; } +/* Show processlist command dump the binlog state. + * + * Input: + * output_info - (OUT) the output proc_info + * output_len - (IN) output proc_info's length + * thd - (IN) the thread + * input_msg - (IN) the input proc_info + * log_file_name - (IN) binlog file name + * log_pos - (IN) binlog position + */ +static void processlist_show_binlog_state(char *output_info, + int output_len, + THD *thd, + const char *input_msg, + const char *log_file_name, + my_off_t log_pos) { + DBUG_ENTER("processlist_show_binlog_state"); + + /* Point to input_msg in case "show processlist" access it before the copy + * is finished. + */ + thd_proc_info(thd, input_msg); + + if (snprintf(output_info, output_len, "%s :%s:%lld:", input_msg, + log_file_name + dirname_length(log_file_name), + log_pos) > 0) { + thd_proc_info(thd, output_info); + } + + DBUG_VOID_RETURN; +} + +static void repl_cleanup(ushort flags) { + if (flags & BINLOG_MIRROR_CLIENT) { + /* One less mirror binlog client. */ + thread_safe_sub(rpl_mirror_binlog_clients, 1, &LOCK_stats); + } +} /* TODO: Clean up loop to only have one call to send_file() @@ -319,6 +365,11 @@ LOG_INFO linfo; char *log_file_name = linfo.log_file_name; char search_file_name[FN_REFLEN], *name; + + /* This buffer should be enough for "comments + :file_name:file_pos:". */ + char binlog_state_msg[FN_REFLEN + 100]; + int binlog_state_msg_len = FN_REFLEN + 100; + IO_CACHE log; File file = -1; String* packet = &thd->packet; @@ -335,6 +386,15 @@ bzero((char*) &log,sizeof(log)); + sql_print_information("Start %s binlog_dump to slave_server(%d), pos(%s, %lu)", + "asynchronous", + thd->server_id, log_ident, (ulong)pos); + + if (flags & BINLOG_MIRROR_CLIENT) { + /* One more mirror binlog clients. */ + thread_safe_increment(rpl_mirror_binlog_clients, &LOCK_stats); + } + #ifndef DBUG_OFF if (opt_sporadic_binlog_dump_fail && (binlog_dump_count++ % 2)) { @@ -344,7 +404,7 @@ } #endif - if (!mysql_bin_log.is_open()) + if (!mysql_bin_log.is_log_open()) { errmsg = "Binary log is not open"; my_errno= ER_MASTER_FATAL_ERROR_READING_BINLOG; @@ -529,6 +589,12 @@ } #endif + /* Update the binlog sending state. */ + processlist_show_binlog_state( + binlog_state_msg, binlog_state_msg_len, thd, + "Send binlog events to slave", + log_file_name, pos); + if ((*packet)[EVENT_TYPE_OFFSET+1] == FORMAT_DESCRIPTION_EVENT) { binlog_can_be_corrupted= test((*packet)[FLAGS_OFFSET+1] & @@ -634,6 +700,13 @@ } if (!thd->killed) { + /* Update the binlog sending state. */ + processlist_show_binlog_state( + binlog_state_msg, binlog_state_msg_len, thd, + "Has sent all binlog to slave; " + "waiting for binlog to be updated", + log_file_name, pos); + /* Note that the following call unlocks lock_log */ mysql_bin_log.wait_for_update(thd, 0); } @@ -650,7 +723,12 @@ if (read_packet) { - thd_proc_info(thd, "Sending binlog event to slave"); + // thd_proc_info(thd, "Sending binlog event to slave"); + /* Update the binlog sending state. */ + processlist_show_binlog_state(binlog_state_msg, + binlog_state_msg_len, thd, + "Sending binlog event to slave", + log_file_name, pos); if (my_net_write(net, (char*)packet->ptr(), packet->length()) ) { errmsg = "Failed on my_net_write()"; @@ -685,10 +763,21 @@ } else { + char old_log_file_name[FN_REFLEN]; bool loop_breaker = 0; /* need this to break out of the for loop from switch */ - thd_proc_info(thd, "Finished reading one binlog; switching to next binlog"); + // thd_proc_info(thd, "Finished reading one binlog; switching to next binlog"); + /* Update the binlog sending state. */ + processlist_show_binlog_state( + binlog_state_msg, binlog_state_msg_len, thd, + "Finished reading one binlog; switching to next binlog", + log_file_name, pos); + + /* Keep the old fileename. */ + strmake(old_log_file_name, log_file_name, + sizeof(old_log_file_name) - 1); + switch (mysql_bin_log.find_next_log(&linfo, 1)) { case LOG_INFO_EOF: loop_breaker = (flags & BINLOG_DUMP_NON_BLOCK); @@ -706,6 +795,16 @@ end_io_cache(&log); (void) my_close(file, MYF(MY_WME)); + + /* A sanity check that we can not serve the same binlog twice because + * the filenames are stored in a .index file. + */ + if (strcmp(old_log_file_name, log_file_name) >= 0) { + errmsg = "Re-serving an already served binlog file."; + my_errno = ER_MASTER_FATAL_ERROR_READING_BINLOG; + goto err; + } + /* Call fake_rotate_event() in case the previous log (the one which @@ -733,6 +832,8 @@ end_io_cache(&log); (void)my_close(file, MYF(MY_WME)); + repl_cleanup(flags); + send_eof(thd); thd_proc_info(thd, "Waiting to finalize termination"); pthread_mutex_lock(&LOCK_thread_count); @@ -743,6 +844,7 @@ err: thd_proc_info(thd, "Waiting to finalize termination"); end_io_cache(&log); + repl_cleanup(flags); /* Exclude iteration through thread list this is needed for purge_logs() - it will iterate through @@ -1316,7 +1418,7 @@ Format_description_log_event *description_event= new Format_description_log_event(3); /* MySQL 4.0 by default */ - if (mysql_bin_log.is_open()) + if (mysql_bin_log.is_log_open()) { LEX_MASTER_INFO *lex_mi= &thd->lex->mi; SELECT_LEX_UNIT *unit= &thd->lex->unit; @@ -1456,7 +1558,7 @@ DBUG_RETURN(TRUE); protocol->prepare_for_resend(); - if (mysql_bin_log.is_open()) + if (mysql_bin_log.is_log_open()) { LOG_INFO li; mysql_bin_log.get_current_log(&li); @@ -1497,7 +1599,7 @@ Protocol *protocol= thd->protocol; DBUG_ENTER("show_binlogs"); - if (!mysql_bin_log.is_open()) + if (!mysql_bin_log.is_log_open()) { my_message(ER_NO_BINARY_LOGGING, ER(ER_NO_BINARY_LOGGING), MYF(0)); return 1; @@ -1606,6 +1708,235 @@ DBUG_RETURN(0); } + +/* make_master: Make the current database a primary and starts the + * binlog logging for all updates. + * + * The function handles the following sql commands: + * . MAKE MASTER MASTER_LOG_FILE='replication_log', MASTER_SERVER_ID=1, + * [WITH BINLOG]; + * . MAKE MASTER MASTER_LOG_FILE='replication_log', MASTER_SERVER_ID=1, + * INDEX='replication_log.index' [WITH BINLOG]; + * . MAKE MASTER REVOKE SESSION; + * . MAKE MASTER REVOKE SESSION WITH KILL; + * . MAKE MASTER GRANT SESSION; + * + * Args: + * thd - the current thread + * binlog_name - binlog's filename + * binlog_indexname - binlog index's filename + * mi - master info struct containing binlog name + * (set when we enable master during runtime) + * + * Return: + * 0 : success + * -1 : failure + */ +int make_master(THD* thd, + const char *binlog_name, + const char *binlog_indexname, + const LEX_MASTER_INFO* mi) { + int error = 0; + + DBUG_ENTER("make_master"); + /* In two mode, we enable the binlog: + * . !mi - LEX is not provided; this is called from startup time + * . mi->log_file_name - binlog is specified in the command + */ + if (!mi || mi->log_file_name) { + /* Get the mutex */ + VOID(pthread_mutex_lock(&LOCK_failover_master)); + + /* If the binlog is already opened, we issue an error. We reuse one + * existing error, which might not be fully accurate. + */ + if (mysql_bin_log.is_log_open()) { + my_error(ER_MASTER_INFO, MYF(0)); + sql_print_error("Replication master log is already open: cannot " + "make another master!"); + error = -1; + } else { + if (!mi) { + /* This opening happens at mysql startup time. */ + if (make_master_open_log(&mysql_bin_log, binlog_name, + 0, max_binlog_size) != 0) { + error = -1; + } + } else { + /* This opening happens during mysql runtime, which is mostly + * requested to do failover. + */ + + error = -1; + if (!is_in_failover()) { + sql_print_error( + "\"make master\" runs only in failover mode. " + "Please run \"make master revoke session (with kill)\""); + } else if (strlen(mi->log_file_name) == 0) { + sql_print_error("Master log filename is not specified correctly."); + } else if (!mi->server_id || mi->server_id == MASTER_INFO_SERVER_ID) { + sql_print_error("\"make master\": invalid server_id(%d)", + mi->server_id); + } else { + /* Open the new log files and delete all existing ones to avoid + * conflicts. + */ + uint32 old_server_id = server_id; + char *binlog_name = NULL; + + /* Set the global master server id. + * We would not change server id for all connection threads. + * All non-super sessions should be blocked by revoke sessions. + * Super-user sessions are responsible for their own operations. + */ + server_id = mi->server_id; + thd->server_id = mi->server_id; + + if (!(binlog_name = my_strdup(mi->log_file_name, MYF(0))) || + make_master_open_index(&binlog_name, mi->log_index_name) != 0 || + make_master_open_log(&mysql_bin_log, binlog_name, + 0, max_binlog_size) != 0) { + sql_print_error("Open master logfile failed."); + thd->server_id = old_server_id; + server_id = old_server_id; + } else if (!mi->with_old_binlog && + mysql_bin_log.reset_logs(thd) != 0) { + sql_print_error("Cleanup existing master logfiles failed."); + thd->server_id = old_server_id; + server_id = old_server_id; + } else { + error = 0; + } + } + if (error == -1) + my_error(ER_MASTER_INFO, MYF(0)); + } + } + + if (error == 0) { + /* indicates that binlog is enabled now */ + using_update_log = 1; + } else if (mysql_bin_log.is_open()) { + mysql_bin_log.close(LOG_CLOSE_INDEX); + } + + /* Release the mutex */ + VOID(pthread_mutex_unlock(&LOCK_failover_master)); + } else { + /* The following actions are related to session management during + * failover operation. We do not want some sessions come in + * during failover and make updates. + * This is invoked for command: MAKE MASTER GRANT/REVOKE SESSION; + */ + if (mi->in_failover) { + set_in_failover(mi->kill_session); + } else { + clear_in_failover(); + } + } + + DBUG_RETURN(error); +} + +static int make_master_open_log(MYSQL_LOG *log, + const char *opt_name, + bool no_auto_events, + ulong max_size) { + char tmp[FN_REFLEN]; + + // get rid of extension + char *p = fn_ext(opt_name); + uint length=(uint) (p-opt_name); + strmake(tmp,opt_name,min(length,FN_REFLEN)); + opt_name=tmp; + + return log->open(opt_name, LOG_BIN, NULL, WRITE_CACHE, 0, + max_size, 0); +} + +int make_master_open_index(char **binlog_name, + const char *binlog_indexname) { + char buf[FN_REFLEN]; + const char *ln; + DBUG_ENTER("make_master_open_index"); + + ln= mysql_bin_log.generate_name(*binlog_name, "-bin", 1, buf); + if (!(*binlog_name) && !binlog_indexname) { + /* + User didn't give us info to name the binlog index file. + Picking `hostname`-bin.index like did in 4.x, causes replication to + fail if the hostname is changed later. So, we would like to instead + require a name. But as we don't want to break many existing setups, we + only give warning, not error. + */ + sql_print_warning("No argument was provided to --log-bin, and " + "--log-bin-index was not used; so replication " + "may break when this MySQL server acts as a " + "master and has his hostname changed!! Please " + "use '--log-bin=%s' to avoid this problem.", ln); + } + if (ln == buf) { + my_free(*binlog_name, MYF(MY_ALLOW_ZERO_PTR)); + *binlog_name = my_strdup(buf, MYF(0)); + } + if (mysql_bin_log.open_index_file(binlog_indexname, ln) != 0) { + DBUG_RETURN(-1); + } + + /* + Used to specify which type of lock we need to use for queries of type + INSERT ... SELECT. This will change when we have row level logging. + */ + using_update_log=1; + + DBUG_RETURN(0); +} + +/* Set the status indicating that we are in failover and deny all non-super + * user access. + * + * Args: + * kill_session - kill all non-super sessions if specified + * + * Return: + * 0 - success + * -1 - failure (caused by not killing all sessions) + */ +static int set_in_failover(bool kill_session) { + failover_deny_access = 1; + + if (kill_session) { + /* If kill session option is specified, we need to kill all non-super + * user sessions. + */ + THD *kill_thd; + + uint error=ER_NO_SUCH_THREAD; + pthread_mutex_lock(&LOCK_thread_count); // For unlink from list + I_List_iterator it(threads); + while ((kill_thd=it++)) { + if (!(kill_thd->main_security_ctx.master_access & SUPER_ACL)) { + pthread_mutex_lock(&kill_thd->LOCK_delete); // Lock from delete + + /* ask the thread to die */ + kill_thd->awake(THD::KILL_CONNECTION); + pthread_mutex_unlock(&kill_thd->LOCK_delete); + } + } + pthread_mutex_unlock(&LOCK_thread_count); + } + return 0; +} + +static void clear_in_failover(void) { + failover_deny_access = 0; +} + +bool is_in_failover(void) { + return failover_deny_access; +} + + #endif /* HAVE_REPLICATION */ diff -r 66cc9e0a6768 sql/sql_repl.h --- a/sql/sql_repl.h Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/sql_repl.h Thu Dec 04 21:46:15 2008 -0800 @@ -38,6 +38,10 @@ int start_slave(THD* thd, MASTER_INFO* mi, bool net_report); int stop_slave(THD* thd, MASTER_INFO* mi, bool net_report); bool change_master(THD* thd, MASTER_INFO* mi); +int make_master(THD* thd, const char *binlog_name, + const char *binlog_indexname, const LEX_MASTER_INFO* mi); +int make_master_open_index(char **binlog_name, const char *binlog_indexname); +bool is_in_failover(void); bool mysql_show_binlog_events(THD* thd); int cmp_master_pos(const char* log_file_name1, ulonglong log_pos1, const char* log_file_name2, ulonglong log_pos2); diff -r 66cc9e0a6768 sql/sql_yacc.yy --- a/sql/sql_yacc.yy Thu Dec 04 21:37:12 2008 -0800 +++ b/sql/sql_yacc.yy Thu Dec 04 21:46:15 2008 -0800 @@ -735,6 +735,7 @@ %token LOOP_SYM %token LOW_PRIORITY %token LT +%token MAKE_SYM %token MAKE_SET_SYM %token MASTER_CONNECT_RETRY_SYM %token MASTER_HOST_SYM @@ -1167,7 +1168,7 @@ query verb_clause create change select do drop insert replace insert2 insert_values update delete truncate rename show describe load alter optimize keycache preload flush - reset purge begin commit rollback savepoint release + make reset purge begin commit rollback savepoint release slave master_def master_defs master_file_def slave_until_opts repair restore backup analyze check start checksum field_list field_list_item field_spec kill column_def key_def @@ -1301,6 +1302,7 @@ | kill | load | lock + | make | optimize | keycache | preload @@ -1428,6 +1430,56 @@ master_defs {} ; + +/* make master */ +make: + MAKE_SYM MASTER_SYM + { + LEX *lex = Lex; + lex->sql_command = SQLCOM_MAKE_MASTER; + bzero((char*) &lex->mi, sizeof(lex->mi)); + } + make_master_defs + { + } + ; + +make_master_defs: + MASTER_LOG_FILE_SYM EQ TEXT_STRING ',' MASTER_SERVER_ID_SYM EQ ulong_num + { + Lex->mi.log_file_name = $3.str; + Lex->mi.server_id = $7; + } + make_master_with_defs {} + | MASTER_LOG_FILE_SYM EQ TEXT_STRING ',' MASTER_SERVER_ID_SYM EQ ulong_num ',' INDEX_SYM EQ TEXT_STRING + { + Lex->mi.log_file_name = $3.str; + Lex->mi.server_id = $7; + Lex->mi.log_index_name = $11.str; + } + make_master_with_defs {} + | GRANT SESSION_SYM + { + Lex->mi.in_failover = 0; + } + | REVOKE SESSION_SYM + { + Lex->mi.in_failover = 1; + } + | REVOKE SESSION_SYM WITH KILL_SYM + { + Lex->mi.in_failover = 1; + Lex->mi.kill_session = 1; + } + ; + +make_master_with_defs: + /* empty */ {} + | WITH BINLOG_SYM + { + /* All old binlogs will be kept after "make master" command. */ + Lex->mi.with_old_binlog = 1; + } master_defs: master_def @@ -8396,6 +8448,7 @@ | HANDLER_SYM {} | HELP_SYM {} | LANGUAGE_SYM {} + | MAKE_SYM {} | NO_SYM {} | OPEN_SYM {} | PREPARE_SYM {}