one rac troubleshooting

登陆节点rac1 后发现：

[root@rac1 ~]# su - oracle
rac1-> sqlplus /nolog

SQL*Plus: Release 10.2.0.1.0 - Production on Mon Mar 16 09:28:53 2009

Copyright (c) 1982, 2005, Oracle. All rights reserved.

SQL> conn / as sysdba
Connected.
SQL> select group#,thread#,status,members from v$log order by 2,1;

GROUP# THREAD# STATUS MEMBERS
---------- ---------- ---------------- ----------
1 1 CURRENT 1
2 1 INACTIVE 1
5 1 INACTIVE 2
3 2 INACTIVE 1
4 2 CURRENT 1
7 2 INACTIVE 2

6 rows selected.

SQL> select * from v$logfile order by 1;

GROUP# STATUS TYPE MEMBER IS_
---------- ------- ------- ---------------------------------------- ---
1 ONLINE /diskc/oradata/demo/redo01.log NO
2 ONLINE /diskc/oradata/demo/redo02.log NO
3 STALE ONLINE /diskc/oradata/demo/redo03.log NO
4 ONLINE /diskc/oradata/demo/redo04.log NO
5 ONLINE /diskc/oradata/demo/redo05_01.log NO
5 ONLINE /diskc/oradata/demo/redo05_02.log NO
7 STALE ONLINE /diskc/oradata/demo/redo07_01.log NO
7 STALE ONLINE /diskc/oradata/demo/redo07_02.log NO

8 rows selected.

登陆节点rac2：

[root@rac2 ~]# su - oracle
rac2-> sqlplus /nolog

SQL*Plus: Release 10.2.0.1.0 - Production on Mon Mar 16 09:40:51 2009

Copyright (c) 1982, 2005, Oracle. All rights reserved.

SQL> conn / as sysdba
Connected to an idle instance.
SQL>

这一个节点上的rac竟然没有启动！！！

开始troubleshooting，不懂rac啊。。。

rac1-> crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.demo.db application ONLINE ONLINE rac1
ora....o1.inst application ONLINE ONLINE rac1
ora....o2.inst application ONLINE OFFLINE
ora....C1.lsnr application ONLINE ONLINE rac1
ora.rac1.gsd application ONLINE ONLINE rac1
ora.rac1.ons application ONLINE ONLINE rac1
ora.rac1.vip application ONLINE ONLINE rac1
ora....C2.lsnr application ONLINE OFFLINE
ora.rac2.gsd application ONLINE OFFLINE
ora.rac2.ons application ONLINE OFFLINE
ora.rac2.vip application ONLINE ONLINE rac1

[root@rac2 bin]# ./crsctl start crs
Attempting to start CRS stack
The CRS stack will be started shortly
[root@rac2 bin]# ./crsctl start resources
Starting resources.
Successfully started CRS resources
[root@rac2 bin]# ./crsctl start ora.rac2.ons
Unexpected parameter: ora.rac2.ons
[root@rac2 bin]#

[root@rac2 ~]# su - oracle
rac2-> srvctl start nodeapps -n rac2
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.gsd' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.ons' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.LISTENER_RAC2.lsnr' has placement error.
rac2->

rac2-> crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy

rac2-> crsctl start resources
Starting resources.
Successfully started CRS resources
rac2-> crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.demo.db application ONLINE ONLINE rac1
ora....o1.inst application ONLINE ONLINE rac1
ora....o2.inst application ONLINE UNKNOWN rac2
ora....C1.lsnr application ONLINE ONLINE rac1
ora.rac1.gsd application ONLINE ONLINE rac1
ora.rac1.ons application ONLINE ONLINE rac1
ora.rac1.vip application ONLINE ONLINE rac1
ora....C2.lsnr application ONLINE UNKNOWN rac2
ora.rac2.gsd application ONLINE UNKNOWN rac2
ora.rac2.ons application ONLINE UNKNOWN rac2
ora.rac2.vip application ONLINE ONLINE rac2
rac2-> pwd
/u01/oracle/
rac2-> cd $ORA_CRS_HOME/bin
rac2-> pwd
/u01/app/oracle/product/10.2.0/crs_1/bin
rac2-> gsdctl stop
rac2-> gsdctl status
rac2-> ./gsdctl status
rac2-> gsdctl start
rac2-> srvctl start nodeapps -n rac2
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.gsd' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.ons' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.LISTENER_RAC2.lsnr' has placement error.
rac2-> exec /usr/bin/ssh-agent $SHELL
rac2-> /usr/bin/ssh-add
Enter passphrase for /u01/oracle//.ssh/id_rsa:
Identity added: /u01/oracle//.ssh/id_rsa (/u01/oracle//.ssh/id_rsa)
Identity added: /u01/oracle//.ssh/id_dsa (/u01/oracle//.ssh/id_dsa)
rac2-> ssh rac1 date
Mon Mar 16 09:54:42 CST 2009
rac2-> ssh rac2 date
Mon Mar 16 09:54:50 CST 2009

两个节点上的时间同步的啊。。。

rac2-> crs_stop -all
Attempting to stop `ora.rac1.gsd` on member `rac1`
Stop of `ora.rac1.gsd` on member `rac1` succeeded.
Attempting to stop `ora.rac1.ons` on member `rac1`
Stop of `ora.rac1.ons` on member `rac1` succeeded.
Attempting to stop `ora.demo.db` on member `rac1`
Stop of `ora.demo.db` on member `rac1` succeeded.
Attempting to stop `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1`
Attempting to stop `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2`
Stop of `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1` succeeded.
Stop of `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2` succeeded.
Attempting to stop `ora.rac2.vip` on member `rac2`
`ora.demo.demo1.inst` is already OFFLINE.
Stop of `ora.rac2.vip` on member `rac2` succeeded.
Attempting to stop `ora.rac1.vip` on member `rac1`
Stop of `ora.rac1.vip` on member `rac1` succeeded.
CRS-0216: Could not stop resource 'ora.demo.demo1.inst'.

rac2-> crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.demo.db application OFFLINE OFFLINE
ora....o1.inst application OFFLINE OFFLINE
ora....o2.inst application OFFLINE OFFLINE
ora....C1.lsnr application OFFLINE OFFLINE
ora.rac1.gsd application OFFLINE OFFLINE
ora.rac1.ons application OFFLINE OFFLINE
ora.rac1.vip application OFFLINE OFFLINE
ora....C2.lsnr application OFFLINE OFFLINE
ora.rac2.gsd application ONLINE UNKNOWN rac2
ora.rac2.ons application ONLINE UNKNOWN rac2
ora.rac2.vip application OFFLINE OFFLINE
rac2-> crs_stop -all
rac2->
rac2-> crs_stat
NAME=ora.demo.db
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.demo.demo1.inst
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.demo.demo2.inst
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.LISTENER_RAC1.lsnr
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.gsd
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.ons
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.vip
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac2.LISTENER_RAC2.lsnr
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac2.gsd
TYPE=application
TARGET=ONLINE
STATE=UNKNOWN on rac2

NAME=ora.rac2.ons
TYPE=application
TARGET=ONLINE
STATE=UNKNOWN on rac2

NAME=ora.rac2.vip
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

rac2-> crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.demo.db application OFFLINE OFFLINE
ora....o1.inst application OFFLINE OFFLINE
ora....o2.inst application OFFLINE OFFLINE
ora....C1.lsnr application OFFLINE OFFLINE
ora.rac1.gsd application OFFLINE OFFLINE
ora.rac1.ons application OFFLINE OFFLINE
ora.rac1.vip application OFFLINE OFFLINE
ora....C2.lsnr application OFFLINE OFFLINE
ora.rac2.gsd application ONLINE UNKNOWN rac2
ora.rac2.ons application ONLINE UNKNOWN rac2
ora.rac2.vip application OFFLINE OFFLINE
rac2-> crs_stop ora.rac2.gsd
Attempting to stop `ora.rac2.gsd` on member `rac2`
Stop of `ora.rac2.gsd` on member `rac2` succeeded.
rac2-> crs_stop ora.rac2.ons
Attempting to stop `ora.rac2.ons` on member `rac2`
Stop of `ora.rac2.ons` on member `rac2` succeeded.
rac2-> crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.demo.db application OFFLINE OFFLINE
ora....o1.inst application OFFLINE OFFLINE
ora....o2.inst application OFFLINE OFFLINE
ora....C1.lsnr application OFFLINE OFFLINE
ora.rac1.gsd application OFFLINE OFFLINE
ora.rac1.ons application OFFLINE OFFLINE
ora.rac1.vip application OFFLINE OFFLINE
ora....C2.lsnr application OFFLINE OFFLINE
ora.rac2.gsd application OFFLINE OFFLINE
ora.rac2.ons application OFFLINE OFFLINE
ora.rac2.vip application OFFLINE OFFLINE
rac2-> crs_start -all
Attempting to start `ora.rac1.vip` on member `rac1`
Attempting to start `ora.rac2.vip` on member `rac2`
Start of `ora.rac1.vip` on member `rac1` succeeded.
Attempting to start `ora.demo.demo1.inst` on member `rac1`
Start of `ora.rac2.vip` on member `rac2` succeeded.
Attempting to start `ora.demo.demo2.inst` on member `rac2`
Start of `ora.demo.demo1.inst` on member `rac1` succeeded.
Attempting to start `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1`
Start of `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1` succeeded.
Start of `ora.demo.demo2.inst` on member `rac2` succeeded.
Attempting to start `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2`
Start of `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2` succeeded.
CRS-1002: Resource 'ora.rac1.ons' is already running on member 'rac1'

Attempting to start `ora.rac1.gsd` on member `rac1`
Start of `ora.rac1.gsd` on member `rac1` succeeded.
CRS-1002: Resource 'ora.rac2.ons' is already running on member 'rac2'

Attempting to start `ora.rac2.gsd` on member `rac2`
Attempting to start `ora.demo.db` on member `rac2`
Start of `ora.rac2.gsd` on member `rac2` succeeded.
Start of `ora.demo.db` on member `rac2` succeeded.
CRS-0223: Resource 'ora.rac1.ons' has placement error.

CRS-0223: Resource 'ora.rac2.ons' has placement error.

rac2->
rac2-> crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.demo.db application ONLINE ONLINE rac2
ora....o1.inst application ONLINE ONLINE rac1
ora....o2.inst application ONLINE ONLINE rac2
ora....C1.lsnr application ONLINE ONLINE rac1
ora.rac1.gsd application ONLINE ONLINE rac1
ora.rac1.ons application ONLINE ONLINE rac1
ora.rac1.vip application ONLINE ONLINE rac1
ora....C2.lsnr application ONLINE ONLINE rac2
ora.rac2.gsd application ONLINE ONLINE rac2
ora.rac2.ons application ONLINE ONLINE rac2
ora.rac2.vip application ONLINE ONLINE rac2
rac2->

SQL> conn / as sysdba
Connected.
SQL> select instance_name,status from v$instance;

INSTANCE_NAME STATUS
---------------- ------------
demo2 OPEN

SQL> select name,open_mode ,log_mode from v$database;

NAME OPEN_MODE LOG_MODE
--------- ---------- ------------
DEMO READ WRITE ARCHIVELOG

SQL> alter system switch logfile;

System altered.

SQL> col member for a40
SQL> select * from v$logfile;

GROUP# STATUS TYPE MEMBER IS_
---------- ------- ------- ---------------------------------------- ---
2 ONLINE /diskc/oradata/demo/redo02.log NO
1 ONLINE /diskc/oradata/demo/redo01.log NO
3 STALE ONLINE /diskc/oradata/demo/redo03.log NO
4 ONLINE /diskc/oradata/demo/redo04.log NO
7 ONLINE /diskc/oradata/demo/redo07_01.log NO
7 ONLINE /diskc/oradata/demo/redo07_02.log NO
5 ONLINE /diskc/oradata/demo/redo05_01.log NO
5 ONLINE /diskc/oradata/demo/redo05_02.log NO

SQL> select group#,thread# from v$log order by 2,1;

GROUP# THREAD#
---------- ----------
1 1
2 1
5 1
3 2
4 2
7 2

6 rows selected.

SQL> alter system switch logfile;

System altered.

SQL> select * from v$logfile;

GROUP# STATUS TYPE MEMBER IS_
---------- ------- ------- ---------------------------------------- ---
2 ONLINE /diskc/oradata/demo/redo02.log NO
1 ONLINE /diskc/oradata/demo/redo01.log NO
3 ONLINE /diskc/oradata/demo/redo03.log NO
4 ONLINE /diskc/oradata/demo/redo04.log NO
7 ONLINE /diskc/oradata/demo/redo07_01.log NO
7 ONLINE /diskc/oradata/demo/redo07_02.log NO
5 ONLINE /diskc/oradata/demo/redo05_01.log NO
5 ONLINE /diskc/oradata/demo/redo05_02.log NO

8 rows selected.

SQL>