ORACLE DBA: ORA-29702: error occurred in Cluster Group Service operation

# problem

SQL> startup;
ORA-29702: error occurred in Cluster Group Service operation

29702, 00000, "error occurred in Cluster Group Service operation"
// *Cause: An unexpected error occurred while performing a CGS operation.
// *Action: Verify that the LMON process is still active. Also,
// check the Oracle LMON trace files for errors.

# solution

cd $ORACLE_HOME/rdbms/lib
make -f ins_rdbms.mk rac_off
make -f ins_rdbms.mk ioracle


[oracle@rac1 ~]$ . oraenv
ORACLE_SID = [RAC1] ? jay
[oracle@rac1 ~]$ sqlplus '/as sysdba'

SQL*Plus: Release 10.2.0.1.0 - Production on Thu Aug 19 11:51:52 2010

Copyright (c) 1982, 2005, Oracle.  All rights reserved.

Connected to an idle instance.

SQL> startup
ORA-29702: error occurred in Cluster Group Service operation


[oracle@rac1 ~]$ cd $ORACLE_HOME/rdbms/lib
[oracle@rac1 lib]$ pwd
/u01/app/oracle/product/10.2.0/db_1/rdbms/lib
[oracle@rac1 lib]$ make -f ins_rdbms.mk rac_off
rm -f /u01/app/oracle/product/10.2.0/db_1/lib/libskgxp10.so
cp /u01/app/oracle/product/10.2.0/db_1/lib//libskgxpd.so /u01/app/oracle/product/10.2.0/db_1/lib/libskgxp10.so
rm -f /u01/app/oracle/product/10.2.0/db_1/lib/libskgxn2.so
cp /u01/app/oracle/product/10.2.0/db_1/lib//libskgxns.so \
      /u01/app/oracle/product/10.2.0/db_1/lib/libskgxn2.so
/usr/bin/ar d /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/libknlopt.a kcsm.o
/usr/bin/ar cr /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/libknlopt.a /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/ksnkcs.o
[oracle@rac1 lib]$ make -f ins_rdbms.mk ioracle
chmod 755 /u01/app/oracle/product/10.2.0/db_1/bin

 - Linking Oracle
rm -f /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/oracle
gcc  -o /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/oracle -L/u01/app/oracle/product/10.2.0/db_1/rdbms/lib/ -L/u01/app/oracle/product/10.2.0/db_1/lib/ -L/u01/app/oracle/product/10.2.0/db_1/lib/stubs/ -L/usr/lib -lirc   -Wl,-E `test -f /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/sskgpsmti.o && echo /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/sskgpsmti.o` /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/opimai.o /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/ssoraed.o /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/ttcsoi.o /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/defopt.o  -Wl,--whole-archive -lperfsrv10 -Wl,--no-whole-archive /u01/app/oracle/product/10.2.0/db_1/lib/nautab.o /u01/app/oracle/product/10.2.0/db_1/lib/naeet.o /u01/app/oracle/product/10.2.0/db_1/lib/naect.o /u01/app/oracle/product/10.2.0/db_1/lib/naedhs.o /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/config.o  -lserver10 -lodm10 -lnnet10 -lskgxp10 -lsnls10 -lnls10  -lcore10 -lsnls10 -lnls10 -lcore10 -lsnls10 -lnls10 -lxml10 -lcore10 -lunls10 -lsnls10 -lnls10 -lcore10 -lnls10 -lhasgen10 -lcore10 -lskgxn2 -locr10 -locrb10 -locrutl10 -lhasgen10 -lcore10 -lskgxn2   -lclient10  -lvsn10 -lcommon10 -lgeneric10 -lknlopt `if /usr/bin/ar tv /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/libknlopt.a | grep xsyeolap.o > /dev/null 2>&1 ; then echo "-loraolap10" ; fi` -lslax10 -lpls10  -lplp10 -lserver10 -lclient10  -lvsn10 -lcommon10 -lgeneric10 -lknlopt -lslax10 -lpls10  -lplp10 -ljox10 -lserver10 -lclsra10 -ldbcfg10 -locijdbcst10 -lwwg  `cat /u01/app/oracle/product/10.2.0/db_1/lib/ldflags`    -lnsslb10 -lncrypt10 -lnsgr10 -lnzjs10 -ln10 -lnnz10 -lnl10 -lnro10 `cat /u01/app/oracle/product/10.2.0/db_1/lib/ldflags`    -lnsslb10 -lncrypt10 -lnsgr10 -lnzjs10 -ln10 -lnnz10 -lnl10 -lmm -lsnls10 -lnls10  -lcore10 -lsnls10 -lnls10 -lcore10 -lsnls10 -lnls10 -lxml10 -lcore10 -lunls10 -lsnls10 -lnls10 -lcore10 -lnls10 `cat /u01/app/oracle/product/10.2.0/db_1/lib/ldflags`    -lnsslb10 -lncrypt10 -lnsgr10 -lnzjs10 -ln10 -lnnz10 -lnl10 -lnro10 `cat /u01/app/oracle/product/10.2.0/db_1/lib/ldflags`    -lnsslb10 -lncrypt10 -lnsgr10 -lnzjs10 -ln10 -lnnz10 -lnl10   -lsnls10 -lnls10  -lcore10 -lsnls10 -lnls10 -lcore10 -lsnls10 -lnls10 -lxml10 -lcore10 -lunls10 -lsnls10 -lnls10 -lcore10 -lnls10 `if /usr/bin/ar tv /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/libknlopt.a | grep "kxmnsd.o" > /dev/null 2>&1 ; then echo " " ; else echo "-lordsdo10"; fi` -lctxc10 -lctx10 -lzx10 -lgx10 -lctx10 -lzx10 -lgx10 -lordimt10 -lsnls10 -lnls10  -lcore10 -lsnls10 -lnls10 -lcore10 -lsnls10 -lnls10 -lxml10 -lcore10 -lunls10 -lsnls10 -lnls10 -lcore10 -lnls10 -lsnls10 -lunls10  -lsnls10 -lnls10  -lcore10 -lsnls10 -lnls10 -lcore10 -lsnls10 -lnls10 -lxml10 -lcore10 -lunls10 -lsnls10 -lnls10 -lcore10 -lnls10 -laio    `cat /u01/app/oracle/product/10.2.0/db_1/lib/sysliblist` -Wl,-rpath,/u01/app/oracle/product/10.2.0/db_1/lib -lm    `cat /u01/app/oracle/product/10.2.0/db_1/lib/sysliblist` -ldl -lm -L/u01/app/oracle/product/10.2.0/db_1/lib
/u01/app/oracle/product/10.2.0/db_1/lib//libcore10.a(lcd.o)(.text+0x8e8): In function `lcdprm':
: warning: the `gets' function is dangerous and should not be used.
mv -f /u01/app/oracle/product/10.2.0/db_1/bin/oracle /u01/app/oracle/product/10.2.0/db_1/bin/oracleO
mv /u01/app/oracle/product/10.2.0/db_1/rdbms/lib/oracle /u01/app/oracle/product/10.2.0/db_1/bin/oracle
chmod 6751 /u01/app/oracle/product/10.2.0/db_1/bin/oracle
[oracle@rac1 lib]$


SQL> startup
ORACLE instance started.

Total System Global Area  444596224 bytes
Fixed Size                  1219904 bytes
Variable Size             130024128 bytes
Database Buffers          310378496 bytes
Redo Buffers                2973696 bytes
Database mounted.
Database opened.
SQL> select name from v$datafile;

NAME
--------------------------------------------------------------------------------/u01/app/oracle/oradata/jay/system01.dbf
/u01/app/oracle/oradata/jay/undotbs01.dbf
/u01/app/oracle/oradata/jay/sysaux01.dbf
/u01/app/oracle/oradata/jay/users01.dbf

SQL> select name from v$database;

NAME
---------
JAY