centOS6.4 oracle11g RAC搭建

安装环境

存储规划
1 本地磁盘

2 共享磁盘

网络规划

DNS服务器的安装

1 安装DNS需要的包

[root@miles ~]# yum install bind-libs bind bind-utils

2 修改DNS的配置文件/etc/named.conf

[root@miles ~]# vi /etc/named.conf 
...
listen-on port 53 { 127.0.0.1;192.168.137.20; };
#53为监听端口号;192.168.137.20为DNS服务器IP;注意后面有分号。
...
allow-query     { localhost;0.0.0.0/0; };
#0.0.0.0/0表示允许任何客户端进行查询。如果设置有问题，当客户端请求域名解析时，会报错“server can't find rac1：REFUSED"。
directory       "/var/named";
#DNS数据库存放目录（正反解析文件存放在这）
...
zone "milesdb.com" IN {
        type master;
        file "milesdb.com";
        allow-update { none; };
};
#file "milesdb.com"指定正向解析文件文件名。

zone "137.168.192.in-addr.arpa" IN {
        type master;
        file "137.168.192.in-addr.arpa";
        allow-update { none; };
};
#file "137.168.192.in-addr.arpa"指向逆向解析文件文件名
...

3 创建正向域名解析文件milesdb.com

[root@miles named]# pwd
/var/named
[root@miles named]# vi milesdb.com
$TTL    86400
@       IN      SOA     mydns   root.mydns (
                                42      ;       serial  (d. adams)
                                3H      ;       refresh
                                15M     ;       retry
                                1W      ;       expiry
                                1D )    ;       minimum
                IN      NS      localhost
localhost       IN      A       127.0.0.1
mydns           IN      A       192.168.137.20
rac1            IN      A       192.168.137.31
rac2            IN      A       192.168.137.32
rac1-vip        IN      A       192.168.137.41
rac2-vip        IN      A       192.168.137.42
myscan          IN      A       192.168.137.51
myscan          IN      A       192.168.137.52
myscan          IN      A       192.168.137.53
rac1-priv1      IN      A       10.10.10.1
rac2-priv1      IN      A       10.10.10.2
rac1-priv2      IN      A       10.10.10.11
rac2-priv2      IN      A       10.10.10.12

4 创建反向解析文件137.168.192.in-addr.arpa

[root@miles named]# vi 137.168.192.in-addr.arpa
$ORIGIN 137.168.192.in-addr.arpa.
$TTL 1H
@       IN      SOA     mydns.milesdb.  root.mydns.milesdb. (   2
                                        3H
                                        1H
                                        1W
                                        1H )
@       IN      NS      milesdb.com.
20      IN      PTR     mydns.milesdb.com.
31      IN      PTR     rac1.milesdb.com.
32      IN      PTR     rac2.milesdb.com.
41      IN      PTR     rac1-vip.milesdb.com.
42      IN      PTR     rac2-vip.milesdb.com.
51      IN      PTR     myscan.milesdb.com.
52      IN      PTR     myscan.milesdb.com.
53      IN      PTR     myscan.milesdb.com.
1       IN      PTR     rac1-priv1.milesdb.com.
2       IN      PTR     rac2-priv1.milesdb.com.
11      IN      PTR     rac1-priv2.milesdb.com.
12      IN      PTR     rac2-priv2.milesdb.com.

5 配置完成，检查
测试配置，如果没有信息，则配置正确

[root@miles ~]# named-checkconf

测试区域文件

[root@miles named]# named-checkzone milesdb.com milesdb.com
zone milesdb.com/IN: loaded serial 42
OK

6 启动DNS服务

[root@miles named]# service named start
Generating /etc/rndc.key:                                  [  OK  ]
Starting named:                                            [  OK  ]

7 设置启动级别

[root@miles named]# chkconfig named on
[root@miles named]# chkconfig --list named
named           0:off   1:off   2:on    3:on    4:on    5:on    6:off

8 配置客户端(RAC节点上)

[root@miles32 ~]# vi /etc/resolv.conf 
...
nameserver 192.168.137.20
search milesdb.com

9 客户端测试DNS

[root@miles31 ~]# nslookup rac1
Server:         192.168.137.20
Address:        192.168.137.20#53

Name:   rac1.milesdb.com
Address: 192.168.137.31

[root@miles31 ~]# nslookup rac2.milesdb.com
Server:         192.168.137.20
Address:        192.168.137.20#53

Name:   rac2.milesdb.com
Address: 192.168.137.32

rac1节点设置

网卡设置
1 将eth0、eth1绑定成bond0

#创建bond0的配置文件
[root@miles31 ~]# vi /etc/sysconfig/network-scripts/ifcfg-bond0
DEVICE=bond0
IPADDR=192.168.137.31
NETMASK=255.255.255.0
USERCTL=no
BOOTPROTO=none
ONBOOT=yes
GATEWAY=192.168.137.1
#创建eth0的配置文件
[root@miles31 ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth0 
DEVICE=eth0
USERCTL=no
ONBOOT=yes
MASTER=bond0
SLAVE=yes
BOOTPROTO=none
#创建eth2的配置文件
[root@miles31 ~]# vi /etc/sysconfig/network-scripts/ifcfg-eth2
DEVICE=eth2
USERCTL=no
ONBOOT=yes
MASTER=bond0
SLAVE=yes
BOOTPROTO=none
#修改模块的配置文件。
#miimon=100，每100毫秒检查一次网络；
#mode 共有7种模式，常用的模式0、1. 0：负载均衡；1：热备模式
[root@miles31 ~]# vi /etc/modprobe.conf
alias bond0 bonding
options bond0 miimon=100 mode=0
#装载模块
[root@miles31 ~]# modprobe bonding
#查看bond模块是否正在运行
[root@miles31 ~]# lsmod|grep bond
bonding               128245  0 
8021q                  25349  1 bonding
ipv6                  317340  286 bonding,ip6t_REJECT,nf_conntrack_ipv6,nf_defrag_ipv6
#重启网络服务
[root@miles31 ~]# service network restart
#查看bond0状态
[root@miles31 ~]# cat /proc/net/bonding/bond0 
Ethernet Channel Bonding Driver: v3.6.0 (September 26, 2009)

Bonding Mode: load balancing (round-robin)
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: eth0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 00:50:56:3a:29:fc
Slave queue ID: 0

Slave Interface: eth2
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 1
Permanent HW addr: 00:50:56:24:16:fe
Slave queue ID: 0

2 设置私有网口的ip地址

#设置eth3
[root@miles31 network-scripts]# vi ifcfg-eth3
DEVICE=eth3
NM_CONTROLLED=yes
ONBOOT=yes
IPADDR=10.10.10.1
NETMASK=255.255.255.0
GATEWAY=192.168.137.1
BOOTPROTO=none
TYPE=Ethernet
IPV6INIT=no
USERCTL=no

#设置eth4
[root@miles31 network-scripts]# vi ifcfg-eth4
DEVICE=eth4
NM_CONTROLLED=yes
ONBOOT=yes
IPADDR=10.10.10.11
NETMASK=255.255.255.0
GATEWAY=192.168.137.1
BOOTPROTO=none
TYPE=Ethernet
IPV6INIT=no
USERCTL=no
#重启网络
[root@miles31 network-scripts]# service network restart

检查交换空间

[root@miles31 network-scripts]# free -m
             total       used       free     shared    buffers     cached
Mem:          1861        667       1194          0         24        165
-/+ buffers/cache:        477       1384
Swap:         4095          0       4095

检查物理内存（至少2Gb）

[root@miles31 network-scripts]# grep MemTotal /proc/meminfo 
MemTotal:        1906552 kB

检查临时目录大小（至少1GB）

[root@miles31 tmp]# df -h /tmp
Filesystem      Size  Used Avail Use% Mounted on
/dev/sda3        16G  9.7G  5.1G  66% /

查看操作系统架构

[root@miles31 tmp]# uname -m
x86_64

检查操作系统版本

#发行版6.5
[root@miles31 tmp]# lsb_release -a
LSB Version:    :base-4.0-amd64:base-4.0-noarch:core-4.0-amd64:core-4.0-noarch:graphics-4.0-amd64:graphics-4.0-noarch:printing-4.0-amd64:printing-4.0-noarch
Distributor ID: CentOS
Description:    CentOS release 6.5 (Final)
Release:        6.5
Codename:       Final

查看内核版本

[root@miles31 tmp]# uname -r
2.6.32-431.el6.x86_64

包检查（ＯＵＩ会提示）

binutils.x86_64 0:2.20.51.0.2-5.43.el6 
compat-libstdc++-33-3.2.3-69.el6.x86_64
compat-libstdc++-296-2.96-144.el6.i686
...

配置内核参数
shmmax:表示单个共享内存段的最大值，以字节为单位，一般为物理内存的一半，大一点也行。
shmmin:表示单个共享内存段的最小值，默认为1字节。
shmall:表示整个系统范围内可用内存页的总量，单位是页，在32位系统上一页等有4KB，也就是4096字节。计算公式为shmmax/PAGE_SIZE。
shmmni:表示整个系统范围内内存段的最大数量，一般为4096。
file-max:表示文件句柄的最大数量。
ip_local_port_range:表示端口的范围。在监听器帮组客户端进程和服务器进程建立连接时，会用到指定范围内的端口。范围从9000-65500.
sem:表示设置的信号量，semmsl、semmns、semopm、semmni这四个参数大小固定。
rmem_default:表示接收套接字缓冲区大小的默认值（字节）。
rmem_max:表示接收套接字缓冲区大小的最大值（字节）。
wmem_default:表示发送套接字缓冲区大小的默认值（字节）。
wmem_max:表示发送套接字缓冲区大小的最大值（字节）。

[root@miles31 ~]# vi /etc/sysctl.conf
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = 310972
kernel.shmmax = 1273741824
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048586
#使生效
[root@miles31 ~]# sysctl -p

设置时间同步
可以使用NTP和CTSS。NTP为第三方提供的时间同步机制。CTSS是oracle自己提供的，作为GI的一部分进行安装，并被OHAS启动。
这里使用CTSS，所以禁用NTP

[root@miles31 ~]# ps -ef | grep ntp

[root@miles31 ~]# service ntpd stop

[root@miles31 ~]# chkconfig ntpd off
#移走NTP的配置文件
[root@miles31 ~]# mv /etc/ntp.conf /etc/ntp.confbak
删除NTP的进程ID文件
[root@miles31 ~]# rm /var/run/ntpd.pid

SSH的设置

#修改参数LoginGraceTime，防止远程节点关闭SSH连接
#LoginGraceTime：表示如果用户不能成功登陆，在切断连接之前服务器等等的时间（秒），0表示没有限制
[root@miles31 ~]# vi /etc/ssh/sshd_config 
LoginGraceTime 0

关闭linux安全增强

[root@miles31 ~]# vi /etc/selinux/config 
SELINUX=disabled

启用Name Service Cache Daemon（nscd）
nscd用于缓存命名服务相关的数据，使oracle集群能够更好的容忍网络失败。如果使用NAS或NFS，则要启用nscd。

[root@miles31 ~]# chkconfig --list nscd
nscd            0:off   1:off   2:off   3:off   4:off   5:off   6:off
#设置启动级别，使nscd能伴随系统一起启动
[root@miles31 ~]# chkconfig --level 35 nscd on
[root@miles31 ~]# service nscd start
Starting nscd:                                             [  OK  ]
[root@miles31 ~]# service nscd status
nscd (pid 11953) is running...

安装ASMLib
ASMLib用于标记分区，简化磁盘管理，是ASM的辅助。被ASMLib标记过的分区才能被ASM使用。

ASMLib安装和配置

#查看操作系统内核版本
[root@miles31 software]# uname -rm
2.6.32-431.el6.x86_64 x86_64
#下载需要的包
#下载地址：http://www.oracle.com/technetwork/server-storage/linux/asmlib/rhel6-1940776.html
[root@miles31 software]# ll
total 2648
-rw-r--r--. 1 root root   13300 Dec 19 15:52 oracleasmlib-2.0.4-1.el6.x86_64.rpm
-rw-r--r--. 1 root root   74984 Dec 19 15:52 oracleasm-support-2.1.8-1.el6.x86_64.rpm
#oracle不再支持el6以后的asm包，所以这里使用RedHat提供的包
[root@miles31 software]# yum install kmod-oracleasm
[root@miles31 software]# rpm -Uvh oracleasm*
#配置ASMLib,通过该配置实现ASMLib配置文件/etc/sysconfig/oracleasm和加载点/dev/oracleasm的创建
[root@miles31 software]# /usr/sbin/oracleasm configure -i
Configuring the Oracle ASM library driver.
...
Default user to own the driver interface []: grid
Default group to own the driver interface []: dba
Start Oracle ASM library driver on boot (y/n) [n]: y
Scan for Oracle ASM disks on boot (y/n) [y]: y
Writing Oracle ASM library driver configuration: done

创建用户和组

#创建组
[root@miles31 dev]# groupadd -g 1000 oinstall
[root@miles31 dev]# groupadd -g 1031 dba
[root@miles31 dev]# groupadd -g 1020 asmdba
#查看组是否存在
[root@miles31 dev]# grep oinstall /etc/group
oinstall:x:1000:
[root@miles31 dev]# grep dba /etc/group
dba:x:1031:
asmdba:x:1020:

#创建用户
[root@miles31 software]# useradd -u 1101 -g oinstall -G dba,asmdba oracle
[root@miles31 software]# useradd -u 1100 -g oinstall -G dba,asmdba grid
#查看用户是否存在
[root@miles31 software]# id -a oracle
uid=1101(oracle) gid=1000(oinstall) groups=1031(dba),1020(asmdba),1000(oinstall)
[root@miles31 software]# id -a grid
uid=1100(grid) gid=1000(oinstall) groups=1031(dba),1020(asmdba),1000(oinstall)

环境变量配置

#grid用户
[root@miles31 software]# su - grid
[grid@miles31 ~]$ vi .bash_profile 
export PATH
export EDITOR=vi
export ORACLE_SID=+ASM1
export ORACLE_BASE=/u01/11g/grid_base
export ORACLE_HOME=/u01/11g/grid_home
export GRID_HOME=$ORACLE_HOME
export GRID_BASE=$ORACLE_BASE
export ORACLE_TERM=xterm
export ORACLE_HOSTNAME=rac1.milesdb.com
export DB_UNIQUE_NAME=milesdb
export CVUQDISK_GRP=oinstall
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export THREADS_FLAG=native
export PATH=$ORACLE_HOME/bin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/X11R6/bin
umask 022
#oracle用户
[grid@miles31 ~]$ su - oracle
Password: 
[oracle@miles31 ~]$ vi .bash_profile 
export EDITOR=vi
export ORACLE_TERM=vt100
export ORACLE_HOSTNAME=rac1.milesdb.com
export DB_UNIQUE_NAME=milesdb
export CVUQDISK_GRP=oinstall
export ORACLE_SID=prod1
export ORACLE_BASE=/u01/11g/db_base
export ORACLE_HOME=$ORACLE_BASE/db_home
export LD_LIBRARY_PATH=$ORACLE_HOME/lib
export PATH=$ORACLE_HOME/bin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/X11R6/bin
umask 022

修改用户资源限制

#nofile：打开文件的最大数量（推荐：软限制>=1024;硬限制>=65536)
#nproc:一个用户的有效进程数（推荐值：软限制>=2047;硬限制>=16384）
#stack ：表示堆栈段尺寸（推荐值：软限制>=10240KB;硬限制>=10240KB）
[root@miles31 software]# vi /etc/security/limits.conf 
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
oracle soft stack 10240
grid soft nproc 2047
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
grid soft stack 10240
#查看oracle用户打开的最大进程数
[oracle@miles31 ~]$ ulimit -Hu
16384
[oracle@miles31 ~]$ ulimit -Su
2047
#最大文件数
[oracle@miles31 ~]$ ulimit -Hn
65536
[oracle@miles31 ~]$ ulimit -Sn
1024
#堆栈设置
[oracle@miles31 ~]$ ulimit -Hs
unlimited
[oracle@miles31 ~]$ ulimit -Ss
10240

下载并解压安装文件
oracle11g相关文件下载地址

cvuqdisk的安装
如果没有cvuqdisk，集群验证工具CVU将不能发现共享磁盘，并报告”Package cvuqdisk not installed”。

[root@miles31 u01]# cd database/
[root@miles31 database]# cd rpm/
[root@miles31 rpm]# ll
total 8
-rwxr-xr-x 1 root root 8173 Jul 15  2009 cvuqdisk-1.0.7-1.rpm
[root@miles31 rpm]# rpm -ivh cvuqdisk-1.0.7-1.rpm 
Preparing...                ########################################### [100%]
Using default group oinstall to install package
   1:cvuqdisk               ########################################### [100%]

rac2节点设置
与rac1类似，只需修改ip，hostname，环境变量即可。

创建ASMLib磁盘

#rac1节点
[root@miles31 u01]# service oracleasm
Usage: /etc/init.d/oracleasm {start|stop|restart|enable|disable|configure|createdisk|deletedisk|querydisk|listdisks|scandisks|status}
#列出操作系统分区
[root@miles31 u01]#  fdisk -l |grep sdc
Disk /dev/sdc: 21.5 GB, 21474836480 bytes
/dev/sdc1               1         132     1060258+  83  Linux
/dev/sdc2             133         264     1060290   83  Linux
/dev/sdc3             265         396     1060290   83  Linux
/dev/sdc4             397        2610    17783955    5  Extended
/dev/sdc5             397        1115     5775336   83  Linux
/dev/sdc6            1116        1834     5775336   83  Linux
/dev/sdc7            1835        2096     2104483+  83  Linux
/dev/sdc8            2097        2358     2104483+  83  Linux
#标记磁盘
[root@miles31 u01]# oracleasm createdisk OCR_VOTE1 /dev/sdc1 
Writing disk header: done
Instantiating disk: done
[root@miles31 u01]# oracleasm createdisk OCR_VOTE2 /dev/sdc2
Writing disk header: done
Instantiating disk: done
[root@miles31 u01]# oracleasm createdisk OCR_VOTE3 /dev/sdc3
Writing disk header: done
Instantiating disk: done
[root@miles31 u01]# oracleasm createdisk DATA1 /dev/sdc5
Writing disk header: done
Instantiating disk: done
[root@miles31 u01]# oracleasm createdisk DATA2 /dev/sdc6
Writing disk header: done
Instantiating disk: done
[root@miles31 u01]# oracleasm createdisk RCY1 /dev/sdc7
Writing disk header: done
Instantiating disk: done
[root@miles31 u01]# oracleasm createdisk RCY2 /dev/sdc8
Writing disk header: done
Instantiating disk: done
#列出ASM磁盘和物理设备的对应关系
[root@miles31 u01]# oracleasm listdisks | xargs oracle querydisk -p
xargs: oracle: No such file or directory
You have new mail in /var/spool/mail/root
[root@miles31 u01]# oracleasm listdisks | xargs oracleasm querydisk -p
Disk "DATA1" is a valid ASM disk
/dev/sdc5: LABEL="DATA1" TYPE="oracleasm" 
Disk "DATA2" is a valid ASM disk
/dev/sdc6: LABEL="DATA2" TYPE="oracleasm" 
Disk "OCR_VOTE1" is a valid ASM disk
/dev/sdc1: LABEL="OCR_VOTE1" TYPE="oracleasm" 
Disk "OCR_VOTE2" is a valid ASM disk
/dev/sdc2: LABEL="OCR_VOTE2" TYPE="oracleasm" 
Disk "OCR_VOTE3" is a valid ASM disk
/dev/sdc3: LABEL="OCR_VOTE3" TYPE="oracleasm" 
Disk "RCY1" is a valid ASM disk
/dev/sdc7: LABEL="RCY1" TYPE="oracleasm" 
Disk "RCY2" is a valid ASM disk
/dev/sdc8: LABEL="RCY2" TYPE="oracleasm" 
[root@miles31 u01]# oracleasm querydisk -p DATA1
Disk "DATA1" is a valid ASM disk
/dev/sdc5: LABEL="DATA1" TYPE="oracleasm" 
#rac2节点
#扫描
[root@miles32 ~]# oracleasm scandisks
[root@miles32 ~]# oracleasm listdisks
DATA1
DATA2
OCR_VOTE1
OCR_VOTE2
OCR_VOTE3
RCY1
RCY2

目录及权限

[root@miles31 u01]# mkdir -p /u01/11g/grid_base
[root@miles31 u01]# mkdir -p /u01/11g/grid_home
[root@miles31 u01]# mkdir -p /u01/11g/db_base/db_home

[root@miles31 u01]# chown grid:oinstall /u01/11g
[root@miles31 u01]# chown -R grid:oinstall /u01/11g/grid_*
[root@miles31 u01]# chown -R oracle:oinstall /u01/11g/db_base

[root@miles31 u01]# chmod -R 755 /u01/11g

VNC的配置
服务端VNC配置

[root@miles31 ~]# yum install vnc-server
#启动VNC
#1对应端口号为5901
#第一次启动会创建配置文件/root/.vnc/xstartup
[root@miles31 ~]# vncserver :1
A VNC server is already running as :1
#查看VNC状态
[root@miles31 ~]# ps -ef |grep vnc
root       2585      1  0 19:07 ?        00:00:00 /usr/bin/Xvnc :1 -desktop miles31:1 (root) -httpd /usr/share/vnc/classes -auth /root/.Xauthority -geometry 1920x1080 -rfbwait 30000 -rfbauth /root/.vnc/passwd -rfbport 5901 -fp catalogue:/etc/X11/fontpath.d -pn
#修改密码
[root@miles31 ~]# vncpasswd
#设置自动启动
[root@miles31 ~]# chkconfig vncserver on
[root@miles31 ~]# chkconfig --list vncserver
vncserver       0:off   1:off   2:on    3:on    4:on    5:on    6:off

#如果需要可以修改VNC使用的桌面类型
[root@miles31 ~]# vi /root/.vnc/xstartup 
    #修改最后一行改为“startkde &” 可以使用KDE桌面。
#修改后重启
[root@miles31 ~]# vncserver -kill :1
Killing Xvnc process ID 2585
[root@miles31 ~]# vncserver :1

New 'miles31:1 (root)' desktop is miles31:1

Starting applications specified in /root/.vnc/xstartup
Log file is /root/.vnc/miles31:1.log

客户端登录
请自行安装VNC viewer

节点间对等访问
因为安装期间，OUI使用ssh和scp远程运行命令，因此需要配置对等访问。其实，在11g中，OUI界面会自动帮助我们建立对等，但是这里还是介绍一下手动配置。

#rac1
[grid@miles31 ~]$ mkdir ~/.ssh
[grid@miles31 ~]$ chmod -R 700 .ssh
#一路回车
[grid@miles31 ~]$ ssh-keygen -t dsa
#之后在oracle用户，和rac2节点进行相同操作

#rac1
[grid@miles31 ~]$ cd .ssh/
[grid@miles31 .ssh]$ cat id_dsa.pub >> authorized_keys
[grid@miles31 .ssh]$ scp authorized_keys miles32:/home/grid/.ssh/
grid@miles32's password: 
authorized_keys                                                                                                          100%  602     0.6KB/s   00:00    

#rac2
[grid@miles32 ~]$ cd .ssh/
#把rac2节点的key内容追加到authorized_keys中
[grid@miles32 .ssh]$ cat id_dsa.pub >> authorized_keys 
[grid@miles32 .ssh]$ scp authorized_keys miles31:/home/grid/.ssh/
The authenticity of host 'miles31 (192.168.137.31)' can't be established.
RSA key fingerprint is c3:c6:9b:8a:c7:5e:80:95:44:f6:ff:60:8e:65:44:48.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'miles31,192.168.137.31' (RSA) to the list of known hosts.
grid@miles31's password: 
authorized_keys                                                                                                          100% 1204     1.2KB/s   00:00    

#oracle用户进行相同操作

#测试连通性
#rac1-》rac2
[grid@miles31 grid]$ ssh rac2 date
The authenticity of host 'rac2 (192.168.137.32)' can't be established.
RSA key fingerprint is c3:c6:9b:8a:c7:5e:80:95:44:f6:ff:60:8e:65:44:48.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'rac2' (RSA) to the list of known hosts.
Tue Dec 22 13:42:20 CST 2015
#rac1-》rac1
[grid@miles31 grid]$ ssh rac1 date
The authenticity of host 'rac1 (192.168.137.31)' can't be established.
RSA key fingerprint is c3:c6:9b:8a:c7:5e:80:95:44:f6:ff:60:8e:65:44:48.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'rac1' (RSA) to the list of known hosts.
Tue Dec 22 13:42:02 CST 2015
#rac2-》rac1
[grid@miles32 ~]$ ssh rac1 date
The authenticity of host 'rac1 (192.168.137.31)' can't be established.
RSA key fingerprint is c3:c6:9b:8a:c7:5e:80:95:44:f6:ff:60:8e:65:44:48.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'rac1' (RSA) to the list of known hosts.
Tue Dec 22 13:43:12 CST 2015
#rac2-》rac2
[grid@miles32 ~]$ ssh rac2 date
The authenticity of host 'rac2 (192.168.137.32)' can't be established.
RSA key fingerprint is c3:c6:9b:8a:c7:5e:80:95:44:f6:ff:60:8e:65:44:48.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'rac2' (RSA) to the list of known hosts.
Tue Dec 22 13:43:22 CST 2015


#oracle用户执行相同操作

安装GI前最后检查
使用CVU（集群验证工具）进行检查。

#安装目录执行
[grid@miles31 grid]$ ./runcluvfy.sh stage -pre crsinst -n miles31,miles32 -fixup -verbose

安装集群软件GI
VNC连接

[root@miles31 ~]# who am i
root     pts/2        2015-12-22 11:22 (:1.)
[root@miles31 ~]# export DISPLAY=localhost:1.0
[root@miles31 ~]# xhost +
[root@miles31 ~]# su - grid
#进入软件安装目录
[grid@miles31 grid]$ ./runInstaller 

[root@miles31 ~]# /u01/11g/oraInventory/orainstRoot.sh 
Changing permissions of /u01/11g/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.

Changing groupname of /u01/11g/oraInventory to oinstall.
The execution of the script is complete.

[root@miles32 u01]# /u01/11g/oraInventory/orainstRoot.sh 
Creating the Oracle inventory pointer file (/etc/oraInst.loc)
Changing permissions of /u01/11g/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.

Changing groupname of /u01/11g/oraInventory to oinstall.
The execution of the script is complete.

[root@miles31 software]# /u01/11g/grid_home/root.sh
...
Checking swap space: must be greater than 500 MB.   Actual 3958 MB    Passed
The inventory pointer is located at /etc/oraInst.loc
The inventory is located at /u01/11g/oraInventory
'UpdateNodeList' was successful.

[root@miles32 software]# /u01/11g/grid_home/root.sh
...
Checking swap space: must be greater than 500 MB.   Actual 4077 MB    Passed
The inventory pointer is located at /etc/oraInst.loc
The inventory is located at /u01/11g/oraInventory

执行root.sh脚本遇到的问题
1

#执行root.sh时
Using configuration parameter file: /u01/11g/grid_home/crs/install/crsconfig_params
Creating trace directory
/u01/11g/grid_home/bin/clscfg.bin: error while loading shared libraries: libcap.so.1: cannot open shared object file: No such file or directory
Failed to create keys in the OLR, rc = 127, 32512
OLR configuration failed
#下载compat-libcap1相关包进行安装
[root@miles32 u01]# rpm -ivh compat-libcap1-1.10-1.x86_64.rpm 
Preparing...                ########################################### [100%]
   1:compat-libcap1         ########################################### [100%]
#重新执行root.sh前，需要执行rootcrs.pl -delete -force -verbose删除配置
[root@miles31 ~]# /u01/11g/grid_home/crs/install/rootcrs.pl -delete -force -verbose

2

#执行root.sh时
Adding daemon to inittab
CRS-4124: Oracle High Availability Services startup failed.
CRS-4000: Command Start failed, or completed with errors.
ohasd failed to start: Inappropriate ioctl for device
ohasd failed to start at /u01/11g/grid_home/crs/install/rootcrs.pl line 443.

#首先删除配置
[root@miles31 ~]# /u01/11g/grid_home/crs/install/rootcrs.pl -deconfig  -verbose -force
#重新执行root.sh
[root@miles31 ~]# /u01/11g/grid_home/root.sh
#注意在root.sh执行过程中，打开另一个窗口，不断刷下面的命令，直到能执行；并且在root.sh执行过程中，这个命令保持运行
[root@miles31 ~]# /bin/dd if=/var/tmp/.oracle/npohasd of=/dev/null bs=1024 count=1
/bin/dd: opening `/var/tmp/.oracle/npohasd': No such file or directory
[root@miles31 ~]# /bin/dd if=/var/tmp/.oracle/npohasd of=/dev/null bs=1024 count=1
/bin/dd: opening `/var/tmp/.oracle/npohasd': No such file or directory
[root@miles32 ~]# /bin/dd if=/var/tmp/.oracle/npohasd of=/dev/null bs=1024 count=1

#当root.sh,显示如下信息时，就可以停止上面的命令了
Checking swap space: must be greater than 500 MB.   Actual 3958 MB    Passed
The inventory pointer is located at /etc/oraInst.loc
The inventory is located at /u01/11g/oraInventory
'UpdateNodeList' was successful.

#这个过程在每个节点都要重复，这个11.2.0.1 RAC在linux6.1以上安装时的一个bug

GI安装完毕后

#每个节点上检查集群状态
[grid@miles31 ~]$ crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online

[grid@miles32 11g]$ crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online

#备份root.sh脚本

#查看vip使用情况
#rac1上51、52为两个scan vip，41为普通vip
[root@miles31 grid_home]# ifconfig -a
bond0     Link encap:Ethernet  HWaddr 00:50:56:3A:29:FC  
          inet addr:192.168.137.31  Bcast:192.168.137.255  Mask:255.255.255.0
          inet6 addr: fe80::250:56ff:fe3a:29fc/64 Scope:Link
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
          RX packets:1948843 errors:0 dropped:0 overruns:0 frame:0
          TX packets:2655206 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:2488038059 (2.3 GiB)  TX bytes:3820934957 (3.5 GiB)

bond0:1   Link encap:Ethernet  HWaddr 00:50:56:3A:29:FC  
          inet addr:192.168.137.41  Bcast:192.168.137.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

bond0:3   Link encap:Ethernet  HWaddr 00:50:56:3A:29:FC  
          inet addr:192.168.137.51  Bcast:192.168.137.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

bond0:4   Link encap:Ethernet  HWaddr 00:50:56:3A:29:FC  
          inet addr:192.168.137.52  Bcast:192.168.137.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
...
#rac2上53为scan vip；42为普通vip
bond0     Link encap:Ethernet  HWaddr 00:50:56:22:E9:F9  
          inet addr:192.168.137.32  Bcast:192.168.137.255  Mask:255.255.255.0
          inet6 addr: fe80::250:56ff:fe22:e9f9/64 Scope:Link
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1
          RX packets:2610390 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1915950 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:3784279547 (3.5 GiB)  TX bytes:2485947496 (2.3 GiB)

bond0:1   Link encap:Ethernet  HWaddr 00:50:56:22:E9:F9  
          inet addr:192.168.137.53  Bcast:192.168.137.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

bond0:2   Link encap:Ethernet  HWaddr 00:50:56:22:E9:F9  
          inet addr:192.168.137.42  Bcast:192.168.137.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MASTER MULTICAST  MTU:1500  Metric:1

#查看监听使用情况
#rac1上两个scan监听，一个本地监听
[root@miles31 grid_home]#  ps -ef |grep -i LISTENE
grid      15085      1  0 12:58 ?        00:00:00 /u01/11g/grid_home/bin/tnslsnr LISTENER_SCAN2 -inherit
grid      15096      1  0 12:59 ?        00:00:00 /u01/11g/grid_home/bin/tnslsnr LISTENER_SCAN3 -inherit
grid      15561      1  0 13:04 ?        00:00:00 /u01/11g/grid_home/bin/tnslsnr LISTENER -inherit
root      17007   3243  0 13:36 pts/0    00:00:00 grep -i LISTENE
#rac2上一个本地、一个scan监听
[root@miles32 grid_home]# ps -ef |grep -i LISTENE
grid      13198      1  0 13:03 ?        00:00:00 /u01/11g/grid_home/bin/tnslsnr LISTENER_SCAN1 -inherit
grid      13618      1  0 13:04 ?        00:00:00 /u01/11g/grid_home/bin/tnslsnr LISTENER -inherit
root      14678   3257  0 13:35 pts/1    00:00:00 grep -i LISTENE

创建ASM实例
安装完GI后，接下来要安装oracle RAC软件和创建数据库。由于数据放在ASM上，所以要先建好ASM和磁盘组。这里用ASMCA创建。

#在刚才的VNC窗口中执行
[grid@miles31 ~]$ asmca

#查看diskgroup状态，两个节点都查看一下
[grid@miles31 ~]$ sqlplus / as sysasm
...
SQL> select name,state from v$asm_diskgroup;

NAME                           STATE
------------------------------ -----------
OCR_VOTE                       MOUNTED
DATA                           MOUNTED
RCY                            MOUNTED

安装Oracle RAC软件

#安装前，使用CVU进行检查
[oracle@miles31 grid]$ ./runcluvfy.sh stage -pre dbinst -fixup -n miles31,miles32 -osdba dba -verbose

#连接VNC进行安装
[oracle@miles31 database]$ ./runInstaller 

执行脚本

[root@miles31 rpm]# /u01/11g/oraInventory/orainstRoot.sh
Changing permissions of /u01/11g/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.

Changing groupname of /u01/11g/oraInventory to oinstall.
The execution of the script is complete.

[root@miles32 rpm]# /u01/11g/oraInventory/orainstRoot.sh
Creating the Oracle inventory pointer file (/etc/oraInst.loc)
Changing permissions of /u01/11g/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.

Changing groupname of /u01/11g/oraInventory to oinstall.
The execution of the script is complete.

[root@miles31 rpm]# /u01/11g/db_base/db_home/root.sh
Running Oracle 11g root.sh script...

The following environment variables are set as:
    ORACLE_OWNER= oracle
    ORACLE_HOME=  /u01/11g/db_base/db_home

Enter the full pathname of the local bin directory: [/usr/local/bin]: 
The file "dbhome" already exists in /usr/local/bin.  Overwrite it? (y/n) 
[n]: y
   Copying dbhome to /usr/local/bin ...
The file "oraenv" already exists in /usr/local/bin.  Overwrite it? (y/n) 
[n]: y
   Copying oraenv to /usr/local/bin ...
The file "coraenv" already exists in /usr/local/bin.  Overwrite it? (y/n) 
[n]: y
   Copying coraenv to /usr/local/bin ...

Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root.sh script.
Now product-specific root actions will be performed.
Finished product-specific root actions.

[root@miles32 rpm]# /u01/11g/db_base/db_home/root.sh
Running Oracle 11g root.sh script...

The following environment variables are set as:
    ORACLE_OWNER= oracle
    ORACLE_HOME=  /u01/11g/db_base/db_home

Enter the full pathname of the local bin directory: [/usr/local/bin]: 
The file "dbhome" already exists in /usr/local/bin.  Overwrite it? (y/n) 
[n]: y
   Copying dbhome to /usr/local/bin ...
The file "oraenv" already exists in /usr/local/bin.  Overwrite it? (y/n) 
[n]: y
   Copying oraenv to /usr/local/bin ...
The file "coraenv" already exists in /usr/local/bin.  Overwrite it? (y/n) 
[n]: y
   Copying coraenv to /usr/local/bin ...

Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root.sh script.
Now product-specific root actions will be performed.
Finished product-specific root actions.

创建数据库

#使用dbca创建
[oracle@miles31 db_home]$ dbca

安装完成后

#查看实例状态
[grid@miles31 ~]$ srvctl status database -d milesdb
Instance milesdb1 is running on node miles31
Instance milesdb2 is running on node miles32