As the root user, install the device-mapper-multipath package using the yum package manager[root@vzwc1 ~]# cat /etc/redhat-release Red Hat Enterprise Linux Server release 6.4 (Santiago)[root@vzwc1 ~]# uname -px86_64[root@vzwc1 ~]# yum install device-mapper-multipath Copy the multipath.conf file found within /usr/share/doc/device-mapper-multipath-0.4.9/ to /etc/[root@vzwc1 ~]# cp /usr/share/doc/device-mapper-multipath-0.4.9/multipath.conf /etc/multipath.conf Capture the scsi id of the local disks on the system.[root@vzwc2 ~]# for i in b c d e f g h i j> do> scsi_id –whitelisted –replace-whitespace –device=/dev/sd$i> done1ATA_VBOX_HARDDISK_VBda7290c1-228b4bfb1ATA_VBOX_HARDDISK_VBcb76adf7-43d27c311ATA_VBOX_HARDDISK_VB2077c81c-7cfe59221ATA_VBOX_HARDDISK_VB7282bfc8-696017261ATA_VBOX_HARDDISK_VBa0072176-9e9ab0a01ATA_VBOX_HARDDISK_VB2a7b18e2-040538061ATA_VBOX_HARDDISK_VB40939671-89967d5e1ATA_VBOX_HARDDISK_VB664cb850-2065525c1ATA_VBOX_HARDDISK_VBa498ea69-53cdb110[root@vzwc2 ~]# Enter the above values into /etc/multipath and give friendly aliases to uuid[root@vzwc1 ~]# cat /etc/multipath.confblacklist { devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*" devnode "^hd[a-z]" wwid 1ATA_VBOX_HARDDISK_VB82d3887f-a892e124 #exclude sda wwid}defaults { user_friendly_names no getuid_callout "/sbin/scsi_id –whitelisted –replace-whitespace –device=/dev/%n"}multipaths { multipath { wwid 1ATA_VBOX_HARDDISK_VBda7290c1-228b4bfb alias ocrvotedisk1 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VBcb76adf7-43d27c31 alias ocrvotedisk2 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VB2077c81c-7cfe5922 alias ocrvotedisk3 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VB7282bfc8-69601726 alias ocrvotedisk4 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VBa0072176-9e9ab0a0 alias ocrvotedisk5 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VB2a7b18e2-04053806 alias data1 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VB40939671-89967d5e alias data2 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VB664cb850-2065525c alias fra1 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 } multipath { wwid 1ATA_VBOX_HARDDISK_VBa498ea69-53cdb110 alias fra2 path_grouping_policy multibus path_selector "round-robin 0" failback immediate rr_weight priorities no_path_retry 5 }} Restart multipathd service[root@vzwc1 ~]# service multipathd restartokStopping multipathd daemon: [ OK ]Starting multipathd daemon: [ OK ][root@vzwc1 ~]# chkconfig multipathd on Check if multipath is setup properly[root@vzwc2 ~]# multipath -llfra2 (1ATA_VBOX_HARDDISK_VBa498ea69-53cdb110) dm-8 ATA,VBOX HARDDISKsize=12G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 11:0:0:0 sdj 8:144 active ready runningocrvotedisk5 (1ATA_VBOX_HARDDISK_VBa0072176-9e9ab0a0) dm-4 ATA,VBOX HARDDISKsize=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 7:0:0:0 sdf 8:80 active ready runningfra1 (1ATA_VBOX_HARDDISK_VB664cb850-2065525c) dm-7 ATA,VBOX HARDDISKsize=12G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 10:0:0:0 sdi 8:128 active ready runningocrvotedisk4 (1ATA_VBOX_HARDDISK_VB7282bfc8-69601726) dm-3 ATA,VBOX HARDDISKsize=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 6:0:0:0 sde 8:64 active ready runningocrvotedisk3 (1ATA_VBOX_HARDDISK_VB2077c81c-7cfe5922) dm-2 ATA,VBOX HARDDISKsize=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 5:0:0:0 sdd 8:48 active ready runningocrvotedisk2 (1ATA_VBOX_HARDDISK_VBcb76adf7-43d27c31) dm-1 ATA,VBOX HARDDISKsize=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 4:0:0:0 sdc 8:32 active ready runningocrvotedisk1 (1ATA_VBOX_HARDDISK_VBda7290c1-228b4bfb) dm-0 ATA,VBOX HARDDISKsize=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 3:0:0:0 sdb 8:16 active ready runningdata2 (1ATA_VBOX_HARDDISK_VB40939671-89967d5e) dm-6 ATA,VBOX HARDDISKsize=20G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 9:0:0:0 sdh 8:112 active ready runningdata1 (1ATA_VBOX_HARDDISK_VB2a7b18e2-04053806) dm-5 ATA,VBOX HARDDISKsize=20G features='1 queue_if_no_path' hwhandler='0' wp=rw-+- policy='round-robin 0' prio=1 status=active – 8:0:0:0 sdg 8:96 active ready running [root@vzwc1 ~]# ls -l /dev/mapper/total 0crw-rw—- 1 root root 10, 236 Sep 24 13:56 controllrwxrwxrwx 1 root root 7 Sep 24 13:57 data1 -> ../dm-5lrwxrwxrwx 1 root root 7 Sep 24 13:57 data2 -> ../dm-6lrwxrwxrwx 1 root root 7 Sep 24 13:56 fra1 -> ../dm-7lrwxrwxrwx 1 root root 7 Sep 24 13:56 fra2 -> ../dm-8lrwxrwxrwx 1 root root 7 Sep 24 13:57 ocrvotedisk1 -> ../dm-0lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk2 -> ../dm-1lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk3 -> ../dm-2lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk4 -> ../dm-3lrwxrwxrwx 1 root root 7 Sep 24 13:56 ocrvotedisk5 -> ../dm-4 [root@vzwc1 ~]# for i in ocrvotedisk1 ocrvotedisk2 ocrvotedisk3 ocrvotedisk4 ocrvotedisk5 data1 data2 fra1 fra2> do> printf "%s %sn" "$i" "$(ls -ll /dev/mapper/$i)"> doneocrvotedisk1 lrwxrwxrwx 1 root root 7 Sep 24 13:57 /dev/mapper/ocrvotedisk1 -> ../dm-0ocrvotedisk2 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk2 -> ../dm-1ocrvotedisk3 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk3 -> ../dm-2ocrvotedisk4 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk4 -> ../dm-3ocrvotedisk5 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/ocrvotedisk5 -> ../dm-4data1 lrwxrwxrwx 1 root root 7 Sep 24 13:57 /dev/mapper/data1 -> ../dm-5data2 lrwxrwxrwx 1 root root 7 Sep 24 13:57 /dev/mapper/data2 -> ../dm-6fra1 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/fra1 -> ../dm-7fra2 lrwxrwxrwx 1 root root 7 Sep 24 13:56 /dev/mapper/fra2 -> ../dm-8Determine partitioned alias for target device[root@vzwc1 ~]# dmsetup ls|sortdata1 (252:5)data2 (252:6)fra1 (252:7)fra2 (252:8)ocrvotedisk1 (252:0)ocrvotedisk2 (252:1)ocrvotedisk3 (252:2)ocrvotedisk4 (252:3)ocrvotedisk5 (252:4) Edit /etc/udev/rules.d/12-dm-permissions.rules file to set the above device ownership as grid:asmadmin[root@vzwc1 ~]# cat /etc/udev/rules.d/12-dm-permissions.rulesENV{DM_NAME}=="ocrvotedisk1", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="ocrvotedisk2", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="ocrvotedisk3", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="ocrvotedisk4", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="ocrvotedisk5", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="data1", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="data2", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="fra1", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"ENV{DM_NAME}=="fra2", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}" Restart multipathd service[root@vzwc1 ~]# /etc/init.d/multipathd restartokStopping multipathd daemon: [ OK ]Starting multipathd daemon: [ OK ] Check if permissions are correct[root@vzwc1 ~]# ls -l /dev/dm-*brw-rw—- 1 grid asmadmin 252, 0 Sep 24 16:35 /dev/dm-0brw-rw—- 1 grid asmadmin 252, 1 Sep 24 16:35 /dev/dm-1brw-rw—- 1 grid asmadmin 252, 2 Sep 24 16:35 /dev/dm-2brw-rw—- 1 grid asmadmin 252, 3 Sep 24 16:35 /dev/dm-3brw-rw—- 1 grid asmadmin 252, 4 Sep 24 16:35 /dev/dm-4brw-rw—- 1 grid asmadmin 252, 5 Sep 24 16:35 /dev/dm-5brw-rw—- 1 grid asmadmin 252, 6 Sep 24 16:35 /dev/dm-6brw-rw—- 1 grid asmadmin 252, 7 Sep 24 16:35 /dev/dm-7brw-rw—- 1 grid asmadmin 252, 8 Sep 24 16:35 /dev/dm-8 Check if oracle ASM symbolic links exists for multipath devices[root@vzwc2 ~]# ls -l /dev/iscsi/*lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-data1 -> ../dm-5lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-data2 -> ../dm-6lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-fra1 -> ../dm-7lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-fra2 -> ../dm-8lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk1 -> ../dm-0lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk2 -> ../dm-1lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk3 -> ../dm-2lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk4 -> ../dm-3lrwxrwxrwx 1 root root 7 Sep 24 16:38 /dev/iscsi/oraasm-ocrvotedisk5 -> ../dm-4 Configure ASM disk change discovery path!1756814819288-ec5a3bdc-1278-446d-979e-32b1151aafcf.jpg
[grid@vzwc2 ~]$ crsctl query css votedisk## STATE File Universal Id File Name Disk group– —– —————– ——— ——— 1. ONLINE 1374f266ca874f15bfde7ca23d073e5f (/dev/mapper/ocrvotedisk1) [CRSDG] 2. ONLINE 4145a926ff0d4f6dbf63e2c3695c580e (/dev/mapper/ocrvotedisk2) [CRSDG] 3. ONLINE 6f3ab4bbe0c14f97bfb20c91820d898b (/dev/mapper/ocrvotedisk3) [CRSDG] 4. ONLINE 8c973dd3f1da4f0dbf2080e04fcf9bd7 (/dev/mapper/ocrvotedisk4) [CRSDG] 5. ONLINE 1bed4a3c6f624fabbfbe2564e81df250 (/dev/mapper/ocrvotedisk5) [CRSDG]Located 5 voting disk(s).[grid@vzwc2 ~]$ sqlplus / as sysasmSQL*Plus: Release 11.2.0.4.0 Production on Wed Sep 24 17:17:22 2014Copyright (c) 1982, 2013, Oracle. All rights reserved.
Connected to:Oracle Database 11g Enterprise Edition Release 11.2.0.4.0 – 64bit ProductionWith the Real Application Clusters and Automatic Storage Management optionsSQL> select * from v$active_instances;INST_NUMBER INST_NAME———– ——————– 1 vzwc1:+ASM1 2 vzwc2:+ASM2SQL> select path from v$asm_disk;PATH——————————————————————————–/dev/mapper/data2/dev/mapper/fra2/dev/mapper/fra1/dev/mapper/data1/dev/mapper/ocrvotedisk2/dev/mapper/ocrvotedisk3/dev/mapper/ocrvotedisk1/dev/mapper/ocrvotedisk4/dev/mapper/ocrvotedisk59 rows selected. .九月 24th, 2014 | Tags: ASM, Oracle | Category: ASM, Database, Oracle, RAC