Wednesday 27 May 2015

Solaris 11.2 breaks my zpool import of clone with missing cache disk

1.       Assign disks to machine
root@prodmachine:~# sanlun lun show |grep zpooltest
zamgnasvm01          /vol/unixprod_vol99_zpooltest/unixprod_vol99_zpooltest_data01  /dev/rdsk/c0t600A0980383034716124465434593156d0s2 qlc3       FCP        1g      C
zamgnasvm01          /vol/unixprod_vol99_zpooltest/unixprod_vol99_zpooltest_data02  /dev/rdsk/c0t600A0980383034716124465434593157d0s2 qlc2       FCP        1g      C
zamgnasvm01          /vol/unixprod_vol99_zpooltest/unixprod_vol99_zpooltest_log01   /dev/rdsk/c0t600A0980383034716124465434593158d0s2 qlc2       FCP        1g      C
zamgnasvm01          /vol/unixprod_vol99_zpooltest/unixprod_vol99_zpooltest_cache01 /dev/rdsk/c0t600A0980383034716124465434593159d0s2 qlc1       FCP        1g      C
root@prodmachine:~#

2.       Create a zpool
root@prodmachine:~# zpool status zpooltest
  pool: zpooltest
 state: ONLINE
  scan: none requested
config:

        NAME                                       STATE     READ WRITE CKSUM
        zpooltest                                  ONLINE       0     0     0
          mirror-0                                 ONLINE       0     0     0
            c0t600A0980383034716124465434593156d0  ONLINE       0     0     0
            c0t600A0980383034716124465434593157d0  ONLINE       0     0     0
        logs
          c0t600A0980383034716124465434593158d0    ONLINE       0     0     0
        cache
          c0t600A0980383034716124465434593159d0    ONLINE       0     0     0

errors: No known data errors
root@prodmachine:~#

3.       Use SAN replication to replicate disks to remote machine
4.       Check remote machine’s BE
root@drmachine:~# beadm list
BE                 Active Mountpoint Space  Policy Created
--                 ------ ---------- -----  ------ -------
solaris-5          -      -          53.83M static 2014-07-08 12:43
solaris-5-backup-1 NR     /          11.78G static 2014-10-13 14:34
solaris-7          -      -          15.05G static 2015-02-26 14:43
solaris-8          -      -          1.77G  static 2015-04-08 10:39
root@drmachine:~# pkg list entire
NAME (PUBLISHER)                                  VERSION                    IFO
entire                                            0.5.11-0.175.1.19.0.6.0    i--
root@drmachine:~#
5.       Only assign the data lun (i.e. no mirror, no log, no cache)
root@drmachine:~# sanlun lun show |grep zpooltest
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_data01 /dev/rdsk/c0t600A0980443355634D3F46644573306Fd0s2 qlc2       FCP        1g      C
root@drmachine:~#
6.       Import the pool (with –f and –m options) RESULT: SUCCESS
root@drmachine:~# zpool import -R / -f -m zpooltest
root@drmachine:~# zpool status zpooltest
  pool: zpooltest
 state: DEGRADED
status: One or more devices are unavailable in response to persistent errors.
        Sufficient replicas exist for the pool to continue functioning in a
        degraded state.
action: Determine if the device needs to be replaced, and clear the errors
        using 'zpool clear' or 'fmadm repaired', or replace the device
        with 'zpool replace'.
        Run 'zpool status -v' to see device specific details.
  scan: none requested
config:

        NAME                                       STATE     READ WRITE CKSUM
        zpooltest                                  DEGRADED     0     0     0
          mirror-0                                 DEGRADED     0     0     0
            c0t600A0980443355634D3F46644573306Fd0  ONLINE       0     0     0
            4167588681570226322                    UNAVAIL      0     0     0
        logs
          16316075685438834122                     UNAVAIL      0     0     0
        cache
          c0t600A0980383034716124465434593159d0    UNAVAIL      0     0     0

errors: No known data errors
root@drmachine:~#
7.       Change to different BE and reboot
root@drmachine:~# beadm activate solaris-7
root@drmachine:~# init 6
root@drmachine:~# beadm list
BE                 Active Mountpoint Space   Policy Created
--                 ------ ---------- -----   ------ -------
solaris-5          -      -          53.83M  static 2014-07-08 12:43
solaris-5-backup-1 -      -          1.45G   static 2014-10-13 14:34
solaris-7          NR     /          25.94G  static 2015-02-26 14:43
solaris-8          -      -          1.77G   static 2015-04-08 10:39
root@drmachine:~# pkg list entire
NAME (PUBLISHER)                                  VERSION                    IFO
entire                                            0.5.11-0.175.2.5.0.5.0     i--
root@drmachine:~#
8.       Import the pool (with –f and –m) RESULT: FAILURE
root@drmachine:~# zpool import -R / -f -m zpooltest
cannot import 'zpooltest': one or more devices is currently unavailable
root@drmachine:~#
9.       Assign the mirror disk to the machine and retry import RESULT: FAILURE
root@drmachine:~# sanlun lun show
controller(7mode)/                                                                                          device                                            host                  lun
vserver(Cmode)       lun-pathname                                                                           filename                                          adapter    protocol   size    mode
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_data01 /dev/rdsk/c0t600A0980443355634D3F46644573306Fd0s2 qlc2       FCP        1g      C
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_data02 /dev/rdsk/c0t600A0980443355634D3F466445733070d0s2 qlc0       FCP        1g      C
root@drmachine:~# zpool import -R / -f -m zpooltest
cannot import 'zpooltest': one or more devices is currently unavailable
root@drmachine:~#
10.   Assign the log device disk to the machine and retry import RESULT: SUCCESS
root@drmachine:~# sanlun lun show
controller(7mode)/                                                                                          device                                            host                  lun
vserver(Cmode)       lun-pathname                                                                           filename                                          adapter    protocol   size    mode
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_data01 /dev/rdsk/c0t600A0980443355634D3F46644573306Fd0s2 qlc2       FCP        1g      C
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_data02 /dev/rdsk/c0t600A0980443355634D3F466445733070d0s2 qlc0       FCP        1g      C
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_log01  /dev/rdsk/c0t600A0980443355634D3F466445733071d0s2 qlc0       FCP        1g      C
root@drmachine:~# zpool import -R / -f -m zpooltest
root@drmachine:~#
11.    Export  the zpool, unassign the log device and assign the cache device disk to the machine and retry import. RESULT: FAILURE
root@drmachine:~# sanlun lun show
controller(7mode)/                                                                                           device                                            host                  lun
vserver(Cmode)       lun-pathname                                                                            filename                                          adapter    protocol   size    mode
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_data01  /dev/rdsk/c0t600A0980443355634D3F46644573306Fd0s2 qlc2       FCP        1g      C
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_data02  /dev/rdsk/c0t600A0980443355634D3F466445733070d0s2 qlc0       FCP        1g      C
zamgnasvmdr01        /vol/zamgnasvm01_unixprod_vol99_zpooltest_mirror_CLONE/unixprod_vol99_zpooltest_cache01 /dev/rdsk/c0t600A0980443355634D3F466445733072d0s2 qlc0       FCP        1g      C
root@drmachine:~# zpool import -R / -f -m zpooltest
cannot import 'zpooltest': one or more devices is currently unavailable
root@drmachine:~#




Got a call logged with Oracle to sort this out. Fixed with Solaris 11.3.