トップ 差分 一覧 ソース 検索 ヘルプ PDF RSS ログイン

FreeBSD gmirror 障害復旧手順例 201805

  • FreeBSD-10 でディスク障害が発生した。
  • FreeBSD gmirror GPT 3T HDD トリプルミラーでの障害
  • 障害復旧手順の記録
  • zfs ではない

作業概要

  • gmirror ada0 ada1 ada2 のトリプルミラー
  • ada1 が故障
  • ada1 を入れ替え
  • ada1 のパーティション設定
  • ada1 の起動コード書き込み
  • トリプルミラー再開

FreeBSD

  • /boot/loader.conf
geom_mirror_load="YES"
  • /etc/fstab
# Device        Mountpoint      FStype  Options Dump    Pass#
/dev/mirror/p2  /               ufs     rw      1       1
/dev/mirror/p3  none            swap    sw      0       0
  • gmirror 正常時
root@ns4:/home/hoge # gmirror status -s
mirror/p1  COMPLETE  ada0p1 (ACTIVE)
mirror/p1  COMPLETE  ada2p1 (ACTIVE)
mirror/p1  COMPLETE  ada1p1 (ACTIVE)
mirror/p2  COMPLETE  ada0p2 (ACTIVE)
mirror/p2  COMPLETE  ada2p2 (ACTIVE)
mirror/p2  COMPLETE  ada1p2 (ACTIVE)
mirror/p3  COMPLETE  ada0p3 (ACTIVE)
mirror/p3  COMPLETE  ada2p3 (ACTIVE)
mirror/p3  COMPLETE  ada1p3 (ACTIVE)

作業記録

状況確認

  • gmirror 故障状況
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  DEGRADED  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
mirror/p2  DEGRADED  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
mirror/p3  DEGRADED  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
  • パーティション情報
root@ns4:/home/hoge # gpart list
Geom name: ada0
modified: false
state: OK
fwheads: 16
fwsectors: 63
last: 5860533134
first: 34
entries: 128
scheme: GPT
Providers:
1. Name: ada0p1
  Mediasize: 65536 (64K)
  Sectorsize: 512
  Stripesize: 4096
  Stripeoffset: 0
  Mode: r1w1e1
  rawuuid: fd22e7d8-f409-11e3-9b6a-0022cfe54d22
  rawtype: 83bd6b9d-7f41-11dc-be0b-001560b84f0f
  label: (null)
  length: 65536
  offset: 20480
  type: freebsd-boot
  index: 1
  end: 167
  start: 40
2. Name: ada0p2
  Mediasize: 2996813361152 (2.7T)
  Sectorsize: 512
  Stripesize: 4096
  Stripeoffset: 0
  Mode: r1w1e1
  rawuuid: fd2317ba-f409-11e3-9b6a-0022cfe54d22
  rawtype: 516e7cb6-6ecf-11d6-8ff8-00022d09712b
  label: (null)
  length: 2996813361152
  offset: 86016
  type: freebsd-ufs
  index: 2
  end: 5853151263
  start: 168
3. Name: ada0p3
  Mediasize: 3779517440 (3.5G)
  Sectorsize: 512
  Stripesize: 4096
  Stripeoffset: 0
  Mode: r1w1e1
  rawuuid: fd25cafb-f409-11e3-9b6a-0022cfe54d22
  rawtype: 516e7cb5-6ecf-11d6-8ff8-00022d09712b
  label: (null)
  length: 3779517440
  offset: 2996813447168
  type: freebsd-swap
  index: 3
  end: 5860533133
  start: 5853151264
Consumers:
1. Name: ada0
   Mediasize: 3000592982016 (2.7T)
   Sectorsize: 512
   Stripesize: 4096
   Stripeoffset: 0
   Mode: r3w3e6

Geom name: ada2
modified: false
state: OK
fwheads: 16
fwsectors: 63
last: 5860533134
first: 34
entries: 128
scheme: GPT
Providers:
1. Name: ada2p1
  Mediasize: 65536 (64K)
  Sectorsize: 512
  Stripesize: 4096
  Stripeoffset: 0
  Mode: r1w1e1
  rawuuid: 461f6d0d-f3c7-11e3-a841-0022cfe54d22
  rawtype: 83bd6b9d-7f41-11dc-be0b-001560b84f0f
  label: (null)
  length: 65536
  offset: 20480
  type: freebsd-boot
  index: 1
  end: 167
  start: 40
2. Name: ada2p2
  Mediasize: 2996813361152 (2.7T)
  Sectorsize: 512
  Stripesize: 4096
  Stripeoffset: 0
  Mode: r1w1e1
  rawuuid: 461fe080-f3c7-11e3-a841-0022cfe54d22
  rawtype: 516e7cb6-6ecf-11d6-8ff8-00022d09712b
  label: (null)
  length: 2996813361152
  offset: 86016
  type: freebsd-ufs
  index: 2
  end: 5853151263
  start: 168
3. Name: ada2p3
  Mediasize: 3779517440 (3.5G)
  Sectorsize: 512
  Stripesize: 4096
  Stripeoffset: 0
  Mode: r1w1e1
  rawuuid: 468c041d-f3c7-11e3-a841-0022cfe54d22
  rawtype: 516e7cb5-6ecf-11d6-8ff8-00022d09712b
  label: (null)
  length: 3779517440
  offset: 2996813447168
  type: freebsd-swap
  index: 3
  end: 5860533133
  start: 5853151264
Consumers:
1. Name: ada2
  Mediasize: 3000592982016 (2.7T)
  Sectorsize: 512
  Stripesize: 4096
  Stripeoffset: 0
  Mode: r3w3e6

GPT パーティション情報の確認

  • ada0
root@ns4:/home/hoge # gpart show ada0
=>        34  5860533101  ada0  GPT  (2.7T)
          34           6        - free -  (3.0K)
          40         128     1  freebsd-boot  (64K)
         168  5853151096     2  freebsd-ufs  (2.7T)
  5853151264     7381870     3  freebsd-swap  (3.5G)
  5860533134           1        - free -  (512B)
  • ada2
root@ns4:/home/hoge # gpart show ada2
=>        34  5860533101  ada2  GPT  (2.7T)
          34           6        - free -  (3.0K)
          40         128     1  freebsd-boot  (64K)
         168  5853151096     2  freebsd-ufs  (2.7T)
  5853151264     7381870     3  freebsd-swap  (3.5G)
  5860533134           1        - free -  (512B)

ada1 設定

ada1 GPT パーティション作成

  • ada1 GPT パーティションなしを確認
root@ns4:/home/hoge # gpart show ada1
gpart: No such geom: ada1.
  • ada1 GPT 管理情報作成
root@ns4:/home/hoge # gpart create -s gpt /dev/ada1
ada1 created
  • ada1 GPT 初期状況確認
root@ns4:/home/hoge # gpart show /dev/ada1
=>        34  5860533101  ada1  GPT  (2.7T)
         34  5860533101        - free -  (2.7T)

ada1 FreeBSD pmbr インストール

pmbr
Proactive Master Boot Record
root@ns4:/home/hoge # ls -l /boot/pmbr
-r--r--r--  1 root  wheel  512 Jan 17  2014 /boot/pmbr
root@ns4:/home/hoge # gpart bootcode -b /boot/pmbr ada1
bootcode written to ada1
  • MBR ツールによる破壊を防止するための疑似MBR情報+GPT起動コード

ada1 FreeBSD bootcode インストール

  • コマンド
ls -l /boot/gptboot
gpart add -b 40 -s 128 -t freebsd-boot ada1
gpart bootcode -p /boot/gptboot -i 1 ada1
  • ada1 GPT 情報より freebsd-boot 領域のサイズは 128
  • ログ
root@ns4:/home/hoge # ls -l /boot/gptboot
-r--r--r--  1 root  wheel  16231 Feb 21 22:02 /boot/gptboot
root@ns4:/home/hoge # gpart show ada1
=>        34  5860533101  ada1  GPT  (2.7T)
          34  5860533101        - free -  (2.7T)
root@ns4:/home/hoge # ls -l /boot/gptboot
-r--r--r--  1 root  wheel  16231 Feb 21 22:02 /boot/gptboot
root@ns4:/home/hoge # gpart add -b 40 -s 128 -t freebsd-boot ada1
ada1p1 added
root@ns4:/home/hoge # gpart bootcode -p /boot/gptboot -i 1 ada1
partcode written to ada1p1
root@ns4:/home/hoge # gpart show ada1
=>        34  5860533101  ada1  GPT  (2.7T)
          34           6        - free -  (3.0K)
          40         128     1  freebsd-boot  (64K)
         168  5860532967        - free -  (2.7T)
  • (参考) GPT パーティションの除去
  • コマンド
gpart delete -i 1 ada1
  • ログ
root@ns4:/home/hoge # gpart show ada1
=>        34  5860533101  ada1  GPT  (2.7T)
          34           6        - free -  (3.0K)
          40  5860533088     1  freebsd-boot  (2.7T)
  5860533128           7        - free -  (3.5K)
root@ns4:/home/hoge # gpart delete -i 1 ada1
ada1p1 deleted
root@ns4:/home/hoge # gpart show ada1
=>        34  5860533101  ada1  GPT  (2.7T)
          34  5860533101        - free -  (2.7T)

ada1 GPT パーティション作成

  • コマンド
gpart add -t freebsd-ufs -s 5853151096 ada1
gpart add -t freebsd-swap -s 7381870 -a 2 ada1
  • ログ
root@ns4:/home/hoge # gpart add -t freebsd-ufs -s 5853151096 ada1
ada1p2 added
root@ns4:/home/hoge # gpart add -t freebsd-swap -s 7381870 -a 2 ada1
ada1p3 added
root@ns4:/home/hoge # gpart show ada1
=>        34  5860533101  ada1  GPT  (2.7T)
          34           6        - free -  (3.0K)
          40         128     1  freebsd-boot  (64K)
         168  5853151096     2  freebsd-ufs  (2.7T)
  5853151264     7381870     3  freebsd-swap  (3.5G)
  5860533134           1        - free -  (512B)
  • ada0 ada2 と同じパーティション情報を ada1 に作成した。
  • gpart backup と gpart restore でパーティション情報を復旧するべき(要確認)

GPT パーティション状況確認

gpart show ada0
gpart show ada1
gpart show ada2
  • 同じパーティション情報であることを確認する。

gmirror 再構築

  • 単純に insert はできない。
root@ns4:/home/hoge # gmirror insert p1 /dev/ada1p1
gmirror: Not all disks connected.
  • DEGRADED 確認
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  DEGRADED  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
mirror/p2  DEGRADED  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
mirror/p3  DEGRADED  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
  • p1 DEGRADED 状況を解消
root@ns4:/home/hoge # gmirror forget p1
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  COMPLETE  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
mirror/p2  DEGRADED  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
mirror/p3  DEGRADED  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
  • p1 に ada1p1 を追加
root@ns4:/home/hoge # gmirror insert p1 /dev/ada1p1
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  COMPLETE  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
                     ada1p1 (ACTIVE)
mirror/p2  DEGRADED  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
mirror/p3  DEGRADED  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
  • p3 に ada1p3 を追加
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  COMPLETE  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
                     ada1p1 (ACTIVE)
mirror/p2  DEGRADED  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
mirror/p3  DEGRADED  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
root@ns4:/home/hoge # gmirror forget p3
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  COMPLETE  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
                     ada1p1 (ACTIVE)
mirror/p2  DEGRADED  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
mirror/p3  COMPLETE  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
  • p3 が DEGRADED から COMPLETE に変わった。
root@ns4:/home/hoge # gmirror insert p3 /dev/ada1p3
  • 新規ディスクの ada1p3 を p3 に追加
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  COMPLETE  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
                     ada1p1 (ACTIVE)
mirror/p2  DEGRADED  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
mirror/p3  DEGRADED  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
                     ada1p3 (SYNCHRONIZING, 3%)
  • p3 の同期が開始された。
  • p3 同期中
root@ns4:/home/hoge # gmirror status p3
     Name    Status  Components
mirror/p3  DEGRADED  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
                     ada1p3 (SYNCHRONIZING, 41%)
  • p3 同期完了
root@ns4:/home/hoge # gmirror status p3
     Name    Status  Components
mirror/p3  COMPLETE  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
                     ada1p3 (ACTIVE)
  • メッセージ確認
> grep GEOM_MIRROR /var/log/messages
May 28 18:10:30 ns4 kernel: GEOM_MIRROR: Device p1: rebuilding provider ada1p1.
May 28 18:10:30 ns4 kernel: GEOM_MIRROR: Device p1: rebuilding provider ada1p1 finished.
May 28 18:28:14 ns4 kernel: GEOM_MIRROR: Device p3: rebuilding provider ada1p3.
May 28 18:29:11 ns4 kernel: GEOM_MIRROR: Device p3: rebuilding provider ada1p3 finished.
May 28 18:32:14 ns4 kernel: GEOM_MIRROR: Device p2: rebuilding provider ada1p2.
May 29 03:29:06 ns4 kernel: GEOM_MIRROR: Device p2: rebuilding provider ada1p2 finished.
  • 復旧完了状況確認
root@ns4:/home/hoge # gmirror status
     Name    Status  Components
mirror/p1  COMPLETE  ada0p1 (ACTIVE)
                     ada2p1 (ACTIVE)
                     ada1p1 (ACTIVE)
mirror/p2  COMPLETE  ada0p2 (ACTIVE)
                     ada2p2 (ACTIVE)
                     ada1p2 (ACTIVE)
mirror/p3  COMPLETE  ada0p3 (ACTIVE)
                     ada2p3 (ACTIVE)
                     ada1p3 (ACTIVE)

起動確認

  • 3台のHDDを入れ替えて3台とも起動できることを確認する。
  • shutdown
  • HDD1-->HDD2-->HDD3-->HDD1
  • 起動確認
  • shutdown
  • HDD1-->HDD2-->HDD3-->HDD1
  • 起動確認
  • shutdown
  • HDD1-->HDD2-->HDD3-->HDD1
  • 起動確認
  • pmbr は正常か? bootcode は正常か?