1. Исходные данные и описание задачи

Текущая схема сети:


Задача: Для исходной схемы DMVPN (Phase 2) необходимо подключить SPOKE-2, который использует 2 интерфейса для выхода в интернет с помощью MultiWAN, для доступа к локальной подсети 192.0.2.128/25. В качестве динамической маршрутизации в DMVPN необходимо использовать BGP. Firewall отключен.

Исходные данные: Для выхода в интернет SPOKE-2 провайдер выделил IP-адреса из подсети  203.0.113.12/30 и 203.0.113.8/30. MultiWAN будет использоваться в режиме балансировки.

Используемые алгоритмы для IKE SA:

 - IKE version: 1
 - Authentication algorithm: sha2-384
 - Encryption algorithm: aes256cbc
 - DH-group 21

Используемые алгоритмы для IPsec SA:

 - Authentication algorithm: sha2-256
 - Encryption algorithm: aes256cbc
 - PFS DH-group 19
 - Protocol: ESP

Исходная конфигурация HUB:

HUB config 
hostname HUB

router bgp log-neighbor-changes

router bgp 65000
  router-id 198.51.100.1
  peer-group Cloud1
    remote-as 65000
    route-reflector-client
    update-source 198.51.100.1
    address-family ipv4 unicast
      enable
    exit
  exit
  listen-range 198.51.100.0/25
    peer-group Cloud1
    enable
  exit
  enable
exit

interface gigabitethernet 1/0/1
  ip firewall disable
  ip address 203.0.113.2/30
exit

tunnel gre 1
  key 10
  ttl 255
  mtu 1400
  multipoint
  ip firewall disable
  local address 203.0.113.2
  ip address 198.51.100.1/25
  ip tcp adjust-mss 1340
  ip nhrp ipsec ipsec_for_spokes dynamic
  ip nhrp multicast dynamic
  ip nhrp enable
  enable
exit

security ike proposal ike_proposal
  authentication algorithm sha2-384
  encryption algorithm aes256
  dh-group 21
exit

security ike policy ike_policy
  pre-shared-key ascii-text encrypted 8CB5107EA7005AFF
  proposal ike_proposal
exit

security ike gateway ike_for_spokes
  ike-policy ike_policy
  local address 203.0.113.2
  local network 203.0.113.2/32 protocol gre 
  remote address any
  remote network any protocol gre 
  mode policy-based
exit

security ipsec proposal ipsec_proposal
  authentication algorithm sha2-256
  encryption algorithm aes256
  pfs dh-group 19
exit

security ipsec policy ipsec_policy
  proposal ipsec_proposal
exit

security ipsec vpn ipsec_for_spokes
  type transport
  ike establish-tunnel route
  ike gateway ike_for_spokes
  ike ipsec-policy ipsec_policy
  enable
exit

ip route 0.0.0.0/0 203.0.113.1

Исходная конфигурация SPOKE-1:

SPOKE-1 config 
hostname SPOKE-1

router bgp log-neighbor-changes

router bgp 65000
  router-id 198.51.100.2
  neighbor 198.51.100.1
    description "Cloud_1"
    remote-as 65000
    address-family ipv4 unicast
      enable
    exit
    enable
  exit
  address-family ipv4 unicast
    network 192.0.2.0/25
  exit
  enable
exit

interface gigabitethernet 1/0/1
  ip firewall disable
  ip address 203.0.113.6/30
exit
interface gigabitethernet 1/0/3
  ip firewall disable
  ip address 192.0.2.1/25
exit

tunnel gre 1
  key 10
  ttl 255
  mtu 1400
  multipoint
  ip firewall disable
  local address 203.0.113.6
  ip address 198.51.100.2/25
  ip tcp adjust-mss 1340
  ip nhrp holding-time 90
  ip nhrp map 198.51.100.1 203.0.113.2
  ip nhrp nhs 198.51.100.1
  ip nhrp ipsec ipsec_for_hub static
  ip nhrp ipsec ipsec_for_spokes dynamic
  ip nhrp multicast nhs
  ip nhrp enable
  enable
exit

security ike proposal ike_proposal
  authentication algorithm sha2-384
  encryption algorithm aes256
  dh-group 21
exit

security ike policy ike_policy
  pre-shared-key ascii-text encrypted 8CB5107EA7005AFF
  proposal ike_proposal
exit

security ike gateway ike_for_hub
  ike-policy ike_policy
  local address 203.0.113.6
  local network 203.0.113.6/32 protocol gre 
  remote address 203.0.113.2
  remote network 203.0.113.2/32 protocol gre 
  mode policy-based
exit
security ike gateway ike_for_spokes
  ike-policy ike_policy
  local address 203.0.113.6
  local network 203.0.113.6/32 protocol gre 
  remote address any
  remote network any protocol gre 
  mode policy-based
exit

security ipsec proposal ipsec_proposal
  authentication algorithm sha2-256
  encryption algorithm aes256
  pfs dh-group 19
exit

security ipsec policy ipsec_policy
  proposal ipsec_proposal
exit

security ipsec vpn ipsec_for_hub
  type transport
  ike establish-tunnel route
  ike gateway ike_for_hub
  ike ipsec-policy ipsec_policy
  enable
exit
security ipsec vpn ipsec_for_spokes
  type transport
  ike establish-tunnel route
  ike gateway ike_for_spokes
  ike ipsec-policy ipsec_policy
  enable
exit

ip route 0.0.0.0/0 203.0.113.5

Вывод оперативной информации со стороны HUB:

HUB 
HUB# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_spokes                  203.0.113.2       203.0.113.6       0xc67e4e0a9f3804c0   0xd9e59bb52a0bc755   Established   
HUB# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.2           203.0.113.6        gre 1       00:01:14    00,16:19:35      dynamic           LCP         
HUB# show bgp neighbors 
BGP neighbor is 198.51.100.2
    BGP state:                          Established
    Type:                               Dynamic neighbor
    Listen range prefix:                198.51.100.0/25
    Neighbor address:                   198.51.100.2
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.2
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop route-reflector AS4
    Source address:                     198.51.100.1
    Weight:                             10
    Hold timer:                         118/180
    Keepalive timer:                    38/60
    Peer group:                         Cloud1
    RR client:                          Yes
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Outgoing route-map:               out_to_Cloud1
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               Yes
    Uptime (d,h:m:s):                   00,16:37:26
HUB# show ip route bgp 
B     * 192.0.2.0/25       [170/0]           via 198.51.100.2 on gre 1         [bgp65000 18:47:58] (i)

Вывод оперативной информации со стороны SPOKE-1:

SPOKE-1 
SPOKE-1# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub                     203.0.113.6       203.0.113.2       0xc67e4e0a9f3804c0   0xd9e59bb52a0bc755   Established   
SPOKE-1# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:12      static            RULCP        
SPOKE-1# show bgp neighbors 
BGP neighbor is 198.51.100.1
    Description:                        Cloud_1
    BGP state:                          Established
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.1
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.1
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.2
    Weight:                             0
    Hold timer:                         118/180
    Keepalive timer:                    49/60
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,16:38:56


2. Решение задачи

2.1 Подключение интерфейсов SPOKE-2

Исходя из исходных данных и описания задачи необходимо ораганизовать подключение линков к WAN и к LAN сетям на SPOKE-2 согласно следующей схеме:

Настроим интерфейсы на SPOKE-2 согласно схеме:

Более подробная настройка MultiWAN описана в статье Технология MultiWAN

SPOKE-2# configure terminal 
SPOKE-2(config)# 
SPOKE-2(config)# interface gigabitethernet 1/0/1
SPOKE-2(config-if-gi)# ip firewall disable
SPOKE-2(config-if-gi)# ip address 203.0.113.14/30
SPOKE-2(config-if-gi)# wan load-balance nexthop 203.0.113.13
SPOKE-2(config-if-gi)# wan load-balance enable
SPOKE-2(config-if-gi)# exit
SPOKE-2(config)# interface gigabitethernet 1/0/2
SPOKE-2(config-if-gi)# ip firewall disable
SPOKE-2(config-if-gi)# ip address 203.0.113.10/30
SPOKE-2(config-if-gi)# wan load-balance nexthop 203.0.113.9
SPOKE-2(config-if-gi)# wan load-balance enable
SPOKE-2(config-if-gi)# exit
SPOKE-2(config)# interface gigabitethernet 1/0/3
SPOKE-2(config-if-gi)# ip firewall disable
SPOKE-2(config-if-gi)# ip address 192.0.2.129/25
SPOKE-2(config-if-gi)# exit
SPOKE-2(config)# ip route 0.0.0.0/0 wan load-balance rule 1
SPOKE-2(config)# 
SPOKE-2(config)# wan load-balance rule 1
SPOKE-2(config-wan-rule)# outbound interface gigabitethernet 1/0/1
SPOKE-2(config-wan-rule)# outbound interface gigabitethernet 1/0/2
SPOKE-2(config-wan-rule)# enable
SPOKE-2(config-wan-rule)# exit
SPOKE-2(config)# exit
SPOKE-2# commit
SPOKE-2# confirm

В результате получим следующую таблицу маршрутизации после подключения интерфейсов согласно схеме и наличии IP-связанности со всеми интерфейсами:

SPOKE-2# show ip route
Codes: C - connected, S - static, R - RIP derived,
       O - OSPF derived, IA - OSPF inter area route,
       E1 - OSPF external type 1 route, E2 - OSPF external type 2 route,
       B - BGP derived, D - DHCP derived, K - kernel route, V - VRRP route,
       i - IS-IS, L1 - IS-IS level-1, L2 - IS-IS level-2, ia - IS-IS inter area,
       H - NHRP, * - FIB route
S     * 0.0.0.0/0          [1/0]             multipath                         [static 19:17:10] 
                                             via 203.0.113.9 on gi1/0/2 weight 1
                                             via 203.0.113.13 on gi1/0/1 weight 1
C     * 203.0.113.8/30     [0/0]             dev gi1/0/2                       [direct 2025-05-14] 
C     * 192.0.2.128/25     [0/0]             dev gi1/0/3                       [direct 18:55:23] 
C     * 203.0.113.12/30    [0/0]             dev gi1/0/1                       [direct 2025-05-14] 


2.2 Подключение SPOKE-2 к Cloud1 и настройка local PBR

Поскольку на SPOKE-2 используются два интерфейса для выхода в WAN, то в сторону HUB для резервировани будет построенно два tunnel GRE, то есть добавится Cloud 2. На данный момент подключим SPOKE-2 к HUB в рамках Cloud 1 через интерфейс gi1/0/1 согласно следующей схеме:

Поскольку MultiWAN используется в режиме балансировки, то GRE-пакеты могут передаваться с любого интерфейса.
Для того, чтобы гарантировать передачу GRE-пакетов с интерфейсов, на которых используется соответствующий local address - необходимо настроить локальный PBR.
Локальный PBR включается с помощью команды ip local policy route-map <NAME. В приложенном примере снизу выделена конфигурация PBR.

1) Настроим local PBR:

SPOKE-2# configure terminal 
SPOKE-2(config)#
SPOKE-2(config)# ip access-list extended LOCAL_1
SPOKE-2(config-acl)# rule 1
SPOKE-2(config-acl-rule)# action permit
SPOKE-2(config-acl-rule)# match source-address 203.0.113.14 255.255.255.255
SPOKE-2(config-acl-rule)# enable
SPOKE-2(config-acl-rule)# exit
SPOKE-2(config-acl)# exit
SPOKE-2(config)# ip access-list extended LOCAL_2
SPOKE-2(config-acl)# rule 1
SPOKE-2(config-acl-rule)# action permit
SPOKE-2(config-acl-rule)# match source-address 203.0.113.10 255.255.255.255
SPOKE-2(config-acl-rule)# enable
SPOKE-2(config-acl-rule)# exit
SPOKE-2(config-acl)# exit
SPOKE-2(config)# 
SPOKE-2(config)# route-map PBR_LOCAL
SPOKE-2(config-route-map)# rule 1
SPOKE-2(config-route-map-rule)# match ip access-group LOCAL_1
SPOKE-2(config-route-map-rule)# action set ip next-hop verify-availability 203.0.113.13 1
SPOKE-2(config-route-map-rule)# exit
SPOKE-2(config-route-map)# rule 2
SPOKE-2(config-route-map-rule)# match ip access-group LOCAL_2
SPOKE-2(config-route-map-rule)# action set ip next-hop verify-availability 203.0.113.9 1
SPOKE-2(config-route-map-rule)# exit
SPOKE-2(config-route-map)# exit
SPOKE-2(config)# 
SPOKE-2(config)# ip local policy route-map PBR_LOCAL
SPOKE-2(config)# exit
SPOKE-2# 

2) Настроим tunnel gre 1 и router bgp 65000:

SPOKE-2# configure terminal 
SPOKE-2(config)# 
SPOKE-2(config)# tunnel gre 1
SPOKE-2(config-gre)# key 10
SPOKE-2(config-gre)# ttl 255
SPOKE-2(config-gre)# mtu 1400
SPOKE-2(config-gre)# multipoint
SPOKE-2(config-gre)# ip firewall disable
SPOKE-2(config-gre)# local address 203.0.113.14
SPOKE-2(config-gre)# ip address 198.51.100.3/25
SPOKE-2(config-gre)# ip tcp adjust-mss 1340
SPOKE-2(config-gre)# ip nhrp holding-time 90
SPOKE-2(config-gre)# ip nhrp map 198.51.100.1 203.0.113.2
SPOKE-2(config-gre)# ip nhrp nhs 198.51.100.1
SPOKE-2(config-gre)# ip nhrp multicast nhs
SPOKE-2(config-gre)# ip nhrp enable
SPOKE-2(config-gre)# enable
SPOKE-2(config-gre)# exit
SPOKE-2(config)# 
SPOKE-2(config)# router bgp 65000
SPOKE-2(config-bgp)# router-id 198.51.100.3
SPOKE-2(config-bgp)# neighbor 198.51.100.1
SPOKE-2(config-bgp-neighbor)# update-source 198.51.100.3 
SPOKE-2(config-bgp-neighbor)# description "Cloud_1"
SPOKE-2(config-bgp-neighbor)# remote-as 65000
SPOKE-2(config-bgp-neighbor)# address-family ipv4 unicast
SPOKE-2(config-bgp-neighbor-af)# enable
SPOKE-2(config-bgp-neighbor-af)# exit
SPOKE-2(config-bgp-neighbor)# enable
SPOKE-2(config-bgp-neighbor)# exit
SPOKE-2(config-bgp)# address-family ipv4 unicast
SPOKE-2(config-bgp-af)# network 192.0.2.128/25
SPOKE-2(config-bgp-af)# exit
SPOKE-2(config-bgp)# enable
SPOKE-2(config-bgp)# exit
SPOKE-2(config)# exit
SPOKE-2# 

3) Настроим IPsec:

SPOKE-2# configure terminal
SPOKE-2(config)# 
SPOKE-2(config)# security ike proposal ike_proposal
SPOKE-2(config-ike-proposal)# authentication algorithm sha2-384
SPOKE-2(config-ike-proposal)# encryption algorithm aes256
SPOKE-2(config-ike-proposal)# dh-group 21
SPOKE-2(config-ike-proposal)# exit
SPOKE-2(config)# 
SPOKE-2(config)# security ike policy ike_policy
SPOKE-2(config-ike-policy)# pre-shared-key ascii-text encrypted 8CB5107EA7005AFF
SPOKE-2(config-ike-policy)# proposal ike_proposal
SPOKE-2(config-ike-policy)# exit
SPOKE-2(config)# 
SPOKE-2(config)# security ike gateway ike_for_hub_cloud1
SPOKE-2(config-ike-gw)# ike-policy ike_policy
SPOKE-2(config-ike-gw)# local address 203.0.113.14
SPOKE-2(config-ike-gw)# local network 203.0.113.14/32 protocol gre 
SPOKE-2(config-ike-gw)# remote address 203.0.113.2
SPOKE-2(config-ike-gw)# remote network 203.0.113.2/32 protocol gre 
SPOKE-2(config-ike-gw)# mode policy-based
SPOKE-2(config-ike-gw)# exit
SPOKE-2(config)# 
SPOKE-2(config)# security ike gateway ike_for_spokes_cloud1
SPOKE-2(config-ike-gw)# ike-policy ike_policy
SPOKE-2(config-ike-gw)# local address 203.0.113.14
SPOKE-2(config-ike-gw)# local network 203.0.113.14/32 protocol gre 
SPOKE-2(config-ike-gw)# remote address any
SPOKE-2(config-ike-gw)# remote network any protocol gre 
SPOKE-2(config-ike-gw)# mode policy-based
SPOKE-2(config-ike-gw)# exit
SPOKE-2(config)# 
SPOKE-2(config)# security ipsec proposal ipsec_proposal
SPOKE-2(config-ipsec-proposal)# authentication algorithm sha2-256
SPOKE-2(config-ipsec-proposal)# encryption algorithm aes256
SPOKE-2(config-ipsec-proposal)# pfs dh-group 19
SPOKE-2(config-ipsec-proposal)# exit
SPOKE-2(config)# 
SPOKE-2(config)# security ipsec policy ipsec_policy
SPOKE-2(config-ipsec-policy)# proposal ipsec_proposal
SPOKE-2(config-ipsec-policy)# exit
SPOKE-2(config)# 
SPOKE-2(config)# security ipsec vpn ipsec_for_hub_cloud1
SPOKE-2(config-ipsec-vpn)# type transport
SPOKE-2(config-ipsec-vpn)# ike establish-tunnel route
SPOKE-2(config-ipsec-vpn)# ike gateway ike_for_hub_cloud1
SPOKE-2(config-ipsec-vpn)# ike ipsec-policy ipsec_policy
SPOKE-2(config-ipsec-vpn)# enable
SPOKE-2(config-ipsec-vpn)# exit
SPOKE-2(config)# security ipsec vpn ipsec_for_spokes_cloud1
SPOKE-2(config-ipsec-vpn)# type transport
SPOKE-2(config-ipsec-vpn)# ike establish-tunnel route
SPOKE-2(config-ipsec-vpn)# ike gateway ike_for_spokes_cloud1
SPOKE-2(config-ipsec-vpn)# ike ipsec-policy ipsec_policy
SPOKE-2(config-ipsec-vpn)# enable
SPOKE-2(config-ipsec-vpn)# exit
SPOKE-2(config)# 
SPOKE-2(config)# tunnel gre 1
SPOKE-2(config-gre)# ip nhrp ipsec ipsec_for_hub_cloud1 static
SPOKE-2(config-gre)# ip nhrp ipsec ipsec_for_spokes_cloud1 dynamic
SPOKE-2(config-gre)# exit
SPOKE-2(config)# exit
SPOKE-2# 
SPOKE-2# commit
SPOKE-2# confirm

Убедимся, что после применения конфигурации и наличии IP-связанности между SPOKE и HUB построился IPsec-туннель, SPOKE зарегистрировался на HUB, BGP построился и SPOKE-2 получил маршрутную информацию от HUB:
Вывод оперативной информации со стороны SPOKE-2:

SPOKE-2# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub_cloud1              203.0.113.14      203.0.113.2       0x330c380391230fb5   0x12ffc921858ba965   Established  
SPOKE-2# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:09      static            RULCP       
SPOKE-2# show bgp neighbors 
BGP neighbor is 198.51.100.1
    Description:                        Cloud_1
    BGP state:                          Established
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.1
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.1
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.3
    Weight:                             0
    Hold timer:                         168/180
    Keepalive timer:                    26/60
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:14:11
SPOKE-2# show bgp ipv4 unicast neighbor 198.51.100.1 routes 
Status codes: u - unicast, b - broadcast, m - multicast, a - anycast
              * - valid, > - best
Origin codes: i - IGP, e - EGP, ? - incomplete

     Network              Next Hop             Metric  LocPrf      Weight Path        
*> u 192.0.2.0/25         198.51.100.2         --      100         0      i
SPOKE-2# show ip route bgp 
B     * 192.0.2.0/25       [170/0]           via 198.51.100.2 on gre 1         [bgp65000 09:52:16 from 198.51.100.1] (i)

Вывод оперативной информации со стороны HUB:

HUB# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_spokes                  203.0.113.2       203.0.113.14      0x330c380391230fb5   0x12ffc921858ba965   Established   
ipsec_for_spokes                  203.0.113.2       203.0.113.6       0xc67e4e0a9f3804c0   0xd9e59bb52a0bc755   Established   
HUB# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.2           203.0.113.6        gre 1       00:01:14    00,16:19:35      dynamic           LCP          
198.51.100.3           203.0.113.14       gre 1       00:01:06    00,00:21:34      dynamic           LCP          
HUB# show bgp neighbors 
BGP neighbor is 198.51.100.2
    BGP state:                          Established
    Type:                               Dynamic neighbor
    Listen range prefix:                198.51.100.0/25
    Neighbor address:                   198.51.100.2
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.2
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop route-reflector AS4
    Source address:                     198.51.100.1
    Weight:                             10
    Hold timer:                         118/180
    Keepalive timer:                    38/60
    Peer group:                         Cloud1
    RR client:                          Yes
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Outgoing route-map:               out_to_Cloud1
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               Yes
    Uptime (d,h:m:s):                   00,16:37:26
BGP neighbor is 198.51.100.3
    BGP state:                          Established
    Type:                               Dynamic neighbor
    Listen range prefix:                198.51.100.0/25
    Neighbor address:                   198.51.100.3
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.3
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop route-reflector AS4
    Source address:                     198.51.100.1
    Weight:                             10
    Hold timer:                         144/180
    Keepalive timer:                    8/60
    Peer group:                         Cloud1
    RR client:                          Yes
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Outgoing route-map:               out_to_Cloud1
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               Yes
    Uptime (d,h:m:s):                   00,00:21:39
HUB# show ip route bgp 
B     * 192.0.2.0/25       [170/0]           via 198.51.100.2 on gre 1         [bgp65000 18:47:58] (i)
B     * 192.0.2.128/25     [170/0]           via 198.51.100.3 on gre 1         [bgp65000 09:44:19] (i)

Поскольку DMVPN Cloud 1 работает в режиме Phase 2, то пустим пинг от SPOKE-2 в сторону SPOKE-1 и убедимся, что построится IPsec-туннель и динамический GRE-туннель между двумя SPOKE.
Вывод оперативной информации со стороны SPOKE-2:

SPOKE-2# ping 192.0.2.1
PING 192.0.2.1 (192.0.2.1) 56 bytes of data.
!!!!!
--- 192.0.2.1 ping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 4005ms
rtt min/avg/max/mdev = 1.268/1.686/3.213/0.765 ms
SPOKE-2# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub_cloud1              203.0.113.14      203.0.113.2       0x330c380391230fb5   0x12ffc921858ba965   Established   
ipsec_for_spokes_cloud1           203.0.113.14      203.0.113.6       0x5dbbfd62a68b683f   0x87e71980a245d90d   Established  
SPOKE-2# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:24      static            RULCP        
198.51.100.2           203.0.113.6        gre 1       00:01:13    00,00:00:16      cached            ULCP     

Вывод оперативной информации со стороны SPOKE-1:

SPOKE-1# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub                     203.0.113.6       203.0.113.2       0xc67e4e0a9f3804c0   0xd9e59bb52a0bc755   Established   
ipsec_for_spokes                  203.0.113.6       203.0.113.14      0x5dbbfd62a68b683f   0x87e71980a245d90d   Established   
SPOKE-1# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:01      static            RULCP        
198.51.100.3           203.0.113.14       gre 1       00:01:10    00,00:00:19      cached            ULCP


2.3 Настройка Cloud 2 между SPOKE-2 и HUB для резервирования

Cloud 1 является основным для подключения всех SPOKE и работает в рамках Phase 2. Поскольку со стороны SPOKE-2 используется 2-ой интерфейс для резервирования, то необходимо настроить tunnel gre 2, а именно организовать Cloud 2.
Cloud 2 будет работать исключительно в рамках Phase 1, когда трафик ходит только через HUB между SPOKE, поскольку Cloud 1 не сможет динамически построить туннели с Cloud 2. Схема будет выглядить следующим образом:


1) Настроим tunnel gre 2 и IPsec на HUB и на SPOKE.
Конфигурация IPsec на HUB будет использоваться исходная. Конфигурация tunnel gre 2 на HUB будет следующей:

HUB# configure terminal 
HUB(config)#
HUB(config)# tunnel gre 2
HUB(config-gre)# key 20
HUB(config-gre)# ttl 255
HUB(config-gre)# mtu 1400
HUB(config-gre)# multipoint
HUB(config-gre)# ip firewall disable
HUB(config-gre)# local address 203.0.113.2
HUB(config-gre)# ip address 198.51.100.129/25
HUB(config-gre)# ip tcp adjust-mss 1340
HUB(config-gre)# ip nhrp ipsec ipsec_for_spokes dynamic
HUB(config-gre)# ip nhrp multicast dynamic
HUB(config-gre)# ip nhrp enable
HUB(config-gre)# enable
HUB(config-gre)# exit
HUB(config)# exit 
HUB# 
HUB# commit
HUB# confirm 

Поскольку Cloud 2 будет работать в рамках Phase 1, то на SPOKE-2 необходимо настроить IPsec только в сторону HUB. Конфигурации ike-policy и ipsec-policy будут использоваться исходные - ike_policy и ipsec_policy.

SPOKE-2# configure terminal 
SPOKE-2(config)# 
SPOKE-2(config)# tunnel gre 2
SPOKE-2(config-gre)# key 20
SPOKE-2(config-gre)# ttl 255
SPOKE-2(config-gre)# mtu 1400
SPOKE-2(config-gre)# ip firewall disable
SPOKE-2(config-gre)# local address 203.0.113.10
SPOKE-2(config-gre)# remote address 203.0.113.2
SPOKE-2(config-gre)# ip address 198.51.100.131/25
SPOKE-2(config-gre)# ip tcp adjust-mss 1340
SPOKE-2(config-gre)# ip nhrp holding-time 90
SPOKE-2(config-gre)# ip nhrp map 198.51.100.129 203.0.113.2
SPOKE-2(config-gre)# ip nhrp nhs 198.51.100.129
SPOKE-2(config-gre)# ip nhrp multicast nhs
SPOKE-2(config-gre)# ip nhrp enable
SPOKE-2(config-gre)# enable
SPOKE-2(config-gre)# exit
SPOKE-2(config)#
SPOKE-2(config)# security ike gateway ike_for_hub_cloud2
SPOKE-2(config-ike-gw)# ike-policy ike_policy
SPOKE-2(config-ike-gw)# local address 203.0.113.10
SPOKE-2(config-ike-gw)# local network 203.0.113.10/32 protocol gre 
SPOKE-2(config-ike-gw)# remote address 203.0.113.2
SPOKE-2(config-ike-gw)# remote network 203.0.113.2/32 protocol gre 
SPOKE-2(config-ike-gw)# mode policy-based
SPOKE-2(config-ike-gw)# exit
SPOKE-2(config)# 
SPOKE-2(config)# security ipsec vpn ipsec_for_hub_cloud2
SPOKE-2(config-ipsec-vpn)# type transport
SPOKE-2(config-ipsec-vpn)# ike establish-tunnel route
SPOKE-2(config-ipsec-vpn)# ike gateway ike_for_hub_cloud2
SPOKE-2(config-ipsec-vpn)# ike ipsec-policy ipsec_policy
SPOKE-2(config-ipsec-vpn)# enable
SPOKE-2(config-ipsec-vpn)# exit
SPOKE-2(config)# 
SPOKE-2(config)# tunnel gre 2
SPOKE-2(config-gre)# ip nhrp ipsec ipsec_for_hub_cloud2 static
SPOKE-2(config-gre)# exit
SPOKE-2(config)# exit
SPOKE-2# commit
SPOKE-2# confirm

Проверим построение IPsec-туннеля и регистрацию SPOKE-2 для Cloud 2.
Вывод оперативной информации со стороны SPOKE-2:

SPOKE-2# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub_cloud1              203.0.113.14      203.0.113.2       0x330c380391230fb5   0x12ffc921858ba965   Established  
ipsec_for_hub_cloud2              203.0.113.10      203.0.113.2       0xa5c1dab09b3f1fb7   0x8b5b047f52c3dec5   Established   
SPOKE-2# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:23      static            RULCP        
198.51.100.129         203.0.113.2        gre 2       --          00,00:00:22      static            RULCNP   

Вывод оперативной информации со стороны HUB:

HUB# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_spokes                  203.0.113.2       203.0.113.14      0x330c380391230fb5   0x12ffc921858ba965   Established   
ipsec_for_spokes                  203.0.113.2       203.0.113.6       0xc67e4e0a9f3804c0   0xd9e59bb52a0bc755   Established   
ipsec_for_spokes                  203.0.113.2       203.0.113.10      0xa5c1dab09b3f1fb7   0x8b5b047f52c3dec5   Established  
HUB# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.2           203.0.113.6        gre 1       00:01:24    00,00:11:27      dynamic           LCP          
198.51.100.3           203.0.113.14       gre 1       00:01:21    00,00:05:50      dynamic           LCP          
198.51.100.131         203.0.113.10       gre 2       00:01:21    00,00:05:49      dynamic           LCP   

2) Далее настроим маршрутизацию на SPOKE-2 и HUB. 
Для Cloud 2 также будет использоваться AS 65000. На SPOKE-2 настроим BGP для neighbor 198.51.100.129 аналогично neighbor 198.51.100.1. 
Поскольку Cloud 1 является основным, то необходимо сделать маршруты от Cloud 1 более приоритетными, чем маршруты от Cloud 2. Для этого настроим атрибут weigth в конфигурации neighbor 198.51.100.1.

SPOKE-2# configure terminal
SPOKE-2(config)# router bgp 65000
SPOKE-2(config-bgp)# neighbor 198.51.100.1
SPOKE-2(config-bgp-neighbor)# weight 10
SPOKE-2(config-bgp-neighbor)# exit
SPOKE-2(config-bgp)# neighbor 198.51.100.129
SPOKE-2(config-bgp-neighbor)# description "Cloud_2"
SPOKE-2(config-bgp-neighbor)# update-source 198.51.100.131
SPOKE-2(config-bgp-neighbor)# remote-as 65000
SPOKE-2(config-bgp-neighbor)# address-family ipv4 unicast
SPOKE-2(config-bgp-neighbor-af)# enable
SPOKE-2(config-bgp-neighbor-af)# exit
SPOKE-2(config-bgp-neighbor)# enable
SPOKE-2(config-bgp-neighbor)# exit
SPOKE-2(config-bgp)# exit
SPOKE-2(config)# exit 
SPOKE-2# 
SPOKE-2# commit
SPOKE-2# confirm

На HUB необходимо учитывать то, что Cloud 2 будет работать в рамках Phase 1 и маршрутная информация должна быть через HUB. Для решения данной задачи необходимо:
 - включить next-hop-self all в конфигурации AFI для peer-group Cloud2.
 - настроить route-map out_Cloud1, в котором полученные маршруты от Cloud2 будут анонсироваться с next-hop самого HUB, для peer-group Cloud1

Поскольку Cloud 1 является основным, то необходимо сделать маршруты от Cloud 1 более приоритетными, чем маршруты от Cloud 2. Для этого настроим атрибут weigth в конфигурации peer-group Cloud1.

HUB# configure terminal 
HUB(config)# 
HUB(config)# object-group network from_Cloud2
HUB(config-object-group-network)# ip address-range 198.51.100.129-198.51.100.254

HUB(config-object-group-network)# exit
HUB(config)# 
HUB(config)# route-map out_Cloud1
HUB(config-route-map)# rule 1
HUB(config-route-map-rule)# match ip bgp next-hop object-group from_Cloud2
HUB(config-route-map-rule)# action set ip bgp-next-hop 198.51.100.1
HUB(config-route-map-rule)# exit
HUB(config-route-map)# exit
HUB(config)# 
HUB(config)# router bgp log-neighbor-changes
HUB(config)# 
HUB(config)# router bgp 65000
HUB(config-bgp)# router-id 198.51.100.1
HUB(config-bgp)# peer-group Cloud1
HUB(config-bgp-group)# weight 10
HUB(config-bgp-group)# address-family ipv4 unicast
HUB(config-bgp-group-af)# route-map out_Cloud1 out
HUB(config-bgp-group-af)# exit
HUB(config-bgp-group)# exit
HUB(config-bgp)# peer-group Cloud2
HUB(config-bgp-group)# remote-as 65000
HUB(config-bgp-group)# update-source 198.51.100.129
HUB(config-bgp-group)# address-family ipv4 unicast
HUB(config-bgp-group-af)# next-hop-self all
HUB(config-bgp-group-af)# enable
HUB(config-bgp-group-af)# exit
HUB(config-bgp-group)# exit
HUB(config-bgp)# listen-range 198.51.100.128/25
HUB(config-bgp-listen)# peer-group Cloud2
HUB(config-bgp-listen)# enable
HUB(config-bgp-listen)# exit
HUB(config-bgp)# exit
HUB(config)# exit 
HUB# 
HUB# commit
HUB# confirm 

Вывод оперативной информации по BGP-соединению со стороны SPOKE-2 и HUB:

SPOKE-2# show bgp neighbors 198.51.100.129 
BGP neighbor is 198.51.100.129
    Description:                        Cloud_2
    BGP state:                          Established
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.129
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.1
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.131
    Weight:                             0
    Hold timer:                         95/180
    Keepalive timer:                    42/60
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:05:55

HUB# show bgp neighbors 198.51.100.131 
BGP neighbor is 198.51.100.131
    BGP state:                          Established
    Type:                               Dynamic neighbor
    Listen range prefix:                198.51.100.128/25
    Neighbor address:                   198.51.100.131
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.3
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.129
    Weight:                             0
    Hold timer:                         108/180
    Keepalive timer:                    21/60
    Peer group:                         Cloud2
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    Yes
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:05:44

3. Итоговые конфигурации маршрутизаторов и проверка переключения канала на SPOKE-2

Схема:

3.1 Итоговые конфигурации маршрутизаторов

Конфигурация HUB:

HUB 
hostname HUB

object-group network from_Cloud2
  ip address-range 198.51.100.129-198.51.100.254
exit

route-map out_Cloud1
  rule 1
    match ip bgp next-hop object-group from_Cloud2
    action set ip bgp-next-hop 198.51.100.1
  exit
exit

router bgp log-neighbor-changes

router bgp 65000
  router-id 198.51.100.1
  peer-group Cloud1
    remote-as 65000
    weight 10
    route-reflector-client
    update-source 198.51.100.1
    address-family ipv4 unicast
      route-map out_Cloud1 out
      enable
    exit
  exit
  peer-group Cloud2
    remote-as 65000
    update-source 198.51.100.129
    address-family ipv4 unicast
      next-hop-self all
      enable
    exit
  exit
  listen-range 198.51.100.0/25
    peer-group Cloud1
    enable
  exit
  listen-range 198.51.100.128/25
    peer-group Cloud2
    enable
  exit
  enable
exit

interface gigabitethernet 1/0/1
  ip firewall disable
  ip address 203.0.113.2/30
exit

tunnel gre 1
  key 10
  ttl 255
  mtu 1400
  multipoint
  ip firewall disable
  local address 203.0.113.2
  ip address 198.51.100.1/25
  ip tcp adjust-mss 1340
  ip nhrp ipsec ipsec_for_spokes dynamic
  ip nhrp multicast dynamic
  ip nhrp enable
  enable
exit
tunnel gre 2
  key 20
  ttl 255
  mtu 1400
  multipoint
  ip firewall disable
  local address 203.0.113.2
  ip address 198.51.100.129/25
  ip tcp adjust-mss 1340
  ip nhrp ipsec ipsec_for_spokes dynamic
  ip nhrp multicast dynamic
  ip nhrp enable
  enable
exit

security ike proposal ike_proposal
  authentication algorithm sha2-384
  encryption algorithm aes256
  dh-group 21
exit

security ike policy ike_policy
  pre-shared-key ascii-text encrypted 8CB5107EA7005AFF
  proposal ike_proposal
exit

security ike gateway ike_for_spokes
  ike-policy ike_policy
  local address 203.0.113.2
  local network 203.0.113.2/32 protocol gre 
  remote address any
  remote network any protocol gre 
  mode policy-based
exit

security ipsec proposal ipsec_proposal
  authentication algorithm sha2-256
  encryption algorithm aes256
  pfs dh-group 19
exit

security ipsec policy ipsec_policy
  proposal ipsec_proposal
exit

security ipsec vpn ipsec_for_spokes
  type transport
  ike establish-tunnel route
  ike gateway ike_for_spokes
  ike ipsec-policy ipsec_policy
  enable
exit

ip route 0.0.0.0/0 203.0.113.1

Конфигурация SPOKE-1:

SPOKE-1 
hostname SPOKE-1

router bgp log-neighbor-changes

router bgp 65000
  router-id 198.51.100.2
  neighbor 198.51.100.1
    description "Cloud_1"
    remote-as 65000
    address-family ipv4 unicast
      enable
    exit
    enable
  exit
  address-family ipv4 unicast
    network 192.0.2.0/25
  exit
  enable
exit

interface gigabitethernet 1/0/1
  ip firewall disable
  ip address 203.0.113.6/30
exit
interface gigabitethernet 1/0/3
  ip firewall disable
  ip address 192.0.2.1/25
exit

tunnel gre 1
  key 10
  ttl 255
  mtu 1400
  multipoint
  ip firewall disable
  local address 203.0.113.6
  ip address 198.51.100.2/25
  ip tcp adjust-mss 1340
  ip nhrp holding-time 90
  ip nhrp map 198.51.100.1 203.0.113.2
  ip nhrp nhs 198.51.100.1
  ip nhrp ipsec ipsec_for_hub static
  ip nhrp ipsec ipsec_for_spokes dynamic
  ip nhrp multicast nhs
  ip nhrp enable
  enable
exit

security ike proposal ike_proposal
  authentication algorithm sha2-384
  encryption algorithm aes256
  dh-group 21
exit

security ike policy ike_policy
  pre-shared-key ascii-text encrypted 8CB5107EA7005AFF
  proposal ike_proposal
exit

security ike gateway ike_for_hub
  ike-policy ike_policy
  local address 203.0.113.6
  local network 203.0.113.6/32 protocol gre 
  remote address 203.0.113.2
  remote network 203.0.113.2/32 protocol gre 
  mode policy-based
exit
security ike gateway ike_for_spokes
  ike-policy ike_policy
  local address 203.0.113.6
  local network 203.0.113.6/32 protocol gre 
  remote address any
  remote network any protocol gre 
  mode policy-based
exit

security ipsec proposal ipsec_proposal
  authentication algorithm sha2-256
  encryption algorithm aes256
  pfs dh-group 19
exit

security ipsec policy ipsec_policy
  proposal ipsec_proposal
exit

security ipsec vpn ipsec_for_hub
  type transport
  ike establish-tunnel route
  ike gateway ike_for_hub
  ike ipsec-policy ipsec_policy
  enable
exit
security ipsec vpn ipsec_for_spokes
  type transport
  ike establish-tunnel route
  ike gateway ike_for_spokes
  ike ipsec-policy ipsec_policy
  enable
exit

ip route 0.0.0.0/0 203.0.113.5

Конфигурация SPOKE-2:

SPOKE-2 
hostname SPOKE-2

ip access-list extended LOCAL_1
  rule 1
    action permit
    match source-address 203.0.113.14 255.255.255.255
    enable
  exit
exit
ip access-list extended LOCAL_2
  rule 1
    action permit
    match source-address 203.0.113.10 255.255.255.255
    enable
  exit
exit

route-map PBR_LOCAL
  rule 1
    match ip access-group LOCAL_1
    action set ip next-hop verify-availability 203.0.113.13 1
  exit
  rule 2
    match ip access-group LOCAL_2
    action set ip next-hop verify-availability 203.0.113.9 1
  exit
exit

ip local policy route-map PBR_LOCAL

router bgp log-neighbor-changes

router bgp 65000
  router-id 198.51.100.3
  neighbor 198.51.100.1
    description "Cloud_1"
    remote-as 65000
    weight 10
    update-source 198.51.100.3
    address-family ipv4 unicast
      enable
    exit
    enable
  exit
  neighbor 198.51.100.129
    description "Cloud_2"
    remote-as 65000
    update-source 198.51.100.131
    address-family ipv4 unicast
      enable
    exit
    enable
  exit
  address-family ipv4 unicast
    network 192.0.2.128/25
  exit
  enable
exit

interface gigabitethernet 1/0/1
  ip firewall disable
  ip address 203.0.113.14/30
  wan load-balance nexthop 203.0.113.13
  wan load-balance enable
exit
interface gigabitethernet 1/0/2
  ip firewall disable
  ip address 203.0.113.10/30
  wan load-balance nexthop 203.0.113.9
  wan load-balance enable
exit
interface gigabitethernet 1/0/3
  ip firewall disable
  ip address 192.0.2.129/25
exit

tunnel gre 1
  key 10
  ttl 255
  mtu 1400
  multipoint
  ip firewall disable
  local address 203.0.113.14
  ip address 198.51.100.3/25
  ip tcp adjust-mss 1340
  ip nhrp holding-time 90
  ip nhrp map 198.51.100.1 203.0.113.2
  ip nhrp nhs 198.51.100.1
  ip nhrp ipsec ipsec_for_hub_cloud1 static
  ip nhrp ipsec ipsec_for_spokes_cloud1 dynamic
  ip nhrp multicast nhs
  ip nhrp enable
  enable
exit
tunnel gre 2
  key 20
  ttl 255
  mtu 1400
  ip firewall disable
  local address 203.0.113.10
  remote address 203.0.113.2
  ip address 198.51.100.131/25
  ip tcp adjust-mss 1340
  ip nhrp holding-time 90
  ip nhrp map 198.51.100.129 203.0.113.2
  ip nhrp nhs 198.51.100.129
  ip nhrp ipsec ipsec_for_hub_cloud2 static
  ip nhrp multicast nhs
  ip nhrp enable
  enable
exit

security ike proposal ike_proposal
  authentication algorithm sha2-384
  encryption algorithm aes256
  dh-group 21
exit

security ike policy ike_policy
  pre-shared-key ascii-text encrypted 8CB5107EA7005AFF
  proposal ike_proposal
exit

security ike gateway ike_for_hub_cloud1
  ike-policy ike_policy
  local address 203.0.113.14
  local network 203.0.113.14/32 protocol gre 
  remote address 203.0.113.2
  remote network 203.0.113.2/32 protocol gre 
  mode policy-based
exit
security ike gateway ike_for_hub_cloud2
  ike-policy ike_policy
  local address 203.0.113.10
  local network 203.0.113.10/32 protocol gre 
  remote address 203.0.113.2
  remote network 203.0.113.2/32 protocol gre 
  mode policy-based
exit
security ike gateway ike_for_spokes_cloud1
  ike-policy ike_policy
  local address 203.0.113.14
  local network 203.0.113.14/32 protocol gre 
  remote address any
  remote network any protocol gre 
  mode policy-based
exit

security ipsec proposal ipsec_proposal
  authentication algorithm sha2-256
  encryption algorithm aes256
  pfs dh-group 19
exit

security ipsec policy ipsec_policy
  proposal ipsec_proposal
exit

security ipsec vpn ipsec_for_hub_cloud1
  type transport
  ike establish-tunnel route
  ike gateway ike_for_hub_cloud1
  ike ipsec-policy ipsec_policy
  enable
exit
security ipsec vpn ipsec_for_hub_cloud2
  type transport
  ike establish-tunnel route
  ike gateway ike_for_hub_cloud2
  ike ipsec-policy ipsec_policy
  enable
exit
security ipsec vpn ipsec_for_spokes_cloud1
  type transport
  ike establish-tunnel route
  ike gateway ike_for_spokes_cloud1
  ike ipsec-policy ipsec_policy
  enable
exit

ip route 0.0.0.0/0 wan load-balance rule 1

wan load-balance rule 1
  outbound interface gigabitethernet 1/0/1
  outbound interface gigabitethernet 1/0/2
  enable
exit


3.2 Вывод оперативной информации

При наличии IP-связанности, получим следующие состояния IPsec, NHRP и BGP:

HUB 
HUB# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_spokes                  203.0.113.2       203.0.113.10      0xafe0e288bee0cf81   0xc841dbf8737f4177   Established   
ipsec_for_spokes                  203.0.113.2       203.0.113.14      0x88373d172b0acc01   0x24437c3d5fa8316f   Established   
ipsec_for_spokes                  203.0.113.2       203.0.113.6       0x00b134361442f9f1   0x4fd04d0d08ecb05a   Established   

HUB# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.2           203.0.113.6        gre 1       00:01:10    00,00:09:37      dynamic           LCP          
198.51.100.3           203.0.113.14       gre 1       00:01:01    00,00:09:46      dynamic           LCP          
198.51.100.131         203.0.113.10       gre 2       00:01:00    00,00:09:47      dynamic           LCP          

HUB# show bgp neighbors 
BGP neighbor is 198.51.100.131
    BGP state:                          Established
    Type:                               Dynamic neighbor
    Listen range prefix:                198.51.100.128/25
    Neighbor address:                   198.51.100.131
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.3
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.129
    Weight:                             0
    Hold timer:                         136/180
    Keepalive timer:                    7/60
    Peer group:                         Cloud2
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    Yes
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:09:49
BGP neighbor is 198.51.100.3
    BGP state:                          Established
    Type:                               Dynamic neighbor
    Listen range prefix:                198.51.100.0/25
    Neighbor address:                   198.51.100.3
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.3
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop route-reflector AS4
    Source address:                     198.51.100.1
    Weight:                             10
    Hold timer:                         143/180
    Keepalive timer:                    22/60
    Peer group:                         Cloud1
    RR client:                          Yes
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Outgoing route-map:               out_Cloud1
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               Yes
    Uptime (d,h:m:s):                   00,00:09:48
BGP neighbor is 198.51.100.2
    BGP state:                          Established
    Type:                               Dynamic neighbor
    Listen range prefix:                198.51.100.0/25
    Neighbor address:                   198.51.100.2
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.2
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop route-reflector AS4
    Source address:                     198.51.100.1
    Weight:                             10
    Hold timer:                         151/180
    Keepalive timer:                    1/60
    Peer group:                         Cloud1
    RR client:                          Yes
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Outgoing route-map:               out_Cloud1
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               Yes
    Uptime (d,h:m:s):                   00,00:09:40 

HUB# show bgp ipv4 unicast 
Status codes: u - unicast, b - broadcast, m - multicast, a - anycast
              * - valid, > - best
Origin codes: i - IGP, e - EGP, ? - incomplete

     Network              Next Hop             Metric  LocPrf      Weight Path        
*> u 192.0.2.0/25         198.51.100.2         --      100         10     i
*> u 192.0.2.128/25       198.51.100.3         --      100         10     i
*  u 192.0.2.128/25       198.51.100.131       --      100         0      i
SPOKE-1 
SPOKE-1# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub                     203.0.113.6       203.0.113.2       0x00b134361442f9f1   0x4fd04d0d08ecb05a   Established   

SPOKE-1# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:22      static            RULCP        

SPOKE-1# show bgp neighbors 
BGP neighbor is 198.51.100.1
    Description:                        Cloud_1
    BGP state:                          Established
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.1
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.1
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.2
    Weight:                             0
    Hold timer:                         143/180
    Keepalive timer:                    21/60
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:14:22

SPOKE-1# show bgp ipv4 unicast 
Status codes: u - unicast, b - broadcast, m - multicast, a - anycast
              * - valid, > - best
Origin codes: i - IGP, e - EGP, ? - incomplete

     Network              Next Hop             Metric  LocPrf      Weight Path        
*> u 192.0.2.128/25       198.51.100.3         --      100         0      i
SPOKE-2 
SPOKE-2# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub_cloud2              203.0.113.10      203.0.113.2       0xafe0e288bee0cf81   0xc841dbf8737f4177   Established   
ipsec_for_hub_cloud1              203.0.113.14      203.0.113.2       0x88373d172b0acc01   0x24437c3d5fa8316f   Established   

SPOKE-2# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:09      static            RULCP        
198.51.100.129         203.0.113.2        gre 2       --          00,00:00:10      static            RULCNP       

SPOKE-2# show bgp neighbors 
BGP neighbor is 198.51.100.1
    Description:                        Cloud_1
    BGP state:                          Established
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.1
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.1
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.3
    Weight:                             0
    Hold timer:                         136/180
    Keepalive timer:                    10/60
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:12:03
BGP neighbor is 198.51.100.129
    Description:                        Cloud_2
    BGP state:                          Established
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.129
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.1
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.131
    Weight:                             0
    Hold timer:                         140/180
    Keepalive timer:                    14/60
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:12:05
 
SPOKE-2# show bgp ipv4 unicast 
Status codes: u - unicast, b - broadcast, m - multicast, a - anycast
              * - valid, > - best
Origin codes: i - IGP, e - EGP, ? - incomplete

     Network              Next Hop             Metric  LocPrf      Weight Path        
*> u 192.0.2.0/25         198.51.100.2         --      100         10     i
*  u 192.0.2.0/25         198.51.100.129       --      100         0      i

В результате, при передаче трафика от LAN 2 в LAN 1 построится динамический туннель в рамках Cloud 1:

SPOKE-1# ping 192.0.2.129 source ip 192.0.2.1 
PING 192.0.2.129 (192.0.2.129) from 192.0.2.1 : 56 bytes of data.
!!!!!
--- 192.0.2.129 ping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 4003ms
rtt min/avg/max/mdev = 1.262/1.674/3.196/0.762 ms
SPOKE-1# ping 192.0.2.129 source ip 192.0.2.1 
PING 192.0.2.129 (192.0.2.129) from 192.0.2.1 : 56 bytes of data.
!!!!!
--- 192.0.2.129 ping statistics ---
5 packets transmitted, 5 received, 0% packet loss, time 4005ms
rtt min/avg/max/mdev = 1.281/1.329/1.412/0.057 ms
SPOKE-1# traceroute 192.0.2.129 source ip 192.0.2.1 
traceroute to 192.0.2.129 (192.0.2.129), 30 hops max, 60 byte packets
 1  192.0.2.129 (192.0.2.129)  1.592 ms  1.442 ms  1.490 ms
SPOKE-1# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:21      static            RULCP        
198.51.100.3           203.0.113.14       gre 1       00:01:13    00,00:00:16      cached            ULCP         
SPOKE-1# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub                     203.0.113.6       203.0.113.2       0x00b134361442f9f1   0x4fd04d0d08ecb05a   Established  
ipsec_for_spokes                  203.0.113.6       203.0.113.14      0x83fd91542bc5a2bd   0x960fa3b9866b4180   Established   


SPOKE-2# traceroute 192.0.2.1 source ip 192.0.2.129 
traceroute to 192.0.2.1 (192.0.2.1), 30 hops max, 60 byte packets
 1  192.0.2.1 (192.0.2.1)  1.430 ms  1.667 ms  1.585 ms

SPOKE-2# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:24      static            RULCP        
198.51.100.2           203.0.113.6        gre 1       00:00:50    00,00:00:39      cached            ULCP         
198.51.100.129         203.0.113.2        gre 2       --          00,00:00:24      static            RLCNP        
SPOKE-2# show security ipsec vpn status 
Name                              Local host        Remote host       Initiator spi        Responder spi        State         
-------------------------------   ---------------   ---------------   ------------------   ------------------   -----------   
ipsec_for_hub_cloud2              203.0.113.10      203.0.113.2       0xafe0e288bee0cf81   0xc841dbf8737f4177   Established   
ipsec_for_hub_cloud1              203.0.113.14      203.0.113.2       0x88373d172b0acc01   0x24437c3d5fa8316f   Established   
ipsec_for_spokes_cloud1           203.0.113.14      203.0.113.6       0x83fd91542bc5a2bd   0x960fa3b9866b4180   Established 


3.3 Переключение канала на SPOKE-2

Рассмотрим случай, когда 203.0.113.13 перестал быть доступным согласно схеме:

В результате недоступности 203.0.113.13 на SPOKE-1:
 - маршрут по умолчанию будет доступен только через 203.0.113.9:

2025-05-16T14:47:17+00:00 %WAN-I-INSTANCE: IP interface gigabitethernet 1/0/1 last check target 203.0.113.13 failure
2025-05-16T14:47:17+00:00 %WAN-I-INSTANCE: IP interface gigabitethernet 1/0/1 changed state to inactive

SPOKE-2# show ip route static 
S     * 0.0.0.0/0          [1/0]             via 203.0.113.9 on gi1/0/2        [static 14:47:17] 

 - по истечению таймеров упадет BGP-сессия и NHRP-соседство с HUB и с SPOKE-1 в рамках Cloud 1. В результате чего маршрут до 192.0.2.0/25 будет доступен через Cloud 2, а именно через HUB с IP 198.51.100.129:

2025-05-16T14:48:28+00:00 %BGP-W-NEIG: BGP 65000 (198.51.100.1): Error: Hold timer expired
2025-05-16T14:48:28+00:00 %BGP-W-NEIG: BGP 65000 (198.51.100.1): Session closed

SPOKE-2# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          --               static            RLP          
198.51.100.129         203.0.113.2        gre 2       --          00,00:00:05      static            RLCNP        
SPOKE-2# show bgp neighbors 
BGP neighbor is 198.51.100.1
    Description:                        Cloud_1
    BGP state:                          Active
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.1
    Neighbor AS:                        65000
    Connect delay:                      3/5
    Last error:                         Socket: No route to host
BGP neighbor is 198.51.100.129
    Description:                        Cloud_2
    BGP state:                          Established
    Type:                               Static neighbor
    Neighbor address:                   198.51.100.129
    Neighbor AS:                        65000
    Neighbor ID:                        198.51.100.1
    Neighbor caps:                      refresh enhanced-refresh restart-aware AS4
    Session:                            internal multihop AS4
    Source address:                     198.51.100.131
    Weight:                             0
    Hold timer:                         146/180
    Keepalive timer:                    17/60
    RR client:                          No
    Address family ipv4 unicast:       
      Send-label:                       No
      Default originate:                No
      Default information originate:    No
      Preference:                       170
      Remove private AS:                No
      Next-hop self:                    No
      Next-hop unchanged:               No
    Uptime (d,h:m:s):                   00,00:54:06
SPOKE-2# show bgp ipv4 unicast 
Status codes: u - unicast, b - broadcast, m - multicast, a - anycast
              * - valid, > - best
Origin codes: i - IGP, e - EGP, ? - incomplete

     Network              Next Hop             Metric  LocPrf      Weight Path        
*> u 192.0.2.0/25         198.51.100.129       --      100         0      i

На SPOKE-1 также пропадет соединение со SPOKE-2 и изменится маршрутная информация. Подсеть 192.0.2.128/25 будет доступна через HUB, а именно через IP-адрес 198.51.100.1:

SPOKE-1# show ip nhrp peers 
 Flags: E - unique, R - nhs, U - used, L - lower-up
        C - connected, G - group, Q - qos, N - nat
        P - protected, I - Redirect-ignored, X - undefined

Tunnel address         NBMA address       Tunnel      Expire      Created          Type              Flags        
                                                      (h:m:s)     (d,h:m:s)                                       
--------------------   ----------------   ---------   ---------   --------------   ---------------   ----------   
198.51.100.1           203.0.113.2        gre 1       --          00,00:00:01      static            RULCP        
SPOKE-1# show bgp ipv4 unicast 
Status codes: u - unicast, b - broadcast, m - multicast, a - anycast
              * - valid, > - best
Origin codes: i - IGP, e - EGP, ? - incomplete

     Network              Next Hop             Metric  LocPrf      Weight Path        
*> u 192.0.2.128/25       198.51.100.1         --      100         0      i

Трафик между LAN будет передаваться через HUB между Cloud 1 и Cloud 2:

SPOKE-1# traceroute 192.0.2.129 source ip 192.0.2.1 
traceroute to 192.0.2.129 (192.0.2.129), 30 hops max, 60 byte packets
 1  198.51.100.1 (198.51.100.1)  0.873 ms  0.595 ms  0.491 ms
 2  192.0.2.129 (192.0.2.129)  2.112 ms  2.311 ms  2.247 ms

SPOKE-2# traceroute 192.0.2.1 source ip 192.0.2.129 
traceroute to 192.0.2.1 (192.0.2.1), 30 hops max, 60 byte packets
 1  198.51.100.129 (198.51.100.129)  0.929 ms  0.672 ms  0.588 ms
 2  192.0.2.1 (192.0.2.1)  1.655 ms  1.677 ms  1.855 ms



  • Нет меток