routeserver-vmss-selfcontained.azcli

##############################################################
#
# This script demonstrates how to configure dynamically BGP
#   adjacencies on Azure Route Server to the instances of a
#   VMSS when it scales in and out.
#
# As opposed to routesserver-vmss.azcli, there are no external
#   components running any automation, but all logic runs inside
#   the VMSS instances.
#
# One application of this setup is using the VMSS as control
#   plane, injecting routes with some other resource as next
#   hop, such as Azure Firewall
#
#
# Jose Moreno, April 2022
##############################################################

# Control
healthcheck_script_url='https://raw.githubusercontent.com/erjosito/azcli/master/routeserver-vmss-selfcontained-healthcheck.py'
housekeeping_script_url='https://raw.githubusercontent.com/erjosito/azcli/master/routeserver-vmss-selfcontained-config.sh'
routes_url='https://raw.githubusercontent.com/erjosito/azcli/master/routeserver-vmss-selfcontained-routes.txt'
autorepair=extension # can be 'extension' or 'probe'
custom_lb=no

# Variables
rg=rsvmss
location=westeurope
vnet_name=hub
vnet_prefix=10.1.0.0/16
gw_subnet_name=GatewaySubnet
gw_subnet_prefix=10.1.0.0/24
vpngw_asn=65501
rs_subnet_name=RouteServersubnet
rs_subnet_prefix=10.1.1.0/24
azfw_subnet_name=AzureFirewallSubnet
azfw_subnet_prefix=10.1.3.0/24
azfw_name=testfw
nva_subnet_name=nva
nva_subnet_prefix=10.1.2.0/24
vm_subnet_name=vm
vm_subnet_prefix=10.1.10.0/24
vm_size=Standard_B1s
publisher=Canonical
offer=UbuntuServer
sku=18.04-LTS
# version=$(az vm image list -p $publisher -f $offer -s $sku --all --query '[0].version' -o tsv 2>/dev/null)
version=latest
nva_asn=65001
nva_name=nva
nva_pip=${nva_name}-pip
nva_cloudinit_file=/tmp/nva_cloudinit.txt
azurevm_name=azurevm
azurevm_pip_name="${azurevm_name}-pip"

# Auxiliary function to get the first IP of a subnet (default gateway)
function first_ip(){
    subnet=$1
    IP=$(echo $subnet | cut -d/ -f 1)
    IP_HEX=$(printf '%.2X%.2X%.2X%.2X\n' `echo $IP | sed -e 's/\./ /g'`)
    NEXT_IP_HEX=$(printf %.8X `echo $(( 0x$IP_HEX + 1 ))`)
    NEXT_IP=$(printf '%d.%d.%d.%d\n' `echo $NEXT_IP_HEX | sed -r 's/(..)/0x\1 /g'`)
    echo "$NEXT_IP"
}

# Auxiliary function to wait until a resource has finished creation (either Successful or Failed)
function wait_until_finished {
     wait_interval=15
     resource_id=$1
     resource_name=$(echo $resource_id | cut -d/ -f 9)
     echo "Waiting for resource $resource_name to finish provisioning..."
     start_time=`date +%s`
     state=$(az resource show --id $resource_id --query properties.provisioningState -o tsv)
     until [[ "$state" == "Succeeded" ]] || [[ "$state" == "Failed" ]] || [[ -z "$state" ]]
     do
        sleep $wait_interval
        state=$(az resource show --id $resource_id --query properties.provisioningState -o tsv)
     done
     if [[ -z "$state" ]]
     then
        echo "Something really bad happened..."
     else
        run_time=$(expr `date +%s` - $start_time)
        ((minutes=${run_time}/60))
        ((seconds=${run_time}%60))
        echo "Resource $resource_name provisioning state is $state, wait time $minutes minutes and $seconds seconds"
     fi
}

# Create Vnets and subnets
echo "Creating RG and VNet..."
az group create -n $rg -l $location -o none
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix --subnet-name $vm_subnet_name --subnet-prefix $vm_subnet_prefix -l $location -o none
az network vnet subnet create --vnet-name $vnet_name -g $rg -n $gw_subnet_name --address-prefix $gw_subnet_prefix -o none
az network vnet subnet create --vnet-name $vnet_name -g $rg -n $rs_subnet_name --address-prefix $rs_subnet_prefix -o none
az network vnet subnet create --vnet-name $vnet_name -g $rg -n $nva_subnet_name --address-prefix $nva_subnet_prefix -o none
az network vnet subnet create --vnet-name $vnet_name -g $rg -n $azfw_subnet_name --address-prefix $azfw_subnet_prefix -o none

# Create Log Analytics workspace
logws_name=$(az monitor log-analytics workspace list -g $rg --query '[].name' -o tsv 2>/dev/null)  # Retrieve the WS name if it already existed
if [[ -z "$logws_name" ]]
then
    logws_name=log$RANDOM
    az monitor log-analytics workspace create -n $logws_name -g $rg -o none
fi
logws_id=$(az resource list -g $rg -n $logws_name --query '[].id' -o tsv)
logws_customer_id=$(az monitor log-analytics workspace show -n $logws_name -g $rg --query customerId -o tsv)
logws_key=$(az monitor log-analytics workspace get-shared-keys -g $rg -n $logws_name --query primarySharedKey -o tsv)

# Configure a RT in the NVA subnet so that it doesnt learn its own routes
nva_rt_name=nva
echo "Creating RT for VMSS..."
az network route-table create -n $nva_rt_name -g $rg -l $location --disable-bgp-route-propagation -o none
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $nva_subnet_name --route-table $nva_rt_name -o none

# Configure a RT in the VM subnet to provide connectivity to the PC where these commands are running
echo "Creating RT for VM..."
vm_rt_name=vm
az network route-table create -n $vm_rt_name -g $rg -l $location -o none
myip=$(curl -s4 ifconfig.co) && echo $myip
az network route-table route create --route-table-name $vm_rt_name -g $rg --address-prefix "${myip}/32" --name "TestPC" --next-hop-type Internet -o none
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $vm_subnet_name --route-table $vm_rt_name -o none

# Deploy RS (no --no-wait option)
echo "Creating Route Server..."
rs_subnet_id=$(az network vnet subnet show -n $rs_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
az network public-ip create -n rs-pip -g $rg --sku Standard --allocation-method Static -o none
az network routeserver create -n rs -g $rg --hosted-subnet $rs_subnet_id -l $location --public-ip-address rs-pip -o none
rs_asn=$(az network routeserver show -n rs -g $rg --query 'virtualRouterAsn' -o tsv) && echo $rs_asn
rs_ip1=$(az network routeserver show -n rs -g $rg --query 'virtualRouterIps[0]' -o tsv) && echo $rs_ip1
rs_ip2=$(az network routeserver show -n rs -g $rg --query 'virtualRouterIps[1]' -o tsv) && echo $rs_ip2

# Create VM for testing purposes
echo "Creating test VM..."
az network nsg create -n "${azurevm_name}-nsg" -g $rg -o none
az network nsg rule create -n SSH --nsg-name "${azurevm_name}-nsg" -g $rg --priority 1000 --destination-port-ranges 22 --access Allow --protocol Tcp -o none
az network nsg rule create -n ICMP --nsg-name "${azurevm_name}-nsg" -g $rg --priority 1030 --destination-port-ranges '*' --access Allow --protocol Icmp -o none
az vm create -n $azurevm_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys --nsg "${azurevm_name}-nsg" \
    --public-ip-address $azurevm_pip_name --vnet-name $vnet_name --size $vm_size --subnet $vm_subnet_name -o none
azurevm_pip_ip=$(az network public-ip show -n $azurevm_pip_name --query ipAddress -o tsv -g $rg) && echo $azurevm_pip_ip
azurevm_nic_id=$(az vm show -n $azurevm_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
azurevm_private_ip=$(az network nic show --ids $azurevm_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $azurevm_private_ip

# Create NSG for NVA
echo "Creating NSG for VMSS NVA..."
az network nsg create -n "${nva_name}-nsg" -g $rg -o none
az network nsg rule create -n SSH --nsg-name "${nva_name}-nsg" -g $rg --priority 1000 --destination-port-ranges 22 --access Allow --protocol Tcp -o none
# az network nsg rule create -n IKE --nsg-name "${nva_name}-nsg" -g $rg --priority 1010 --destination-port-ranges 4500 --access Allow --protocol Udp -o none  # Not required since no IPsec
# az network nsg rule create -n IPsec --nsg-name "${nva_name}-nsg" -g $rg --priority 1020 --destination-port-ranges 500 --access Allow --protocol Udp -o none  # Not required since no IPsec
az network nsg rule create -n ICMP --nsg-name "${nva_name}-nsg" -g $rg --priority 1030 --source-address-prefixes '*' --destination-address-prefixes '*' --destination-port-ranges '*' --access Allow --protocol Icmp -o none
az network nsg rule create -n Webin --nsg-name "${nva_name}-nsg" -g $rg --priority 1040 --source-address-prefixes 'VirtualNetwork' --destination-port-ranges 8080 --access Allow --protocol Tcp -o none  # To troubleshoot the HTTP healthchecks
az network nsg rule create -n ICMPout --nsg-name "${nva_name}-nsg" -g $rg --priority 1130 --source-address-prefixes '*' --destination-address-prefixes '*' --destination-port-ranges '*' --access Allow --protocol Icmp  --direction Outbound -o none

# Create Azure Firewall
echo "Creating Azure Firewall..."
azfw_pip_name="${azfw_name}-pip"
azfw_policy_name="${azfw_name}-policy"
az network firewall policy create -n $azfw_policy_name -g $rg --sku Standard -o none
az network public-ip create -n $azfw_pip_name -g $rg --sku Standard --allocation-method Static -o none
az network firewall create -n $azfw_name -g $rg -l $location --policy $azfw_policy_name -o none
azfw_id=$(az network firewall show -n $azfw_name -g $rg -o tsv --query id)
az network firewall ip-config create -f $azfw_name -n azfw-ipconfig -g $rg --public-ip-address $azfw_pip_name --vnet-name $vnet_name
az network firewall update -n $azfw_name -g $rg -o none
azfw_private_ip=$(az network firewall show -n $azfw_name -g $rg -o tsv --query 'ipConfigurations[0].privateIpAddress')
echo "Azure Firewall created with private IP $azfw_private_ip"

# Enable firewall logs
azfw_id=$(az network firewall show -n $azfw_name -g $rg -o tsv --query id)
az monitor diagnostic-settings create -n mydiag --resource $azfw_id --workspace $logws_id -o none \
    --metrics '[{"category": "AllMetrics", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false }, "timeGrain": null}]' \
    --logs '[{"category": "AzureFirewallApplicationRule", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false}}, 
            {"category": "AzureFirewallNetworkRule", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false}}]'

# Allow-all rule
echo "Creating firewall rules..."
az network firewall policy rule-collection-group create -n myrcg --policy-name $azfw_policy_name -g $rg --priority 1000 -o none
# az network firewall policy rule-collection-group collection rule add --rule-type NetworkRule -g $rg --rcg-name myrcg --policy-name $azfw_policy_name \
#     --action Allow --collection-priority 1020 --collection-name testnetrules --name permitany --source-addresses '*' --destination-addresses '*' \
#     --ip-protocols Any --destination-ports '*' -o none
az network firewall policy rule-collection-group collection add-filter-collection --rule-type NetworkRule -g $rg --rcg-name myrcg --policy-name $azfw_policy_name \
    --action Allow --collection-priority 1010 --name allowany --rule-name allowany --source-addresses '*' --destination-addresses '*' \
    --ip-protocols Any --destination-ports '*' -o none

# Create managed identity for VMSS
id_name="$nva_name"
id_id=$(az identity show -n $id_name -g $rg --query id -o tsv)
if [[ -z "$id_id" ]]; then
    echo "Creating managed identity and assigning contributor role to RG..."
    az identity create -n $id_name -g $rg -o none
    id_id=$(az identity show -n $id_name -g $rg --query id -o tsv)
    rg_id=$(az group show -n $rg --query id -o tsv)
    az role assignment create --scope $rg_id --assignee $id_id --role 'Contributor' -o none
else
    echo "Found managed identity $id_id"
fi

# Create Cloudinit file that will be used by the VMSS with Bird, python and Flask (Python/Flask required to provide "smart" HTTP probes)
# Azure CLI is installed to to run housekeeping tasks
echo "Creating cloudinit file for VMSS..."
nva_default_gw=$(first_ip "$nva_subnet_prefix")
cat <<EOF > $nva_cloudinit_file
#cloud-config
packages:
  - bird
  - python3-pip
  - jq
runcmd:
  - [ pip3, install, flask ]
  - [ wget, "$healthcheck_script_url", "-P", "/root/" ]
  - [ wget, "$housekeeping_script_url", "-P", "/root/" ]
  - [ chmod, "755", "/root/routeserver-vmss-selfcontained-config.sh" ]
  - [ wget, "https://aka.ms/InstallAzureCLIDeb", "-O", "/root/install_azcli.sh" ]
  - bash /root/install_azcli.sh
  - /root/routeserver-vmss-selfcontained-config.sh
  - [ systemctl, restart, bird ]
  - python3 /root/routeserver-vmss-selfcontained-healthcheck.py &
write_files:
- path: /etc/crontab
  append: true
  content: |
    */5 * * * * root /root/routeserver-vmss-selfcontained-config.sh
- content: |
    $routes_url
  path: /root/routes_url
- content: |
    log syslog all;
    protocol device {
            scan time 10;
    }
    protocol direct {
        disabled;
    }
    protocol kernel {
        preference 254;
        learn;
        merge paths on;
        import filter {
            reject;
        };
        export filter {
            reject;
        };
    }
    protocol static {
        import all;
        # Example: 
        # route 0.0.0.0/0 via $nva_default_gw;
        # Routes advertised --DONT CHANGE THIS LINE--
    }
    filter TO_RS {
        bgp_next_hop = $azfw_private_ip;
        accept;
    }
    protocol bgp rs0 {
        description "RouteServer instance 0";
        multihop;
        local as $nva_asn;
        neighbor $rs_ip1 as $rs_asn;
            import filter {accept;};
            export filter TO_RS;
    }
    protocol bgp rs1 {
        description "Route Server instance 1";
        multihop;
        local as $nva_asn;
        neighbor $rs_ip2 as $rs_asn;
            import filter {accept;};
            export filter TO_RS;
    }
  path: /etc/bird/bird.conf.template
EOF

# Create a Standard LB for instance monitoring and self healing
if [[ "$custom_lb" == "yes" ]]; then
    echo "Creating Azure LB..."
    lb_name=$nva_name
    lb_probe_name=$nva_name
    nva_subnet_id=$(az network vnet subnet show -n $nva_subnet_name --vnet-name $vnet_name -g $rg --query id -o tsv)
    # az network lb create -n $lb_name -g $rg  --sku Standard --subnet $nva_subnet_id -o none
    az network lb create -n $lb_name -g $rg  --sku Standard -o none
    # az network lb probe create -n $lb_probe_name --lb-name $lb_name -g $rg --protocol tcp --port 179 --interval 30 --threshold 3 -o none
    az network lb probe create -n $lb_probe_name --lb-name $lb_name -g $rg --protocol http --port 8080 --path '/api/healthcheck' --interval 30 --threshold 3 -o none
fi

# Create a VMSS using previous cloudinit file
# The value of parameter automaticRepairsPolicy.repairAction is invalid. !!!!!!
echo "Creating VMSS..."
if [[ "$custom_lb" == "yes" ]]; then
    az vmss create -n $nva_name -g $rg -l $location --image "${publisher}:${offer}:${sku}:${version}" --generate-ssh-keys \
        --vnet-name $vnet_name --subnet $nva_subnet_name --assign-identity $id_id -z 1 2 3 --lb-sku Standard \
        --load-balancer $lb_name --health-probe $lb_probe_name --automatic-repairs-grace-period 30 --automatic-repairs-action Replace \
        --vm-sku ${vm_size} --custom-data "$nva_cloudinit_file" --nsg "${nva_name}-nsg" --instance-count 1 -o none
    echo 'Creating inbound NAT rule...'
    lb_frontend_name=$(az network lb frontend-ip list --lb-name $lb_name -g $rg --query '[0].name' -o tsv)
    lb_backend_name=$(az network lb address-pool list --lb-name $lb_name -g $rg --query '[0].name' -o tsv)
    az network lb inbound-nat-rule create --backend-port 22 --lb-name $lb_name --name ssh --protocol Tcp -g $rg --backend-pool-name $lb_backend_name --frontend-ip-name $lb_frontend_name --frontend-port-range-start 1022 --frontend-port-range-end 1030
else
    az vmss create -n $nva_name -g $rg -l $location --image "${publisher}:${offer}:${sku}:${version}" --generate-ssh-keys \
        --vnet-name $vnet_name --subnet $nva_subnet_name --assign-identity $id_id -z 1 2 3 \
        --vm-sku ${vm_size} --custom-data "$nva_cloudinit_file" --nsg "${nva_name}-nsg" --instance-count 1 -o none
fi
az vmss list-instance-connection-info -n $nva_name -g $rg -o table

# Configure Autorepair
if [[ "$autorepair" == "extension" ]]; then
    # Configure App Health Extension
    echo "Enabling Application Health Extension in the VMSS..."
    cat <<EOF > /tmp/health_extension.json
{
    "protocol": "http",
    "port": 8080,
    "requestPath": "/api/healthcheck",
    "intervalInSeconds": 30,
    "numberOfProbes": 1
}
EOF
    az vmss extension set -n ApplicationHealthLinux --publisher Microsoft.ManagedServices --version 1.0 -g $rg --vmss-name $nva_name --settings /tmp/health_extension.json -o none
    az vmss update-instances -n $nva_name -g $rg --instance-ids '*' -o none
    az vmss update -n $nva_name -g $rg --enable-automatic-repairs true --automatic-repairs-grace-period 30 -o none
elif [[ "$autorepair" == "probe" ]] && [[ "custom_lb" == "yes" ]]; then
    # Get ALB associated to VMSS and create a LB rule and healthprobe
    vmss_lb_name=$(az vmss show -n $nva_name -g $rg --query 'virtualMachineProfile.networkProfile.networkInterfaceConfigurations[0].ipConfigurations[0].loadBalancerBackendAddressPools[0].id' -o tsv | cut -d/ -f9)
    vmss_lb_probe_name=$nva_name
    az network lb probe create -n $vmss_lb_probe_name --lb-name $vmss_lb_name -g $rg --protocol http --port 8080 --path '/api/healthcheck' --interval 30 --threshold 3 -o none
    vmss_lb_probe_id=$(az network lb probe show -n $vmss_lb_probe_name --lb-name $vmss_lb_name -g $rg --query id -o tsv)
    vmss_lb_frontend_name=$(az network lb frontend-ip list --lb-name $vmss_lb_name -g $rg --query '[0].name' -o tsv)
    vmss_lb_backend_name=$(az network lb address-pool list --lb-name $vmss_lb_name -g $rg --query '[0].name' -o tsv)
    az network lb rule create -n port8080 --lb-name $vmss_lb_name -g $rg -o none \
        --protocol tcp --frontend-port 8080 --backend-port 8080 --frontend-ip-name $vmss_lb_frontend_name --backend-pool-name $vmss_lb_backend_name --probe-name $vmss_lb_probe_name
    az vmss update -n $nva_name -g $rg --set "virtualMachineProfile.networkProfile.healthProbe={\"id\": \"$vmss_lb_probe_id\"}" -o none
    az vmss update-instances -n $nva_name -g $rg --instance-ids '*' -o none
    az vmss update -n $nva_name -g $rg --enable-automatic-repairs true --automatic-repairs-grace-period 30 -o none
    az vmss update -n $nva_name -g $rg --set 'automaticRepairsPolicy.repairAction=Replace' -o none
    az vmss show -n $nva_name -g $rg --query 'automaticRepairsPolicy'
fi 

# Install Azure Monitor and Log Analytics extension
echo "Installing Azure Monitor and Log Analytics extension..."
if [[ -n "$logws_customer_id" ]] && [[ -n "$logws_key" ]]; then
    az vmss extension set --vmss-name $nva_name -g $rg -n AzureMonitorLinuxAgent  --publisher Microsoft.Azure.Monitor -o none
    az vmss extension set --vmss-name $nva_name -g $rg -n OmsAgentForLinux --publisher Microsoft.EnterpriseCloud.Monitoring -o none \
        --protected-settings "{\"workspaceKey\":\"$logws_key\"}" --settings "{\"workspaceId\":\"$logws_customer_id\",\"skipDockerProviderInstall\": true}"
    az vmss update-instances -n $nva_name -g $rg --instance-ids '*' -o none
else
    echo "Please set the Az Monitor workspace variables"
fi

# Scale NVA VMSS in and out
az vmss scale -n $nva_name -g $rg --new-capacity 3 -o none

# See adjacencies in RS
az network routeserver peering list --routeserver rs -g $rg -o table

# See adjacencies in all NVA instances
vm_list=$(az vmss list-instance-connection-info -n $nva_name -g $rg | grep ':')
while IFS= read -r vm; do
    vm=$(echo $vm | cut -d\" -f 4)
    echo "Processing $vm..."
    vm_ip=$(echo $vm | cut -d: -f 1)
    vm_port=$(echo $vm | cut -d: -f 2)
    echo "Connecting to IP $vm_ip port $vm_port..."
    ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_ip -p $vm_port "sudo birdc show protocols"
done <<< "$vm_list"

# SSH to first VMSS instance
vm1=$(az vmss list-instance-connection-info -n $nva_name -g $rg | grep ':' | head -1)
vm=$(echo $vm1 | cut -d\" -f 4)
vm_ip=$(echo $vm | cut -d: -f 1)
vm_port=$(echo $vm | cut -d: -f 2)
echo "Connecting to VM $vm on IP $vm_ip and port $vm_port..."
ssh -o BatchMode=yes -o StrictHostKeyChecking=no $vm_ip -p $vm_port

# See effective routes in VM
azurevm_nic_id=$(az vm show -n $azurevm_name -g "$rg" --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic show-effective-route-table --ids $azurevm_nic_id -o table

# Learned routes from ARS for all BGP peers
peer_list=$(az network routeserver peering list --routeserver rs -g $rg --query '[].name' -o tsv)
while IFS= read -r peer; do
    echo "Learned routes from $peer:"
    az network routeserver peering list-learned-routes -n $peer --routeserver rs -g $rg --query 'RouteServiceRole_IN_0' -o table
done <<< "$peer_list"

# Ping from VM
azurevm_pip_ip=$(az network public-ip show -n $azurevm_pip_name --query ipAddress -o tsv -g $rg) && echo $azurevm_pip_ip
az network public-ip list -o table -g $rg
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $azurevm_pip_ip "curl -s4 ifconfig.co"  # The public IP should be the VMSS' public ALB

#################
# Other options #
#################

# ER Gateway
echo "Creating ER Gateway..."
az network public-ip create -g $rg -n ergw_pip --allocation-method Dynamic --sku Basic -l $location -o none
az network vnet-gateway create -g $rg -n ergw --gateway-type ExpressRoute --sku Standard -l $location --vnet $vnet_name --public-ip-addresses ergw_pip -o none --no-wait
# Delete
# az network vnet-gateway delete -g $rg -n ergw --no-wait -o none

# Spokes
for spoke_id in {1..2}; do
    spoke_vnet_name="spoke${spoke_id}"
    spoke_vnet_prefix="10.1${spoke_id}.0.0/16"
    spoke_vm_subnet_name=vm
    spoke_vm_subnet_prefix="10.1${spoke_id}.0.0/24"
    echo "Creating VNet ${spoke_vnet_name} with prefix ${spoke_vnet_prefix}..."
    az network vnet create -g $rg -n $spoke_vnet_name --address-prefix $spoke_vnet_prefix --subnet-name $spoke_vm_subnet_name --subnet-prefix $spoke_vm_subnet_prefix -o none
    az network vnet peering create -n hubtospoke${spoke_id} -g $rg --vnet-name $vnet_name --remote-vnet $spoke_vnet_name --allow-vnet-access --allow-forwarded-traffic --allow-gateway-transit -o none
    az network vnet peering create -n spoke${spoke_id}tohub -g $rg --vnet-name $spoke_vnet_name --remote-vnet $vnet_name --allow-vnet-access --allow-forwarded-traffic --use-remote-gateways -o none
    spoke_vm_name="$spoke_vnet_name"
    az vm create -n $spoke_vm_name -g $rg -l $location --image ubuntuLTS --generate-ssh-keys --nsg "${spoke_vm_name}-nsg" \
        --public-ip-address "${spoke_vm_name}-pip" --vnet-name $spoke_vnet_name --size $vm_size --subnet $spoke_vm_subnet_name --no-wait -o none
done

# NSG Flow Logs and Az FW diag settings to storage
storage_account_name=$(az storage account list -g $rg --query '[].name' -o tsv 2>/dev/null)  # Retrieve the storage account name if it already existed
if [[ -z "$storage_account_name" ]]
then
    storage_account_name=log$RANDOM
    az storage account create -n $storage_account_name -g $rg --sku Standard_LRS --kind StorageV2 -l $location -o none
fi
nsg_list=$(az network nsg list -g $rg -o tsv --query '[].name')
while IFS= read -r nsg_name; do
    echo "Creating flow log definition for NSG ${nsg_name}..."
    az network watcher flow-log create -l $location -n "${nsg_name}-${location}" -g $rg \
        --nsg $nsg_name --storage-account $storage_account_name --log-version 2 --retention 7 \
        --workspace $logws_id --interval 10 --traffic-analytics true -o none
done <<< "$nsg_list"
azfw_id=$(az network firewall show -n $azfw_name -g $rg -o tsv --query id)
az monitor diagnostic-settings create -n storagediag --resource $azfw_id --storage-account $storage_account_name -o none \
    --metrics '[{"category": "AllMetrics", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false }, "timeGrain": null}]' \
    --logs '[{"category": "AzureFirewallApplicationRule", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false}}, 
            {"category": "AzureFirewallNetworkRule", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false}}]'

######################
# Azure Monitor Logs #
######################

# AzFW Log Categories
query='AzureDiagnostics 
| where ResourceType == "AZUREFIREWALLS" 
| distinct Category '
az monitor log-analytics query -w $logws_customer_id --analytics-query $query -o tsv
# AzFW Net rules
query_net='AzureDiagnostics
| where ResourceType == "AZUREFIREWALLS" 
| where Category == "AzureFirewallNetworkRule"
| where TimeGenerated >= ago(5m) 
| project TimeGenerated, msg_s
| take 100 '
az monitor log-analytics query -w $logws_customer_id --analytics-query $query_net -o tsv
# VM logs
query_syslog='Syslog 
| top 100 by TimeGenerated desc '
az monitor log-analytics query -w $logws_customer_id --analytics-query $query_syslog -o tsv

###############
# Danger Zone #
###############

# az vmss delete -n $nva_name -g $rg
# az network routeserver delete -n rs -g $rg -y
# az network lb delete -n $lb_name -g $rg
# az network firewall delete -n $azfw_name -g $rg --no-wait
# az group delete -y --no-wait -n $rg