plink_udr.azcli

#######################
# Test infrastructure #
#######################

# This script is using a JSON template to deploy private endpoints using a specific API version
plink_template_file="/tmp/plink_template.json"
cat <<EOF > ${plink_template_file}
{
  "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "location": {
      "type": "string",
      "defaultValue": "[resourceGroup().location]"
    },
    "privateEndpointName": {
      "type": "string"
    },
    "subnetId": {
      "type": "string"
    },
    "privateLinkResourceId": {
      "type": "string"
    },
    "privateLinkResourceGroupId": {
      "type": "string"
    }
  },
  "resources": [
    {
      "name": "[parameters('privateEndpointName')]",
      "type": "Microsoft.Network/privateEndpoints",
      "apiVersion": "2020-05-01",
      "location": "[parameters('location')]",
      "properties": {
        "subnet": {
          "id": "[parameters('subnetId')]"
        },
        "privateLinkServiceConnections": [
          {
            "name": "MyPrivateEndpointConnection",
            "properties": {
              "privateLinkServiceId": "[parameters('privateLinkResourceId')]",
              "groupIds": [
                "[parameters('privateLinkResourceGroupId')]"
              ]
            }
          }
        ]
      }
    }
  ]
}
EOF


# Resource group
rg=plinkudr
location=westeurope
az group create -n $rg -l $location
# Azure SQL
sql_server_name=myserver$RANDOM
# sql_server_name=$(az sql server list -g $rg --query '[0].name' -o tsv)  # For an already existing server
sql_db_name=mydb
sql_username=azure
sql_password=Microsoft123!
az sql server create -n $sql_server_name -g $rg -l $location --admin-user $sql_username --admin-password $sql_password
az sql db create -n $sql_db_name -s $sql_server_name -g $rg -e Basic -c 5 --no-wait
# Optionally configure database as serverless SKU
# az sql db update -g $rg -s $sql_server_name -n $sql_db_name --edition GeneralPurpose --min-capacity 1 --capacity 4 --family Gen5 --compute-model Serverless --auto-pause-delay 1440
sql_server_fqdn=$(az sql server show -n $sql_server_name -g $rg -o tsv --query fullyQualifiedDomainName)
# Create Vnet
vnet_name=myvnet
vnet_prefix=192.168.0.0/16
subnet_sql_name=sql
subnet_sql_prefix=192.168.2.0/24
subnet_vm_name=vm
subnet_vm_prefix=192.168.13.0/24
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix -l $location
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_sql_name --address-prefix $subnet_sql_prefix
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_vm_name --address-prefix $subnet_vm_prefix

# Create VM with troubleshooting web page (installed through CSE)
vm_name=apivm
vm_nsg_name=${vm_name}-nsg
vm_pip_name=${vm_name}-pip
vm_disk_name=${vm_name}-disk0
vm_sku=Standard_B2ms
publisher=Canonical
offer=UbuntuServer
sku=18.04-LTS
image_urn=$(az vm image list -p $publisher -f $offer -s $sku -l $location --query '[0].urn' -o tsv)
az vm create -n $vm_name -g $rg -l $location --image $image_urn --size $vm_sku --generate-ssh-keys \
  --os-disk-name $vm_disk_name --os-disk-size-gb 32 \
  --vnet-name $vnet_name --subnet $subnet_vm_name \
  --nsg $vm_nsg_name --nsg-rule SSH --public-ip-address $vm_pip_name
# Add rule to NSG on port 8080
az network nsg rule create -n TCP8080 --nsg-name $vm_nsg_name -g $rg \
  --protocol Tcp --access Allow --priority 105 --direction Inbound \
  --destination-port-ranges 8080
# Install app, this will take a while (a bunch of apt updates, installs, etc).
# You might have to Ctrl-C this, it hangs when executing the app (for some reason i am not able to run it as a background task)
script_url=https://raw.githubusercontent.com/erjosito/whoami/master/api-vm/cse.sh
script_command='./cse.sh'
az vm extension set -n customScript --vm-name $vm_name -g $rg --publisher Microsoft.Azure.Extensions \
  --protected-settings "{\"fileUris\": [\"${script_url}\"],\"commandToExecute\": \"${script_command}\"}"
# Set environment variables
# command="export SQL_SERVER_USERNAME=${sql_username} && export SQL_SERVER_PASSWORD=${sql_password}"
# az vm run-command invoke -n $vm_name -g $rg --command-id RunShellScript --scripts "${command}"
# az vm run-command invoke -n $vm_name -g $rg --command-id RunShellScript --scripts 'export SQL_SERVER_USERNAME=$1 && export SQL_SERVER_PASSWORD=$2' \
#    --parameters $sql_username $sql_password
# Get private IP
vm_private_ip=$(az vm list-ip-addresses -n $vm_name -g $rg --query '[0].virtualMachine.network.privateIpAddresses[0]' -o tsv)
# Get public IP
vm_pip_ip=$(az network public-ip show -n $vm_pip_name -g $rg --query ipAddress -o tsv)
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "ip a"
echo "You can SSH to $vm_pip_ip"
# Send a probe to the app
curl -s ${vm_pip_ip}:8080/api/healthcheck

# Optionally, explore VM API endpoints
curl -s ${vm_pip_ip}:8080/api/ip
curl -s ${vm_pip_ip}:8080/api/printenv

#####################
#     SQL Server    #
#####################

# Create private link endpoint for SQL Server
sql_endpoint_name=sqlep
sql_server_id=$(az sql server show -n $sql_server_name -g $rg -o tsv --query id)
az network vnet subnet update -n $subnet_sql_name -g $rg --vnet-name $vnet_name --disable-private-endpoint-network-policies true
sql_subnet_id=$(az network vnet subnet show -n $subnet_sql_name --vnet-name $vnet_name -g $rg --query id -o tsv) && echo $sql_subnet_id
#az network private-endpoint create -n $sql_endpoint_name -g $rg --vnet-name $vnet_name --subnet $subnet_sql_name --private-connection-resource-id $sql_server_id --group-id sqlServer --connection-name sqlConnection
az deployment group create -n sqlendpoint -g $rg --template-file ${plink_template_file} \
  --parameters "{ \"privateEndpointName\": { \"value\": \"${sql_endpoint_name}\" }, \"subnetId\": { \"value\": \"${sql_subnet_id}\" }, \"privateLinkResourceId\": { \"value\": \"${sql_server_id}\" }, \"privateLinkResourceGroupId\": { \"value\": \"sqlServer\" } }"
sql_nic_id=$(az network private-endpoint show -n $sql_endpoint_name -g $rg --query 'networkInterfaces[0].id' -o tsv)
sql_endpoint_ip=$(az network nic show --ids $sql_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv)
echo "Private IP address for SQL server ${sql_server_name}: ${sql_endpoint_ip}"

# Optionally, verify that the endpoints resolve to the public IP from your PC
nslookup ${sql_server_fqdn}
nslookup ${sql_server_name}.privatelink.database.windows.net

# Create private DNS zone
dns_zone_name=privatelink.database.windows.net
az network private-dns zone create -n $dns_zone_name -g $rg
az network private-dns link vnet create -g $rg -z $dns_zone_name -n myDnsLink --virtual-network $vnet_name --registration-enabled false

# Linking private endpoint with DNS zone to automatically create the A record
az network private-endpoint dns-zone-group create --endpoint-name $sql_endpoint_name -g $rg -n myzonegroup --zone-name zone1 --private-dns-zone $dns_zone_name

# Manual creation of A record
# az network private-dns record-set a create -n $sql_server_name -z $dns_zone_name -g $rg
# az network private-dns record-set a add-record --record-set-name $sql_server_name -z $dns_zone_name -g $rg -a $sql_endpoint_ip

# DNS resolution verification (DNS TTL might take some minutes to expire)
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${sql_server_fqdn}"

# Test SQL query
curl "http://${vm_pip_ip}:8080/api/sqlsrcip?SQL_SERVER_FQDN=${sql_server_fqdn}&SQL_SERVER_USERNAME=${sql_username}&SQL_SERVER_PASSWORD=${sql_password}"

# Check effective routes generated by private link
# If using the UDR functionality, at the time of this testing there is a /32 route for the private endpoint
# Still marked as "Active", since there is no other overlapping UDR
# Default   Active   192.168.2.4/32    InterfaceEndpoint
vm_nic_id=$(az vm show -n $vm_name -g $rg --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic show-effective-route-table --ids $vm_nic_id -o table

#####################
#  Storage account  #
#####################

# Create Azure Storage account
storage_account_name=plink$RANDOM
storage_container_name=test
storage_blob_name=test.txt
az storage account create -n $storage_account_name -g $rg --sku Standard_LRS --kind StorageV2
storage_account_key=$(az storage account keys list -n $storage_account_name -g $rg --query '[0].value' -o tsv)
az storage container create -n $storage_container_name --public-access container \
    --auth-mode key --account-name $storage_account_name --account-key $storage_account_key
echo 'Hello world!' >"/tmp/${storage_blob_name}"
az storage blob upload -n "$storage_blob_name" -c "$storage_container_name" -f "/tmp/${storage_blob_name}" \
    --auth-mode key --account-name "$storage_account_name" --account-key "${storage_account_key}"

# Create storage endpoint (in the SQL Subnet)
storage_endpoint_name=storageep
storage_account_id=$(az storage account show -n $storage_account_name -g $rg -o tsv --query id)
# az network vnet subnet update -n $subnet_sql_name -g $rg --vnet-name $vnet_name --disable-private-endpoint-network-policies true
# az network private-endpoint create -n $storage_endpoint_name -g $rg --vnet-name $vnet_name --subnet $subnet_sql_name --private-connection-resource-id $storage_account_id --group-id blob --connection-name blob
az deployment group create -n sqlendpoint -g $rg --template-file ${plink_template_file} \
  --parameters "{ \"privateEndpointName\": { \"value\": \"${storage_endpoint_name}\" }, \"subnetId\": { \"value\": \"${sql_subnet_id}\" }, \"privateLinkResourceId\": { \"value\": \"${storage_account_id}\" }, \"privateLinkResourceGroupId\": { \"value\": \"blob\" } }"
storage_nic_id=$(az network private-endpoint show -n $storage_endpoint_name -g $rg --query 'networkInterfaces[0].id' -o tsv)
storage_endpoint_ip=$(az network nic show --ids $storage_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv)
echo "Private IP address for Storage Account ${storage_account_name}: ${storage_endpoint_ip}"

# Create private DNS
dns_zone_name=privatelink.blob.core.windows.net
az network private-dns zone create -n $dns_zone_name -g $rg
az network private-dns link vnet create -g $rg -z $dns_zone_name -n myDnsLink --virtual-network $vnet_name --registration-enabled false

# Linking private endpoint with DNS zone to automatically create the A record
az network private-endpoint dns-zone-group create --endpoint-name $storage_endpoint_name -g $rg -n myzonegroup --zone-name zone1 --private-dns-zone $dns_zone_name

# Manual creation of A record
# az network private-dns record-set a create -n $storage_account_name -z $dns_zone_name -g $rg
# az network private-dns record-set a add-record --record-set-name $storage_account_name -z $dns_zone_name -g $rg -a $storage_endpoint_ip

# Verify DNS resolution for both endpoints
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${storage_account_name}.privatelink.blob.core.windows.net"
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${sql_server_fqdn}"

# Download file
# ssh $vm_pip_ip "wget https://${storage_account_name}.blob.core.windows.net/test/test.txt --no-check-certificate"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "curl -s https://${storage_account_name}.blob.core.windows.net/test/test.txt"


##################
# Azure Firewall #
##################

# Create Azure Firewall
azfw_name=myazfw
azfw_pip_name=myazfw-pip
subnet_azfw_name=AzureFirewallSubnet
subnet_azfw_prefix=192.168.15.0/24
logws_name=log$RANDOM
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_azfw_name --address-prefix $subnet_azfw_prefix
az network public-ip create -g $rg -n $azfw_pip_name --sku standard --allocation-method static -l $location
azfw_ip=$(az network public-ip show -g $rg -n $azfw_pip_name --query ipAddress -o tsv)
az network firewall create -n $azfw_name -g $rg -l $location
azfw_id=$(az network firewall show -n $azfw_name -g $rg -o tsv --query id)
az monitor log-analytics workspace create -n $logws_name -g $rg
logws_id=$(az monitor log-analytics workspace show -n $logws_name -g $rg --query id -o tsv)
logws_customerid=$(az monitor log-analytics workspace show -n $logws_name -g $rg --query customerId -o tsv)
az monitor diagnostic-settings create -n mydiag --resource $azfw_id --workspace $logws_id \
    --metrics '[{"category": "AllMetrics", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false }, "timeGrain": null}]' \
    --logs '[{"category": "AzureFirewallApplicationRule", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false}}, 
            {"category": "AzureFirewallNetworkRule", "enabled": true, "retentionPolicy": {"days": 0, "enabled": false}}]'
az network firewall ip-config create -f $azfw_name -n azfw-ipconfig -g $rg --public-ip-address $azfw_pip_name --vnet-name $vnet_name
az network firewall update -n $azfw_name -g $rg
azfw_private_ip=$(az network firewall show -n $azfw_name -g $rg -o tsv --query 'ipConfigurations[0].privateIpAddress')
# az network firewall application-rule create -f $azfw_name -g $rg -c AllowAll --protocols Http=8080 Https=443 --target-fqdns "*" --source-addresses $vnet_prefix -n Allow-all --priority 200 --action Allow
az network firewall network-rule create -f $azfw_name -g $rg -c VnetTraffic --protocols Any --destination-addresses $vnet_prefix --destination-ports '*' --source-addresses $vnet_prefix \
  -n Allow-Vnet-Traffic --priority 210 --action Allow

# Create client Route Table (VM subnet)
vm_rt_name=vmrt
az network route-table create -n $vm_rt_name -g $rg -l $location
vm_rt_id=$(az network route-table show -n $vm_rt_name -g $rg --query id -o tsv)
# az network route-table route create -n sqlendpoint --route-table-name $vm_rt_name -g $rg --next-hop-type VirtualAppliance --address-prefix "${sql_endpoint_ip}/32" --next-hop-ip-address $azfw_private_ip
# az network route-table route create -n storageendpoint --route-table-name $vm_rt_name -g $rg --next-hop-type VirtualAppliance --address-prefix "${storage_endpoint_ip}/32" --next-hop-ip-address $azfw_private_ip
az network route-table route create --route-table $vm_rt_name -n endpointsubnet -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_sql_prefix}" --next-hop-ip-address $nva_private_ip
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $subnet_vm_name --route-table $vm_rt_id
# Remove route
# az network route-table route delete -n sqlendpoint --route-table-name $vm_rt_name -g $rg
az network route-table route list --route-table-name $vm_rt_name -g $rg -o table

# Check effective routes generated with route table attached
vm_nic_id=$(az vm show -n $vm_name -g $rg --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic show-effective-route-table --ids $vm_nic_id -o table

# Create server Route Table (actually not required)
# ep_rt_name=plinkrt
# az network route-table create -n $ep_rt_name -g $rg -l $location
# ep_rt_id=$(az network route-table show -n $ep_rt_name -g $rg --query id -o tsv)
# az network route-table route create -n vmsubnet --route-table-name $ep_rt_name -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_vm_prefix}" --next-hop-ip-address $azfw_private_ip
# az network route-table route create -n vmip --route-table-name $ep_rt_name -g $rg --next-hop-type VirtualAppliance --address-prefix "${vm_private_ip}/32" --next-hop-ip-address $azfw_private_ip
# az network vnet subnet update -g $rg --vnet-name $vnet_name -n $subnet_sql_name --route-table $ep_rt_id
# az network route-table route list --route-table-name $ep_rt_name -g $rg -o table

# Test SQL query
curl "http://${vm_pip_ip}:8080/api/sqlsrcip?SQL_SERVER_FQDN=${sql_server_fqdn}&SQL_SERVER_USERNAME=${sql_username}&SQL_SERVER_PASSWORD=${sql_password}"

# Show Azure Firewall logs
query_nw_rule='AzureDiagnostics
| where Category == "AzureFirewallNetworkRule"
| parse msg_s with Protocol " request from " SourceIP ":" SourcePortInt:int " to " TargetIP ":" TargetPortInt:int *
| parse msg_s with * ". Action: " Action1a
| parse msg_s with * " was " Action1b " to " NatDestination
| parse msg_s with Protocol2 " request from " SourceIP2 " to " TargetIP2 ". Action: " Action2
| extend SourcePort = tostring(SourcePortInt),TargetPort = tostring(TargetPortInt)
| extend Action = case(Action1a == "", case(Action1b == "",Action2,Action1b), Action1a),Protocol = case(Protocol == "", Protocol2, Protocol),SourceIP = case(SourceIP == "", SourceIP2, SourceIP),TargetIP = case(TargetIP == "", TargetIP2, TargetIP),SourcePort = case(SourcePort == "", "N/A", SourcePort),TargetPort = case(TargetPort == "", "N/A", TargetPort),NatDestination = case(NatDestination == "", "N/A", NatDestination)
| project TimeGenerated, msg_s, Protocol, SourceIP,SourcePort,TargetIP,TargetPort,Action, NatDestination'
# This might take a while to work
az monitor log-analytics query -w $logws_customerid --analytics-query $query_nw_rule -o tsv

# Customized SNAT behavior of AzFW
az network firewall update -n $azfw_name -g $rg --private-ranges $subnet_vm_prefix

# Test SQL query
curl "http://${vm_pip_ip}:8080/api/sqlsrcip?SQL_SERVER_FQDN=${sql_server_fqdn}&SQL_SERVER_USERNAME=${sql_username}&SQL_SERVER_PASSWORD=${sql_password}"

# Customized SNAT behavior of AzFW: revert to default
az network firewall update -n $azfw_name -g $rg --private-ranges IANAPrivateRanges

#############
# Verifying #
#############

# Full test
echo "Route Table at the client (VM) side:"
az network route-table route list --route-table-name $vm_rt_name -g $rg -o table
# echo "Route Table at the server (private link endpoint) side:"
# az network route-table route list --route-table-name $ep_rt_name -g $rg -o table
# Test DNS resolution
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${sql_server_fqdn}"
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${storage_account_name}.blob.core.windows.net"
# Test SQL Server
curl "http://${vm_pip_ip}:8080/api/sqlsrcip?SQL_SERVER_FQDN=${sql_server_fqdn}&SQL_SERVER_USERNAME=${sql_username}&SQL_SERVER_PASSWORD=${sql_password}"
# Test Storage Account
ssh $vm_pip_ip "curl -s https://${storage_account_name}.blob.core.windows.net/test/test.txt"

# Optionally add/remove routes
az network route-table route list --route-table-name $vm_rt_name -g $rg -o table
# az network route-table route create --route-table $vm_rt_name -n sqlendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${sql_endpoint_ip}/32" --next-hop-ip-address $azfw_private_ip
# az network route-table route create --route-table $vm_rt_name -n storageendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${storage_endpoint_ip}/32" --next-hop-ip-address $azfw_private_ip
az network route-table route delete --route-table $vm_rt_name -n sqlendpoint -g $rg
az network route-table route delete --route-table $vm_rt_name -n storageendpoint -g $rg
# az network route-table route list --route-table $ep_rt_name -g $rg -o table
# az network route-table route create --route-table $ep_rt_name -n vmsubnet -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_vm_prefix}" --next-hop-ip-address $azfw_private_ip
# az network route-table route create --route-table $ep_rt_name -n vmip -g $rg --next-hop-type VirtualAppliance --address-prefix "${vm_private_ip}/32" --next-hop-ip-address $azfw_private_ip
# az network route-table route delete --route-table $ep_rt_name -n vmsubnet -g $rg
# az network route-table route delete --route-table $ep_rt_name -n vmip -g $rg
# Verify subnets
az network vnet subnet show -g $rg --vnet-name $vnet_name -n $subnet_vm_name --query 'routeTable.id' -o tsv
az network vnet subnet show -g $rg --vnet-name $vnet_name -n $subnet_sql_name --query 'routeTable.id' -o tsv

#################
#   Linux NVA   #
#################

# Create Ubuntu VM in a new subnet, to be used as NVA
subnet_nva_name=nva
subnet_nva_prefix=192.168.16.0/24
nva_name=nva
nva_nsg_name=${nva_name}-nsg
nva_pip_name=${nva_name}-pip
nva_disk_name=${nva_name}-disk0
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_nva_name --address-prefix $subnet_nva_prefix
nva_sku=Standard_B2ms
publisher=Canonical
offer=UbuntuServer
sku=18.04-LTS
image_urn=$(az vm image list -p $publisher -f $offer -s $sku -l $location --query '[0].urn' -o tsv)
# Deploy VM
az vm create -n $nva_name -g $rg -l $location --image $image_urn --size $nva_sku --generate-ssh-keys \
  --os-disk-name $nva_disk_name --os-disk-size-gb 32 \
  --vnet-name $vnet_name --subnet $subnet_nva_name \
  --nsg $nva_nsg_name --nsg-rule SSH --public-ip-address $nva_pip_name
# Enable IP forwarding
nva_nic_id=$(az vm show -n $nva_name -g $rg --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic update --ids $nva_nic_id --ip-forwarding true
# Connect to VM
nva_pip_ip=$(az network public-ip show -n $nva_pip_name -g $rg --query ipAddress -o tsv)
echo "You can SSH to $nva_pip_ip"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $nva_pip_ip "sudo sysctl -w net.ipv4.ip_forward=1"
# Get private IP
nva_private_ip=$(az network nic show --ids $nva_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv)
echo "NVA provisioned with private IP $nva_private_ip"

# Create client Route Table in the VM subnet if required
vm_rt_name=vmrt
vm_rt_id=$(az network route-table show -n $vm_rt_name -g $rg --query id -o tsv)
if [[ -z "$vm_rt_id" ]]
then
    echo "Creating route table $vm_rt_name...."
    az network route-table create -n $vm_rt_name -g $rg -l $location
    vm_rt_id=$(az network route-table show -n $vm_rt_name -g $rg --query id -o tsv)
    az network vnet subnet update -g $rg --vnet-name $vnet_name -n $subnet_vm_name --route-table $vm_rt_id
else
    echo "Route table $vm_rt_name already exists, no need to create one"
fi

# Configure client/server routes
# az network route-table route update --route-table $vm_rt_name -n sqlendpoint -g $rg --next-hop-ip-address $nva_private_ip
# az network route-table route update --route-table $vm_rt_name -n storageendpoint -g $rg --next-hop-ip-address $nva_private_ip
# az network route-table route update --route-table $ep_rt_name -n vmip -g $rg --next-hop-ip-address $nva_private_ip
# az network route-table route update --route-table $ep_rt_name -n vmsubnet -g $rg --next-hop-ip-address $nva_private_ip
# az network route-table route create --route-table $vm_rt_name -n sqlendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${sql_endpoint_ip}/32" --next-hop-ip-address $nva_private_ip
# az network route-table route create --route-table $vm_rt_name -n storageendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${storage_endpoint_ip}/32" --next-hop-ip-address $nva_private_ip
az network route-table route create --route-table $vm_rt_name -n endpointsubnet -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_sql_prefix}" --next-hop-ip-address $nva_private_ip

# Optionally add/remove routes
az network route-table route list --route-table-name $vm_rt_name -g $rg -o table
# az network route-table route create --route-table $vm_rt_name -n sqlendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${sql_endpoint_ip}/32" --next-hop-ip-address $nva_private_ip
# az network route-table route create --route-table $vm_rt_name -n storageendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${storage_endpoint_ip}/32" --next-hop-ip-address $nva_private_ip
az network route-table route delete --route-table $vm_rt_name -n sqlendpoint -g $rg
az network route-table route delete --route-table $vm_rt_name -n storageendpoint -g $rg
# az network route-table route list --route-table $ep_rt_name -g $rg -o table
# az network route-table route create --route-table $ep_rt_name -n vmsubnet -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_vm_prefix}" --next-hop-ip-address $nva_private_ip
# az network route-table route create --route-table $ep_rt_name -n vmip -g $rg --next-hop-type VirtualAppliance --address-prefix "${vm_private_ip}/32" --next-hop-ip-address $nva_private_ip
# az network route-table route delete --route-table $ep_rt_name -n vmsubnet -g $rg
# az network route-table route delete --route-table $ep_rt_name -n vmip -g $rg

# Inspect effective routes in VM
# The /32 routes should now be marked as invalid:
# User      Active   192.168.2.0/24    VirtualAppliance   192.168.16.4
# Default   Invalid  192.168.2.4/32    InterfaceEndpoint
# Default   Invalid  192.168.2.5/32    InterfaceEndpoint
vm_nic_id=$(az vm show -n $vm_name -g $rg --query 'networkProfile.networkInterfaces[0].id' -o tsv)
az network nic show-effective-route-table --ids $vm_nic_id -o table

# Quick test
echo "Testing SQL endpoint..."
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${sql_server_fqdn}"
curl "http://${vm_pip_ip}:8080/api/sqlsrcip?SQL_SERVER_FQDN=${sql_server_fqdn}&SQL_SERVER_USERNAME=${sql_username}&SQL_SERVER_PASSWORD=${sql_password}"
echo "Testing Storage endpoint..."
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${storage_account_name}.privatelink.blob.core.windows.net"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "curl -s https://${storage_account_name}.blob.core.windows.net/test/test.txt"


# Optionally, configure SNAT in the NVA
# Enable SNAT
ssh  -n -o BatchMode=yes -o StrictHostKeyChecking=no $nva_pip_ip "sudo iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE"
# Disable SNAT
ssh  -n -o BatchMode=yes -o StrictHostKeyChecking=no $nva_pip_ip "sudo iptables -t nat -D POSTROUTING -o eth0 -j MASQUERADE"

# You can SSH into the NVA and use tcpdump to inspect packets
ssh $nva_pip_ip "sudo tcpdump not host $nva_private_ip"

# Full test and diagnostics
echo "Route Table at the client (VM) side:"
az network route-table route list --route-table-name $vm_rt_name -g $rg -o table
echo "Route Table at the server (private link endpoint) side:"
az network route-table route list --route-table-name $ep_rt_name -g $rg -o table
# Test DNS resolution
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${sql_server_fqdn}"
curl -s "http://${vm_pip_ip}:8080/api/dns?fqdn=${storage_account_name}.blob.core.windows.net"
# Test SQL Server
curl "http://${vm_pip_ip}:8080/api/sqlsrcip?SQL_SERVER_FQDN=${sql_server_fqdn}&SQL_SERVER_USERNAME=${sql_username}&SQL_SERVER_PASSWORD=${sql_password}"
# Test Storage Account
ssh $vm_pip_ip "curl -s https://${storage_account_name}.blob.core.windows.net/test/test.txt"

# Optionally add/remove routes
az network route-table route list --route-table-name $vm_rt_name -g $rg -o table
az network route-table route create --route-table $vm_rt_name -n sqlendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${sql_endpoint_ip}/32" --next-hop-ip-address $nva_private_ip
az network route-table route create --route-table $vm_rt_name -n storageendpoint -g $rg --next-hop-type VirtualAppliance --address-prefix "${storage_endpoint_ip}/32" --next-hop-ip-address $nva_private_ip
az network route-table route delete --route-table $vm_rt_name -n sqlendpoint
az network route-table route delete --route-table $vm_rt_name -n storageendpoint
az network route-table route list --route-table $ep_rt_name -g $rg -o table
az network route-table route create --route-table $ep_rt_name -n vmsubnet -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_vm_prefix}" --next-hop-ip-address $nva_private_ip
az network route-table route create --route-table $ep_rt_name -n vmip -g $rg --next-hop-type VirtualAppliance --address-prefix "${vm_private_ip}/32" --next-hop-ip-address $nva_private_ip
az network route-table route delete --route-table $ep_rt_name -n vmsubnet
az network route-table route delete --route-table $ep_rt_name -n vmip
# Verify subnets
az network vnet subnet show -g $rg --vnet-name $vnet_name -n $subnet_vm_name --query 'routeTable.id' -o tsv
az network vnet subnet show -g $rg --vnet-name $vnet_name -n $subnet_sql_name --query 'routeTable.id' -o tsv

##################################################
# Web app with vnet integration and private link #
##################################################

# Create subnets
subnet_webapp_be_name=webapp-be
subnet_webapp_be_prefix=192.168.5.0/24
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_webapp_be_name --address-prefix $subnet_webapp_be_prefix
subnet_webapp_fe_name=webapp-fe
subnet_webapp_fe_prefix=192.168.6.0/24
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_webapp_fe_name --address-prefix $subnet_webapp_fe_prefix

# Create webapp
svcplan_name=webappplan
app_name_api=api-$RANDOM
app_name_web=web-$RANDOM
az appservice plan create -n $svcplan_name -g $rg --sku B1 --is-linux
az webapp create -n $app_name_api -g $rg -p $svcplan_name --deployment-container-image-name erjosito/sqlapi:1.0
az webapp config appsettings set -n $app_name_api -g $rg --settings "WEBSITES_PORT=8080" "SQL_SERVER_USERNAME=$sql_username" "SQL_SERVER_PASSWORD=$sql_password" "SQL_SERVER_FQDN=${sql_server_fqdn}"
az webapp restart -n $app_name_api -g $rg
app_url_api=$(az webapp show -n $app_name_api -g $rg --query defaultHostName -o tsv)
# Note: it might take some seconds/minutes for the web app to come up and answer successfully the following command
curl "http://${app_url_api}/api/healthcheck"

# SQL Server firewall rules
sqlapi_webapp_source_ip=$(curl -s http://${app_url_api}/api/ip | jq -r .my_public_ip)
az sql server firewall-rule create -g $rg -s $sql_server_name -n webapp-sqlapi-source --start-ip-address $sqlapi_webapp_source_ip --end-ip-address $sqlapi_webapp_source_ip
az sql server firewall-rule list -g $rg -s $sql_server_name -o table
echo "Testing the SQL server over its public endpoint"
curl -s "http://${app_url_api}/api/sqlsrcip"

# Web app vnet integration
az webapp vnet-integration add -n $app_name_api -g $rg --vnet $vnet_name --subnet $subnet_webapp_be_name
# az webapp vnet-integration list -n $app_name_api -g $rg -o table
# new_webapp_source_ip=$(curl -s http://${app_url_api}/api/ip | jq -r .my_public_ip)
# echo "After vnet integration, the egress IP address for the web app is ${new_webapp_source_ip} (the old one was ${old_webapp_source_ip})"
echo "Configuring DNS resolution for web app..."
az webapp config appsettings set -n $app_name_api -g $rg --settings "WEBSITE_DNS_SERVER=168.63.129.16" "WEBSITE_VNET_ROUTE_ALL=1"
az webapp restart -n $app_name_api -g $rg
echo "Testing the SQL server over its private endpoint"
curl -s "http://${app_url_api}/api/dns?fqdn=${sql_server_fqdn}"
curl -s "http://${app_url_api}/api/sqlsrcip"

# Private link for web app
webapp_endpoint_name=mywebep
svcplan_id=$(az appservice plan show -n $svcplan_name -g $rg -o tsv --query id)
webapp_id=$(az webapp show -n $app_name_api -g $rg -o tsv --query id)
az network vnet subnet update -n $subnet_webapp_fe_name -g $rg --vnet-name $vnet_name --disable-private-endpoint-network-policies true
# Private link endpoints only work for premium SKU
az appservice plan update -n $svcplan_name -g $rg --sku P1V2
group_id=sites
# az network private-link-resource list -n $app_name_api -g $rg --type Microsoft.Web -o table  # DOES NOT WORK
# az network private-endpoint create -n $webapp_endpoint_name -g $rg --vnet-name $vnet_name --subnet $subnet_webapp_fe_name --private-connection-resource-id $webapp_id --group-id $group_id --connection-name webappConnection
fe_subnet_id=$(az network vnet subnet show -n $subnet_webapp_fe_name --vnet-name $vnet_name -g $rg --query id -o tsv) && echo $fe_subnet_id
az deployment group create -n webappendpoint -g $rg --template-file ${plink_template_file} \
  --parameters "{ \"privateEndpointName\": { \"value\": \"${webapp_endpoint_name}\" }, \"subnetId\": { \"value\": \"${fe_subnet_id}\" }, \"privateLinkResourceId\": { \"value\": \"${webapp_id}\" }, \"privateLinkResourceGroupId\": { \"value\": \"$group_id\" } }"
webapp_nic_id=$(az network private-endpoint show -n $webapp_endpoint_name -g $rg --query 'networkInterfaces[0].id' -o tsv)
webapp_endpoint_ip=$(az network nic show --ids $webapp_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv) && echo $webapp_endpoint_ip

# Private DNS Zone
dns_zone_name=privatelink.azurewebsites.net
az network private-dns zone create -n $dns_zone_name -g $rg
az network private-dns link vnet create -g $rg -z $dns_zone_name -n myDnsLink --virtual-network $vnet_name --registration-enabled false
az network private-endpoint dns-zone-group create --endpoint-name $webapp_endpoint_name -g $rg -n myzonegroup --zone-name zone1 --private-dns-zone $dns_zone_name
az network private-endpoint dns-zone-group list --endpoint-name $webapp_endpoint_name -g $rg -o table
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "nslookup $app_url_api"

# Verify frontend private link and backend connectivity to SQL using privatelink
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "curl -s http://${app_url_api}/api/healthcheck"
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "curl -s http://${app_url_api}/api/sqlsrcip"

# Add routing to the VM RT to send traffic through the AzFW/NVA
az network route-table route create --route-table $vm_rt_name -n webappfe -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_webapp_fe_prefix}" --next-hop-ip-address $nva_private_ip
az network route-table route list --route-table $vm_rt_name -g $rg -o table

# The source IP as seen by the app should now be the AzFW/NVA (doing SNAT), instead of the client VM
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "curl -s http://${app_url_api}/api/ip" | jq -r '.["x-forwarded-for"]'

# RT for Vnet Integration (webapp -> SQL) to go through the AzFW
webapp_rt_name=webapprt
az network route-table create -n $webapp_rt_name -g $rg -l $location
webapp_rt_id=$(az network route-table show -n $webapp_rt_name -g $rg --query id -o tsv)
az network vnet subnet update -g $rg --vnet-name $vnet_name -n $subnet_webapp_be_name --route-table $webapp_rt_id
az network route-table route create --route-table $webapp_rt_name -n endpointsubnet -g $rg --next-hop-type VirtualAppliance --address-prefix "${subnet_sql_prefix}" --next-hop-ip-address $nva_private_ip

# The source IP as seen by the SQL Server should be again the AzFW/NVA, instead of the webapp backend vnet
ssh -n -o BatchMode=yes -o StrictHostKeyChecking=no $vm_pip_ip "curl -s http://${app_url_api}/api/sqlsrcip"

#######################
# Application Gateway #
#######################

# Variables
rg=plinkappgw
location=westeurope
appgw_rg=appgw
appgw_name=appgw
appgw_vnet=appgw
vm_size=Standard_B1ms
testvm_name=testvm
testvm_pip_name="${testvm_name}-pip"

# Create VNet and test VM
az group create -n $rg -l $location
vnet_name=myvnet
vnet_prefix=192.168.0.0/16
subnet_ep_name=pe
subnet_ep_prefix=192.168.2.0/24
subnet_vm_name=vm
subnet_vm_prefix=192.168.13.0/24
az network vnet create -g $rg -n $vnet_name --address-prefix $vnet_prefix -l $location
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_ep_name --address-prefix $subnet_ep_prefix
az network vnet subnet create -g $rg --vnet-name $vnet_name -n $subnet_vm_name --address-prefix $subnet_vm_prefix
az vm create -n $testvm_name -g $rg -l $location --image UbuntuLTS --size $vm_size --generate-ssh-keys --vnet-name $vnet_name --subnet $subnet_vm_name --public-ip-address $testvm_pip_name
testvm_pip=$(az network public-ip show -n $testvm_pip_name -g $rg --query ipAddress -o tsv) && echo $testvm_pip
ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$testvm_pip" "ip a"

# Install web server on VM
script_url=https://raw.githubusercontent.com/erjosito/whoami/master/api-vm/cse.sh
script_command='./cse.sh'
az vm extension set -n customScript --vm-name $testvm_name -g $rg --publisher Microsoft.Azure.Extensions \
  --protected-settings "{\"fileUris\": [\"${script_url}\"],\"commandToExecute\": \"${script_command}\"}"
ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$testvm_pip" "curl -s http://localhost:8080/api/healthcheck"

# Azure CLI creation experience
# https://docs.microsoft.com/en-us/cli/azure/network/application-gateway/private-link?view=azure-cli-latest
appgw_pls_name=appgwpls
frontend_ip_name=$(az network application-gateway frontend-ip list -g $appgw_rg --gateway-name $appgw_name -o tsv --query '[0].name') && echo $frontend_ip_name
subnet_appgw_id=$(az network application-gateway show -n $appgw_name -g $appgw_rg --query 'gatewayIpConfigurations[0].subnet.id' -o tsv) && echo $subnet_appgw_id
subnet_ep_id=$(az network vnet subnet show -n $subnet_ep_name --vnet-name $vnet_name -g $rg --query id -o tsv) && echo $subnet_ep_id
subnet_appgwep_id=$(az network vnet subnet show -n ep --vnet-name $appgw_vnet -g $appgw_rg --query id -o tsv) && echo $subnet_appgwep_id
az network vnet subnet update --ids $subnet_appgwep_id --disable-private-link-service-network-policies true
az network vnet subnet update --ids $subnet_appgwep_id --disable-private-link-service-network-policies true
# This command up to now only supports subnets in the same vnet as the appgw
az network application-gateway private-link add -n $appgw_pls_name -g $appgw_rg --gateway-name $appgw_name --frontend-ip $frontend_ip_name --subnet $subnet_appgw_id
# The previous command creates something like this (inside of the appgw config):
  # "privateLinkConfigurations": [
  #   {
  #     "etag": "W/\"ce0d005d-2b6d-443c-923a-e94d521cf2bb\"",
  #     "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/applicationGateways/appgw/privateLinkConfigurations/appgwpls",
  #     "ipConfigurations": [
  #       {
  #         "etag": "W/\"ce0d005d-2b6d-443c-923a-e94d521cf2bb\"",
  #         "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/applicationGateways/appgw/privateLinkConfigurations/appgwpls/ipConfigurations/PrivateLinkDefaultIPConfiguration",
  #         "name": "PrivateLinkDefaultIPConfiguration",
  #         "primary": null,
  #         "privateIpAddress": null,
  #         "privateIpAllocationMethod": "Dynamic",
  #         "provisioningState": "Succeeded",
  #         "resourceGroup": "appgw",
  #         "subnet": {
  #           "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/virtualNetworks/appgw/subnets/ep",
  #           "resourceGroup": "appgw"
  #         },
  #         "type": "Microsoft.Network/applicationGateways/privateLinkConfigurations/ipConfigurations"
  #       }
  #     ],
  #     "name": "appgwpls",
  #     "provisioningState": "Succeeded",
  #     "resourceGroup": "appgw",
  #     "type": "Microsoft.Network/applicationGateways/privateLinkConfigurations"
  #   }
  # ],
appgw_pls_ipconfig=plsipconfig
az network application-gateway private-link ip-config add --name $appgw_pls_ipconfig --private-link $appgw_pls_name --gateway-name $appgw_name -g $appgw_rg
# The previous command creates something like this (inside of the appgw config):
  # "privateLinkConfigurations": [
  #   {
  #     "etag": "W/\"7882bd7c-6ecf-4d1a-991c-1cacd9177f94\"",
  #     "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/applicationGateways/appgw/privateLinkConfigurations/appgwpls",
  #     "ipConfigurations": [
  #       {
  #         "etag": "W/\"7882bd7c-6ecf-4d1a-991c-1cacd9177f94\"",
  #         "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/applicationGateways/appgw/privateLinkConfigurations/appgwpls/ipConfigurations/PrivateLinkDefaultIPConfiguration",
  #         "name": "PrivateLinkDefaultIPConfiguration",
  #         "primary": null,
  #         "privateIpAddress": null,
  #         "privateIpAllocationMethod": "Dynamic",
  #         "provisioningState": "Succeeded",
  #         "resourceGroup": "appgw",
  #         "subnet": {
  #           "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/virtualNetworks/appgw/subnets/ep",
  #           "resourceGroup": "appgw"
  #           },
  #           "type": "Microsoft.Network/applicationGateways/privateLinkConfigurations/ipConfigurations"
  #       },
  #       {
  #         "etag": "W/\"7882bd7c-6ecf-4d1a-991c-1cacd9177f94\"",
  #         "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/applicationGateways/appgw/privateLinkConfigurations/appgwpls/ipConfigurations/plsipconfig",
  #         "name": "plsipconfig",
  #         "primary": false,
  #         "privateIpAddress": null,
  #         "privateIpAllocationMethod": "Dynamic",
  #         "provisioningState": "Succeeded",
  #         "resourceGroup": "appgw",
  #         "subnet": {
  #           "id": "/subscriptions/e7da9914-9b05-4891-893c-546cb7b0422e/resourceGroups/appgw/providers/Microsoft.Network/virtualNetworks/appgw/subnets/ep",
  #           "resourceGroup": "appgw"
  #         },
  #         "type": "Microsoft.Network/applicationGateways/privateLinkConfigurations/ipConfigurations"
  #       }
  #     ],
  #     "name": "appgwpls",
  #     "provisioningState": "Succeeded",
  #     "resourceGroup": "appgw",
  #     "type": "Microsoft.Network/applicationGateways/privateLinkConfigurations"
  #   }
  # ],


plink_template_file="/tmp/plink_template.json"
cat <<'EOF' > ${plink_template_file}
{
  "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "contentVersion": "1.0.0.0",
  "parameters": {
    "location": {
      "type": "string",
      "defaultValue": "[resourceGroup().location]"
    },
    "privateEndpointName": {
      "type": "string"
    },
    "subnetId": {
      "type": "string"
    },
    "privateLinkServiceResourceId": {
      "type": "string"
    },
    "privateLinkResourceGroupId": {
      "type": "string",
      "defaultValue": ""
    }
  },
  "resources": [
    {
      "name": "[parameters('privateEndpointName')]",
      "type": "Microsoft.Network/privateEndpoints",
      "apiVersion": "2021-05-01",
      "location": "[parameters('location')]",
      "properties": {
        "subnet": {
          "id": "[parameters('subnetId')]"
        },
        "privateLinkServiceConnections": [
          {
            "name": "MyPrivateEndpointConnection",
            "properties": {
              "privateLinkServiceId": "[parameters('privateLinkServiceResourceId')]",
              "groupIds": [
                "[parameters('privateLinkResourceGroupId')]"
              ]
            }
          }
        ]
      }
    }
  ]
}
EOF

# Create endpoint
appgw_ep_name=appgwep
appgw_id=$(az network application-gateway show -n $appgw_name -g $appgw_rg -o tsv --query id) && echo $appgw_id
az network vnet subnet update -n $subnet_ep_name -g $rg --vnet-name $vnet_name --disable-private-endpoint-network-policies true
#az network private-endpoint create -n $sql_endpoint_name -g $rg --vnet-name $vnet_name --subnet $subnet_sql_name --private-connection-resource-id $sql_server_id --group-id sqlServer --connection-name sqlConnection
appgw_pls_id=$(az network application-gateway private-link show -n $appgw_pls_name --gateway-name $appgw_name -g $appgw_rg --query id -o tsv) && echo $appgw_pls_id
az deployment group create -n $appgw_ep_name -g $rg --template-file ${plink_template_file} \
  --parameters "privateEndpointName=${appgw_ep_name}" "subnetId=${subnet_ep_id}" "privateLinkServiceResourceId=${appgw_id}" "privateLinkResourceGroupId=appGatewayFrontendIP"
appgw_ep_nic_id=$(az network private-endpoint show -n $appgw_ep_name -g $rg --query 'networkInterfaces[0].id' -o tsv)
appgw_ep_ip=$(az network nic show --ids $appgw_ep_nic_id --query 'ipConfigurations[0].privateIpAddress' -o tsv)
echo "Private IP address for App Gateway ${appgw_name} in VNet ${vnet_name}: ${appgw_ep_ip}"

# Test
ssh -n -o StrictHostKeyChecking=no -o BatchMode=yes "$testvm_pip" "curl http://$appgw_ep_ip"


###############
# Danger zone #
###############

# Cleanup
# az group delete -n $rg -y --no-wait