如何使用Azure的Terraform强制删除/释放网络服务组？

我定期从Azure中删除时收到此错误。

│ Error: deleting Network Security Group "multisqlite-bacalhau-eastus-ssh-nsg" (Resource Group "multisqlite-bacalhau-eastus-rg"): network.SecurityGroupsClient#Delete: Failure sending request: StatusCode=400 -- Original Error: Code="NetworkSecurityGroupOldReferencesNotCleanedUp" Message="Network security group multisqlite-bacalhau-eastus-ssh-nsg cannot be deleted because old references for the following Nics: (\n/subscriptions/72ac7288-fb92-4ad6-83bc-5cfd361f47ef/resourceGroups/multisqlite-bacalhau-eastus-rg/providers/Microsoft.Network/networkSecurityGroups/multisqlite-bacalhau-eastus-ssh-nsg:/subscriptions/72ac7288-fb92-4ad6-83bc-5cfd361f47ef/resourceGroups/multisqlite-bacalhau-eastus-rg/providers/Microsoft.Network/networkInterfaces/multisqlite-bacalhau-eastus-nic) and Subnet: (\n/subscriptions/72ac7288-fb92-4ad6-83bc-5cfd361f47ef/resourceGroups/multisqlite-bacalhau-eastus-rg/providers/Microsoft.Network/networkSecurityGroups/multisqlite-bacalhau-eastus-ssh-nsg:) have not been released yet." Details=[]

原因并不明显--我看到它说“还没有发布”，但是当我转到Web UI时，它让我正确地删除它（terraform完成）。有什么我能做的吗？这是整个地形

terraform {
  required_version = ">=1.0"
  required_providers {
    azurerm = {
      source  = "hashicorp/azurerm"
      version = "~>3.0"
    }
    random = {
      source  = "hashicorp/random"
      version = "~>3.0"
    }
  }
}
provider "azurerm" {
  tenant_id       = var.tenantId
  subscription_id = var.subscriptionId
  client_id       = var.clientId
  client_secret   = var.clientSecret
  features {
    resource_group {
      prevent_deletion_if_contains_resources = false
    }
  }
}


data "cloudinit_config" "user_data" {

  for_each = var.locations

  # Azure VMs apparently require it to be b64 encoded
  gzip          = true
  base64_encode = true

  part {
    filename     = "cloud-config.yaml"
    content_type = "text/cloud-config"

    content = templatefile("cloud-init/init-vm.yml", {
      app_name : var.app_tag,

      bacalhau_service : filebase64("${path.root}/node_files/bacalhau.service"),
      ipfs_service : base64encode(file("${path.module}/node_files/ipfs.service")),
      start_bacalhau : filebase64("${path.root}/node_files/start-bacalhau.sh"),
      sensor_data_generator_py : filebase64("${path.root}/node_files/sensor_data_generator.py"),
      requirements_txt : filebase64("${path.root}/node_files/requirements.txt"),

      # Need to do the below to remove spaces and newlines from public key
      ssh_key : compact(split("\n", file(var.public_key)))[0],
      tailscale_key : var.tailscale_key,
      node_name : "${var.app_tag}-${each.key}-vm",
      username : var.username,
      region : each.value.region,
      zone : each.value.region,
      project_id : "${var.app_tag}",
    })
  }
}

resource "azurerm_resource_group" "rg" {
  for_each = var.locations
  name     = "${var.app_tag}-${each.key}-rg"
  location = each.key
}

resource "azurerm_virtual_network" "vnet" {
  for_each            = var.locations
  name                = "${var.app_tag}-${each.key}-vnet"
  address_space       = ["10.0.0.0/16"]
  location            = azurerm_resource_group.rg[each.key].location
  resource_group_name = azurerm_resource_group.rg[each.key].name
}

resource "azurerm_subnet" "internal" {
  for_each             = var.locations
  name                 = "${var.app_tag}-${each.key}-internal-vnet"
  resource_group_name  = azurerm_resource_group.rg[each.key].name
  virtual_network_name = azurerm_virtual_network.vnet[each.key].name
  address_prefixes     = ["10.0.2.0/24"]
}

resource "azurerm_public_ip" "public_ip_allocator" {
  for_each            = var.locations
  name                = "${var.app_tag}-${each.key}-public-ip"
  resource_group_name = azurerm_resource_group.rg[each.key].name
  location            = azurerm_resource_group.rg[each.key].location
  allocation_method   = "Dynamic"
}

data "azurerm_public_ip" "public_ip" {
  for_each            = var.locations
  name                = azurerm_public_ip.public_ip_allocator[each.key].name
  resource_group_name = azurerm_linux_virtual_machine.instance[each.key].resource_group_name
}

resource "azurerm_network_interface" "nic" {
  for_each            = var.locations
  name                = "${var.app_tag}-${each.key}-nic"
  location            = azurerm_resource_group.rg[each.key].location
  resource_group_name = azurerm_resource_group.rg[each.key].name

  ip_configuration {
    name                          = "${var.app_tag}-${each.key}-ipConfiguration"
    subnet_id                     = azurerm_subnet.internal[each.key].id
    private_ip_address_allocation = "Dynamic"
    public_ip_address_id          = azurerm_public_ip.public_ip_allocator[each.key].id
  }
}

resource "azurerm_network_security_group" "nsg" {
  for_each            = var.locations
  name                = "${var.app_tag}-${each.key}-ssh-nsg"
  location            = azurerm_resource_group.rg[each.key].location
  resource_group_name = azurerm_resource_group.rg[each.key].name

  security_rule {
    name                       = "allow_ssh_sg"
    priority                   = 100
    direction                  = "Inbound"
    access                     = "Allow"
    protocol                   = "Tcp"
    source_port_range          = "*"
    destination_port_range     = "22"
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }
}

resource "azurerm_network_interface_security_group_association" "association" {
  for_each                  = var.locations
  network_interface_id      = azurerm_network_interface.nic[each.key].id
  network_security_group_id = azurerm_network_security_group.nsg[each.key].id
}

# Create a random password
resource "random_password" "password" {
  length           = 16
  special          = true
  override_special = "_%@"
}

resource "azurerm_linux_virtual_machine" "instance" {
  for_each              = var.locations
  name                  = "${var.app_tag}-${each.key}-vm"
  location              = azurerm_resource_group.rg[each.key].location
  resource_group_name   = azurerm_resource_group.rg[each.key].name
  network_interface_ids = [azurerm_network_interface.nic[each.key].id]
  size                  = "Standard_D2s_v3"
  computer_name         = "${var.app_tag}-${each.key}-vm"
  admin_username        = var.username

  custom_data = data.cloudinit_config.user_data[each.key].rendered

  admin_ssh_key {
    username   = var.username
    public_key = file(var.public_key)
  }

  os_disk {
    caching              = "ReadWrite"
    storage_account_type = "Standard_LRS"
  }

  source_image_reference {
    publisher = "canonical"
    offer     = "0001-com-ubuntu-server-jammy"
    sku       = "22_04-lts-gen2"
    version   = "latest"
  }
}

locals {
  # Get just one of the bootstrap IP from data.azurerm_public_ip.public_ip where it is in the bootstrap region
  bootstrap_ip = data.azurerm_public_ip.public_ip[var.bootstrap_region].ip_address

  # Get all the regions except the bootstrap region and put them in a set
  non_bootstrap_regions = toset([for region in keys(var.locations) : region if region != var.bootstrap_region])
}

resource "null_resource" "copy-bacalhau-bootstrap-to-local" {
  depends_on = [azurerm_linux_virtual_machine.instance]

  connection {
    host        = local.bootstrap_ip
    port        = 22
    user        = var.username
    private_key = file(var.private_key)
  }

  provisioner "remote-exec" {
    inline = [
      "echo 'SSHD is now alive.'",
      "timeout 300 bash -c 'until [[ -s /run/bacalhau.run ]]; do sleep 1; done' && echo 'Bacalhau is now alive.'",
    ]
  }

  provisioner "local-exec" {
    command = "ssh -o StrictHostKeyChecking=no ${var.username}@${local.bootstrap_ip} 'sudo cat /run/bacalhau.run' > ${var.bacalhau_run_file}"
  }
}

resource "null_resource" "copy-to-node-if-worker" {
  // Only run this on worker nodes, not the bootstrap node
  for_each = local.non_bootstrap_regions

  depends_on = [null_resource.copy-bacalhau-bootstrap-to-local]

  connection {
    host        = data.azurerm_public_ip.public_ip[each.value].ip_address
    port        = 22
    user        = var.username
    private_key = file(var.private_key)
  }

  provisioner "file" {
    destination = "/home/${var.username}/bacalhau-bootstrap"
    content     = file(var.bacalhau_run_file)
  }

  provisioner "remote-exec" {
    inline = [
      "sudo mv /home/${var.username}/bacalhau-bootstrap /etc/bacalhau-bootstrap",
      "sudo systemctl daemon-reload",
      "sudo systemctl restart bacalhau.service",
    ]
  }
}

我已经创建了与之关联的NSG和NIC。
代码：

resource "azurerm_virtual_network" "example" {
  name                = "kexamplenetwork"
  address_space       = ["10.0.0.0/16"]
  location            = data.azurerm_resource_group.example.location
  resource_group_name = data.azurerm_resource_group.example.name
}

resource "azurerm_subnet" "example" {
  name                 = "internal"
  resource_group_name = data.azurerm_resource_group.example.name
  virtual_network_name = azurerm_virtual_network.example.name
  address_prefixes     = ["10.0.2.0/24"]
}

resource "azurerm_network_security_group" "example" {
  name                = "kexample-nsg"
 location            = data.azurerm_resource_group.example.location
  resource_group_name = data.azurerm_resource_group.example.name
    security_rule {
    name                       = "test123"
    priority                   = 100
    direction                  = "Inbound"
    access                     = "Allow"
    protocol                   = "Tcp"
    source_port_range          = "*"
    destination_port_range     = "*"
    source_address_prefix      = "*"
    destination_address_prefix = "*"
  }
}

resource "azurerm_network_interface" "example" {
  name                = "kexample-nic"
 location            = data.azurerm_resource_group.example.location
  resource_group_name = data.azurerm_resource_group.example.name

  ip_configuration {
    name                          = "testconfiguration1"
    subnet_id                     = azurerm_subnet.example.id
    private_ip_address_allocation = "Dynamic"
  }
}

resource "azurerm_network_interface_security_group_association" "example" {
  network_interface_id      = azurerm_network_interface.example.id
  network_security_group_id = azurerm_network_security_group.example.id
}

我使用下面的命令删除了nsg，它被删除了。

terraform destroy -target='resourcetype.name'*
terraform destroy -target='azurerm_network_security_group.example'

但请注意，它先删除与NSG的网络接口关联，然后删除NSG。

因此，请确保首先解除NICS和子网到nsg的关联，或解除NIC关联，因为它们依赖于nsg引用。

在powershell中，检查与nsg关联或连接的所有子网：

az network nsg show -n "kexample-nsg" -g “rg” --query 'subnets[].id' -o tsv

az network nsg show -n "kexample-nsg" -g “rg”

那么所有那些id都可以用来改变nsg

az network vnet subnet update --ids [] (resource IDs space-delimited) --network-security-group "kexample-nsg"

或az network vnet子网更新-g networks --name subnetName --remove networkSecurityGroup --vnet-name vnetName --订阅订阅
或者可以解除NIC的关联：

Get-AzureRmNetworkInterface -ResourceGroupName “rg” -Name "examplenic" 
$nic.NetworkSecurityGroup = $null
Set-AzureRmNetworkInterface -NetworkInterface $nic

您也可以查看此内容bad request - Azure - Cannot delete Network Security Group - Stack Overflow

如何使用Azure的Terraform强制删除/释放网络服务组？

1条答案

相关问题

热门标签

最新问答