disable broken devices on Disk Agent registration

ydb-platform · Nov 5, 2024 · 32dc4c8 · 32dc4c8
1 parent 3346ef3
commit 32dc4c8
Show file tree

Hide file tree

Showing 28 changed files with 738 additions and 101 deletions.
diff --git a/cloud/blockstore/config/disk.proto b/cloud/blockstore/config/disk.proto
@@ -261,7 +261,11 @@ message TDiskAgentConfig
 
     // List of device UUIDs with suspended I/O.
     // I/O operations for such a device will result in errors.
+    // Is used for the config cache file only.
     repeated string DevicesWithSuspendedIO = 35;
+
+    // Disable devices that have been recognized as broken by the DR
+    optional bool DisableBrokenDevices = 36;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp b/cloud/blockstore/libs/rdma_test/rdma_test_environment.cpp
@@ -14,23 +14,27 @@ namespace NCloud::NBlockStore::NStorage {
 TRdmaTestEnvironment::TRdmaTestEnvironment(size_t deviceSize, ui32 poolSize)
     : Storage(std::make_shared<TMemoryTestStorage>(deviceSize))
 {
-    THashMap<TString, TStorageAdapterPtr> devices;
-    devices[Device_1] = std::make_shared<TStorageAdapter>(
-        Storage,
-        4_KB,                // storageBlockSize
-        true,                // normalize,
-        TDuration::Zero(),   // maxRequestDuration
-        TDuration::Zero()    // shutdownTimeout
-    );
+    THashMap<TString, TStorageAdapterPtr> devices{
+        {Device_1,
+         std::make_shared<TStorageAdapter>(
+             Storage,
+             4_KB,                // storageBlockSize
+             true,                // normalize,
+             TDuration::Zero(),   // maxRequestDuration
+             TDuration::Zero()    // shutdownTimeout
+             )}};
+
     TVector<TString> uuids;
     for (const auto& [key, value]: devices) {
         uuids.push_back(key);
     }
-    auto deviceClient = std::make_shared<TDeviceClient>(
+
+    DeviceClient = std::make_shared<TDeviceClient>(
         TDuration::MilliSeconds(100),
         uuids,
         Logging->CreateLog("BLOCKSTORE_DISK_AGENT"));
-    deviceClient->AcquireDevices(
+
+    DeviceClient->AcquireDevices(
         uuids,
         ClientId,
         TInstant::Now(),
@@ -60,7 +64,7 @@ TRdmaTestEnvironment::TRdmaTestEnvironment(size_t deviceSize, ui32 poolSize)
         std::move(oldRequestCounters),
         Logging,
         Server,
-        std::move(deviceClient),
+        DeviceClient,
         std::move(devices));
 
     RdmaTarget->Start();

diff --git a/cloud/blockstore/libs/rdma_test/rdma_test_environment.h b/cloud/blockstore/libs/rdma_test/rdma_test_environment.h
@@ -36,6 +36,8 @@ struct TRdmaTestEnvironment
         "console",
         TLogSettings{TLOG_RESOURCES});
 
+    std::shared_ptr<TDeviceClient> DeviceClient;
+
     TRdmaTestEnvironment(size_t deviceSize = 4_MB, ui32 poolSize = 1);
 
     virtual ~TRdmaTestEnvironment();

diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_disable.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_disable.cpp
@@ -26,6 +26,7 @@ void TDiskAgentActor::HandleDisableConcreteAgent(
     if (record.DeviceUUIDsSize()) {
         for (const auto& d: record.GetDeviceUUIDs()) {
             State->DisableDevice(d);
+            State->ReportDisabledDeviceError(d);
         }
     } else {
         HandlePoisonPill(nullptr, ctx);

diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_io.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_io.cpp
@@ -178,17 +178,6 @@ void TDiskAgentActor::PerformIO(
             started);
     };
 
-    if (State->IsDeviceDisabled(deviceUUID)) {
-        LOG_INFO(ctx, TBlockStoreComponents::DISK_AGENT,
-            "Dropped %s request to device %s, session %s",
-            TMethod::Name,
-            deviceUUID.c_str(),
-            clientId.c_str());
-        State->ReportDisabledDeviceError(deviceUUID);
-        replyError(E_IO, "Device disabled");
-        return;
-    }
-
     LOG_TRACE(ctx, TBlockStoreComponents::DISK_AGENT,
         "%s [%s / %s]",
         TMethod::Name,

diff --git a/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_register.cpp b/cloud/blockstore/libs/storage/disk_agent/disk_agent_actor_register.cpp
@@ -87,6 +87,9 @@ void TRegisterActor::HandleRegisterAgentResponse(
 
     auto response = std::make_unique<TEvDiskAgentPrivate::TEvRegisterAgentResponse>(
         msg->GetError());
+    response->DevicesToSuspendIO.assign(
+        msg->Record.GetDevicesToSuspendIO().cbegin(),
+        msg->Record.GetDevicesToSuspendIO().cend());
     NCloud::Reply(ctx, *RequestInfo, std::move(response));
 }
 
@@ -144,6 +147,7 @@ void TDiskAgentActor::HandleRegisterAgentResponse(
     if (!HasError(msg->GetError())) {
         RegistrationState = ERegistrationState::Registered;
         LOG_INFO(ctx, TBlockStoreComponents::DISK_AGENT, "Register completed");
+        State->UpdateDevicesWithSuspendedIO(msg->DevicesToSuspendIO);
     } else {
         LOG_WARN(ctx, TBlockStoreComponents::DISK_AGENT,
             "Register failed: %s. Try later", FormatError(msg->GetError()).c_str());