Skip to content

Commit 31f2dc3

Browse files
DevenAhluwaliaDevenAhluwalia
authored
[FileCopy Protocol] Bootstrap Controller (#2974)
* Init BootstrapController (cherry picked from commit 1ce57b3) * Init BootstrapController (cherry picked from commit e991679) * NA * NA * Incorporated PR review comments * ++ Test suite * PR comments WIP * PR comments WIP * PR comments WIP
1 parent 1b57345 commit 31f2dc3

File tree

13 files changed

+1089
-20
lines changed

13 files changed

+1089
-20
lines changed

‎ambry-api/src/main/java/com/github/ambry/clustermap/StateModelListenerType.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -47,5 +47,11 @@ public enum StateModelListenerType {
4747
* The partition state change listener owned by file copy manager. It takes actions when new replica is added (OFFLINE ->
4848
* BOOTSTRAP)
4949
*/
50-
FileCopyManagerListener
50+
FileCopyManagerListener,
51+
52+
/**
53+
* The partition state change listener owned by Bootstrap Controller.
54+
* It takes actions when Offline -> Bootstrap state transition is called for a partition.
55+
*/
56+
BootstrapControllerListener
5157
}

‎ambry-api/src/main/java/com/github/ambry/clustermap/StateTransitionException.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ public enum TransitionErrorCode {
7878
/**
7979
* If File Based Replication Protocol fails at some point for specific replica.
8080
*/
81-
FileCopyProtocolFailure
81+
FileCopyProtocolFailure,
82+
83+
/**
84+
* If Bootstap Controller fails in pre-filecopy steps for specific replica.
85+
*/
86+
BootstrapControllerFailure
8287
}
8388
}

‎ambry-api/src/main/java/com/github/ambry/config/StoreConfig.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -680,6 +680,7 @@ public class StoreConfig {
680680
@Config("store.reshuffle.disks.on.reorder")
681681
@Default("false")
682682
public final boolean storeReshuffleDisksOnReorder;
683+
683684
public final static String storeReshuffleDisksOnReorderName = "store.reshuffle.disks.on.reorder";
684685

685686
/**
@@ -890,4 +891,4 @@ public StoreConfig(VerifiableProperties verifiableProperties) {
890891
storeBootstrapInProgressFile = verifiableProperties.getString(STORE_BOOTSTRAP_IN_PROGRESS_FILE, "bootstrap_in_progress");
891892
storeFileCopyCompletedFileName = verifiableProperties.getString(STORE_FILE_COPY_COMPLETED_FILE_NAME, "file_copy_completed");
892893
}
893-
}
894+
}

‎ambry-api/src/main/java/com/github/ambry/server/StoreManager.java

+17
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
package com.github.ambry.server;
1515

16+
import com.github.ambry.clustermap.ClusterParticipant;
1617
import com.github.ambry.clustermap.PartitionId;
1718
import com.github.ambry.clustermap.ReplicaId;
1819
import com.github.ambry.store.Store;
@@ -21,6 +22,7 @@
2122
import java.nio.file.FileStore;
2223
import java.util.Collection;
2324
import java.util.List;
25+
import java.util.regex.Pattern;
2426

2527

2628
/**
@@ -123,4 +125,19 @@ public interface StoreManager {
123125
* @return {@code true} if disabling was successful. {@code false} if not.
124126
*/
125127
boolean controlCompactionForBlobStore(PartitionId id, boolean enabled);
128+
129+
/**
130+
* Check if a file exists in the store directory.
131+
* @return {@code true} if the file exists, {@code false} otherwise.
132+
*/
133+
boolean isFileExists(PartitionId partitionId, String fileName);
134+
135+
/**
136+
* Check if files exist for a given pattern in the store directory.
137+
* @param partitionId
138+
* @param pattern
139+
* @return {@code true} if the files exist, {@code false} otherwise.
140+
* @throws IOException
141+
*/
142+
boolean isFilesExistForPattern(PartitionId partitionId, Pattern pattern) throws IOException;
126143
}

‎ambry-api/src/main/java/com/github/ambry/store/ReplicationProtocolTransitionType.java

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/**
2+
* Copyright 2024 LinkedIn Corp. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
*/
14+
15+
package com.github.ambry.store;
16+
17+
/**
18+
* The types of hydration protocol transitions that can happen.
19+
* The possible options are :- Blob based and File copy based hydration
20+
*/
21+
public enum ReplicationProtocolTransitionType {
22+
/**
23+
* Pre restart protocol: NA
24+
* Bootstrap status: NA
25+
* Post restart protocol: Blob based
26+
*/
27+
NEW_PARTITION_TO_BLOB_BASED_HYDRATION,
28+
29+
/**
30+
* Pre restart protocol: NA
31+
* Bootstrap status: NA
32+
* Post restart protocol: File based
33+
*/
34+
NEW_PARTITION_TO_FILE_BASED_HYDRATION,
35+
36+
/**
37+
* Pre restart protocol: Blob based
38+
* Bootstrap status: Complete
39+
* Post restart protocol: File based
40+
*/
41+
BLOB_BASED_HYDRATION_COMPLETE_TO_FILE_BASED_HYDRATION,
42+
43+
/**
44+
* Pre restart protocol: File based
45+
* Bootstrap status: Complete
46+
* Post restart protocol: Blob based
47+
*/
48+
FILE_BASED_HYDRATION_COMPLETE_TO_BLOB_BASED_HYDRATION,
49+
50+
/**
51+
* Pre restart protocol: Blob based
52+
* Bootstrap status: Complete
53+
* Post restart protocol: Blob based
54+
*/
55+
BLOB_BASED_HYDRATION_COMPLETE_TO_BLOB_BASED_HYDRATION,
56+
57+
/**
58+
* Pre restart protocol: File based
59+
* Bootstrap status: Complete
60+
* Post restart protocol: File based
61+
*/
62+
FILE_BASED_HYDRATION_COMPLETE_TO_FILE_BASED_HYDRATION,
63+
64+
/**
65+
* Pre restart protocol: Blob based
66+
* Bootstrap status: InComplete
67+
* Post restart protocol: Blob based
68+
*/
69+
BLOB_BASED_HYDRATION_INCOMPLETE_TO_BLOB_BASED_HYDRATION,
70+
71+
/**
72+
* Pre restart protocol: File based
73+
* Bootstrap status: InComplete
74+
* Post restart protocol: File based
75+
*/
76+
FILE_BASED_HYDRATION_INCOMPLETE_TO_FILE_BASED_HYDRATION,
77+
78+
/**
79+
* Pre restart protocol: Blob based
80+
* Bootstrap status: InComplete
81+
* Post restart protocol: File based
82+
*/
83+
BLOB_BASED_HYDRATION_INCOMPLETE_TO_FILE_BASED_HYDRATION,
84+
85+
/**
86+
* Pre restart protocol: File based
87+
* Bootstrap status: InComplete
88+
* Post restart protocol: Blob based
89+
*/
90+
FILE_BASED_HYDRATION_INCOMPLETE_TO_BLOB_BASED_HYDRATION
91+
}

‎ambry-cloud/src/main/java/com/github/ambry/cloud/CloudStorageManager.java

+14
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@
1414
package com.github.ambry.cloud;
1515

1616
import com.github.ambry.clustermap.ClusterMap;
17+
import com.github.ambry.clustermap.ClusterParticipant;
1718
import com.github.ambry.clustermap.PartitionId;
19+
import com.github.ambry.clustermap.PartitionStateChangeListener;
1820
import com.github.ambry.clustermap.ReplicaId;
21+
import com.github.ambry.clustermap.StateModelListenerType;
1922
import com.github.ambry.config.VerifiableProperties;
2023
import com.github.ambry.server.ServerErrorCode;
2124
import com.github.ambry.server.StoreManager;
@@ -27,6 +30,7 @@
2730
import java.util.Map;
2831
import java.util.concurrent.locks.ReadWriteLock;
2932
import java.util.concurrent.locks.ReentrantReadWriteLock;
33+
import java.util.regex.Pattern;
3034
import org.slf4j.Logger;
3135
import org.slf4j.LoggerFactory;
3236

@@ -128,6 +132,16 @@ public boolean controlCompactionForBlobStore(PartitionId id, boolean enabled) {
128132
throw new UnsupportedOperationException("Method not supported");
129133
}
130134

135+
@Override
136+
public boolean isFileExists(PartitionId partitionId, String fileName) {
137+
throw new UnsupportedOperationException("Method not supported");
138+
}
139+
140+
@Override
141+
public boolean isFilesExistForPattern(PartitionId partitionId, Pattern allLogSegmentFilesPattern) {
142+
throw new UnsupportedOperationException("Method not supported");
143+
}
144+
131145
@Override
132146
public List<PartitionId> setBlobStoreStoppedState(List<PartitionId> partitionIds, boolean markStop) {
133147
throw new UnsupportedOperationException("Method not supported");

‎ambry-clustermap/src/test/java/com/github/ambry/clustermap/ReplicaTest.java

-17
Original file line numberDiff line numberDiff line change
@@ -20,23 +20,6 @@
2020
import static com.github.ambry.clustermap.TestUtils.*;
2121
import static org.junit.Assert.*;
2222

23-
24-
// Permits Replica to be constructed with a null Partition
25-
class TestReplica extends Replica {
26-
public TestReplica(HardwareLayout hardwareLayout, JSONObject jsonObject) throws JSONException {
27-
super(hardwareLayout, null, jsonObject);
28-
}
29-
30-
public TestReplica(TestHardwareLayout hardwareLayout, Disk disk) throws JSONException {
31-
super(null, disk, hardwareLayout.clusterMapConfig);
32-
}
33-
34-
@Override
35-
public void validatePartition() {
36-
// Null OK
37-
}
38-
}
39-
4023
/**
4124
* Tests {@link Replica} class.
4225
*/

‎ambry-clustermap/src/test/java/com/github/ambry/clustermap/TestReplica.java

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/**
2+
* Copyright 2024 LinkedIn Corp. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
*/
14+
15+
package com.github.ambry.clustermap;
16+
17+
import org.json.JSONException;
18+
import org.json.JSONObject;
19+
20+
21+
// Permits Replica to be constructed with a null Partition
22+
public class TestReplica extends Replica {
23+
public TestReplica(HardwareLayout hardwareLayout, JSONObject jsonObject) throws JSONException {
24+
super(hardwareLayout, null, jsonObject);
25+
}
26+
27+
public TestReplica(TestUtils.TestHardwareLayout hardwareLayout, Disk disk) throws JSONException {
28+
super(null, disk, hardwareLayout.clusterMapConfig);
29+
}
30+
31+
@Override
32+
public void validatePartition() {
33+
// Null OK
34+
}
35+
}

0 commit comments

Comments
 (0)