1)attention_alg 2)no_attention 3)scale

Author: closest-git

main_tabular_data.py

@@ -285,18 +285,20 @@ def Fold_learning(fold_n,data,config,visual):
     parser = argparse.ArgumentParser()
     # parser.add_argument('--use-gpu', action='store_true')
     parser.add_argument('--data_root',required=True)
-    parser.add_argument('--dataset', default='CLICK',help="MICROSOFT,YAHOO,YEAR,CLICK,HIGGS")
+    parser.add_argument('--dataset', default='CLICK',help="MICROSOFT,YAHOO,YEAR,CLICK,HIGGS,EPSILON")
     parser.add_argument('--iterations', default=1000, type=int)
     parser.add_argument('--model', default="QForest", help='QForest,GBDT,LinearRegressor')
-    parser.add_argument('--learning_rate', default="0.01", type=float)
+    parser.add_argument('--learning_rate', default="0.001", type=float)
     parser.add_argument('--subsample', default="1", type=float)
-    parser.add_argument('--QF_fit', default="0", type=int)
+    parser.add_argument('--QF_fit', default="1", type=int)
+    parser.add_argument('--attention', default="eca_response", type=str)
+    parser.add_argument('--scale', default="medium",help='small，medium，large', type=str)
     args = parser.parse_args()
     dataset = args.dataset
     data = quantum_forest.TabularDataset(dataset,data_path=args.data_root, random_state=1337, quantile_transform=True, quantile_noise=1e-3)
     #data = quantum_forest.TabularDataset(dataset,data_path=data_root, random_state=1337, quantile_transform=True)
 
-    config = quantum_forest.QForest_config(data,0.002,feat_info="attention")   #,feat_info="importance","attention"
+    config = quantum_forest.QForest_config(data,0.002)   #,feat_info="importance","attention"
     random_state = 42
     config.device = quantum_forest.OnInitInstance(random_state)
     config.model=args.model      #"QForest"            "GBDT" "LinearRegressor"    
@@ -306,19 +308,26 @@ def Fold_learning(fold_n,data,config,visual):
     config.bagging_fraction = args.subsample
     config.nMostEpochs = args.iterations
     config.QF_fit = args.QF_fit
+    config.attention_alg = args.attention
+    if args.scale == "small":
+        config.depth, config.batch_size, config.nTree = 4, 256, 256
+    elif args.scale == "medium":
+        config.depth, config.batch_size, config.nTree = 5, 512, 1024
+    elif args.scale == "large":
+        config.depth, config.batch_size, config.nTree = 5, 512, 2048
 
     if dataset=="YAHOO" or dataset=="MICROSOFT" or dataset=="CLICK" or dataset=="HIGGS" or dataset=="EPSILON":
         config,visual = quantum_forest.InitExperiment(config, 0)
         data.onFold(0,config,pkl_path=f"{args.data_root}{dataset}/FOLD_Quantile_.pickle")
         Fold_learning(0,data, config,visual)
-    else:
+    else:   #"YEAR"
         nFold = 5 if dataset != "HIGGS" else 20
         folds = KFold(n_splits=nFold, shuffle=True)
         index_sets=[]
         for fold_n, (train_index, valid_index) in enumerate(folds.split(data.X)):
             index_sets.append(valid_index)
         for fold_n in range(len(index_sets)):
-            config, visual = InitExperiment(config, fold_n)
+            config, visual = quantum_forest.InitExperiment(config, fold_n)
             train_list=[]
             for i in range(nFold):
                 if i==fold_n:           #test
@@ -330,7 +339,7 @@ def Fold_learning(fold_n,data,config,visual):
             train_index=np.concatenate(train_list)
             print(f"train={len(train_index)} valid={len(valid_index)} test={len(index_sets[fold_n])}")
 
-            data.onFold(fold_n,config,train_index=train_index, valid_index=valid_index,test_index=index_sets[fold_n],pkl_path=f"{data_root}{dataset}/FOLD_{fold_n}.pickle")
+            data.onFold(fold_n,config,train_index=train_index, valid_index=valid_index,test_index=index_sets[fold_n],pkl_path=f"{args.data_root}{dataset}/FOLD_{fold_n}.pickle")
             Fold_learning(fold_n,data,config,visual)
             break
 

python-package/quantum_forest/DifferentiableTree.py

@@ -37,7 +37,7 @@ def init_attention(self, feat_info=None):
         self.feat_map = random.choices(population = list(range(self.in_features)),k = self.nGateFuncs) #weights = weight,
         #self.feat_map[0]=1
         self.init_attention_func=nn.init.eye
-        if self.no_attention:
+        if False:   #self.no_attention:
             self.feat_weight = nn.Parameter(torch.Tensor(self.nGateFuncs).uniform_(), requires_grad=True)
             pass
         elif False:  #仅用于对比
@@ -63,7 +63,7 @@ def init_attention(self, feat_info=None):
             self.feat_attention = nn.Parameter( feat_val, requires_grad=True )
         #self.feat_W = nn.Parameter(torch.Tensor(self.in_features).uniform_(), requires_grad=True)
 
-        print(f"====== init_attention f={self.init_attention_func.__name__} no_attention={self.no_attention}")
+        print(f"====== init_attention f={self.init_attention_func.__name__} attention={self.config.attention_alg}")
 
     #weights computed as entmax over the learnable feature selection matrix F ∈ R d×n
     def get_attention_value(self,input):
@@ -226,20 +226,25 @@ def __init__(self, in_features, num_trees,config, flatten_output=True,feat_info=
         if self.config.path_way=="TREE_map":
             self.nFeature = 2**depth-1
         self.nGateFuncs = self.num_trees*self.nFeature
-        if False:
+        if self.config.attention_alg == "eca_response":
             self.att_reponse = eca_reponse(self.num_trees)
-            #self.att_input = eca_input(self.in_features)
+        elif self.config.attention_alg == "eca_input":
+            self.att_input = eca_input(self.in_features)
+        elif self.config.attention_alg == "":
+            print("!!! Empty attention_alg.Please try \"--attention=eca_response\" !!!\n")
+        else:
+            raise ValueError( f'INVALID self.config.attention_alg = {self.config.attention_alg}' )
 
-        if self.config.attention_alg == "weight":
-            if False and isAdptiveAlpha:      #可以试试，就是时间太长
-                self.entmax_alpha = nn.Parameter(torch.tensor(1.5, requires_grad=True))
-                self.attention_func = entmax.entmax_bisect    #sparsemax, entmax15, entmax_bisect
-            else:
-                self.attention_func = entmax15
+        
+        if False and isAdptiveAlpha:      #可以试试，就是时间太长
+            self.entmax_alpha = nn.Parameter(torch.tensor(1.5, requires_grad=True))
+            self.attention_func = entmax.entmax_bisect    #sparsemax, entmax15, entmax_bisect
         else:
+            self.attention_func = entmax15
+        if self.config.attention_alg == "excitation_max":       #失败的尝试
             self.attention_func = excitation_max(in_features,self.nGateFuncs,self.num_trees)
         self.bin_func = "05_01"     #"05_01"        "entmoid15" "softmax"
-        self.no_attention = config.no_attention
+        # self.no_attention = config.no_attention
         self.threshold_init_beta, self.threshold_init_cutoff = threshold_init_beta, threshold_init_cutoff
         self.init_responce_func = initialize_response_
         self.response_mean,self.response_std = 0,0
@@ -309,13 +314,13 @@ def forward(self, input):
         if hasattr(self,"att_input"):
             input = self.att_input(input)
         # new input shape: [batch_size, in_features]
-        if self.no_attention:
-            feature_values = input[:, self.feat_map]  # torch.index_select(input.flatten(), 0, self.feat_select)
-            #feature_values = torch.einsum('bf,f->bf',feature_values,self.feat_weight)
-            feature_values = feature_values.reshape(-1, self.num_trees, self.depth)
-            assert feature_values.shape[0] == input.shape[0]
-        else:
-            feature_values = self.get_attention_value(input)
+        # if self.no_attention:
+        #     feature_values = input[:, self.feat_map]  # torch.index_select(input.flatten(), 0, self.feat_select)
+        #     #feature_values = torch.einsum('bf,f->bf',feature_values,self.feat_weight)
+        #     feature_values = feature_values.reshape(-1, self.num_trees, self.depth)
+        #     assert feature_values.shape[0] == input.shape[0]
+        # else:
+        feature_values = self.get_attention_value(input)
         # ^--[batch_size, num_trees, depth]
 
         threshold_logits = (feature_values - self.feature_thresholds) * torch.exp(-self.log_temperatures)
@@ -421,13 +426,13 @@ def initialize(self, input, eps=1e-6):
             print(f"!!!!!! DeTree::initialize@{self.__repr__()} has only {nSamp} sampls. This may cause instability.\n")
 
         with torch.no_grad():
-            if self.no_attention:
-                feature_values = input[:, self.feat_map]                #torch.index_select(input.flatten(), 0, self.feat_select)
-                feature_values = torch.einsum('bf,f->bf', feature_values, self.feat_weight)
-                feature_values=feature_values.reshape(-1,self.num_trees,self.depth)
-                assert feature_values.shape[0]==input.shape[0]
-            else:
-                feature_values = self.get_attention_value(input)
+            # if self.no_attention:
+            #     feature_values = input[:, self.feat_map]                #torch.index_select(input.flatten(), 0, self.feat_select)
+            #     feature_values = torch.einsum('bf,f->bf', feature_values, self.feat_weight)
+            #     feature_values=feature_values.reshape(-1,self.num_trees,self.depth)
+            #     assert feature_values.shape[0]==input.shape[0]
+            # else:
+            feature_values = self.get_attention_value(input)
 
             # initialize thresholds: sample random percentiles of data
             percentiles_q = 100 * np.random.beta(self.threshold_init_beta, self.threshold_init_beta,
@@ -450,8 +455,8 @@ def __repr__(self):
             f_info = self.attention_func.__repr__()
             f_name = "excitation_max"
             f_init = ""
-        else:
-            f_info = 0 if self.no_attention else self.feat_attention.shape[0]
+        else:            
+            f_info = self.feat_attention.shape[0]
             f_name = self.attention_func.__name__
             f_init = self.init_attention_func.__name__
         main_str = "{}(F={},f={},B={}, T={},D={}, response_dim={}, " \

python-package/quantum_forest/QForest.py

@@ -21,7 +21,7 @@ def __init__(self,data, lr_base, nLayer=1,choice_func="r_0.5",feat_info = None,r
         self.rDrop = 0
         self.custom_legend = None
         self.feat_info = feat_info
-        self.no_attention = False
+        # self.no_attention = False       #always False since 5/28/2020
         self.max_features = None
         self.input_dropout = 0        #YAHOO 0.59253-0.59136 没啥用
         self.num_layers = 1
@@ -46,7 +46,7 @@ def __init__(self,data, lr_base, nLayer=1,choice_func="r_0.5",feat_info = None,r
 
         self.err_relative = False
         self.task = "train"
-        self.attention_alg = "weight"       #
+        self.attention_alg = "eca_reponse"       # 'eca_input'
 
         self.nMostEpochs = 1000
         self.depth, self.batch_size, self.nTree, self.response_dim, self.nLayers = 5, 256, 256, 8, 1
@@ -121,7 +121,7 @@ def env_title(self):
     def main_str(self):
         main_str = f"{self.data_set}_ layers={self.nLayer} depth={self.depth} batch={self.batch_size} nTree={self.nTree} response_dim={self.response_dim} " \
             f"\nmax_out={self.max_out} choice=[{self.choice_func}] feat_info={self.feat_info}" \
-            f"\nNO_ATTENTION={self.no_attention} reg_L1={self.reg_L1} path_way={self.path_way}"
+            f"\nATTENTION={self.config.attention_alg} reg_L1={self.reg_L1} path_way={self.path_way}"
         #if self.isFC:       main_str+=" [FC]"
         if self.custom_legend is not None:
             main_str = main_str + f"_{self.custom_legend}"

python-package/quantum_forest/tabular_data.py

@@ -156,7 +156,7 @@ def problem(self):
     def onFold(self,fold,config,pkl_path=None, train_index=None, valid_index=None, test_index=None):
         if pkl_path is not None:
             print("====== onFold pkl_path={} ......".format(pkl_path))
-        if False and pkl_path is not None and os.path.isfile(pkl_path):
+        if pkl_path is not None and os.path.isfile(pkl_path):
             with open(pkl_path, "rb") as fp:
                 [self.X_train,self.y_train,self.X_valid, self.y_valid,self.X_test,self.y_test,\
             self.quantile_noise,self.Y_trans_method,self.accu_scale,self.Y_mu_0, self.Y_std_0,self.zero_feats] = pickle.load(fp)

test_LGB.bat

@@ -0,0 +1,9 @@
+E:
+cd  E:\fengnaixing\cys\QuantumForest
+python main_tabular_data.py --data_root="E:/fengnaixing/cys/Datasets/" --dataset=YAHOO --model=GBDT
+python main_tabular_data.py --data_root="E:/fengnaixing/cys/Datasets/" --dataset=CLICK --model=GBDT
+python main_tabular_data.py --data_root="E:/fengnaixing/cys/Datasets/" --dataset=MICROSOFT --model=GBDT
+python main_tabular_data.py --data_root="E:/fengnaixing/cys/Datasets/" --dataset=YEAR --model=GBDT
+python main_tabular_data.py --data_root="E:/fengnaixing/cys/Datasets/" --dataset=HIGGS --model=GBDT
+python main_tabular_data.py --data_root="E:/fengnaixing/cys/Datasets/" --dataset=EPSILON --model=GBDT
+:: python main_tabular_data.py --data_root=“E:/fengnaixing/cys/Datasets/” --model=GBDT --dataset=MICROSOFTtes

test_QF.bat

@@ -0,0 +1,24 @@
+::@echo off
+E:
+cd  E:\fengnaixing\cys\QuantumForest
+:: https://stackoverflow.com/questions/18462169/how-to-loop-through-array-in-batch
+::set DATA[0]=YAHOO
+::set DATA[1]=CLICK
+::set DATA[2]=MICROSOFT
+::set DATA[3]=YEAR
+::set DATA[4]=HIGGS
+::set DATA[5]=EPSILON
+set "cmd=python main_tabular_data.py --data_root=../Datasets/"
+::set "param=--model=GBDT"
+
+%cmd% --dataset=YAHOO %param%
+%cmd% --dataset=CLICK %param%
+%cmd% --dataset=MICROSOFT %param%
+%cmd% --dataset=YEAR %param%
+%cmd% --dataset=HIGGS %param%
+%cmd% --dataset=EPSILON %param%
+:: --model=GBDT --dataset=MICROSOFT --learning_rate=0.001
+:: --attention=""
+:: --scale="large"
+:: C:/Users/fengnaixing/test_QF.bat
+:: python main_tabular_data.py --data_root=../Datasets/ --dataset=YAHOO --model=GBDT