Dev (#287)

* Add Weight48IntegerQuantizer

* Add naive qoq int w4a8

configs/quantization/methods/RTN/rtn_w_a_wint4aint8.yml

@@ -0,0 +1,41 @@
+base:
+    seed: &seed 42
+model:
+    type: Llama
+    path: /mnt/nvme1/yongyang/models/llama2-7b
+    torch_dtype: auto
+eval:
+    eval_pos: [pretrain, fake_quant]
+    name: wikitext2
+    download: False
+    path: /mnt/nvme0/yongyang/llm_datasets/llmc/eval/wikitext2
+    seq_len: 2048
+    # For 7B / 13B model eval, bs can be set to "1", and inference_per_block can be set to "False".
+    # For 70B model eval, bs can be set to "20", and inference_per_block can be set to "True".
+    bs: 1
+    inference_per_block: False
+quant:
+    method: RTN
+    weight:
+        quant_type: int-quant
+        bit: 48
+        bit4:
+            symmetric: False
+            granularity: per_group
+            group_size: 128
+            scales_bit: 8
+            scales_symmetric: True
+            zeros_bit: 8
+            zeros_symmetric: True
+        bit8:
+            symmetric: True
+            granularity: per_channel
+            int_range: [-120, 120]
+    act:
+        quant_type: int-quant
+        bit: 8
+        symmetric: True
+        granularity: per_token
+save:
+    save_fake: False
+    save_path: /path/to/save/

llmc/compression/quantization/base_blockwise_quantization.py

@@ -23,7 +23,7 @@
                            _TRANSFORMERS_LN_TYPES_, EffcientFakeQuantLinear,
                            FakeQuantLinear, LlmcActFn, OriginFloatLinear,
                            RotateLinear)
-from .quant import FloatQuantizer, IntegerQuantizer
+from .quant import FloatQuantizer, IntegerQuantizer, Weight48IntegerQuantizer
 from .utils import check_do_quant, check_w_only, get_aquantizer, get_wquantizer
 
 
@@ -175,22 +175,32 @@ def set_quant_config(self):
         self.tp = self.quant_config.get('tp', 1)
         self.quant_config['weight']['tp'] = self.tp
 
-        # select quant module
-        self.quant_type = self.quant_config.get('quant_type', 'int-quant')
-        if self.quant_type == 'int-quant':
-            self.quant_module = IntegerQuantizer
-        elif self.quant_type == 'float-quant':
-            self.quant_module = FloatQuantizer
-        logger.info(f'The used Quant Module is {self.quant_module}')
-
-        # set weight quant config
-        self.wquantizer = self.quant_module(**self.quant_config['weight'])
+        # select quantizer
+        # weight
+        quant_type = self.quant_config['weight'].get('quant_type', 'int-quant')
+        if quant_type == 'int-quant':
+            if self.quant_config['weight']['bit'] == 48:
+                self.weight_quant_module = Weight48IntegerQuantizer
+            else:
+                self.weight_quant_module = IntegerQuantizer
+        elif quant_type == 'float-quant':
+            self.weight_quant_module = FloatQuantizer
+        logger.info(f'The used Weight Quant Module is {self.weight_quant_module}')
+        self.wquantizer = self.weight_quant_module(**self.quant_config['weight'])
 
-        # set act quant config
+        # act
         if 'act' in self.quant_config:
             self.w_only = False
+            quant_type = self.quant_config['act'].get('quant_type', 'int-quant')
+            if quant_type == 'int-quant':
+                if self.quant_config['act']['bit'] == 48:
+                    self.act_quant_module = Weight48IntegerQuantizer
+                else:
+                    self.act_quant_module = IntegerQuantizer
+            elif quant_type == 'float-quant':
+                self.act_quant_module = FloatQuantizer
             self.quant_config['act']['tp'] = self.tp
-            self.aquantizer = self.quant_module(**self.quant_config['act'])
+            self.aquantizer = self.act_quant_module(**self.quant_config['act'])
             self.act_static = self.quant_config['act'].get('static', False)
             if self.act_static:
                 assert (