change 5 pytorch/huggingface models to fp16 (#11894)

Author: JinheTang

python/llm/example/GPU/HuggingFace/LLM/codellama/generate.py

@@ -47,7 +47,7 @@
                                                  optimize_model=False,
                                                  trust_remote_code=True,
                                                  use_cache=True)
-    model = model.to('xpu')
+    model = model.half().to('xpu')
 
     # Load tokenizer
     tokenizer = CodeLlamaTokenizer.from_pretrained(model_path,

python/llm/example/GPU/HuggingFace/LLM/internlm/generate.py

@@ -47,7 +47,7 @@
                                                  optimize_model=False,
                                                  trust_remote_code=True,
                                                  use_cache=True)
-    model = model.to('xpu')
+    model = model.half().to('xpu')
 
     # Load tokenizer
     tokenizer = AutoTokenizer.from_pretrained(model_path,

python/llm/example/GPU/HuggingFace/LLM/solar/generate.py

@@ -47,7 +47,7 @@
                                                  load_in_4bit=True,
                                                  trust_remote_code=True,
                                                  use_cache=True)
-    model = model.to('xpu')
+    model = model.half().to('xpu')
 
     # Load tokenizer
     tokenizer = AutoTokenizer.from_pretrained(model_path,

python/llm/example/GPU/PyTorch-Models/Model/codellama/generate.py

@@ -50,7 +50,7 @@
     # This will allow the memory-intensive embedding layer to utilize the CPU instead of iGPU.
     model = optimize_model(model)
 
-    model = model.to('xpu')
+    model = model.half().to('xpu')
 
     # Load tokenizer
     tokenizer = CodeLlamaTokenizer.from_pretrained(model_path, trust_remote_code=True)

python/llm/example/GPU/PyTorch-Models/Model/internlm2/generate.py

@@ -46,7 +46,7 @@
                                                  use_cache=True)
     model = optimize_model(model)
 
-    model = model.to('xpu')
+    model = model.half().to('xpu')
 
     # Load tokenizer
     tokenizer = AutoTokenizer.from_pretrained(model_path,

python/llm/example/GPU/PyTorch-Models/Model/mistral/generate.py

@@ -49,7 +49,7 @@
     # This will allow the memory-intensive embedding layer to utilize the CPU instead of iGPU.
     model = optimize_model(model)
 
-    model = model.to('xpu')
+    model = model.half().to('xpu')
 
     # Load tokenizer
     tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

python/llm/example/GPU/PyTorch-Models/Model/solar/generate.py

@@ -49,7 +49,7 @@
     # When running LLMs on Intel iGPUs for Windows users, we recommend setting `cpu_embedding=True` in the optimize_model function.
     # This will allow the memory-intensive embedding layer to utilize the CPU instead of iGPU.
     model = optimize_model(model)
-    model = model.to('xpu')
+    model = model.half().to('xpu')
 
     # Load tokenizer
     tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)