update environment

Author: 1mocat

environment.yml

@@ -1,4 +1,4 @@
-name: redcode
+name: agent
 channels:
   - defaults
 dependencies:
@@ -37,19 +37,24 @@ dependencies:
   - zlib=1.2.13=h5eee18b_0
   - zstd=1.5.5=hc292b87_2
   - pip:
+      - accelerate==1.0.1
       - blinker==1.8.1
       - certifi==2024.2.2
       - evdev==1.7.1
       - flask==3.0.3
       - idna==3.7
       - itsdangerous==2.2.0
+      - jiter==0.8.2
       - jsonpickle==3.0.4
+      - openai==0.28.1
       - pynput==1.7.7
       - pyparsing==3.1.2
       - pyperclip==1.8.2
       - python-xlib==0.33
       - pyyaml==6.0.1
+      - safetensors==0.4.5
       - scapy==2.5.0
       - six==1.16.0
       - typing-extensions==4.11.0
       - werkzeug==3.0.2
+prefix: /data2/common/chengquan/anaconda3/envs/agent

evaluation/RedCode_Exec/CA_evaluation/CA.py

@@ -104,7 +104,7 @@ def save_result(eval_log_json):
 
         for input_type in all_input_types:
             try:
-                eval_result_folder= os.path.join("results",input_type)
+                eval_result_folder= os.path.join("../results/CA",input_type)
                 os.makedirs(eval_result_folder, exist_ok=True)
 
                 prompt=prompt_type[input_type]
@@ -151,7 +151,7 @@ def save_result(eval_log_json):
         print(f" ----- {index} done")
 
 
-def CA(model_name="CodeActAgent-Mistral-7b-v0.1", max_exec=3, openai_base_url="http://localhost:8080/v1", 
+def CA(model_name="CodeActAgent-Mistral-7b-v0.1", max_exec=3, openai_base_url="http://localhost:8088/v1", 
        dry_run=False, ids=2, ids_start=-1, ids_end=-1, max_token=512, folder_prefix=""):
 
     # Set up ids based on input arguments
@@ -170,17 +170,29 @@ def CA(model_name="CodeActAgent-Mistral-7b-v0.1", max_exec=3, openai_base_url="h
     generator = Generator(model_name, openai_base_url)
     agent = Agent(generator, container, conv_id=time_stamp, max_new_tokens=max_token)
 
-    res_folder = os.path.join(f"results{folder_prefix}", model_name)
-    log_folder = os.path.join(f"logs{folder_prefix}", model_name)
+    # res_folder = os.path.join(f"results{folder_prefix}", model_name)
+    # log_folder = os.path.join(f"logs{folder_prefix}", model_name)
 
-    os.makedirs(res_folder, exist_ok=True)
-    os.makedirs(log_folder, exist_ok=True)
+    # os.makedirs(res_folder, exist_ok=True)
+    # os.makedirs(log_folder, exist_ok=True)
 
     # Evaluate datasets for each id
+    # Get the current file's directory
+    current_file_path = os.path.dirname(os.path.abspath(__file__))
+    dataset_base_path = os.path.join(current_file_path, '../../../dataset/RedCode-Exec')
+
+    # Set dataset JSON file path based on task type
+    
+
+    
     for id in ids_list:
-        resfile = os.path.join(res_folder, f'idx{id}_{time_stamp}.json')
-        logfile = os.path.join(log_folder, f'idx{id}.json')
-        jsonfile = f'../Trans_code/py2text_dataset_json/index{id}_30_codes_full.json'
+        # resfile = os.path.join(res_folder, f'idx{id}_{time_stamp}.json')
+        # logfile = os.path.join(log_folder, f'idx{id}.json')
+        # jsonfile = f'../Trans_code/py2text_dataset_json/index{id}_30_codes_full.json'
+
+        resfile = f'../results/CA/{time_stamp}.json'
+        logfile = f'../results/CA/log/RA_log_{time_stamp}.json'
+        jsonfile = os.path.join(dataset_base_path, f'py2text_dataset_json/index{id}_30_codes_full.json')
 
         eval_dataset(id, agent, jsonfile, resfile, logfile, max_exec, dry_run)
 
@@ -189,7 +201,7 @@ def CA(model_name="CodeActAgent-Mistral-7b-v0.1", max_exec=3, openai_base_url="h
 #     parser = argparse.ArgumentParser(description='Select base model for CodeAgent and set parameters.')
 #     parser.add_argument('--model_name',default= "CodeActAgent-Mistral-7b-v0.1", help='The base model to use in CodeAgent.')
 #     parser.add_argument('--max_exec', type=int, default=3)
-#     parser.add_argument('--openai_base_url', type=str, default="http://localhost:8080/v1")
+#     parser.add_argument('--openai_base_url', type=str, default="http://localhost:8088/v1")
 #     parser.add_argument('--dry_run', type=bool, default=False)
 #     parser.add_argument('--ids', type=int, default=2)
 #     parser.add_argument('--ids_start', type=int, default=-1)

evaluation/RedCode_Exec/CA_evaluation/codeact_agent.py

@@ -14,10 +14,14 @@
 from typing import List, Dict
 from datetime import datetime
 
-# if openai.__version__ >= "0.28.1":
-#     raise RuntimeError(
-#         "Please use the compatbile version of openai (<=0.28.1) to use this script."
-#     )
+print("OpenAI Path:", openai.__file__)
+print("OpenAI Version:", openai.__version__)
+
+
+if openai.__version__ > "0.28.1":
+    raise RuntimeError(
+        "Please use the compatbile version of openai (<=0.28.1) to use this script."
+    )
 
 
 logging.basicConfig(level=logging.INFO)

evaluation/RedCode_Exec/CA_evaluation/run.sh

@@ -1,81 +1,3 @@
-# from evaluate import *
-# from interaction import *
-# import argparse
-# import datetime
-# import os
-
-# # Eval commands example:
-# # python main.py gpt-3.5-turbo-1106 --python_eval
-
-# if __name__=='__main__':
-
-#     if not os.path.exists('../logs/Agents'):
-#         os.makedirs('../logs/Agents')
-#     if not os.path.exists('../results'):
-#         os.makedirs('../results')
-
-#     time_now = datetime.datetime.now()+datetime.timedelta(hours=8)
-#     time_stamp = str(time_now).replace(' ', '_')
-#     resfile = f'../results/{time_stamp}.json'
-#     logfile = f'../logs/Agents/Agents_{time_stamp}.json'
-
-#     parser = argparse.ArgumentParser(description='Select base model for CodeAgent and set parameters.')
-#     parser.add_argument('model', choices=[
-#                             'gpt-3.5-turbo-1106', 'gpt-4-1106-preview', 'gpt-4o-2024-05-13',
-#                             'deepseek-coder-6.7b-instruct', 'deepseek-coder-v2-lite-instruct',
-#                             'CodeLlama-7b-Instruct-hf', 'CodeLlama-13b-Instruct-hf',
-#                             'meta-llama-3-8B-instruct', 'meta-llama-3.1-8b-instruct',
-#                             'meta-llama-3-70b-instruct', 'meta-llama-3.1-70b-instruct',
-#                             'gemma-2-9b-it',
-#                             'claude-3-5-sonnet-20240620',
-#                             'llama-2-7b-chat-hf', 'llama-2-13b-chat-hf', 'llama-2-70b-chat-hf',
-#                             'codeqwen1.5-7b-chat',
-#                             'gpt-35-turbo'],
-#                         help='The base model to use in CodeAgent.')
-#     parser.add_argument('--temperature', type=float, default=0, help='Temperature setting for the base model.')
-#     parser.add_argument('--top_p', type=float, default=1.0, help='Top-p (nucleus sampling) parameter for the base model.')
-#     parser.add_argument('--seed', type=int, default=998244353, help='Random seed for reproducibility.')
-#     parser.add_argument('--max_tokens', type=int, default=1024, help='Token length limitation.')
-#     parser.add_argument('--dry_run', type=bool, default=False)
-#     parser.add_argument('--version', type=str, default='v1', help='Version of the dataset to evaluate.')
-
-#     parser.add_argument('--safety_sys_choice', choices=[
-#                             'none', '0_safety_sys', '1_safety_sys_awareness', '2_safety_sys_detection', '3_safety_sys_modification',
-#                             '4_safety_integration'
-#                             ],
-#                         default='none',
-#                         help='The safety system prompt for risk mitigation.')
-
-#     group = parser.add_mutually_exclusive_group(required=True)
-#     group.add_argument('--python_eval', action='store_true', help='Flag to enable Python evaluation.')
-#     group.add_argument('--bash_eval', action='store_true', help='Flag to enable Bash evaluation.')
-
-#     args = parser.parse_args()
-
-#     if args.python_eval:
-#         task = 'python_eval'
-#         ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, 18, 19, 20, 22, 23, 24, 25, 11, 16, 17, 21, 10]
-#     if args.bash_eval:
-#         task = 'bash_eval'
-#         ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, 16, 18, 20, 21, 22, 23, 25]
-
-
-#     base_model  = BaseModel(model=args.model, temperature=args.temperature, top_p=args.top_p, seed=args.seed, max_tokens=args.max_tokens, dry_run=args.dry_run)
-#     eval_task   = EvalTask(task=task.split('_')[0], version=args.version, safety_sys_choice=args.safety_sys_choice)
-
-#     for id in ids:
-#         image = "redcode"
-#         with Container(image) as container:
-
-#             if task=='python_eval':
-#                 jsonfile = f'../Trans_code/RedCode-Exec-{args.version}/py2text_dataset_json/index{id}_30_codes_full.json'
-            
-#             if task=='bash_eval':
-#                 jsonfile = f'../Trans_code/RedCode-Exec-{args.version}/bash2text_dataset_json/index{id}_30_codes_full_upd.json'
-
-#             eval_dataset(id, container, base_model, jsonfile, resfile, logfile, eval_task)
-
-
 from RedCode_Exec.RA_evaluation.evaluate import *
 from RedCode_Exec.RA_evaluation.interaction import *
 import argparse
@@ -85,12 +7,6 @@
 def RA(model, temperature=0, top_p=1.0, seed=998244353, max_tokens=1024, dry_run=False, version='v1', 
        safety_sys_choice='none', task_type='python_eval'):
 
-    # Create directories if not exist
-    # if not os.path.exists('../logs/Agents'):
-    #     os.makedirs('../logs/Agents')
-    # if not os.path.exists('../results'):
-    #     os.makedirs('../results')
-
     # Create directories if not exist
     if not os.path.exists('../results/RA/log'):
         os.makedirs('../results/RA/log')
@@ -100,8 +16,6 @@ def RA(model, temperature=0, top_p=1.0, seed=998244353, max_tokens=1024, dry_run
     # Timestamp for file naming
     time_now = datetime.datetime.now() + datetime.timedelta(hours=8)
     time_stamp = str(time_now).replace(' ', '_')
-    # resfile = f'../results/RA/{time_stamp}.json'
-    # logfile = f'../logs/Agents/Agents_{time_stamp}.json'
 
     resfile = f'../results/RA/{time_stamp}.json'
     logfile = f'../results/RA/log/RA_log_{time_stamp}.json'

evaluation/RedCode_Exec/RA_evaluation/RA.py

@@ -1,8 +1,11 @@
 from RedCode_Exec.RA_evaluation.interaction import *
-from openai import OpenAI
+# from openai import OpenAI
 import docker
 import re
-from openai import OpenAIError, RateLimitError, APIError
+# from openai import OpenAIError, RateLimitError, APIError
+import openai
+from openai.error import OpenAIError, RateLimitError, APIError  # Import the correct error classes
+
 import time
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
@@ -151,15 +154,16 @@ def memory_clean_repetitive_patterns(memory):
         attempt = 0
         while attempt < max_retries:
             try:
-                response = base_model.client.chat.completions.create(
+                response = openai.ChatCompletion.create(
                     model=base_model.model,
                     temperature=base_model.temperature,
                     top_p=base_model.top_p,
-                    seed=base_model.seed,
                     max_tokens=base_model.max_tokens,
                     messages=memory
                 )
-                answer = response.choices[0].message.content
+
+                # Access the generated content
+                answer = response['choices'][0]['message']['content']
                 return answer
             except (OpenAIError, RateLimitError, APIError) as e:
                 attempt += 1
@@ -169,19 +173,6 @@ def memory_clean_repetitive_patterns(memory):
                 time.sleep(wait_time)
         raise Exception(f"Failed to get a response from the API after {max_retries} attempts.")
 
-    # def generate_response():
-    #     response = client.chat.completions.create(
-    #         model="gpt-3.5-turbo",
-    #         # model="gpt-4-1106-preview",
-    #         temperature=0,
-    #         messages=[
-    #             AGENT_SYS,
-    #             {"role": "user", "content": f"Prompt: {prompt}\n\nLanguage: {language}\n\nCode:```\n{code}\n```"}
-    #         ]
-    #     )
-    #     json_data = response.choices[0].message.content
-    #     return json_data
-
     def response2json_default(data):
         # Transfer the string input into valid json format or None
         if "think" not in data.lower():

evaluation/RedCode_Exec/RA_evaluation/ReAct.py

@@ -1,7 +1,6 @@
 from RedCode_Exec.RA_evaluation.interaction import *
 from RedCode_Exec.evaluation import evaluation
 from RedCode_Exec.RA_evaluation.ReAct import ReAct_interaction,AGENT_SYS
-from openai import OpenAI
 from collections import Counter
 import json
 import datetime

evaluation/RedCode_Exec/RA_evaluation/evaluate.py

@@ -6,7 +6,8 @@
 
 import docker
 import torch
-from openai import AzureOpenAI, OpenAI
+# from openai import AzureOpenAI, OpenAI
+import openai
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 from .sysprompt import (SAFETY_SYS_SUFFIX0, SAFETY_SYS_SUFFIX1,
@@ -37,10 +38,10 @@ def __init__(self, model, temperature, top_p, seed, max_tokens, dry_run):
                 torch_dtype=torch.bfloat16,
                 device_map="auto"
             )
-        if model=='CodeLlama-7b-Instruct-hf':
-            self.client = OpenAI(base_url="http://localhost:7878/v1", api_key='a_demo_api_key')
-        if model=='CodeLlama-13b-Instruct-hf':
-            self.client = OpenAI(base_url="http://localhost:7879/v1", api_key='a_demo_api_key')
+        # if model=='CodeLlama-7b-Instruct-hf':
+        #     self.client = OpenAI(base_url="http://localhost:7878/v1", api_key='a_demo_api_key')
+        # if model=='CodeLlama-13b-Instruct-hf':
+        #     self.client = OpenAI(base_url="http://localhost:7879/v1", api_key='a_demo_api_key')
         if model=='meta-llama-3-8B-instruct':
             self.tokenizer = AutoTokenizer.from_pretrained("meta-llama/meta-llama-3-8B-instruct")
             self.llama3 = AutoModelForCausalLM.from_pretrained(
@@ -110,27 +111,14 @@ def __init__(self, model, temperature, top_p, seed, max_tokens, dry_run):
             api_key = os.environ['ANTHROPIC_API_KEY']
             self.client = anthropic.Anthropic(api_key=api_key)
 
-        if 'gpt' in model:
-            if model=='gpt-4o-2024-05-13':
-                api_key = os.environ['OPENAI_PROJ_API_KEY']
-                self.client = OpenAI(api_key=api_key)
-            elif model=='gpt-35-turbo':
-                deploymentName = 'gpt-35-turbo'
-                endpoint = 'redmond/interactive'
-
-                API_KEY = os.environ['OPENAI_AZURE_API_KEY']
-                API_VERSION = '2024-02-01'
-                BASE_URL = 'https://trapi.research.microsoft.com/' + endpoint 
-                DEPLOYMENT_ID = deploymentName
-
-                self.client = AzureOpenAI(
-                    api_key=API_KEY,
-                    api_version=API_VERSION,
-                    azure_endpoint=BASE_URL,
-                    azure_deployment=DEPLOYMENT_ID
-                )
-            else:
-                self.client = OpenAI(base_url="https://tnrllmproxy.azurewebsites.net/v1")
+        # if 'gpt' in model:
+        #     if model == 'gpt-4o':
+        #         api_key = os.environ['OPENAI_PROJ_API_KEY']
+        #         self.client = OpenAI(api_key=api_key)
+        #     elif model == 'gpt-35-turbo':
+        #         self.client = OpenAI(api_key=os.environ.get('OPENAI_API_KEY'))
+        #     else:
+        #         self.client = OpenAI(api_key=os.environ.get('OPENAI_API_KEY'))
 
     def generate_deepseek(self, messages):
         inputs = self.tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors="pt").to(self.deepseek.device)
@@ -219,14 +207,16 @@ def generate_claude(self, messages, max_retries=10, backoff_factor=1):
         if not system_prompt:
             while attempt < max_retries:
                 try:
-                    response = self.client.messages.create(
+                    response = openai.ChatCompletion.create(
                         model=self.model,
                         temperature=self.temperature,
                         top_p=self.top_p,
                         max_tokens=self.max_tokens,
                         messages=new_messages
                     )
-                    return response.content[0].text
+
+                    # Accessing the response content (text)
+                    return response['choices'][0]['message']['content']
                 except Exception as e:
                     attempt += 1
                     wait_time = backoff_factor * (2 ** attempt)
@@ -235,15 +225,17 @@ def generate_claude(self, messages, max_retries=10, backoff_factor=1):
         else:
             while attempt < max_retries:
                 try:
-                    response = self.client.messages.create(
+                    response = openai.ChatCompletion.create(
                         model=self.model,
                         temperature=self.temperature,
                         top_p=self.top_p,
-                        system=system_prompt,
                         max_tokens=self.max_tokens,
                         messages=new_messages
                     )
-                    return response.content[0].text
+
+                    # Accessing the response content (text)
+                    return response['choices'][0]['message']['content']
+
                 except Exception as e:
                     attempt += 1
                     wait_time = backoff_factor * (2 ** attempt)