fix llama prompt + add longer llama model

joonspk-research · May 20, 2024 · 3121ec6 · 3121ec6
1 parent f16e735
commit 3121ec6
Showing 1 changed file with 11 additions and 9 deletions.
diff --git a/reverie/backend_server/persona/prompt_template/gpt_structure.py b/reverie/backend_server/persona/prompt_template/gpt_structure.py
@@ -21,7 +21,8 @@ def ChatGPT_single_request(prompt):
   temp_sleep()
 
   completion = openai.ChatCompletion.create(
-    model="llama3:8b-instruct-q8_0", 
+    model="llama3-gradient:8b-instruct-1048k-q8_0",
+    # model="llama3:8b-instruct-q8_0", 
     messages=[{"role": "user", "content": prompt}]
   )
   return completion["choices"][0]["message"]["content"]
@@ -47,7 +48,7 @@ def GPT4_request(prompt):
 
   try: 
     completion = openai.ChatCompletion.create(
-    model="llama3:8b-instruct-q8_0", 
+    model="llama3-gradient:8b-instruct-1048k-q8_0",
     messages=[{"role": "user", "content": prompt}]
     )
     return completion["choices"][0]["message"]["content"]
@@ -72,7 +73,7 @@ def ChatGPT_request(prompt):
   # temp_sleep()
   try: 
     completion = openai.ChatCompletion.create(
-    model="gpt-3.5-turbo", 
+    model="llama3-gradient:8b-instruct-1048k-q8_0",
     messages=[{"role": "user", "content": prompt}]
     )
     return completion["choices"][0]["message"]["content"]
@@ -209,9 +210,9 @@ def GPT_request(prompt, gpt_parameter):
   """
   temp_sleep()
   try: 
-    response = openai.Completion.create(
+    response = openai.ChatCompletion.create(
                 model=gpt_parameter["engine"],
-                prompt=prompt,
+                messages=[{"role": "system", "content": prompt}],
                 temperature=gpt_parameter["temperature"],
                 max_tokens=gpt_parameter["max_tokens"],
                 top_p=gpt_parameter["top_p"],
@@ -220,9 +221,10 @@ def GPT_request(prompt, gpt_parameter):
                 stream=gpt_parameter["stream"],
                 stop=gpt_parameter["stop"],)
     return response.choices[0].text
-  except: 
-    print ("TOKEN LIMIT EXCEEDED")
-    return "TOKEN LIMIT EXCEEDED"
+  except Exception as e: 
+    # print ("TOKEN LIMIT EXCEEDED")
+    print(str(e))
+    return "NO GENERATION"
 
 
 def generate_prompt(curr_input, prompt_lib_file): 
@@ -283,7 +285,7 @@ def get_embedding(text, model="nomic-embed-text"):
 
 
 if __name__ == '__main__':
-  gpt_parameter = {"engine": "text-davinci-003", "max_tokens": 50, 
+  gpt_parameter = {"engine": model, "max_tokens": 50, 
                    "temperature": 0, "top_p": 1, "stream": False,
                    "frequency_penalty": 0, "presence_penalty": 0, 
                    "stop": ['"']}